repack.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581
  1. #include "builtin.h"
  2. #include "cache.h"
  3. #include "config.h"
  4. #include "dir.h"
  5. #include "parse-options.h"
  6. #include "run-command.h"
  7. #include "sigchain.h"
  8. #include "strbuf.h"
  9. #include "string-list.h"
  10. #include "strvec.h"
  11. #include "midx.h"
  12. #include "packfile.h"
  13. #include "prune-packed.h"
  14. #include "object-store.h"
  15. #include "promisor-remote.h"
  16. #include "shallow.h"
  17. static int delta_base_offset = 1;
  18. static int pack_kept_objects = -1;
  19. static int write_bitmaps = -1;
  20. static int use_delta_islands;
  21. static char *packdir, *packtmp;
  22. static const char *const git_repack_usage[] = {
  23. N_("git repack [<options>]"),
  24. NULL
  25. };
  26. static const char incremental_bitmap_conflict_error[] = N_(
  27. "Incremental repacks are incompatible with bitmap indexes. Use\n"
  28. "--no-write-bitmap-index or disable the pack.writebitmaps configuration."
  29. );
  30. static int repack_config(const char *var, const char *value, void *cb)
  31. {
  32. if (!strcmp(var, "repack.usedeltabaseoffset")) {
  33. delta_base_offset = git_config_bool(var, value);
  34. return 0;
  35. }
  36. if (!strcmp(var, "repack.packkeptobjects")) {
  37. pack_kept_objects = git_config_bool(var, value);
  38. return 0;
  39. }
  40. if (!strcmp(var, "repack.writebitmaps") ||
  41. !strcmp(var, "pack.writebitmaps")) {
  42. write_bitmaps = git_config_bool(var, value);
  43. return 0;
  44. }
  45. if (!strcmp(var, "repack.usedeltaislands")) {
  46. use_delta_islands = git_config_bool(var, value);
  47. return 0;
  48. }
  49. return git_default_config(var, value, cb);
  50. }
  51. /*
  52. * Remove temporary $GIT_OBJECT_DIRECTORY/pack/.tmp-$$-pack-* files.
  53. */
  54. static void remove_temporary_files(void)
  55. {
  56. struct strbuf buf = STRBUF_INIT;
  57. size_t dirlen, prefixlen;
  58. DIR *dir;
  59. struct dirent *e;
  60. dir = opendir(packdir);
  61. if (!dir)
  62. return;
  63. /* Point at the slash at the end of ".../objects/pack/" */
  64. dirlen = strlen(packdir) + 1;
  65. strbuf_addstr(&buf, packtmp);
  66. /* Hold the length of ".tmp-%d-pack-" */
  67. prefixlen = buf.len - dirlen;
  68. while ((e = readdir(dir))) {
  69. if (strncmp(e->d_name, buf.buf + dirlen, prefixlen))
  70. continue;
  71. strbuf_setlen(&buf, dirlen);
  72. strbuf_addstr(&buf, e->d_name);
  73. unlink(buf.buf);
  74. }
  75. closedir(dir);
  76. strbuf_release(&buf);
  77. }
  78. static void remove_pack_on_signal(int signo)
  79. {
  80. remove_temporary_files();
  81. sigchain_pop(signo);
  82. raise(signo);
  83. }
  84. /*
  85. * Adds all packs hex strings to the fname list, which do not
  86. * have a corresponding .keep file. These packs are not to
  87. * be kept if we are going to pack everything into one file.
  88. */
  89. static void get_non_kept_pack_filenames(struct string_list *fname_list,
  90. const struct string_list *extra_keep)
  91. {
  92. DIR *dir;
  93. struct dirent *e;
  94. char *fname;
  95. if (!(dir = opendir(packdir)))
  96. return;
  97. while ((e = readdir(dir)) != NULL) {
  98. size_t len;
  99. int i;
  100. for (i = 0; i < extra_keep->nr; i++)
  101. if (!fspathcmp(e->d_name, extra_keep->items[i].string))
  102. break;
  103. if (extra_keep->nr > 0 && i < extra_keep->nr)
  104. continue;
  105. if (!strip_suffix(e->d_name, ".pack", &len))
  106. continue;
  107. fname = xmemdupz(e->d_name, len);
  108. if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
  109. string_list_append_nodup(fname_list, fname);
  110. else
  111. free(fname);
  112. }
  113. closedir(dir);
  114. }
  115. static void remove_redundant_pack(const char *dir_name, const char *base_name)
  116. {
  117. struct strbuf buf = STRBUF_INIT;
  118. struct multi_pack_index *m = get_local_multi_pack_index(the_repository);
  119. strbuf_addf(&buf, "%s.pack", base_name);
  120. if (m && midx_contains_pack(m, buf.buf))
  121. clear_midx_file(the_repository);
  122. strbuf_insertf(&buf, 0, "%s/", dir_name);
  123. unlink_pack_path(buf.buf, 1);
  124. strbuf_release(&buf);
  125. }
  126. struct pack_objects_args {
  127. const char *window;
  128. const char *window_memory;
  129. const char *depth;
  130. const char *threads;
  131. const char *max_pack_size;
  132. int no_reuse_delta;
  133. int no_reuse_object;
  134. int quiet;
  135. int local;
  136. };
  137. static void prepare_pack_objects(struct child_process *cmd,
  138. const struct pack_objects_args *args)
  139. {
  140. strvec_push(&cmd->args, "pack-objects");
  141. if (args->window)
  142. strvec_pushf(&cmd->args, "--window=%s", args->window);
  143. if (args->window_memory)
  144. strvec_pushf(&cmd->args, "--window-memory=%s", args->window_memory);
  145. if (args->depth)
  146. strvec_pushf(&cmd->args, "--depth=%s", args->depth);
  147. if (args->threads)
  148. strvec_pushf(&cmd->args, "--threads=%s", args->threads);
  149. if (args->max_pack_size)
  150. strvec_pushf(&cmd->args, "--max-pack-size=%s", args->max_pack_size);
  151. if (args->no_reuse_delta)
  152. strvec_pushf(&cmd->args, "--no-reuse-delta");
  153. if (args->no_reuse_object)
  154. strvec_pushf(&cmd->args, "--no-reuse-object");
  155. if (args->local)
  156. strvec_push(&cmd->args, "--local");
  157. if (args->quiet)
  158. strvec_push(&cmd->args, "--quiet");
  159. if (delta_base_offset)
  160. strvec_push(&cmd->args, "--delta-base-offset");
  161. strvec_push(&cmd->args, packtmp);
  162. cmd->git_cmd = 1;
  163. cmd->out = -1;
  164. }
  165. /*
  166. * Write oid to the given struct child_process's stdin, starting it first if
  167. * necessary.
  168. */
  169. static int write_oid(const struct object_id *oid, struct packed_git *pack,
  170. uint32_t pos, void *data)
  171. {
  172. struct child_process *cmd = data;
  173. if (cmd->in == -1) {
  174. if (start_command(cmd))
  175. die(_("could not start pack-objects to repack promisor objects"));
  176. }
  177. xwrite(cmd->in, oid_to_hex(oid), the_hash_algo->hexsz);
  178. xwrite(cmd->in, "\n", 1);
  179. return 0;
  180. }
  181. static void repack_promisor_objects(const struct pack_objects_args *args,
  182. struct string_list *names)
  183. {
  184. struct child_process cmd = CHILD_PROCESS_INIT;
  185. FILE *out;
  186. struct strbuf line = STRBUF_INIT;
  187. prepare_pack_objects(&cmd, args);
  188. cmd.in = -1;
  189. /*
  190. * NEEDSWORK: Giving pack-objects only the OIDs without any ordering
  191. * hints may result in suboptimal deltas in the resulting pack. See if
  192. * the OIDs can be sent with fake paths such that pack-objects can use a
  193. * {type -> existing pack order} ordering when computing deltas instead
  194. * of a {type -> size} ordering, which may produce better deltas.
  195. */
  196. for_each_packed_object(write_oid, &cmd,
  197. FOR_EACH_OBJECT_PROMISOR_ONLY);
  198. if (cmd.in == -1)
  199. /* No packed objects; cmd was never started */
  200. return;
  201. close(cmd.in);
  202. out = xfdopen(cmd.out, "r");
  203. while (strbuf_getline_lf(&line, out) != EOF) {
  204. char *promisor_name;
  205. int fd;
  206. if (line.len != the_hash_algo->hexsz)
  207. die(_("repack: Expecting full hex object ID lines only from pack-objects."));
  208. string_list_append(names, line.buf);
  209. /*
  210. * pack-objects creates the .pack and .idx files, but not the
  211. * .promisor file. Create the .promisor file, which is empty.
  212. *
  213. * NEEDSWORK: fetch-pack sometimes generates non-empty
  214. * .promisor files containing the ref names and associated
  215. * hashes at the point of generation of the corresponding
  216. * packfile, but this would not preserve their contents. Maybe
  217. * concatenate the contents of all .promisor files instead of
  218. * just creating a new empty file.
  219. */
  220. promisor_name = mkpathdup("%s-%s.promisor", packtmp,
  221. line.buf);
  222. fd = open(promisor_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
  223. if (fd < 0)
  224. die_errno(_("unable to create '%s'"), promisor_name);
  225. close(fd);
  226. free(promisor_name);
  227. }
  228. fclose(out);
  229. if (finish_command(&cmd))
  230. die(_("could not finish pack-objects to repack promisor objects"));
  231. }
  232. #define ALL_INTO_ONE 1
  233. #define LOOSEN_UNREACHABLE 2
  234. int cmd_repack(int argc, const char **argv, const char *prefix)
  235. {
  236. struct {
  237. const char *name;
  238. unsigned optional:1;
  239. } exts[] = {
  240. {".pack"},
  241. {".idx"},
  242. {".bitmap", 1},
  243. {".promisor", 1},
  244. };
  245. struct child_process cmd = CHILD_PROCESS_INIT;
  246. struct string_list_item *item;
  247. struct string_list names = STRING_LIST_INIT_DUP;
  248. struct string_list rollback = STRING_LIST_INIT_NODUP;
  249. struct string_list existing_packs = STRING_LIST_INIT_DUP;
  250. struct strbuf line = STRBUF_INIT;
  251. int i, ext, ret, failed;
  252. FILE *out;
  253. /* variables to be filled by option parsing */
  254. int pack_everything = 0;
  255. int delete_redundant = 0;
  256. const char *unpack_unreachable = NULL;
  257. int keep_unreachable = 0;
  258. struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
  259. int no_update_server_info = 0;
  260. struct pack_objects_args po_args = {NULL};
  261. struct option builtin_repack_options[] = {
  262. OPT_BIT('a', NULL, &pack_everything,
  263. N_("pack everything in a single pack"), ALL_INTO_ONE),
  264. OPT_BIT('A', NULL, &pack_everything,
  265. N_("same as -a, and turn unreachable objects loose"),
  266. LOOSEN_UNREACHABLE | ALL_INTO_ONE),
  267. OPT_BOOL('d', NULL, &delete_redundant,
  268. N_("remove redundant packs, and run git-prune-packed")),
  269. OPT_BOOL('f', NULL, &po_args.no_reuse_delta,
  270. N_("pass --no-reuse-delta to git-pack-objects")),
  271. OPT_BOOL('F', NULL, &po_args.no_reuse_object,
  272. N_("pass --no-reuse-object to git-pack-objects")),
  273. OPT_BOOL('n', NULL, &no_update_server_info,
  274. N_("do not run git-update-server-info")),
  275. OPT__QUIET(&po_args.quiet, N_("be quiet")),
  276. OPT_BOOL('l', "local", &po_args.local,
  277. N_("pass --local to git-pack-objects")),
  278. OPT_BOOL('b', "write-bitmap-index", &write_bitmaps,
  279. N_("write bitmap index")),
  280. OPT_BOOL('i', "delta-islands", &use_delta_islands,
  281. N_("pass --delta-islands to git-pack-objects")),
  282. OPT_STRING(0, "unpack-unreachable", &unpack_unreachable, N_("approxidate"),
  283. N_("with -A, do not loosen objects older than this")),
  284. OPT_BOOL('k', "keep-unreachable", &keep_unreachable,
  285. N_("with -a, repack unreachable objects")),
  286. OPT_STRING(0, "window", &po_args.window, N_("n"),
  287. N_("size of the window used for delta compression")),
  288. OPT_STRING(0, "window-memory", &po_args.window_memory, N_("bytes"),
  289. N_("same as the above, but limit memory size instead of entries count")),
  290. OPT_STRING(0, "depth", &po_args.depth, N_("n"),
  291. N_("limits the maximum delta depth")),
  292. OPT_STRING(0, "threads", &po_args.threads, N_("n"),
  293. N_("limits the maximum number of threads")),
  294. OPT_STRING(0, "max-pack-size", &po_args.max_pack_size, N_("bytes"),
  295. N_("maximum size of each packfile")),
  296. OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects,
  297. N_("repack objects in packs marked with .keep")),
  298. OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
  299. N_("do not repack this pack")),
  300. OPT_END()
  301. };
  302. git_config(repack_config, NULL);
  303. argc = parse_options(argc, argv, prefix, builtin_repack_options,
  304. git_repack_usage, 0);
  305. if (delete_redundant && repository_format_precious_objects)
  306. die(_("cannot delete packs in a precious-objects repo"));
  307. if (keep_unreachable &&
  308. (unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE)))
  309. die(_("--keep-unreachable and -A are incompatible"));
  310. if (write_bitmaps < 0) {
  311. if (!(pack_everything & ALL_INTO_ONE) ||
  312. !is_bare_repository())
  313. write_bitmaps = 0;
  314. }
  315. if (pack_kept_objects < 0)
  316. pack_kept_objects = write_bitmaps > 0;
  317. if (write_bitmaps && !(pack_everything & ALL_INTO_ONE))
  318. die(_(incremental_bitmap_conflict_error));
  319. packdir = mkpathdup("%s/pack", get_object_directory());
  320. packtmp = mkpathdup("%s/.tmp-%d-pack", packdir, (int)getpid());
  321. sigchain_push_common(remove_pack_on_signal);
  322. prepare_pack_objects(&cmd, &po_args);
  323. strvec_push(&cmd.args, "--keep-true-parents");
  324. if (!pack_kept_objects)
  325. strvec_push(&cmd.args, "--honor-pack-keep");
  326. for (i = 0; i < keep_pack_list.nr; i++)
  327. strvec_pushf(&cmd.args, "--keep-pack=%s",
  328. keep_pack_list.items[i].string);
  329. strvec_push(&cmd.args, "--non-empty");
  330. strvec_push(&cmd.args, "--all");
  331. strvec_push(&cmd.args, "--reflog");
  332. strvec_push(&cmd.args, "--indexed-objects");
  333. if (has_promisor_remote())
  334. strvec_push(&cmd.args, "--exclude-promisor-objects");
  335. if (write_bitmaps > 0)
  336. strvec_push(&cmd.args, "--write-bitmap-index");
  337. else if (write_bitmaps < 0)
  338. strvec_push(&cmd.args, "--write-bitmap-index-quiet");
  339. if (use_delta_islands)
  340. strvec_push(&cmd.args, "--delta-islands");
  341. if (pack_everything & ALL_INTO_ONE) {
  342. get_non_kept_pack_filenames(&existing_packs, &keep_pack_list);
  343. repack_promisor_objects(&po_args, &names);
  344. if (existing_packs.nr && delete_redundant) {
  345. if (unpack_unreachable) {
  346. strvec_pushf(&cmd.args,
  347. "--unpack-unreachable=%s",
  348. unpack_unreachable);
  349. strvec_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
  350. } else if (pack_everything & LOOSEN_UNREACHABLE) {
  351. strvec_push(&cmd.args,
  352. "--unpack-unreachable");
  353. } else if (keep_unreachable) {
  354. strvec_push(&cmd.args, "--keep-unreachable");
  355. strvec_push(&cmd.args, "--pack-loose-unreachable");
  356. } else {
  357. strvec_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
  358. }
  359. }
  360. } else {
  361. strvec_push(&cmd.args, "--unpacked");
  362. strvec_push(&cmd.args, "--incremental");
  363. }
  364. cmd.no_stdin = 1;
  365. ret = start_command(&cmd);
  366. if (ret)
  367. return ret;
  368. out = xfdopen(cmd.out, "r");
  369. while (strbuf_getline_lf(&line, out) != EOF) {
  370. if (line.len != the_hash_algo->hexsz)
  371. die(_("repack: Expecting full hex object ID lines only from pack-objects."));
  372. string_list_append(&names, line.buf);
  373. }
  374. fclose(out);
  375. ret = finish_command(&cmd);
  376. if (ret)
  377. return ret;
  378. if (!names.nr && !po_args.quiet)
  379. printf_ln(_("Nothing new to pack."));
  380. close_object_store(the_repository->objects);
  381. /*
  382. * Ok we have prepared all new packfiles.
  383. * First see if there are packs of the same name and if so
  384. * if we can move them out of the way (this can happen if we
  385. * repacked immediately after packing fully.
  386. */
  387. failed = 0;
  388. for_each_string_list_item(item, &names) {
  389. for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
  390. char *fname, *fname_old;
  391. fname = mkpathdup("%s/pack-%s%s", packdir,
  392. item->string, exts[ext].name);
  393. if (!file_exists(fname)) {
  394. free(fname);
  395. continue;
  396. }
  397. fname_old = mkpathdup("%s/old-%s%s", packdir,
  398. item->string, exts[ext].name);
  399. if (file_exists(fname_old))
  400. if (unlink(fname_old))
  401. failed = 1;
  402. if (!failed && rename(fname, fname_old)) {
  403. free(fname);
  404. free(fname_old);
  405. failed = 1;
  406. break;
  407. } else {
  408. string_list_append(&rollback, fname);
  409. free(fname_old);
  410. }
  411. }
  412. if (failed)
  413. break;
  414. }
  415. if (failed) {
  416. struct string_list rollback_failure = STRING_LIST_INIT_DUP;
  417. for_each_string_list_item(item, &rollback) {
  418. char *fname, *fname_old;
  419. fname = mkpathdup("%s/%s", packdir, item->string);
  420. fname_old = mkpathdup("%s/old-%s", packdir, item->string);
  421. if (rename(fname_old, fname))
  422. string_list_append(&rollback_failure, fname);
  423. free(fname);
  424. free(fname_old);
  425. }
  426. if (rollback_failure.nr) {
  427. int i;
  428. fprintf(stderr,
  429. _("WARNING: Some packs in use have been renamed by\n"
  430. "WARNING: prefixing old- to their name, in order to\n"
  431. "WARNING: replace them with the new version of the\n"
  432. "WARNING: file. But the operation failed, and the\n"
  433. "WARNING: attempt to rename them back to their\n"
  434. "WARNING: original names also failed.\n"
  435. "WARNING: Please rename them in %s manually:\n"), packdir);
  436. for (i = 0; i < rollback_failure.nr; i++)
  437. fprintf(stderr, "WARNING: old-%s -> %s\n",
  438. rollback_failure.items[i].string,
  439. rollback_failure.items[i].string);
  440. }
  441. exit(1);
  442. }
  443. /* Now the ones with the same name are out of the way... */
  444. for_each_string_list_item(item, &names) {
  445. for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
  446. char *fname, *fname_old;
  447. struct stat statbuffer;
  448. int exists = 0;
  449. fname = mkpathdup("%s/pack-%s%s",
  450. packdir, item->string, exts[ext].name);
  451. fname_old = mkpathdup("%s-%s%s",
  452. packtmp, item->string, exts[ext].name);
  453. if (!stat(fname_old, &statbuffer)) {
  454. statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
  455. chmod(fname_old, statbuffer.st_mode);
  456. exists = 1;
  457. }
  458. if (exists || !exts[ext].optional) {
  459. if (rename(fname_old, fname))
  460. die_errno(_("renaming '%s' failed"), fname_old);
  461. }
  462. free(fname);
  463. free(fname_old);
  464. }
  465. }
  466. /* Remove the "old-" files */
  467. for_each_string_list_item(item, &names) {
  468. for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
  469. char *fname;
  470. fname = mkpathdup("%s/old-%s%s",
  471. packdir,
  472. item->string,
  473. exts[ext].name);
  474. if (remove_path(fname))
  475. warning(_("failed to remove '%s'"), fname);
  476. free(fname);
  477. }
  478. }
  479. /* End of pack replacement. */
  480. reprepare_packed_git(the_repository);
  481. if (delete_redundant) {
  482. const int hexsz = the_hash_algo->hexsz;
  483. int opts = 0;
  484. string_list_sort(&names);
  485. for_each_string_list_item(item, &existing_packs) {
  486. char *sha1;
  487. size_t len = strlen(item->string);
  488. if (len < hexsz)
  489. continue;
  490. sha1 = item->string + len - hexsz;
  491. if (!string_list_has_string(&names, sha1))
  492. remove_redundant_pack(packdir, item->string);
  493. }
  494. if (!po_args.quiet && isatty(2))
  495. opts |= PRUNE_PACKED_VERBOSE;
  496. prune_packed_objects(opts);
  497. if (!keep_unreachable &&
  498. (!(pack_everything & LOOSEN_UNREACHABLE) ||
  499. unpack_unreachable) &&
  500. is_repository_shallow(the_repository))
  501. prune_shallow(PRUNE_QUICK);
  502. }
  503. if (!no_update_server_info)
  504. update_server_info(0);
  505. remove_temporary_files();
  506. if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0))
  507. write_midx_file(get_object_directory(), 0);
  508. string_list_clear(&names, 0);
  509. string_list_clear(&rollback, 0);
  510. string_list_clear(&existing_packs, 0);
  511. strbuf_release(&line);
  512. return 0;
  513. }