diff options
Diffstat (limited to 'builtin/repack.c')
-rw-r--r-- | builtin/repack.c | 282 |
1 files changed, 196 insertions, 86 deletions
diff --git a/builtin/repack.c b/builtin/repack.c index a5bacc7797..0541c3ce15 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -1,9 +1,13 @@ #include "builtin.h" -#include "cache.h" +#include "alloc.h" #include "config.h" #include "dir.h" +#include "environment.h" +#include "gettext.h" +#include "hex.h" #include "parse-options.h" #include "run-command.h" +#include "server-info.h" #include "sigchain.h" #include "strbuf.h" #include "string-list.h" @@ -32,7 +36,6 @@ static int write_bitmaps = -1; static int use_delta_islands; static int run_update_server_info = 1; static char *packdir, *packtmp_name, *packtmp; -static char *cruft_expiration; static const char *const git_repack_usage[] = { N_("git repack [<options>]"), @@ -92,44 +95,6 @@ static int repack_config(const char *var, const char *value, void *cb) } /* - * Remove temporary $GIT_OBJECT_DIRECTORY/pack/.tmp-$$-pack-* files. - */ -static void remove_temporary_files(void) -{ - struct strbuf buf = STRBUF_INIT; - size_t dirlen, prefixlen; - DIR *dir; - struct dirent *e; - - dir = opendir(packdir); - if (!dir) - return; - - /* Point at the slash at the end of ".../objects/pack/" */ - dirlen = strlen(packdir) + 1; - strbuf_addstr(&buf, packtmp); - /* Hold the length of ".tmp-%d-pack-" */ - prefixlen = buf.len - dirlen; - - while ((e = readdir(dir))) { - if (strncmp(e->d_name, buf.buf + dirlen, prefixlen)) - continue; - strbuf_setlen(&buf, dirlen); - strbuf_addstr(&buf, e->d_name); - unlink(buf.buf); - } - closedir(dir); - strbuf_release(&buf); -} - -static void remove_pack_on_signal(int signo) -{ - remove_temporary_files(); - sigchain_pop(signo); - raise(signo); -} - -/* * Adds all packs hex strings to either fname_nonkept_list or * fname_kept_list based on whether each pack has a corresponding * .keep file or not. Packs without a .keep file are not to be kept @@ -188,7 +153,8 @@ static void remove_redundant_pack(const char *dir_name, const char *base_name) } static void prepare_pack_objects(struct child_process *cmd, - const struct pack_objects_args *args) + const struct pack_objects_args *args, + const char *out) { strvec_push(&cmd->args, "pack-objects"); if (args->window) @@ -211,7 +177,7 @@ static void prepare_pack_objects(struct child_process *cmd, strvec_push(&cmd->args, "--quiet"); if (delta_base_offset) strvec_push(&cmd->args, "--delta-base-offset"); - strvec_push(&cmd->args, packtmp); + strvec_push(&cmd->args, out); cmd->git_cmd = 1; cmd->out = -1; } @@ -220,8 +186,9 @@ static void prepare_pack_objects(struct child_process *cmd, * Write oid to the given struct child_process's stdin, starting it first if * necessary. */ -static int write_oid(const struct object_id *oid, struct packed_git *pack, - uint32_t pos, void *data) +static int write_oid(const struct object_id *oid, + struct packed_git *pack UNUSED, + uint32_t pos UNUSED, void *data) { struct child_process *cmd = data; @@ -247,11 +214,15 @@ static struct { {".idx"}, }; -static unsigned populate_pack_exts(char *name) +struct generated_pack_data { + struct tempfile *tempfiles[ARRAY_SIZE(exts)]; +}; + +static struct generated_pack_data *populate_pack_exts(const char *name) { struct stat statbuf; struct strbuf path = STRBUF_INIT; - unsigned ret = 0; + struct generated_pack_data *data = xcalloc(1, sizeof(*data)); int i; for (i = 0; i < ARRAY_SIZE(exts); i++) { @@ -261,11 +232,11 @@ static unsigned populate_pack_exts(char *name) if (stat(path.buf, &statbuf)) continue; - ret |= (1 << i); + data->tempfiles[i] = register_tempfile(path.buf); } strbuf_release(&path); - return ret; + return data; } static void repack_promisor_objects(const struct pack_objects_args *args, @@ -275,7 +246,7 @@ static void repack_promisor_objects(const struct pack_objects_args *args, FILE *out; struct strbuf line = STRBUF_INIT; - prepare_pack_objects(&cmd, args); + prepare_pack_objects(&cmd, args, packtmp); cmd.in = -1; /* @@ -320,7 +291,7 @@ static void repack_promisor_objects(const struct pack_objects_args *args, line.buf); write_promisor_file(promisor_name, NULL, 0); - item->util = (void *)(uintptr_t)populate_pack_exts(item->string); + item->util = populate_pack_exts(item->string); free(promisor_name); } @@ -355,7 +326,8 @@ static int geometry_cmp(const void *va, const void *vb) } static void init_pack_geometry(struct pack_geometry **geometry_p, - struct string_list *existing_kept_packs) + struct string_list *existing_kept_packs, + const struct pack_objects_args *args) { struct packed_git *p; struct pack_geometry *geometry; @@ -365,6 +337,14 @@ static void init_pack_geometry(struct pack_geometry **geometry_p, geometry = *geometry_p; for (p = get_all_packs(the_repository); p; p = p->next) { + if (args->local && !p->pack_local) + /* + * When asked to only repack local packfiles we skip + * over any packfiles that are borrowed from alternate + * object directories. + */ + continue; + if (!pack_kept_objects) { /* * Any pack that has its pack_keep bit set will appear @@ -478,8 +458,10 @@ static void split_pack_geometry(struct pack_geometry *geometry, int factor) geometry->split = split; } -static struct packed_git *get_largest_active_pack(struct pack_geometry *geometry) +static struct packed_git *get_preferred_pack(struct pack_geometry *geometry) { + uint32_t i; + if (!geometry) { /* * No geometry means either an all-into-one repack (in which @@ -494,7 +476,21 @@ static struct packed_git *get_largest_active_pack(struct pack_geometry *geometry } if (geometry->split == geometry->pack_nr) return NULL; - return geometry->pack[geometry->pack_nr - 1]; + + /* + * The preferred pack is the largest pack above the split line. In + * other words, it is the largest pack that does not get rolled up in + * the geometric repack. + */ + for (i = geometry->pack_nr; i > geometry->split; i--) + /* + * A pack that is not local would never be included in a + * multi-pack index. We thus skip over any non-local packs. + */ + if (geometry->pack[i - 1]->pack_local) + return geometry->pack[i - 1]; + + return NULL; } static void clear_pack_geometry(struct pack_geometry *geometry) @@ -588,6 +584,17 @@ static void midx_included_packs(struct string_list *include, for (i = geometry->split; i < geometry->pack_nr; i++) { struct packed_git *p = geometry->pack[i]; + /* + * The multi-pack index never refers to packfiles part + * of an alternate object database, so we skip these. + * While git-multi-pack-index(1) would silently ignore + * them anyway, this allows us to skip executing the + * command completely when we have only non-local + * packfiles. + */ + if (!p->pack_local) + continue; + strbuf_addstr(&buf, pack_basename(p)); strbuf_strip_suffix(&buf, ".pack"); strbuf_addstr(&buf, ".idx"); @@ -621,7 +628,7 @@ static int write_midx_included_packs(struct string_list *include, { struct child_process cmd = CHILD_PROCESS_INIT; struct string_list_item *item; - struct packed_git *largest = get_largest_active_pack(geometry); + struct packed_git *preferred = get_preferred_pack(geometry); FILE *in; int ret; @@ -642,9 +649,9 @@ static int write_midx_included_packs(struct string_list *include, if (write_bitmaps) strvec_push(&cmd.args, "--bitmap"); - if (largest) + if (preferred) strvec_pushf(&cmd.args, "--preferred-pack=%s", - pack_basename(largest)); + pack_basename(preferred)); if (refs_snapshot) strvec_pushf(&cmd.args, "--refs-snapshot=%s", refs_snapshot); @@ -661,8 +668,39 @@ static int write_midx_included_packs(struct string_list *include, return finish_command(&cmd); } +static void remove_redundant_bitmaps(struct string_list *include, + const char *packdir) +{ + struct strbuf path = STRBUF_INIT; + struct string_list_item *item; + size_t packdir_len; + + strbuf_addstr(&path, packdir); + strbuf_addch(&path, '/'); + packdir_len = path.len; + + /* + * Remove any pack bitmaps corresponding to packs which are now + * included in the MIDX. + */ + for_each_string_list_item(item, include) { + strbuf_addstr(&path, item->string); + strbuf_strip_suffix(&path, ".idx"); + strbuf_addstr(&path, ".bitmap"); + + if (unlink(path.buf) && errno != ENOENT) + warning_errno(_("could not remove stale bitmap: %s"), + path.buf); + + strbuf_setlen(&path, packdir_len); + } + strbuf_release(&path); +} + static int write_cruft_pack(const struct pack_objects_args *args, + const char *destination, const char *pack_prefix, + const char *cruft_expiration, struct string_list *names, struct string_list *existing_packs, struct string_list *existing_kept_packs) @@ -672,8 +710,10 @@ static int write_cruft_pack(const struct pack_objects_args *args, struct string_list_item *item; FILE *in, *out; int ret; + const char *scratch; + int local = skip_prefix(destination, packdir, &scratch); - prepare_pack_objects(&cmd, args); + prepare_pack_objects(&cmd, args, destination); strvec_push(&cmd.args, "--cruft"); if (cruft_expiration) @@ -698,6 +738,10 @@ static int write_cruft_pack(const struct pack_objects_args *args, * By the time it is read here, it contains only the pack(s) * that were just written, which is exactly the set of packs we * want to consider kept. + * + * If `--expire-to` is given, the double-use served by `names` + * ensures that the pack written to `--expire-to` excludes any + * objects contained in the cruft pack. */ in = xfdopen(cmd.in, "w"); for_each_string_list_item(item, names) @@ -710,10 +754,19 @@ static int write_cruft_pack(const struct pack_objects_args *args, out = xfdopen(cmd.out, "r"); while (strbuf_getline_lf(&line, out) != EOF) { + struct string_list_item *item; + if (line.len != the_hash_algo->hexsz) die(_("repack: Expecting full hex object ID lines only " "from pack-objects.")); - string_list_append(names, line.buf); + /* + * avoid putting packs written outside of the repository in the + * list of names + */ + if (local) { + item = string_list_append(names, line.buf); + item->util = populate_pack_exts(line.buf); + } } fclose(out); @@ -745,6 +798,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) struct pack_objects_args cruft_po_args = {NULL}; int geometric_factor = 0; int write_midx = 0; + const char *cruft_expiration = NULL; + const char *expire_to = NULL; struct option builtin_repack_options[] = { OPT_BIT('a', NULL, &pack_everything, @@ -756,7 +811,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) N_("same as -a, pack unreachable cruft objects separately"), PACK_CRUFT), OPT_STRING(0, "cruft-expiration", &cruft_expiration, N_("approxidate"), - N_("with -C, expire objects older than this")), + N_("with --cruft, expire objects older than this")), OPT_BOOL('d', NULL, &delete_redundant, N_("remove redundant packs, and run git-prune-packed")), OPT_BOOL('f', NULL, &po_args.no_reuse_delta, @@ -794,6 +849,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) N_("find a geometric progression with factor <N>")), OPT_BOOL('m', "write-midx", &write_midx, N_("write a multi-pack index of the resulting packs")), + OPT_STRING(0, "expire-to", &expire_to, N_("dir"), + N_("pack prefix to store a pack containing pruned objects")), OPT_END() }; @@ -833,6 +890,18 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (write_bitmaps && !(pack_everything & ALL_INTO_ONE) && !write_midx) die(_(incremental_bitmap_conflict_error)); + if (write_bitmaps && po_args.local && has_alt_odb(the_repository)) { + /* + * When asked to do a local repack, but we have + * packfiles that are inherited from an alternate, then + * we cannot guarantee that the multi-pack-index would + * have full coverage of all objects. We thus disable + * writing bitmaps in that case. + */ + warning(_("disabling bitmap writing, as some objects are not being packed")); + write_bitmaps = 0; + } + if (write_midx && write_bitmaps) { struct strbuf path = STRBUF_INIT; @@ -855,13 +924,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (geometric_factor) { if (pack_everything) die(_("options '%s' and '%s' cannot be used together"), "--geometric", "-A/-a"); - init_pack_geometry(&geometry, &existing_kept_packs); + init_pack_geometry(&geometry, &existing_kept_packs, &po_args); split_pack_geometry(geometry, geometric_factor); } - sigchain_push_common(remove_pack_on_signal); - - prepare_pack_objects(&cmd, &po_args); + prepare_pack_objects(&cmd, &po_args, packtmp); show_progress = !po_args.quiet && isatty(2); @@ -887,7 +954,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) strvec_push(&cmd.args, "--reflog"); strvec_push(&cmd.args, "--indexed-objects"); } - if (has_promisor_remote()) + if (repo_has_promisor_remote(the_repository)) strvec_push(&cmd.args, "--exclude-promisor-objects"); if (!write_midx) { if (write_bitmaps > 0) @@ -934,7 +1001,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) ret = start_command(&cmd); if (ret) - return ret; + goto cleanup; if (geometry) { FILE *in = xfdopen(cmd.in, "w"); @@ -952,14 +1019,18 @@ int cmd_repack(int argc, const char **argv, const char *prefix) out = xfdopen(cmd.out, "r"); while (strbuf_getline_lf(&line, out) != EOF) { + struct string_list_item *item; + if (line.len != the_hash_algo->hexsz) die(_("repack: Expecting full hex object ID lines only from pack-objects.")); - string_list_append(&names, line.buf); + item = string_list_append(&names, line.buf); + item->util = populate_pack_exts(item->string); } + strbuf_release(&line); fclose(out); ret = finish_command(&cmd); if (ret) - return ret; + goto cleanup; if (!names.nr && !po_args.quiet) printf_ln(_("Nothing new to pack.")); @@ -984,49 +1055,81 @@ int cmd_repack(int argc, const char **argv, const char *prefix) cruft_po_args.local = po_args.local; cruft_po_args.quiet = po_args.quiet; - ret = write_cruft_pack(&cruft_po_args, pack_prefix, &names, + ret = write_cruft_pack(&cruft_po_args, packtmp, pack_prefix, + cruft_expiration, &names, &existing_nonkept_packs, &existing_kept_packs); if (ret) - return ret; + goto cleanup; + + if (delete_redundant && expire_to) { + /* + * If `--expire-to` is given with `-d`, it's possible + * that we're about to prune some objects. With cruft + * packs, pruning is implicit: any objects from existing + * packs that weren't picked up by new packs are removed + * when their packs are deleted. + * + * Generate an additional cruft pack, with one twist: + * `names` now includes the name of the cruft pack + * written in the previous step. So the contents of + * _this_ cruft pack exclude everything contained in the + * existing cruft pack (that is, all of the unreachable + * objects which are no older than + * `--cruft-expiration`). + * + * To make this work, cruft_expiration must become NULL + * so that this cruft pack doesn't actually prune any + * objects. If it were non-NULL, this call would always + * generate an empty pack (since every object not in the + * cruft pack generated above will have an mtime older + * than the expiration). + */ + ret = write_cruft_pack(&cruft_po_args, expire_to, + pack_prefix, + NULL, + &names, + &existing_nonkept_packs, + &existing_kept_packs); + if (ret) + goto cleanup; + } } string_list_sort(&names); - for_each_string_list_item(item, &names) { - item->util = (void *)(uintptr_t)populate_pack_exts(item->string); - } - close_object_store(the_repository->objects); /* * Ok we have prepared all new packfiles. */ for_each_string_list_item(item, &names) { + struct generated_pack_data *data = item->util; + for (ext = 0; ext < ARRAY_SIZE(exts); ext++) { - char *fname, *fname_old; + char *fname; fname = mkpathdup("%s/pack-%s%s", packdir, item->string, exts[ext].name); - fname_old = mkpathdup("%s-%s%s", - packtmp, item->string, exts[ext].name); - if (((uintptr_t)item->util) & ((uintptr_t)1 << ext)) { + if (data->tempfiles[ext]) { + const char *fname_old = get_tempfile_path(data->tempfiles[ext]); struct stat statbuffer; + if (!stat(fname_old, &statbuffer)) { statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); chmod(fname_old, statbuffer.st_mode); } - if (rename(fname_old, fname)) - die_errno(_("renaming '%s' failed"), fname_old); + if (rename_tempfile(&data->tempfiles[ext], fname)) + die_errno(_("renaming pack to '%s' failed"), fname); } else if (!exts[ext].optional) - die(_("missing required file: %s"), fname_old); + die(_("pack-objects did not write a '%s' file for pack %s-%s"), + exts[ext].name, packtmp, item->string); else if (unlink(fname) < 0 && errno != ENOENT) die_errno(_("could not unlink: %s"), fname); free(fname); - free(fname_old); } } /* End of pack replacement. */ @@ -1059,10 +1162,13 @@ int cmd_repack(int argc, const char **argv, const char *prefix) refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL, show_progress, write_bitmaps > 0); + if (!ret && write_bitmaps) + remove_redundant_bitmaps(&include, packdir); + string_list_clear(&include, 0); if (ret) - return ret; + goto cleanup; } reprepare_packed_git(the_repository); @@ -1089,6 +1195,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix) strbuf_addstr(&buf, pack_basename(p)); strbuf_strip_suffix(&buf, ".pack"); + if ((p->pack_keep) || + (string_list_has_string(&existing_kept_packs, + buf.buf))) + continue; + remove_redundant_pack(packdir, buf.buf); } strbuf_release(&buf); @@ -1106,7 +1217,6 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (run_update_server_info) update_server_info(0); - remove_temporary_files(); if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) { unsigned flags = 0; @@ -1115,11 +1225,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix) write_midx_file(get_object_directory(), NULL, NULL, flags); } - string_list_clear(&names, 0); +cleanup: + string_list_clear(&names, 1); string_list_clear(&existing_nonkept_packs, 0); string_list_clear(&existing_kept_packs, 0); clear_pack_geometry(geometry); - strbuf_release(&line); - return 0; + return ret; } |