diff --git a/Makefile b/Makefile index a757ed3963..61c1d6d04b 100644 --- a/Makefile +++ b/Makefile @@ -1219,6 +1219,7 @@ LIB_OBJS += odb/source.o LIB_OBJS += odb/source-files.o LIB_OBJS += odb/source-inmemory.o LIB_OBJS += odb/source-loose.o +LIB_OBJS += odb/source-packed.o LIB_OBJS += odb/streaming.o LIB_OBJS += odb/transaction.o LIB_OBJS += oid-array.o @@ -1234,6 +1235,7 @@ LIB_OBJS += pack-refs.o LIB_OBJS += pack-revindex.o LIB_OBJS += pack-write.o LIB_OBJS += packfile.o +LIB_OBJS += packfile-list.o LIB_OBJS += pager.o LIB_OBJS += parallel-checkout.o LIB_OBJS += parse.o diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 446d649904..7aed3ad2a1 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -915,8 +915,8 @@ static void batch_each_object(struct batch_options *opt, for (source = the_repository->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - int ret = packfile_store_for_each_object(files->packed, &oi, - batch_one_object_oi, &payload, &opts); + int ret = odb_source_for_each_object(&files->packed->base, &oi, + batch_one_object_oi, &payload, &opts); if (ret) break; } diff --git a/builtin/grep.c b/builtin/grep.c index 6a09571903..8080d1bf5e 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -1363,7 +1363,7 @@ int cmd_grep(int argc, odb_prepare_alternates(the_repository->objects); for (source = the_repository->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - packfile_store_prepare(files->packed); + odb_source_packed_prepare(files->packed); } } diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 00ffb36394..6e73c85cde 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -10,6 +10,7 @@ #include "trace2.h" #include "odb.h" #include "odb/source.h" +#include "odb/source-files.h" #include "replace-object.h" #include "repository.h" @@ -85,12 +86,12 @@ static int parse_object_dir(const struct option *opt, const char *arg, return 0; } -static struct odb_source *handle_object_dir_option(struct repository *repo) +static struct odb_source_files *handle_object_dir_option(struct repository *repo) { struct odb_source *source = odb_find_source(repo->objects, opts.object_dir); if (!source) source = odb_add_to_alternates_memory(repo->objects, opts.object_dir); - return source; + return odb_source_files_downcast(source); } static struct option common_opts[] = { @@ -167,7 +168,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; - struct odb_source *source; + struct odb_source_files *source; int ret; opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE; @@ -211,7 +212,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, read_packs_from_stdin(&packs); - ret = write_midx_file_only(source, &packs, + ret = write_midx_file_only(source->packed, &packs, opts.preferred_pack, opts.refs_snapshot, opts.incremental_base, opts.flags); @@ -223,7 +224,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } - ret = write_midx_file(source, opts.preferred_pack, + ret = write_midx_file(source->packed, opts.preferred_pack, opts.refs_snapshot, opts.flags); free(opts.refs_snapshot); @@ -237,7 +238,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, struct multi_pack_index *m, *cur; struct multi_pack_index *from_midx = NULL; struct multi_pack_index *to_midx = NULL; - struct odb_source *source; + struct odb_source_files *source; int ret; struct option *options; @@ -282,7 +283,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, FREE_AND_NULL(options); - m = get_multi_pack_index(source); + m = get_multi_pack_index(source->packed); for (cur = m; cur && !(from_midx && to_midx); cur = cur->base_midx) { const char *midx_csum = midx_get_checksum_hex(cur); @@ -305,7 +306,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, die(_("MIDX %s must be an ancestor of %s"), argv[0], argv[1]); } - ret = write_midx_file_compact(source, from_midx, to_midx, + ret = write_midx_file_compact(source->packed, from_midx, to_midx, opts.incremental_base, opts.flags); return ret; @@ -319,7 +320,7 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, static struct option builtin_multi_pack_index_verify_options[] = { OPT_END(), }; - struct odb_source *source; + struct odb_source_files *source; options = add_common_options(builtin_multi_pack_index_verify_options); @@ -337,7 +338,7 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, FREE_AND_NULL(options); - return verify_midx_file(source, opts.flags); + return verify_midx_file(source->packed, opts.flags); } static int cmd_multi_pack_index_expire(int argc, const char **argv, @@ -348,7 +349,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, static struct option builtin_multi_pack_index_expire_options[] = { OPT_END(), }; - struct odb_source *source; + struct odb_source_files *source; options = add_common_options(builtin_multi_pack_index_expire_options); @@ -366,7 +367,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, FREE_AND_NULL(options); - return expire_midx_packs(source, opts.flags); + return expire_midx_packs(source->packed, opts.flags); } static int cmd_multi_pack_index_repack(int argc, const char **argv, @@ -379,7 +380,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, N_("during repack, collect pack-files of smaller size into a batch that is larger than this size")), OPT_END(), }; - struct odb_source *source; + struct odb_source_files *source; options = add_common_options(builtin_multi_pack_index_repack_options); @@ -398,7 +399,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, FREE_AND_NULL(options); - return midx_repack(source, (size_t)opts.batch_size, opts.flags); + return midx_repack(source->packed, (size_t)opts.batch_size, opts.flags); } int cmd_multi_pack_index(int argc, diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 0b20f9b62d..fa227d855a 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1777,7 +1777,8 @@ static int want_object_in_pack_mtime(const struct object_id *oid, odb_prepare_alternates(the_repository->objects); for (source = the_repository->objects->sources; source; source = source->next) { - struct multi_pack_index *m = get_multi_pack_index(source); + struct odb_source_files *files = odb_source_files_downcast(source); + struct multi_pack_index *m = get_multi_pack_index(files->packed); struct pack_entry e; if (m && fill_midx_entry(m, oid, &e)) { @@ -4510,8 +4511,8 @@ static void add_objects_in_unpacked_packs(void) if (!source->local) continue; - if (packfile_store_for_each_object(files->packed, &oi, - add_object_in_unpacked_pack, NULL, &opts)) + if (odb_source_for_each_object(&files->packed->base, &oi, + add_object_in_unpacked_pack, NULL, &opts)) die(_("cannot open pack index")); } } diff --git a/builtin/repack.c b/builtin/repack.c index 1524a9c13a..47966a686b 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -458,6 +458,8 @@ int cmd_repack(int argc, } if (!names.nr) { + struct odb_source_files *files = odb_source_files_downcast(existing.source); + if (!po_args.quiet) printf_ln(_("Nothing new to pack.")); /* @@ -473,7 +475,7 @@ int cmd_repack(int argc, * midx_has_unknown_packs() will make the decision for * us. */ - if (!get_multi_pack_index(existing.source)) + if (!get_multi_pack_index(files->packed)) midx_must_contain_cruft = 1; } @@ -626,10 +628,12 @@ int cmd_repack(int argc, update_server_info(repo, 0); if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) { + struct odb_source_files *files = odb_source_files_downcast(existing.source); unsigned flags = 0; + if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(existing.source, NULL, NULL, flags); + write_midx_file(files->packed, NULL, NULL, flags); } cleanup: diff --git a/commit-graph.c b/commit-graph.c index 0820cf5fb8..9e734b72fb 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -2016,8 +2016,8 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) odb_prepare_alternates(ctx->r->objects); for (source = ctx->r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - packfile_store_for_each_object(files->packed, &oi, add_packed_commits_oi, - ctx, &opts); + odb_source_for_each_object(&files->packed->base, &oi, add_packed_commits_oi, + ctx, &opts); } if (ctx->progress_done < ctx->approx_nr_objects) diff --git a/meson.build b/meson.build index 0947904eec..64720ca3c6 100644 --- a/meson.build +++ b/meson.build @@ -407,6 +407,7 @@ libgit_sources = [ 'odb/source-files.c', 'odb/source-inmemory.c', 'odb/source-loose.c', + 'odb/source-packed.c', 'odb/streaming.c', 'odb/transaction.c', 'oid-array.c', @@ -422,6 +423,7 @@ libgit_sources = [ 'pack-revindex.c', 'pack-write.c', 'packfile.c', + 'packfile-list.c', 'pager.c', 'parallel-checkout.c', 'parse.c', diff --git a/midx-write.c b/midx-write.c index 19e1cd10b7..8c1837f6df 100644 --- a/midx-write.c +++ b/midx-write.c @@ -25,9 +25,9 @@ #define NO_PREFERRED_PACK (~((uint32_t)0)) extern int midx_checksum_valid(struct multi_pack_index *m); -extern void clear_midx_files_ext(struct odb_source *source, const char *ext, +extern void clear_midx_files_ext(struct odb_source_packed *source, const char *ext, const char *keep_hash); -extern void clear_incremental_midx_files_ext(struct odb_source *source, +extern void clear_incremental_midx_files_ext(struct odb_source_packed *source, const char *ext, const struct strvec *keep_hashes); extern int cmp_idx_or_pack_name(const char *idx_or_pack_name, @@ -119,7 +119,7 @@ struct write_midx_context { struct string_list *to_include; struct repository *repo; - struct odb_source *source; + struct odb_source_packed *source; }; static uint32_t midx_pack_perm(struct write_midx_context *ctx, @@ -1107,7 +1107,7 @@ done: return ret; } -static void clear_midx_files(struct odb_source *source, +static void clear_midx_files(struct odb_source_packed *source, const struct strvec *hashes, unsigned incremental) { /* @@ -1237,7 +1237,7 @@ static int midx_hashcmp(const struct multi_pack_index *a, } struct write_midx_opts { - struct odb_source *source; /* non-optional */ + struct odb_source_packed *source; /* non-optional */ struct string_list *packs_to_include; struct string_list *packs_to_drop; @@ -1253,7 +1253,7 @@ struct write_midx_opts { static int write_midx_internal(struct write_midx_opts *opts) { - struct repository *r = opts->source->odb->repo; + struct repository *r = opts->source->base.odb->repo; struct strbuf midx_name = STRBUF_INIT; unsigned char midx_hash[GIT_MAX_RAWSZ]; uint32_t start_pack; @@ -1301,7 +1301,7 @@ static int write_midx_internal(struct write_midx_opts *opts) if (ctx.incremental) strbuf_addf(&midx_name, "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX", - opts->source->path); + opts->source->base.path); else get_midx_filename(opts->source, &midx_name); if (safe_create_leading_directories(r, midx_name.buf)) @@ -1396,7 +1396,7 @@ static int write_midx_internal(struct write_midx_opts *opts) fill_packs_from_midx_range(&ctx, bitmap_order); } else { ctx.to_include = opts->packs_to_include; - for_each_file_in_pack_dir(opts->source->path, add_pack_to_midx, &ctx); + for_each_file_in_pack_dir(opts->source->base.path, add_pack_to_midx, &ctx); } stop_progress(&ctx.progress); @@ -1847,7 +1847,7 @@ cleanup: return result; } -int write_midx_file(struct odb_source *source, +int write_midx_file(struct odb_source_packed *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) @@ -1862,7 +1862,7 @@ int write_midx_file(struct odb_source *source, return write_midx_internal(&opts); } -int write_midx_file_only(struct odb_source *source, +int write_midx_file_only(struct odb_source_packed *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, @@ -1881,7 +1881,7 @@ int write_midx_file_only(struct odb_source *source, return write_midx_internal(&opts); } -int write_midx_file_compact(struct odb_source *source, +int write_midx_file_compact(struct odb_source_packed *source, struct multi_pack_index *from, struct multi_pack_index *to, const char *incremental_base, @@ -1898,7 +1898,7 @@ int write_midx_file_compact(struct odb_source *source, return write_midx_internal(&opts); } -int expire_midx_packs(struct odb_source *source, unsigned flags) +int expire_midx_packs(struct odb_source_packed *source, unsigned flags) { uint32_t i, *count, result = 0; struct string_list packs_to_drop = STRING_LIST_INIT_DUP; @@ -1915,7 +1915,7 @@ int expire_midx_packs(struct odb_source *source, unsigned flags) if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - source->odb->repo, + source->base.odb->repo, _("Counting referenced objects"), m->num_objects); for (i = 0; i < m->num_objects; i++) { @@ -1927,7 +1927,7 @@ int expire_midx_packs(struct odb_source *source, unsigned flags) if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - source->odb->repo, + source->base.odb->repo, _("Finding and deleting unreferenced packfiles"), m->num_packs); for (i = 0; i < m->num_packs; i++) { @@ -2085,9 +2085,9 @@ static void fill_included_packs_batch(struct repository *r, free(pack_info); } -int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags) +int midx_repack(struct odb_source_packed *source, size_t batch_size, unsigned flags) { - struct repository *r = source->odb->repo; + struct repository *r = source->base.odb->repo; int result = 0; uint32_t i, packs_to_repack = 0; unsigned char *include_pack; @@ -2131,7 +2131,7 @@ int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags) strvec_push(&cmd.args, "pack-objects"); - strvec_pushf(&cmd.args, "%s/pack/pack", source->path); + strvec_pushf(&cmd.args, "%s/pack/pack", source->base.path); if (delta_base_offset) strvec_push(&cmd.args, "--delta-base-offset"); diff --git a/midx.c b/midx.c index efbfbb13f4..cc6b94f9dd 100644 --- a/midx.c +++ b/midx.c @@ -17,9 +17,9 @@ #define MIDX_PACK_ERROR ((void *)(intptr_t)-1) int midx_checksum_valid(struct multi_pack_index *m); -void clear_midx_files_ext(struct odb_source *source, const char *ext, +void clear_midx_files_ext(struct odb_source_packed *source, const char *ext, const char *keep_hash); -void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source_packed *source, const char *ext, const struct strvec *keep_hashes); int cmp_idx_or_pack_name(const char *idx_or_pack_name, const char *idx_name); @@ -27,25 +27,25 @@ int cmp_idx_or_pack_name(const char *idx_or_pack_name, const char *midx_get_checksum_hex(const struct multi_pack_index *m) { return hash_to_hex_algop(midx_get_checksum_hash(m), - m->source->odb->repo->hash_algo); + m->source->base.odb->repo->hash_algo); } const unsigned char *midx_get_checksum_hash(const struct multi_pack_index *m) { - return m->data + m->data_len - m->source->odb->repo->hash_algo->rawsz; + return m->data + m->data_len - m->source->base.odb->repo->hash_algo->rawsz; } -void get_midx_filename(struct odb_source *source, struct strbuf *out) +void get_midx_filename(struct odb_source_packed *source, struct strbuf *out) { get_midx_filename_ext(source, out, NULL, NULL); } -void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, +void get_midx_filename_ext(struct odb_source_packed *source, struct strbuf *out, const unsigned char *hash, const char *ext) { - strbuf_addf(out, "%s/pack/multi-pack-index", source->path); + strbuf_addf(out, "%s/pack/multi-pack-index", source->base.path); if (ext) - strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); + strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, source->base.odb->repo->hash_algo), ext); } static int midx_read_oid_fanout(const unsigned char *chunk_start, @@ -99,17 +99,16 @@ static int midx_read_object_offsets(const unsigned char *chunk_start, return 0; } -struct multi_pack_index *get_multi_pack_index(struct odb_source *source) +struct multi_pack_index *get_multi_pack_index(struct odb_source_packed *source) { - struct odb_source_files *files = odb_source_files_downcast(source); - packfile_store_prepare(files->packed); - return files->packed->midx; + odb_source_packed_prepare(source); + return source->midx; } -static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source, +static struct multi_pack_index *load_multi_pack_index_one(struct odb_source_packed *source, const char *midx_name) { - struct repository *r = source->odb->repo; + struct repository *r = source->base.odb->repo; struct multi_pack_index *m = NULL; int fd; struct stat st; @@ -234,23 +233,23 @@ cleanup_fail: return NULL; } -void get_midx_chain_dirname(struct odb_source *source, struct strbuf *buf) +void get_midx_chain_dirname(struct odb_source_packed *source, struct strbuf *buf) { - strbuf_addf(buf, "%s/pack/multi-pack-index.d", source->path); + strbuf_addf(buf, "%s/pack/multi-pack-index.d", source->base.path); } -void get_midx_chain_filename(struct odb_source *source, struct strbuf *buf) +void get_midx_chain_filename(struct odb_source_packed *source, struct strbuf *buf) { get_midx_chain_dirname(source, buf); strbuf_addstr(buf, "/multi-pack-index-chain"); } -void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, +void get_split_midx_filename_ext(struct odb_source_packed *source, struct strbuf *buf, const unsigned char *hash, const char *ext) { get_midx_chain_dirname(source, buf); strbuf_addf(buf, "/multi-pack-index-%s.%s", - hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); + hash_to_hex_algop(hash, source->base.odb->repo->hash_algo), ext); } static int open_multi_pack_index_chain(const struct git_hash_algo *hash_algo, @@ -306,11 +305,11 @@ static int add_midx_to_chain(struct multi_pack_index *midx, return 1; } -static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source, +static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source_packed *source, int fd, struct stat *st, int *incomplete_chain) { - const struct git_hash_algo *hash_algo = source->odb->repo->hash_algo; + const struct git_hash_algo *hash_algo = source->base.odb->repo->hash_algo; struct multi_pack_index *midx_chain = NULL; struct strbuf buf = STRBUF_INIT; int valid = 1; @@ -362,7 +361,7 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source, return midx_chain; } -static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *source) +static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source_packed *source) { struct strbuf chain_file = STRBUF_INIT; struct stat st; @@ -370,7 +369,8 @@ static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *s struct multi_pack_index *m = NULL; get_midx_chain_filename(source, &chain_file); - if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) { + if (open_multi_pack_index_chain(source->base.odb->repo->hash_algo, + chain_file.buf, &fd, &st)) { int incomplete; /* ownership of fd is taken over by load function */ m = load_midx_chain_fd_st(source, fd, &st, &incomplete); @@ -380,7 +380,7 @@ static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *s return m; } -struct multi_pack_index *load_multi_pack_index(struct odb_source *source) +struct multi_pack_index *load_multi_pack_index(struct odb_source_packed *source) { struct strbuf midx_name = STRBUF_INIT; struct multi_pack_index *m; @@ -456,7 +456,7 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m, int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) { - struct odb_source_files *files = odb_source_files_downcast(m->source); + struct odb_source_packed *packed = m->source; struct strbuf pack_name = STRBUF_INIT; struct packed_git *p; @@ -467,10 +467,10 @@ int prepare_midx_pack(struct multi_pack_index *m, if (m->packs[pack_int_id]) return 0; - strbuf_addf(&pack_name, "%s/pack/%s", files->base.path, + strbuf_addf(&pack_name, "%s/pack/%s", packed->base.path, m->pack_names[pack_int_id]); - p = packfile_store_load_pack(files->packed, - pack_name.buf, files->base.local); + p = packfile_store_load_pack(packed, + pack_name.buf, packed->base.local); strbuf_release(&pack_name); if (!p) { @@ -523,7 +523,7 @@ int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m, { int ret = bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup, - m->source->odb->repo->hash_algo->rawsz, + m->source->base.odb->repo->hash_algo->rawsz, result); if (result) *result += m->num_objects_in_base; @@ -554,7 +554,7 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid, n = midx_for_object(&m, n); oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n), - m->source->odb->repo->hash_algo); + m->source->base.odb->repo->hash_algo); return oid; } @@ -734,26 +734,25 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) return 0; } -int prepare_multi_pack_index_one(struct odb_source *source) +int prepare_multi_pack_index_one(struct odb_source_packed *source) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct repository *r = source->odb->repo; + struct repository *r = source->base.odb->repo; prepare_repo_settings(r); if (!r->settings.core_multi_pack_index) return 0; - if (files->packed->midx) + if (source->midx) return 1; - files->packed->midx = load_multi_pack_index(source); + source->midx = load_multi_pack_index(source); - return !!files->packed->midx; + return !!source->midx; } int midx_checksum_valid(struct multi_pack_index *m) { - return hashfile_checksum_valid(m->source->odb->repo->hash_algo, + return hashfile_checksum_valid(m->source->base.odb->repo->hash_algo, m->data, m->data_len); } @@ -776,7 +775,7 @@ static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUS die_errno(_("failed to remove %s"), full_path); } -void clear_midx_files_ext(struct odb_source *source, const char *ext, +void clear_midx_files_ext(struct odb_source_packed *source, const char *ext, const char *keep_hash) { struct clear_midx_data data = { @@ -793,12 +792,12 @@ void clear_midx_files_ext(struct odb_source *source, const char *ext, strbuf_release(&buf); } - for_each_file_in_pack_dir(source->path, clear_midx_file_ext, &data); + for_each_file_in_pack_dir(source->base.path, clear_midx_file_ext, &data); strset_clear(&data.keep); } -void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source_packed *source, const char *ext, const struct strvec *keep_hashes) { struct clear_midx_data data = { @@ -817,7 +816,7 @@ void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext } } - for_each_file_in_pack_subdir(source->path, "multi-pack-index.d", + for_each_file_in_pack_subdir(source->base.path, "multi-pack-index.d", clear_midx_file_ext, &data); strbuf_release(&buf); @@ -826,26 +825,28 @@ void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext void clear_midx_file(struct repository *r) { + struct odb_source_files *files; struct strbuf midx = STRBUF_INIT; - get_midx_filename(r->objects->sources, &midx); - if (r->objects) { struct odb_source *source; for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); + files = odb_source_files_downcast(source); if (files->packed->midx) close_midx(files->packed->midx); files->packed->midx = NULL; } } + files = odb_source_files_downcast(r->objects->sources); + get_midx_filename(files->packed, &midx); + if (remove_path(midx.buf)) die(_("failed to clear multi-pack-index at %s"), midx.buf); - clear_midx_files_ext(r->objects->sources, MIDX_EXT_BITMAP, NULL); - clear_midx_files_ext(r->objects->sources, MIDX_EXT_REV, NULL); + clear_midx_files_ext(files->packed, MIDX_EXT_BITMAP, NULL); + clear_midx_files_ext(files->packed, MIDX_EXT_REV, NULL); strbuf_release(&midx); } @@ -853,28 +854,27 @@ void clear_midx_file(struct repository *r) void clear_incremental_midx_files(struct repository *r, const struct strvec *keep_hashes) { - struct odb_source *source = r->objects->sources; + struct odb_source_files *files; + struct odb_source *source; struct strbuf chain = STRBUF_INIT; - get_midx_chain_filename(source, &chain); - - for (; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); + for (source = r->objects->sources; source; source = source->next) { + files = odb_source_files_downcast(source); if (files->packed->midx) close_midx(files->packed->midx); files->packed->midx = NULL; } + files = odb_source_files_downcast(r->objects->sources); + get_midx_chain_filename(files->packed, &chain); + if (!keep_hashes && remove_path(chain.buf)) die(_("failed to clear multi-pack-index chain at %s"), chain.buf); - clear_incremental_midx_files_ext(r->objects->sources, MIDX_EXT_BITMAP, - keep_hashes); - clear_incremental_midx_files_ext(r->objects->sources, MIDX_EXT_REV, - keep_hashes); - clear_incremental_midx_files_ext(r->objects->sources, MIDX_EXT_MIDX, - keep_hashes); + clear_incremental_midx_files_ext(files->packed, MIDX_EXT_BITMAP, keep_hashes); + clear_incremental_midx_files_ext(files->packed, MIDX_EXT_REV, keep_hashes); + clear_incremental_midx_files_ext(files->packed, MIDX_EXT_MIDX, keep_hashes); strbuf_release(&chain); } @@ -918,9 +918,9 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b) display_progress(progress, _n); \ } while (0) -int verify_midx_file(struct odb_source *source, unsigned flags) +int verify_midx_file(struct odb_source_packed *source, unsigned flags) { - struct repository *r = source->odb->repo; + struct repository *r = source->base.odb->repo; struct pair_pos_vs_id *pairs = NULL; uint32_t i; struct progress *progress = NULL; diff --git a/midx.h b/midx.h index 63853a03a4..939c18e588 100644 --- a/midx.h +++ b/midx.h @@ -37,7 +37,7 @@ struct strvec; "GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL" struct multi_pack_index { - struct odb_source *source; + struct odb_source_packed *source; const unsigned char *data; size_t data_len; @@ -92,16 +92,16 @@ struct multi_pack_index { const char *midx_get_checksum_hex(const struct multi_pack_index *m) /* static buffer */; const unsigned char *midx_get_checksum_hash(const struct multi_pack_index *m); -void get_midx_filename(struct odb_source *source, struct strbuf *out); -void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, +void get_midx_filename(struct odb_source_packed *source, struct strbuf *out); +void get_midx_filename_ext(struct odb_source_packed *source, struct strbuf *out, const unsigned char *hash, const char *ext); -void get_midx_chain_dirname(struct odb_source *source, struct strbuf *out); -void get_midx_chain_filename(struct odb_source *source, struct strbuf *out); -void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, +void get_midx_chain_dirname(struct odb_source_packed *source, struct strbuf *out); +void get_midx_chain_filename(struct odb_source_packed *source, struct strbuf *out); +void get_split_midx_filename_ext(struct odb_source_packed *source, struct strbuf *buf, const unsigned char *hash, const char *ext); -struct multi_pack_index *get_multi_pack_index(struct odb_source *source); -struct multi_pack_index *load_multi_pack_index(struct odb_source *source); +struct multi_pack_index *get_multi_pack_index(struct odb_source_packed *source); +struct multi_pack_index *load_multi_pack_index(struct odb_source_packed *source); int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id); struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id); @@ -123,22 +123,22 @@ int midx_contains_pack(struct multi_pack_index *m, int midx_layer_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); -int prepare_multi_pack_index_one(struct odb_source *source); +int prepare_multi_pack_index_one(struct odb_source_packed *source); /* * Variant of write_midx_file which writes a MIDX containing only the packs * specified in packs_to_include. */ -int write_midx_file(struct odb_source *source, +int write_midx_file(struct odb_source_packed *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); -int write_midx_file_only(struct odb_source *source, +int write_midx_file_only(struct odb_source_packed *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, const char *incremental_base, unsigned flags); -int write_midx_file_compact(struct odb_source *source, +int write_midx_file_compact(struct odb_source_packed *source, struct multi_pack_index *from, struct multi_pack_index *to, const char *incremental_base, @@ -146,9 +146,9 @@ int write_midx_file_compact(struct odb_source *source, void clear_midx_file(struct repository *r); void clear_incremental_midx_files(struct repository *r, const struct strvec *keep_hashes); -int verify_midx_file(struct odb_source *source, unsigned flags); -int expire_midx_packs(struct odb_source *source, unsigned flags); -int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags); +int verify_midx_file(struct odb_source_packed *source, unsigned flags); +int expire_midx_packs(struct odb_source_packed *source, unsigned flags); +int midx_repack(struct odb_source_packed *source, size_t batch_size, unsigned flags); void close_midx(struct multi_pack_index *m); diff --git a/odb/source-files.c b/odb/source-files.c index 5bdd042922..3bc6419dd7 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -29,7 +29,7 @@ static void odb_source_files_free(struct odb_source *source) struct odb_source_files *files = odb_source_files_downcast(source); chdir_notify_unregister(NULL, odb_source_files_reparent, files); odb_source_free(&files->loose->base); - packfile_store_free(files->packed); + odb_source_free(&files->packed->base); odb_source_release(&files->base); free(files); } @@ -38,14 +38,14 @@ static void odb_source_files_close(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); odb_source_close(&files->loose->base); - packfile_store_close(files->packed); + odb_source_close(&files->packed->base); } static void odb_source_files_reprepare(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); odb_source_reprepare(&files->loose->base); - packfile_store_reprepare(files->packed); + odb_source_reprepare(&files->packed->base); } static int odb_source_files_read_object_info(struct odb_source *source, @@ -55,7 +55,7 @@ static int odb_source_files_read_object_info(struct odb_source *source, { struct odb_source_files *files = odb_source_files_downcast(source); - if (!packfile_store_read_object_info(files->packed, oid, oi, flags) || + if (!odb_source_read_object_info(&files->packed->base, oid, oi, flags) || !odb_source_read_object_info(&files->loose->base, oid, oi, flags)) return 0; @@ -67,7 +67,7 @@ static int odb_source_files_read_object_stream(struct odb_read_stream **out, const struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!packfile_store_read_object_stream(out, files->packed, oid) || + if (!odb_source_read_object_stream(out, &files->packed->base, oid) || !odb_source_read_object_stream(out, &files->loose->base, oid)) return 0; return -1; @@ -88,7 +88,7 @@ static int odb_source_files_for_each_object(struct odb_source *source, return ret; } - ret = packfile_store_for_each_object(files->packed, request, cb, cb_data, opts); + ret = odb_source_for_each_object(&files->packed->base, request, cb, cb_data, opts); if (ret) return ret; @@ -103,7 +103,7 @@ static int odb_source_files_count_objects(struct odb_source *source, unsigned long count; int ret; - ret = packfile_store_count_objects(files->packed, flags, &count); + ret = odb_source_count_objects(&files->packed->base, flags, &count); if (ret < 0) goto out; @@ -133,7 +133,7 @@ static int odb_source_files_find_abbrev_len(struct odb_source *source, unsigned len = min_len; int ret; - ret = packfile_store_find_abbrev_len(files->packed, oid, len, &len); + ret = odb_source_find_abbrev_len(&files->packed->base, oid, len, &len); if (ret < 0) goto out; @@ -152,7 +152,7 @@ static int odb_source_files_freshen_object(struct odb_source *source, const struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - if (packfile_store_freshen_object(files->packed, oid) || + if (odb_source_freshen_object(&files->packed->base, oid) || odb_source_freshen_object(&files->loose->base, oid)) return 1; return 0; @@ -269,7 +269,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, CALLOC_ARRAY(files, 1); odb_source_init(&files->base, odb, ODB_SOURCE_FILES, path, local); files->loose = odb_source_loose_new(odb, path, local); - files->packed = packfile_store_new(&files->base); + files->packed = odb_source_packed_new(odb, path, local); files->base.free = odb_source_files_free; files->base.close = odb_source_files_close; diff --git a/odb/source-files.h b/odb/source-files.h index 23a3b4e04b..d7ac3c1c81 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -4,7 +4,7 @@ #include "odb/source.h" struct odb_source_loose; -struct packfile_store; +struct odb_source_packed; /* * The files object database source uses a combination of loose objects and @@ -13,7 +13,7 @@ struct packfile_store; struct odb_source_files { struct odb_source base; struct odb_source_loose *loose; - struct packfile_store *packed; + struct odb_source_packed *packed; }; /* Allocate and initialize a new object source. */ diff --git a/odb/source-packed.c b/odb/source-packed.c new file mode 100644 index 0000000000..42c28fba0e --- /dev/null +++ b/odb/source-packed.c @@ -0,0 +1,764 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "chdir-notify.h" +#include "dir.h" +#include "git-zlib.h" +#include "mergesort.h" +#include "midx.h" +#include "odb/source-packed.h" +#include "odb/streaming.h" +#include "packfile.h" + +static int find_pack_entry(struct odb_source_packed *store, + const struct object_id *oid, + struct pack_entry *e) +{ + struct packfile_list_entry *l; + + odb_source_packed_prepare(store); + if (store->midx && fill_midx_entry(store->midx, oid, e)) + return 1; + + for (l = store->packs.head; l; l = l->next) { + struct packed_git *p = l->pack; + + if (!p->multi_pack_index && packfile_fill_entry(p, oid, e)) { + if (!store->skip_mru_updates) + packfile_list_prepend(&store->packs, p); + return 1; + } + } + + return 0; +} + +static int odb_source_packed_read_object_info(struct odb_source *source, + const struct object_id *oid, + struct object_info *oi, + enum object_info_flags flags) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + struct pack_entry e; + int ret; + + /* + * In case the first read didn't surface the object, we have to reload + * packfiles. This may cause us to discover new packfiles that have + * been added since the last time we have prepared the packfile store. + */ + if (flags & OBJECT_INFO_SECOND_READ) + odb_source_reprepare(source); + + if (!find_pack_entry(packed, oid, &e)) + return 1; + + /* + * We know that the caller doesn't actually need the + * information below, so return early. + */ + if (!oi) + return 0; + + ret = packed_object_info(e.p, e.offset, oi); + if (ret < 0) { + mark_bad_packed_object(e.p, oid); + return -1; + } + + return 0; +} + +static int odb_source_packed_read_object_stream(struct odb_read_stream **out, + struct odb_source *source, + const struct object_id *oid) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + struct pack_entry e; + + if (!find_pack_entry(packed, oid, &e)) + return -1; + + return packfile_read_object_stream(out, oid, e.p, e.offset); +} + +struct odb_source_packed_for_each_object_wrapper_data { + struct odb_source_packed *store; + const struct object_info *request; + odb_for_each_object_cb cb; + void *cb_data; +}; + +static int odb_source_packed_for_each_object_wrapper(const struct object_id *oid, + struct packed_git *pack, + uint32_t index_pos, + void *cb_data) +{ + struct odb_source_packed_for_each_object_wrapper_data *data = cb_data; + + if (data->request) { + off_t offset = nth_packed_object_offset(pack, index_pos); + struct object_info oi = *data->request; + + if (packed_object_info_with_index_pos(pack, offset, + &index_pos, &oi) < 0) { + mark_bad_packed_object(pack, oid); + return -1; + } + + return data->cb(oid, &oi, data->cb_data); + } else { + return data->cb(oid, NULL, data->cb_data); + } +} + +static int match_hash(unsigned len, const unsigned char *a, const unsigned char *b) +{ + do { + if (*a != *b) + return 0; + a++; + b++; + len -= 2; + } while (len > 1); + if (len) + if ((*a ^ *b) & 0xf0) + return 0; + return 1; +} + +static int for_each_prefixed_object_in_midx( + struct odb_source_packed *store, + struct multi_pack_index *m, + const struct odb_for_each_object_options *opts, + struct odb_source_packed_for_each_object_wrapper_data *data) +{ + int ret; + + for (; m; m = m->base_midx) { + uint32_t num, i, first = 0; + int len = opts->prefix_hex_len > m->source->base.odb->repo->hash_algo->hexsz ? + m->source->base.odb->repo->hash_algo->hexsz : opts->prefix_hex_len; + + if (!m->num_objects) + continue; + + num = m->num_objects + m->num_objects_in_base; + + bsearch_one_midx(opts->prefix, m, &first); + + /* + * At this point, "first" is the location of the lowest + * object with an object name that could match "opts->prefix". + * See if we have 0, 1 or more objects that actually match(es). + */ + for (i = first; i < num; i++) { + const struct object_id *current = NULL; + struct object_id oid; + + current = nth_midxed_object_oid(&oid, m, i); + + if (!match_hash(len, opts->prefix->hash, current->hash)) + break; + + if (data->request) { + struct object_info oi = *data->request; + + ret = odb_source_read_object_info(&store->base, current, + &oi, 0); + if (ret) + goto out; + + ret = data->cb(&oid, &oi, data->cb_data); + if (ret) + goto out; + } else { + ret = data->cb(&oid, NULL, data->cb_data); + if (ret) + goto out; + } + } + } + + ret = 0; + +out: + return ret; +} + +static int for_each_prefixed_object_in_pack( + struct odb_source_packed *store, + struct packed_git *p, + const struct odb_for_each_object_options *opts, + struct odb_source_packed_for_each_object_wrapper_data *data) +{ + uint32_t num, i, first = 0; + int len = opts->prefix_hex_len > p->repo->hash_algo->hexsz ? + p->repo->hash_algo->hexsz : opts->prefix_hex_len; + int ret; + + num = p->num_objects; + bsearch_pack(opts->prefix, p, &first); + + /* + * At this point, "first" is the location of the lowest object + * with an object name that could match "bin_pfx". See if we have + * 0, 1 or more objects that actually match(es). + */ + for (i = first; i < num; i++) { + struct object_id oid; + + nth_packed_object_id(&oid, p, i); + if (!match_hash(len, opts->prefix->hash, oid.hash)) + break; + + if (data->request) { + struct object_info oi = *data->request; + + ret = odb_source_read_object_info(&store->base, &oid, &oi, 0); + if (ret) + goto out; + + ret = data->cb(&oid, &oi, data->cb_data); + if (ret) + goto out; + } else { + ret = data->cb(&oid, NULL, data->cb_data); + if (ret) + goto out; + } + } + + ret = 0; + +out: + return ret; +} + +static int odb_source_packed_for_each_prefixed_object( + struct odb_source_packed *store, + const struct odb_for_each_object_options *opts, + struct odb_source_packed_for_each_object_wrapper_data *data) +{ + struct packfile_list_entry *e; + struct multi_pack_index *m; + bool pack_errors = false; + int ret; + + if (opts->flags) + BUG("flags unsupported"); + + store->skip_mru_updates = true; + + m = get_multi_pack_index(store); + if (m) { + ret = for_each_prefixed_object_in_midx(store, m, opts, data); + if (ret) + goto out; + } + + for (e = packfile_store_get_packs(store); e; e = e->next) { + if (e->pack->multi_pack_index) + continue; + + if (open_pack_index(e->pack)) { + pack_errors = true; + continue; + } + + if (!e->pack->num_objects) + continue; + + ret = for_each_prefixed_object_in_pack(store, e->pack, opts, data); + if (ret) + goto out; + } + + ret = 0; + +out: + store->skip_mru_updates = false; + if (!ret && pack_errors) + ret = -1; + return ret; +} + +static int odb_source_packed_for_each_object(struct odb_source *source, + const struct object_info *request, + odb_for_each_object_cb cb, + void *cb_data, + const struct odb_for_each_object_options *opts) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + struct odb_source_packed_for_each_object_wrapper_data data = { + .store = packed, + .request = request, + .cb = cb, + .cb_data = cb_data, + }; + struct packfile_list_entry *e; + int pack_errors = 0, ret; + + if (opts->prefix) + return odb_source_packed_for_each_prefixed_object(packed, opts, &data); + + packed->skip_mru_updates = true; + + for (e = packfile_store_get_packs(packed); e; e = e->next) { + struct packed_git *p = e->pack; + + if ((opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) + continue; + if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY) && + !p->pack_promisor) + continue; + if ((opts->flags & ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS) && + p->pack_keep_in_core) + continue; + if ((opts->flags & ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS) && + p->pack_keep) + continue; + if (open_pack_index(p)) { + pack_errors = 1; + continue; + } + + ret = for_each_object_in_pack(p, odb_source_packed_for_each_object_wrapper, + &data, opts->flags); + if (ret) + goto out; + } + + ret = 0; + +out: + packed->skip_mru_updates = false; + + if (!ret && pack_errors) + ret = -1; + return ret; +} + +static int odb_source_packed_count_objects(struct odb_source *source, + enum odb_count_objects_flags flags UNUSED, + unsigned long *out) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + struct packfile_list_entry *e; + struct multi_pack_index *m; + unsigned long count = 0; + int ret; + + m = get_multi_pack_index(packed); + if (m) + count += m->num_objects + m->num_objects_in_base; + + for (e = packfile_store_get_packs(packed); e; e = e->next) { + if (e->pack->multi_pack_index) + continue; + if (open_pack_index(e->pack)) { + ret = -1; + goto out; + } + + count += e->pack->num_objects; + } + + *out = count; + ret = 0; + +out: + return ret; +} + +static int extend_abbrev_len(const struct object_id *a, + const struct object_id *b, + unsigned *out) +{ + unsigned len = oid_common_prefix_hexlen(a, b); + if (len != hash_algos[a->algo].hexsz && len >= *out) + *out = len + 1; + return 0; +} + +static void find_abbrev_len_for_midx(struct multi_pack_index *m, + const struct object_id *oid, + unsigned min_len, + unsigned *out) +{ + unsigned len = min_len; + + for (; m; m = m->base_midx) { + int match = 0; + uint32_t num, first = 0; + struct object_id found_oid; + + if (!m->num_objects) + continue; + + num = m->num_objects + m->num_objects_in_base; + match = bsearch_one_midx(oid, m, &first); + + /* + * first is now the position in the packfile where we + * would insert the object ID if it does not exist (or the + * position of the object ID if it does exist). Hence, we + * consider a maximum of two objects nearby for the + * abbreviation length. + */ + + if (!match) { + if (nth_midxed_object_oid(&found_oid, m, first)) + extend_abbrev_len(&found_oid, oid, &len); + } else if (first < num - 1) { + if (nth_midxed_object_oid(&found_oid, m, first + 1)) + extend_abbrev_len(&found_oid, oid, &len); + } + if (first > 0) { + if (nth_midxed_object_oid(&found_oid, m, first - 1)) + extend_abbrev_len(&found_oid, oid, &len); + } + } + + *out = len; +} + +static void find_abbrev_len_for_pack(struct packed_git *p, + const struct object_id *oid, + unsigned min_len, + unsigned *out) +{ + int match; + uint32_t num, first = 0; + struct object_id found_oid; + unsigned len = min_len; + + num = p->num_objects; + match = bsearch_pack(oid, p, &first); + + /* + * first is now the position in the packfile where we would insert + * the object ID if it does not exist (or the position of mad->hash if + * it does exist). Hence, we consider a maximum of two objects + * nearby for the abbreviation length. + */ + if (!match) { + if (!nth_packed_object_id(&found_oid, p, first)) + extend_abbrev_len(&found_oid, oid, &len); + } else if (first < num - 1) { + if (!nth_packed_object_id(&found_oid, p, first + 1)) + extend_abbrev_len(&found_oid, oid, &len); + } + if (first > 0) { + if (!nth_packed_object_id(&found_oid, p, first - 1)) + extend_abbrev_len(&found_oid, oid, &len); + } + + *out = len; +} + +static int odb_source_packed_find_abbrev_len(struct odb_source *source, + const struct object_id *oid, + unsigned min_len, + unsigned *out) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + struct packfile_list_entry *e; + struct multi_pack_index *m; + + m = get_multi_pack_index(packed); + if (m) + find_abbrev_len_for_midx(m, oid, min_len, &min_len); + + for (e = packfile_store_get_packs(packed); e; e = e->next) { + if (e->pack->multi_pack_index) + continue; + if (open_pack_index(e->pack) || !e->pack->num_objects) + continue; + + find_abbrev_len_for_pack(e->pack, oid, min_len, &min_len); + } + + *out = min_len; + return 0; +} + +static int odb_source_packed_freshen_object(struct odb_source *source, + const struct object_id *oid) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + struct pack_entry e; + + if (!find_pack_entry(packed, oid, &e)) + return 0; + if (e.p->is_cruft) + return 0; + if (e.p->freshened) + return 1; + if (utime(e.p->pack_name, NULL)) + return 0; + e.p->freshened = 1; + + return 1; +} + +static int odb_source_packed_write_object(struct odb_source *source UNUSED, + const void *buf UNUSED, + unsigned long len UNUSED, + enum object_type type UNUSED, + struct object_id *oid UNUSED, + struct object_id *compat_oid UNUSED, + unsigned flags UNUSED) +{ + return error("packed backend cannot write objects"); +} + +static int odb_source_packed_write_object_stream(struct odb_source *source UNUSED, + struct odb_write_stream *stream UNUSED, + size_t len UNUSED, + struct object_id *oid UNUSED) +{ + return error("packed backend cannot write object streams"); +} + +static int odb_source_packed_begin_transaction(struct odb_source *source UNUSED, + struct odb_transaction **out UNUSED) +{ + return error("packed backend cannot begin transactions"); +} + +static int odb_source_packed_read_alternates(struct odb_source *source UNUSED, + struct strvec *out UNUSED) +{ + return 0; +} + +static int odb_source_packed_write_alternate(struct odb_source *source UNUSED, + const char *alternate UNUSED) +{ + return error("packed backend cannot write alternates"); +} + +void (*report_garbage)(unsigned seen_bits, const char *path); + +static void report_helper(const struct string_list *list, + int seen_bits, int first, int last) +{ + if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX)) + return; + + for (; first < last; first++) + report_garbage(seen_bits, list->items[first].string); +} + +static void report_pack_garbage(struct string_list *list) +{ + int baselen = -1, first = 0, seen_bits = 0; + + if (!report_garbage) + return; + + string_list_sort(list); + + for (size_t i = 0; i < list->nr; i++) { + const char *path = list->items[i].string; + if (baselen != -1 && + strncmp(path, list->items[first].string, baselen)) { + report_helper(list, seen_bits, first, i); + baselen = -1; + seen_bits = 0; + } + if (baselen == -1) { + const char *dot = strrchr(path, '.'); + if (!dot) { + report_garbage(PACKDIR_FILE_GARBAGE, path); + continue; + } + baselen = dot - path + 1; + first = i; + } + if (!strcmp(path + baselen, "pack")) + seen_bits |= 1; + else if (!strcmp(path + baselen, "idx")) + seen_bits |= 2; + } + report_helper(list, seen_bits, first, list->nr); +} + +struct prepare_pack_data { + struct odb_source_packed *source; + struct string_list *garbage; +}; + +static void prepare_pack(const char *full_name, size_t full_name_len, + const char *file_name, void *_data) +{ + struct prepare_pack_data *data = (struct prepare_pack_data *)_data; + size_t base_len = full_name_len; + + if (strip_suffix_mem(full_name, &base_len, ".idx") && + !(data->source->midx && + midx_contains_pack(data->source->midx, file_name))) { + char *trimmed_path = xstrndup(full_name, full_name_len); + packfile_store_load_pack(data->source, + trimmed_path, data->source->base.local); + free(trimmed_path); + } + + if (!report_garbage) + return; + + if (!strcmp(file_name, "multi-pack-index") || + !strcmp(file_name, "multi-pack-index.d")) + return; + if (starts_with(file_name, "multi-pack-index") && + (ends_with(file_name, ".bitmap") || ends_with(file_name, ".rev"))) + return; + if (ends_with(file_name, ".idx") || + ends_with(file_name, ".rev") || + ends_with(file_name, ".pack") || + ends_with(file_name, ".bitmap") || + ends_with(file_name, ".keep") || + ends_with(file_name, ".promisor") || + ends_with(file_name, ".mtimes")) + string_list_append(data->garbage, full_name); + else + report_garbage(PACKDIR_FILE_GARBAGE, full_name); +} + +static void prepare_packed_git_one(struct odb_source_packed *source) +{ + struct string_list garbage = STRING_LIST_INIT_DUP; + struct prepare_pack_data data = { + .source = source, + .garbage = &garbage, + }; + + for_each_file_in_pack_dir(source->base.path, prepare_pack, &data); + + report_pack_garbage(data.garbage); + string_list_clear(data.garbage, 0); +} + +DEFINE_LIST_SORT(static, sort_packs, struct packfile_list_entry, next); + +static int sort_pack(const struct packfile_list_entry *a, + const struct packfile_list_entry *b) +{ + int st; + + /* + * Local packs tend to contain objects specific to our + * variant of the project than remote ones. In addition, + * remote ones could be on a network mounted filesystem. + * Favor local ones for these reasons. + */ + st = a->pack->pack_local - b->pack->pack_local; + if (st) + return -st; + + /* + * Younger packs tend to contain more recent objects, + * and more recent objects tend to get accessed more + * often. + */ + if (a->pack->mtime < b->pack->mtime) + return 1; + else if (a->pack->mtime == b->pack->mtime) + return 0; + return -1; +} + +void odb_source_packed_prepare(struct odb_source_packed *source) +{ + if (source->initialized) + return; + + prepare_multi_pack_index_one(source); + prepare_packed_git_one(source); + + sort_packs(&source->packs.head, sort_pack); + for (struct packfile_list_entry *e = source->packs.head; e; e = e->next) + if (!e->next) + source->packs.tail = e; + + source->initialized = true; +} + +static void odb_source_packed_reprepare(struct odb_source *source) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + packed->initialized = false; + odb_source_packed_prepare(packed); +} + +static void odb_source_packed_reparent(const char *name UNUSED, + const char *old_cwd, + const char *new_cwd, + void *cb_data) +{ + struct odb_source_packed *packed = cb_data; + char *path = reparent_relative_path(old_cwd, new_cwd, + packed->base.path); + free(packed->base.path); + packed->base.path = path; +} + +static void odb_source_packed_close(struct odb_source *source) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + + for (struct packfile_list_entry *e = packed->packs.head; e; e = e->next) { + if (e->pack->do_not_close) + BUG("want to close pack marked 'do-not-close'"); + close_pack(e->pack); + } + if (packed->midx) + close_midx(packed->midx); + packed->midx = NULL; +} + +static void odb_source_packed_free(struct odb_source *source) +{ + struct odb_source_packed *packed = odb_source_packed_downcast(source); + + chdir_notify_unregister(NULL, odb_source_packed_reparent, packed); + + for (struct packfile_list_entry *e = packed->packs.head; e; e = e->next) + free(e->pack); + packfile_list_clear(&packed->packs); + + strmap_clear(&packed->packs_by_path, 0); + odb_source_release(&packed->base); + free(packed); +} + +struct odb_source_packed *odb_source_packed_new(struct object_database *odb, + const char *path, + bool local) +{ + struct odb_source_packed *packed; + + CALLOC_ARRAY(packed, 1); + odb_source_init(&packed->base, odb, ODB_SOURCE_PACKED, path, local); + strmap_init(&packed->packs_by_path); + + packed->base.free = odb_source_packed_free; + packed->base.close = odb_source_packed_close; + packed->base.reprepare = odb_source_packed_reprepare; + packed->base.read_object_info = odb_source_packed_read_object_info; + packed->base.read_object_stream = odb_source_packed_read_object_stream; + packed->base.for_each_object = odb_source_packed_for_each_object; + packed->base.count_objects = odb_source_packed_count_objects; + packed->base.find_abbrev_len = odb_source_packed_find_abbrev_len; + packed->base.freshen_object = odb_source_packed_freshen_object; + packed->base.write_object = odb_source_packed_write_object; + packed->base.write_object_stream = odb_source_packed_write_object_stream; + packed->base.begin_transaction = odb_source_packed_begin_transaction; + packed->base.read_alternates = odb_source_packed_read_alternates; + packed->base.write_alternate = odb_source_packed_write_alternate; + + if (!is_absolute_path(path)) + chdir_notify_register(NULL, odb_source_packed_reparent, packed); + + return packed; +} diff --git a/odb/source-packed.h b/odb/source-packed.h new file mode 100644 index 0000000000..f0724b204c --- /dev/null +++ b/odb/source-packed.h @@ -0,0 +1,94 @@ +#ifndef ODB_SOURCE_PACKED_H +#define ODB_SOURCE_PACKED_H + +#include "odb/source.h" +#include "packfile-list.h" +#include "strmap.h" + +/* + * A store that manages packfiles for a given object database. + */ +struct odb_source_packed { + struct odb_source base; + + /* + * The list of packfiles in the order in which they have been most + * recently used. + */ + struct packfile_list packs; + + /* + * Cache of packfiles which are marked as "kept", either because there + * is an on-disk ".keep" file or because they are marked as "kept" in + * memory. + * + * Should not be accessed directly, but via + * `packfile_store_get_kept_pack_cache()`. The list of packs gets + * invalidated when the stored flags and the flags passed to + * `packfile_store_get_kept_pack_cache()` mismatch. + */ + struct { + struct packed_git **packs; + unsigned flags; + } kept_cache; + + /* The multi-pack index that belongs to this specific packfile store. */ + struct multi_pack_index *midx; + + /* + * A map of packfile names to packed_git structs for tracking which + * packs have been loaded already. + */ + struct strmap packs_by_path; + + /* + * Whether packfiles have already been populated with this store's + * packs. + */ + bool initialized; + + /* + * Usually, packfiles will be reordered to the front of the `packs` + * list whenever an object is looked up via them. This has the effect + * that packs that contain a lot of accessed objects will be located + * towards the front. + * + * This is usually desirable, but there are exceptions. One exception + * is when the looking up multiple objects in a loop for each packfile. + * In that case, we may easily end up with an infinite loop as the + * packfiles get reordered to the front repeatedly. + * + * Setting this field to `true` thus disables these reorderings. + */ + bool skip_mru_updates; +}; + +/* + * Allocate and initialize a new empty packfile store for the given object + * database. + */ +struct odb_source_packed *odb_source_packed_new(struct object_database *odb, + const char *path, + bool local); + +/* + * Cast the given object database source to the packed backend. This will cause + * a BUG in case the source doesn't use this backend. + */ +static inline struct odb_source_packed *odb_source_packed_downcast(struct odb_source *source) +{ + if (source->type != ODB_SOURCE_PACKED) + BUG("trying to downcast source of type '%d' to packed", source->type); + return container_of(source, struct odb_source_packed, base); +} + +/* + * Prepare the source by loading packfiles and multi-pack indices for + * all alternates. This becomes a no-op if the source is already prepared. + * + * It shouldn't typically be necessary to call this function directly, as + * functions that access the source know to prepare it. + */ +void odb_source_packed_prepare(struct odb_source_packed *source); + +#endif diff --git a/odb/source.h b/odb/source.h index 2192a101b8..b9a7642b2c 100644 --- a/odb/source.h +++ b/odb/source.h @@ -17,6 +17,9 @@ enum odb_source_type { /* The "loose" backend that uses loose objects, only. */ ODB_SOURCE_LOOSE, + /* The "packed" backend that uses packfiles. */ + ODB_SOURCE_PACKED, + /* The "in-memory" backend that stores objects in memory. */ ODB_SOURCE_INMEMORY, }; diff --git a/pack-bitmap.c b/pack-bitmap.c index f9af8a96bd..6bfcbc8ce6 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -238,7 +238,7 @@ static uint32_t bitmap_name_hash(struct bitmap_index *index, uint32_t pos) static struct repository *bitmap_repo(struct bitmap_index *bitmap_git) { if (bitmap_is_midx(bitmap_git)) - return bitmap_git->midx->source->odb->repo; + return bitmap_git->midx->source->base.odb->repo; return bitmap_git->pack->repo; } @@ -711,7 +711,8 @@ static int open_midx_bitmap(struct repository *r, odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { - struct multi_pack_index *midx = get_multi_pack_index(source); + struct odb_source_files *files = odb_source_files_downcast(source); + struct multi_pack_index *midx = get_multi_pack_index(files->packed); if (midx && !open_midx_bitmap_1(bitmap_git, midx)) ret = 0; } @@ -3399,7 +3400,8 @@ int verify_bitmap_files(struct repository *r) odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { - struct multi_pack_index *m = get_multi_pack_index(source); + struct odb_source_files *files = odb_source_files_downcast(source); + struct multi_pack_index *m = get_multi_pack_index(files->packed); char *midx_bitmap_name; if (!m) diff --git a/pack-revindex.c b/pack-revindex.c index 1b67863606..62387ae632 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -383,13 +383,13 @@ int load_midx_revindex(struct multi_pack_index *m) * not want to accidentally call munmap() in the middle of the * MIDX. */ - trace2_data_string("load_midx_revindex", m->source->odb->repo, + trace2_data_string("load_midx_revindex", m->source->base.odb->repo, "source", "midx"); m->revindex_data = (const uint32_t *)m->chunk_revindex; return 0; } - trace2_data_string("load_midx_revindex", m->source->odb->repo, + trace2_data_string("load_midx_revindex", m->source->base.odb->repo, "source", "rev"); if (m->has_chain) @@ -401,7 +401,7 @@ int load_midx_revindex(struct multi_pack_index *m) midx_get_checksum_hash(m), MIDX_EXT_REV); - ret = load_revindex_from_disk(m->source->odb->repo->hash_algo, + ret = load_revindex_from_disk(m->source->base.odb->repo->hash_algo, revindex_name.buf, m->num_objects, &m->revindex_map, diff --git a/packfile-list.c b/packfile-list.c new file mode 100644 index 0000000000..01fb913abf --- /dev/null +++ b/packfile-list.c @@ -0,0 +1,86 @@ +#include "git-compat-util.h" +#include "packfile.h" +#include "packfile-list.h" + +void packfile_list_clear(struct packfile_list *list) +{ + struct packfile_list_entry *e, *next; + + for (e = list->head; e; e = next) { + next = e->next; + free(e); + } + + list->head = list->tail = NULL; +} + +static struct packfile_list_entry *packfile_list_remove_internal(struct packfile_list *list, + struct packed_git *pack) +{ + struct packfile_list_entry *e, *prev; + + for (e = list->head, prev = NULL; e; prev = e, e = e->next) { + if (e->pack != pack) + continue; + + if (prev) + prev->next = e->next; + if (list->head == e) + list->head = e->next; + if (list->tail == e) + list->tail = prev; + + return e; + } + + return NULL; +} + +void packfile_list_remove(struct packfile_list *list, struct packed_git *pack) +{ + free(packfile_list_remove_internal(list, pack)); +} + +void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack) +{ + struct packfile_list_entry *entry; + + entry = packfile_list_remove_internal(list, pack); + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->pack = pack; + } + entry->next = list->head; + + list->head = entry; + if (!list->tail) + list->tail = entry; +} + +void packfile_list_append(struct packfile_list *list, struct packed_git *pack) +{ + struct packfile_list_entry *entry; + + entry = packfile_list_remove_internal(list, pack); + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->pack = pack; + } + entry->next = NULL; + + if (list->tail) { + list->tail->next = entry; + list->tail = entry; + } else { + list->head = list->tail = entry; + } +} + +struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, + const struct object_id *oid) +{ + for (; packs; packs = packs->next) + if (find_pack_entry_one(oid, packs->pack)) + return packs->pack; + return NULL; +} diff --git a/packfile-list.h b/packfile-list.h new file mode 100644 index 0000000000..1b05e2aa36 --- /dev/null +++ b/packfile-list.h @@ -0,0 +1,28 @@ +#ifndef PACKFILE_LIST_H +#define PACKFILE_LIST_H + +struct object_id; + +struct packfile_list { + struct packfile_list_entry *head, *tail; +}; + +struct packfile_list_entry { + struct packfile_list_entry *next; + struct packed_git *pack; +}; + +void packfile_list_clear(struct packfile_list *list); +void packfile_list_remove(struct packfile_list *list, struct packed_git *pack); +void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack); +void packfile_list_append(struct packfile_list *list, struct packed_git *pack); + +/* + * Find the pack within the "packs" list whose index contains the object + * "oid". For general object lookups, you probably don't want this; use + * find_pack_entry() instead. + */ +struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, + const struct object_id *oid); + +#endif diff --git a/packfile.c b/packfile.c index 89366abfe3..59cee7925d 100644 --- a/packfile.c +++ b/packfile.c @@ -8,7 +8,6 @@ #include "pack.h" #include "repository.h" #include "dir.h" -#include "mergesort.h" #include "packfile.h" #include "delta.h" #include "hash-lookup.h" @@ -48,89 +47,6 @@ static size_t pack_mapped; #define SZ_FMT PRIuMAX static inline uintmax_t sz_fmt(size_t s) { return s; } -void packfile_list_clear(struct packfile_list *list) -{ - struct packfile_list_entry *e, *next; - - for (e = list->head; e; e = next) { - next = e->next; - free(e); - } - - list->head = list->tail = NULL; -} - -static struct packfile_list_entry *packfile_list_remove_internal(struct packfile_list *list, - struct packed_git *pack) -{ - struct packfile_list_entry *e, *prev; - - for (e = list->head, prev = NULL; e; prev = e, e = e->next) { - if (e->pack != pack) - continue; - - if (prev) - prev->next = e->next; - if (list->head == e) - list->head = e->next; - if (list->tail == e) - list->tail = prev; - - return e; - } - - return NULL; -} - -void packfile_list_remove(struct packfile_list *list, struct packed_git *pack) -{ - free(packfile_list_remove_internal(list, pack)); -} - -void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack) -{ - struct packfile_list_entry *entry; - - entry = packfile_list_remove_internal(list, pack); - if (!entry) { - entry = xmalloc(sizeof(*entry)); - entry->pack = pack; - } - entry->next = list->head; - - list->head = entry; - if (!list->tail) - list->tail = entry; -} - -void packfile_list_append(struct packfile_list *list, struct packed_git *pack) -{ - struct packfile_list_entry *entry; - - entry = packfile_list_remove_internal(list, pack); - if (!entry) { - entry = xmalloc(sizeof(*entry)); - entry->pack = pack; - } - entry->next = NULL; - - if (list->tail) { - list->tail->next = entry; - list->tail = entry; - } else { - list->head = list->tail = entry; - } -} - -struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, - const struct object_id *oid) -{ - for (; packs; packs = packs->next) - if (find_pack_entry_one(oid, packs->pack)) - return packs->pack; - return NULL; -} - void pack_report(struct repository *repo) { fprintf(stderr, @@ -859,7 +775,7 @@ struct packed_git *add_packed_git(struct repository *r, const char *path, return p; } -void packfile_store_add_pack(struct packfile_store *store, +void packfile_store_add_pack(struct odb_source_packed *store, struct packed_git *pack) { if (pack->pack_fd != -1) @@ -869,7 +785,7 @@ void packfile_store_add_pack(struct packfile_store *store, strmap_put(&store->packs_by_path, pack->pack_name, pack); } -struct packed_git *packfile_store_load_pack(struct packfile_store *store, +struct packed_git *packfile_store_load_pack(struct odb_source_packed *store, const char *idx_path, int local) { struct strbuf key = STRBUF_INIT; @@ -885,7 +801,7 @@ struct packed_git *packfile_store_load_pack(struct packfile_store *store, p = strmap_get(&store->packs_by_path, key.buf); if (!p) { - p = add_packed_git(store->source->odb->repo, idx_path, + p = add_packed_git(store->base.odb->repo, idx_path, strlen(idx_path), local); if (p) packfile_store_add_pack(store, p); @@ -895,52 +811,6 @@ struct packed_git *packfile_store_load_pack(struct packfile_store *store, return p; } -void (*report_garbage)(unsigned seen_bits, const char *path); - -static void report_helper(const struct string_list *list, - int seen_bits, int first, int last) -{ - if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX)) - return; - - for (; first < last; first++) - report_garbage(seen_bits, list->items[first].string); -} - -static void report_pack_garbage(struct string_list *list) -{ - int i, baselen = -1, first = 0, seen_bits = 0; - - if (!report_garbage) - return; - - string_list_sort(list); - - for (i = 0; i < list->nr; i++) { - const char *path = list->items[i].string; - if (baselen != -1 && - strncmp(path, list->items[first].string, baselen)) { - report_helper(list, seen_bits, first, i); - baselen = -1; - seen_bits = 0; - } - if (baselen == -1) { - const char *dot = strrchr(path, '.'); - if (!dot) { - report_garbage(PACKDIR_FILE_GARBAGE, path); - continue; - } - baselen = dot - path + 1; - first = i; - } - if (!strcmp(path + baselen, "pack")) - seen_bits |= 1; - else if (!strcmp(path + baselen, "idx")) - seen_bits |= 2; - } - report_helper(list, seen_bits, first, list->nr); -} - void for_each_file_in_pack_subdir(const char *objdir, const char *subdir, each_file_in_pack_dir_fn fn, @@ -983,116 +853,9 @@ void for_each_file_in_pack_dir(const char *objdir, for_each_file_in_pack_subdir(objdir, NULL, fn, data); } -struct prepare_pack_data { - struct odb_source *source; - struct string_list *garbage; -}; - -static void prepare_pack(const char *full_name, size_t full_name_len, - const char *file_name, void *_data) +struct packfile_list_entry *packfile_store_get_packs(struct odb_source_packed *store) { - struct prepare_pack_data *data = (struct prepare_pack_data *)_data; - struct odb_source_files *files = odb_source_files_downcast(data->source); - size_t base_len = full_name_len; - - if (strip_suffix_mem(full_name, &base_len, ".idx") && - !(files->packed->midx && - midx_contains_pack(files->packed->midx, file_name))) { - char *trimmed_path = xstrndup(full_name, full_name_len); - packfile_store_load_pack(files->packed, - trimmed_path, data->source->local); - free(trimmed_path); - } - - if (!report_garbage) - return; - - if (!strcmp(file_name, "multi-pack-index") || - !strcmp(file_name, "multi-pack-index.d")) - return; - if (starts_with(file_name, "multi-pack-index") && - (ends_with(file_name, ".bitmap") || ends_with(file_name, ".rev"))) - return; - if (ends_with(file_name, ".idx") || - ends_with(file_name, ".rev") || - ends_with(file_name, ".pack") || - ends_with(file_name, ".bitmap") || - ends_with(file_name, ".keep") || - ends_with(file_name, ".promisor") || - ends_with(file_name, ".mtimes")) - string_list_append(data->garbage, full_name); - else - report_garbage(PACKDIR_FILE_GARBAGE, full_name); -} - -static void prepare_packed_git_one(struct odb_source *source) -{ - struct string_list garbage = STRING_LIST_INIT_DUP; - struct prepare_pack_data data = { - .source = source, - .garbage = &garbage, - }; - - for_each_file_in_pack_dir(source->path, prepare_pack, &data); - - report_pack_garbage(data.garbage); - string_list_clear(data.garbage, 0); -} - -DEFINE_LIST_SORT(static, sort_packs, struct packfile_list_entry, next); - -static int sort_pack(const struct packfile_list_entry *a, - const struct packfile_list_entry *b) -{ - int st; - - /* - * Local packs tend to contain objects specific to our - * variant of the project than remote ones. In addition, - * remote ones could be on a network mounted filesystem. - * Favor local ones for these reasons. - */ - st = a->pack->pack_local - b->pack->pack_local; - if (st) - return -st; - - /* - * Younger packs tend to contain more recent objects, - * and more recent objects tend to get accessed more - * often. - */ - if (a->pack->mtime < b->pack->mtime) - return 1; - else if (a->pack->mtime == b->pack->mtime) - return 0; - return -1; -} - -void packfile_store_prepare(struct packfile_store *store) -{ - if (store->initialized) - return; - - prepare_multi_pack_index_one(store->source); - prepare_packed_git_one(store->source); - - sort_packs(&store->packs.head, sort_pack); - for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) - if (!e->next) - store->packs.tail = e; - - store->initialized = true; -} - -void packfile_store_reprepare(struct packfile_store *store) -{ - store->initialized = false; - packfile_store_prepare(store); -} - -struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *store) -{ - packfile_store_prepare(store); + odb_source_packed_prepare(store); if (store->midx) { struct multi_pack_index *m = store->midx; @@ -1103,37 +866,6 @@ struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *stor return store->packs.head; } -int packfile_store_count_objects(struct packfile_store *store, - enum odb_count_objects_flags flags UNUSED, - unsigned long *out) -{ - struct packfile_list_entry *e; - struct multi_pack_index *m; - unsigned long count = 0; - int ret; - - m = get_multi_pack_index(store->source); - if (m) - count += m->num_objects + m->num_objects_in_base; - - for (e = packfile_store_get_packs(store); e; e = e->next) { - if (e->pack->multi_pack_index) - continue; - if (open_pack_index(e->pack)) { - ret = -1; - goto out; - } - - count += e->pack->num_objects; - } - - *out = count; - ret = 0; - -out: - return ret; -} - unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, size_t *sizep) { @@ -1599,8 +1331,8 @@ static void add_delta_base_cache(struct packed_git *p, off_t base_offset, hashmap_add(&delta_base_cache, &ent->ent); } -static int packed_object_info_with_index_pos(struct packed_git *p, off_t obj_offset, - uint32_t *maybe_index_pos, struct object_info *oi) +int packed_object_info_with_index_pos(struct packed_git *p, off_t obj_offset, + uint32_t *maybe_index_pos, struct object_info *oi) { struct pack_window *w_curs = NULL; size_t size; @@ -2132,9 +1864,9 @@ int is_pack_valid(struct packed_git *p) return !open_packed_git(p); } -static int fill_pack_entry(const struct object_id *oid, - struct pack_entry *e, - struct packed_git *p) +int packfile_fill_entry(struct packed_git *p, + const struct object_id *oid, + struct pack_entry *e) { off_t offset; @@ -2160,81 +1892,7 @@ static int fill_pack_entry(const struct object_id *oid, return 1; } -static int find_pack_entry(struct packfile_store *store, - const struct object_id *oid, - struct pack_entry *e) -{ - struct packfile_list_entry *l; - - packfile_store_prepare(store); - if (store->midx && fill_midx_entry(store->midx, oid, e)) - return 1; - - for (l = store->packs.head; l; l = l->next) { - struct packed_git *p = l->pack; - - if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) { - if (!store->skip_mru_updates) - packfile_list_prepend(&store->packs, p); - return 1; - } - } - - return 0; -} - -int packfile_store_freshen_object(struct packfile_store *store, - const struct object_id *oid) -{ - struct pack_entry e; - if (!find_pack_entry(store, oid, &e)) - return 0; - if (e.p->is_cruft) - return 0; - if (e.p->freshened) - return 1; - if (utime(e.p->pack_name, NULL)) - return 0; - e.p->freshened = 1; - return 1; -} - -int packfile_store_read_object_info(struct packfile_store *store, - const struct object_id *oid, - struct object_info *oi, - enum object_info_flags flags) -{ - struct pack_entry e; - int ret; - - /* - * In case the first read didn't surface the object, we have to reload - * packfiles. This may cause us to discover new packfiles that have - * been added since the last time we have prepared the packfile store. - */ - if (flags & OBJECT_INFO_SECOND_READ) - packfile_store_reprepare(store); - - if (!find_pack_entry(store, oid, &e)) - return 1; - - /* - * We know that the caller doesn't actually need the - * information below, so return early. - */ - if (!oi) - return 0; - - ret = packed_object_info(e.p, e.offset, oi); - if (ret < 0) { - mark_bad_packed_object(e.p, oid); - return -1; - } - - return 0; -} - -static void maybe_invalidate_kept_pack_cache(struct packfile_store *store, +static void maybe_invalidate_kept_pack_cache(struct odb_source_packed *store, unsigned flags) { if (!store->kept_cache.packs) @@ -2245,7 +1903,7 @@ static void maybe_invalidate_kept_pack_cache(struct packfile_store *store, store->kept_cache.flags = 0; } -struct packed_git **packfile_store_get_kept_pack_cache(struct packfile_store *store, +struct packed_git **packfile_store_get_kept_pack_cache(struct odb_source_packed *store, unsigned flags) { maybe_invalidate_kept_pack_cache(store, flags); @@ -2286,14 +1944,12 @@ struct packed_git **packfile_store_get_kept_pack_cache(struct packfile_store *st int has_object_pack(struct repository *r, const struct object_id *oid) { struct odb_source *source; - struct pack_entry e; odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - int ret = find_pack_entry(files->packed, oid, &e); - if (ret) - return ret; + if (!odb_source_read_object_info(&files->packed->base, oid, NULL, 0)) + return 1; } return 0; @@ -2313,7 +1969,7 @@ int has_object_kept_pack(struct repository *r, const struct object_id *oid, for (; *cache; cache++) { struct packed_git *p = *cache; - if (fill_pack_entry(oid, &e, p)) + if (packfile_fill_entry(p, oid, &e)) return 1; } } @@ -2365,373 +2021,6 @@ int for_each_object_in_pack(struct packed_git *p, return r; } -struct packfile_store_for_each_object_wrapper_data { - struct packfile_store *store; - const struct object_info *request; - odb_for_each_object_cb cb; - void *cb_data; -}; - -static int packfile_store_for_each_object_wrapper(const struct object_id *oid, - struct packed_git *pack, - uint32_t index_pos, - void *cb_data) -{ - struct packfile_store_for_each_object_wrapper_data *data = cb_data; - - if (data->request) { - off_t offset = nth_packed_object_offset(pack, index_pos); - struct object_info oi = *data->request; - - if (packed_object_info_with_index_pos(pack, offset, - &index_pos, &oi) < 0) { - mark_bad_packed_object(pack, oid); - return -1; - } - - return data->cb(oid, &oi, data->cb_data); - } else { - return data->cb(oid, NULL, data->cb_data); - } -} - -static int match_hash(unsigned len, const unsigned char *a, const unsigned char *b) -{ - do { - if (*a != *b) - return 0; - a++; - b++; - len -= 2; - } while (len > 1); - if (len) - if ((*a ^ *b) & 0xf0) - return 0; - return 1; -} - -static int for_each_prefixed_object_in_midx( - struct packfile_store *store, - struct multi_pack_index *m, - const struct odb_for_each_object_options *opts, - struct packfile_store_for_each_object_wrapper_data *data) -{ - int ret; - - for (; m; m = m->base_midx) { - uint32_t num, i, first = 0; - int len = opts->prefix_hex_len > m->source->odb->repo->hash_algo->hexsz ? - m->source->odb->repo->hash_algo->hexsz : opts->prefix_hex_len; - - if (!m->num_objects) - continue; - - num = m->num_objects + m->num_objects_in_base; - - bsearch_one_midx(opts->prefix, m, &first); - - /* - * At this point, "first" is the location of the lowest - * object with an object name that could match "opts->prefix". - * See if we have 0, 1 or more objects that actually match(es). - */ - for (i = first; i < num; i++) { - const struct object_id *current = NULL; - struct object_id oid; - - current = nth_midxed_object_oid(&oid, m, i); - - if (!match_hash(len, opts->prefix->hash, current->hash)) - break; - - if (data->request) { - struct object_info oi = *data->request; - - ret = packfile_store_read_object_info(store, current, - &oi, 0); - if (ret) - goto out; - - ret = data->cb(&oid, &oi, data->cb_data); - if (ret) - goto out; - } else { - ret = data->cb(&oid, NULL, data->cb_data); - if (ret) - goto out; - } - } - } - - ret = 0; - -out: - return ret; -} - -static int for_each_prefixed_object_in_pack( - struct packfile_store *store, - struct packed_git *p, - const struct odb_for_each_object_options *opts, - struct packfile_store_for_each_object_wrapper_data *data) -{ - uint32_t num, i, first = 0; - int len = opts->prefix_hex_len > p->repo->hash_algo->hexsz ? - p->repo->hash_algo->hexsz : opts->prefix_hex_len; - int ret; - - num = p->num_objects; - bsearch_pack(opts->prefix, p, &first); - - /* - * At this point, "first" is the location of the lowest object - * with an object name that could match "bin_pfx". See if we have - * 0, 1 or more objects that actually match(es). - */ - for (i = first; i < num; i++) { - struct object_id oid; - - nth_packed_object_id(&oid, p, i); - if (!match_hash(len, opts->prefix->hash, oid.hash)) - break; - - if (data->request) { - struct object_info oi = *data->request; - - ret = packfile_store_read_object_info(store, &oid, &oi, 0); - if (ret) - goto out; - - ret = data->cb(&oid, &oi, data->cb_data); - if (ret) - goto out; - } else { - ret = data->cb(&oid, NULL, data->cb_data); - if (ret) - goto out; - } - } - - ret = 0; - -out: - return ret; -} - -static int packfile_store_for_each_prefixed_object( - struct packfile_store *store, - const struct odb_for_each_object_options *opts, - struct packfile_store_for_each_object_wrapper_data *data) -{ - struct packfile_list_entry *e; - struct multi_pack_index *m; - bool pack_errors = false; - int ret; - - if (opts->flags) - BUG("flags unsupported"); - - store->skip_mru_updates = true; - - m = get_multi_pack_index(store->source); - if (m) { - ret = for_each_prefixed_object_in_midx(store, m, opts, data); - if (ret) - goto out; - } - - for (e = packfile_store_get_packs(store); e; e = e->next) { - if (e->pack->multi_pack_index) - continue; - - if (open_pack_index(e->pack)) { - pack_errors = true; - continue; - } - - if (!e->pack->num_objects) - continue; - - ret = for_each_prefixed_object_in_pack(store, e->pack, opts, data); - if (ret) - goto out; - } - - ret = 0; - -out: - store->skip_mru_updates = false; - if (!ret && pack_errors) - ret = -1; - return ret; -} - -int packfile_store_for_each_object(struct packfile_store *store, - const struct object_info *request, - odb_for_each_object_cb cb, - void *cb_data, - const struct odb_for_each_object_options *opts) -{ - struct packfile_store_for_each_object_wrapper_data data = { - .store = store, - .request = request, - .cb = cb, - .cb_data = cb_data, - }; - struct packfile_list_entry *e; - int pack_errors = 0, ret; - - if (opts->prefix) - return packfile_store_for_each_prefixed_object(store, opts, &data); - - store->skip_mru_updates = true; - - for (e = packfile_store_get_packs(store); e; e = e->next) { - struct packed_git *p = e->pack; - - if ((opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) - continue; - if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY) && - !p->pack_promisor) - continue; - if ((opts->flags & ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS) && - p->pack_keep_in_core) - continue; - if ((opts->flags & ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS) && - p->pack_keep) - continue; - if (open_pack_index(p)) { - pack_errors = 1; - continue; - } - - ret = for_each_object_in_pack(p, packfile_store_for_each_object_wrapper, - &data, opts->flags); - if (ret) - goto out; - } - - ret = 0; - -out: - store->skip_mru_updates = false; - - if (!ret && pack_errors) - ret = -1; - return ret; -} - -static int extend_abbrev_len(const struct object_id *a, - const struct object_id *b, - unsigned *out) -{ - unsigned len = oid_common_prefix_hexlen(a, b); - if (len != hash_algos[a->algo].hexsz && len >= *out) - *out = len + 1; - return 0; -} - -static void find_abbrev_len_for_midx(struct multi_pack_index *m, - const struct object_id *oid, - unsigned min_len, - unsigned *out) -{ - unsigned len = min_len; - - for (; m; m = m->base_midx) { - int match = 0; - uint32_t num, first = 0; - struct object_id found_oid; - - if (!m->num_objects) - continue; - - num = m->num_objects + m->num_objects_in_base; - match = bsearch_one_midx(oid, m, &first); - - /* - * first is now the position in the packfile where we - * would insert the object ID if it does not exist (or the - * position of the object ID if it does exist). Hence, we - * consider a maximum of two objects nearby for the - * abbreviation length. - */ - - if (!match) { - if (nth_midxed_object_oid(&found_oid, m, first)) - extend_abbrev_len(&found_oid, oid, &len); - } else if (first < num - 1) { - if (nth_midxed_object_oid(&found_oid, m, first + 1)) - extend_abbrev_len(&found_oid, oid, &len); - } - if (first > 0) { - if (nth_midxed_object_oid(&found_oid, m, first - 1)) - extend_abbrev_len(&found_oid, oid, &len); - } - } - - *out = len; -} - -static void find_abbrev_len_for_pack(struct packed_git *p, - const struct object_id *oid, - unsigned min_len, - unsigned *out) -{ - int match; - uint32_t num, first = 0; - struct object_id found_oid; - unsigned len = min_len; - - num = p->num_objects; - match = bsearch_pack(oid, p, &first); - - /* - * first is now the position in the packfile where we would insert - * the object ID if it does not exist (or the position of mad->hash if - * it does exist). Hence, we consider a maximum of two objects - * nearby for the abbreviation length. - */ - if (!match) { - if (!nth_packed_object_id(&found_oid, p, first)) - extend_abbrev_len(&found_oid, oid, &len); - } else if (first < num - 1) { - if (!nth_packed_object_id(&found_oid, p, first + 1)) - extend_abbrev_len(&found_oid, oid, &len); - } - if (first > 0) { - if (!nth_packed_object_id(&found_oid, p, first - 1)) - extend_abbrev_len(&found_oid, oid, &len); - } - - *out = len; -} - -int packfile_store_find_abbrev_len(struct packfile_store *store, - const struct object_id *oid, - unsigned min_len, - unsigned *out) -{ - struct packfile_list_entry *e; - struct multi_pack_index *m; - - m = get_multi_pack_index(store->source); - if (m) - find_abbrev_len_for_midx(m, oid, min_len, &min_len); - - for (e = packfile_store_get_packs(store); e; e = e->next) { - if (e->pack->multi_pack_index) - continue; - if (open_pack_index(e->pack) || !e->pack->num_objects) - continue; - - find_abbrev_len_for_pack(e->pack, oid, min_len, &min_len); - } - - *out = min_len; - return 0; -} - struct add_promisor_object_data { struct repository *repo; struct oidset *set; @@ -2832,37 +2121,6 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l return 0; } -struct packfile_store *packfile_store_new(struct odb_source *source) -{ - struct packfile_store *store; - CALLOC_ARRAY(store, 1); - store->source = source; - strmap_init(&store->packs_by_path); - return store; -} - -void packfile_store_free(struct packfile_store *store) -{ - for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) - free(e->pack); - packfile_list_clear(&store->packs); - - strmap_clear(&store->packs_by_path, 0); - free(store); -} - -void packfile_store_close(struct packfile_store *store) -{ - for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) { - if (e->pack->do_not_close) - BUG("want to close pack marked 'do-not-close'"); - close_pack(e->pack); - } - if (store->midx) - close_midx(store->midx); - store->midx = NULL; -} - struct odb_packed_read_stream { struct odb_read_stream base; struct packed_git *pack; @@ -2986,15 +2244,3 @@ int packfile_read_object_stream(struct odb_read_stream **out, return 0; } - -int packfile_store_read_object_stream(struct odb_read_stream **out, - struct packfile_store *store, - const struct object_id *oid) -{ - struct pack_entry e; - - if (!find_pack_entry(store, oid, &e)) - return -1; - - return packfile_read_object_stream(out, oid, e.p, e.offset); -} diff --git a/packfile.h b/packfile.h index 5729a37018..71a71017ee 100644 --- a/packfile.h +++ b/packfile.h @@ -5,9 +5,10 @@ #include "object.h" #include "odb.h" #include "odb/source-files.h" +#include "odb/source-packed.h" #include "oidset.h" +#include "packfile-list.h" #include "repository.h" -#include "strmap.h" /* in odb.h */ struct object_info; @@ -54,133 +55,18 @@ struct packed_git { char pack_name[FLEX_ARRAY]; /* more */ }; -struct packfile_list { - struct packfile_list_entry *head, *tail; -}; - -struct packfile_list_entry { - struct packfile_list_entry *next; - struct packed_git *pack; -}; - -void packfile_list_clear(struct packfile_list *list); -void packfile_list_remove(struct packfile_list *list, struct packed_git *pack); -void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack); -void packfile_list_append(struct packfile_list *list, struct packed_git *pack); - -/* - * Find the pack within the "packs" list whose index contains the object - * "oid". For general object lookups, you probably don't want this; use - * find_pack_entry() instead. - */ -struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, - const struct object_id *oid); - -/* - * A store that manages packfiles for a given object database. - */ -struct packfile_store { - struct odb_source *source; - - /* - * The list of packfiles in the order in which they have been most - * recently used. - */ - struct packfile_list packs; - - /* - * Cache of packfiles which are marked as "kept", either because there - * is an on-disk ".keep" file or because they are marked as "kept" in - * memory. - * - * Should not be accessed directly, but via - * `packfile_store_get_kept_pack_cache()`. The list of packs gets - * invalidated when the stored flags and the flags passed to - * `packfile_store_get_kept_pack_cache()` mismatch. - */ - struct { - struct packed_git **packs; - unsigned flags; - } kept_cache; - - /* The multi-pack index that belongs to this specific packfile store. */ - struct multi_pack_index *midx; - - /* - * A map of packfile names to packed_git structs for tracking which - * packs have been loaded already. - */ - struct strmap packs_by_path; - - /* - * Whether packfiles have already been populated with this store's - * packs. - */ - bool initialized; - - /* - * Usually, packfiles will be reordered to the front of the `packs` - * list whenever an object is looked up via them. This has the effect - * that packs that contain a lot of accessed objects will be located - * towards the front. - * - * This is usually desirable, but there are exceptions. One exception - * is when the looking up multiple objects in a loop for each packfile. - * In that case, we may easily end up with an infinite loop as the - * packfiles get reordered to the front repeatedly. - * - * Setting this field to `true` thus disables these reorderings. - */ - bool skip_mru_updates; -}; - -/* - * Allocate and initialize a new empty packfile store for the given object - * database source. - */ -struct packfile_store *packfile_store_new(struct odb_source *source); - -/* - * Free the packfile store and all its associated state. All packfiles - * tracked by the store will be closed. - */ -void packfile_store_free(struct packfile_store *store); - -/* - * Close all packfiles associated with this store. The packfiles won't be - * free'd, so they can be re-opened at a later point in time. - */ -void packfile_store_close(struct packfile_store *store); - -/* - * Prepare the packfile store by loading packfiles and multi-pack indices for - * all alternates. This becomes a no-op if the store is already prepared. - * - * It shouldn't typically be necessary to call this function directly, as - * functions that access the store know to prepare it. - */ -void packfile_store_prepare(struct packfile_store *store); - -/* - * Clear the packfile caches and try to look up any new packfiles that have - * appeared since last preparing the packfiles store. - * - * This function must be called under the `odb_read_lock()`. - */ -void packfile_store_reprepare(struct packfile_store *store); - /* * Add the pack to the store so that contained objects become accessible via * the store. This moves ownership into the store. */ -void packfile_store_add_pack(struct packfile_store *store, +void packfile_store_add_pack(struct odb_source_packed *store, struct packed_git *pack); /* * Get all packs managed by the given store, including packfiles that are * referenced by multi-pack indices. */ -struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *store); +struct packfile_list_entry *packfile_store_get_packs(struct odb_source_packed *store); struct repo_for_each_pack_data { struct odb_source *source; @@ -238,54 +124,26 @@ static inline void repo_for_each_pack_data_next(struct repo_for_each_pack_data * ((p) = (eack_pack_data.entry ? eack_pack_data.entry->pack : NULL)); \ repo_for_each_pack_data_next(&eack_pack_data)) -int packfile_store_read_object_stream(struct odb_read_stream **out, - struct packfile_store *store, - const struct object_id *oid); - -/* - * Try to read the object identified by its ID from the object store and - * populate the object info with its data. Returns 1 in case the object was - * not found, 0 if it was and read successfully, and a negative error code in - * case the object was corrupted. - */ -int packfile_store_read_object_info(struct packfile_store *store, - const struct object_id *oid, - struct object_info *oi, - enum object_info_flags flags); - /* * Open the packfile and add it to the store if it isn't yet known. Returns * either the newly opened packfile or the preexisting packfile. Returns a * `NULL` pointer in case the packfile could not be opened. */ -struct packed_git *packfile_store_load_pack(struct packfile_store *store, +struct packed_git *packfile_store_load_pack(struct odb_source_packed *store, const char *idx_path, int local); -int packfile_store_freshen_object(struct packfile_store *store, - const struct object_id *oid); - enum kept_pack_type { KEPT_PACK_ON_DISK = (1 << 0), KEPT_PACK_IN_CORE = (1 << 1), KEPT_PACK_IN_CORE_OPEN = (1 << 2), }; -/* - * Count the number objects contained in the given packfile store. If - * successful, the number of objects will be written to the `out` pointer. - * - * Return 0 on success, a negative error code otherwise. - */ -int packfile_store_count_objects(struct packfile_store *store, - enum odb_count_objects_flags flags, - unsigned long *out); - /* * Retrieve the cache of kept packs from the given packfile store. Accepts a * combination of `kept_pack_type` flags. The cache is computed on demand and * will be recomputed whenever the flags change. */ -struct packed_git **packfile_store_get_kept_pack_cache(struct packfile_store *store, +struct packed_git **packfile_store_get_kept_pack_cache(struct odb_source_packed *store, unsigned flags); struct pack_window { @@ -356,26 +214,6 @@ int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn, void *data, enum odb_for_each_object_flags flags); -/* - * Iterate through all packed objects in the given packfile store and invoke - * the callback function for each of them. If an object info request is given, - * then the object info will be read for every individual object and passed to - * the callback as if `packfile_store_read_object_info()` was called for the - * object. - * - * The flags parameter is a combination of `odb_for_each_object_flags`. - */ -int packfile_store_for_each_object(struct packfile_store *store, - const struct object_info *request, - odb_for_each_object_cb cb, - void *cb_data, - const struct odb_for_each_object_options *opts); - -int packfile_store_find_abbrev_len(struct packfile_store *store, - const struct object_id *oid, - unsigned min_len, - unsigned *out); - /* A hook to report invalid files in pack directory */ #define PACKDIR_FILE_PACK 1 #define PACKDIR_FILE_IDX 2 @@ -454,6 +292,10 @@ off_t nth_packed_object_offset(const struct packed_git *, uint32_t n); */ off_t find_pack_entry_one(const struct object_id *oid, struct packed_git *); +int packfile_fill_entry(struct packed_git *p, + const struct object_id *oid, + struct pack_entry *e); + int is_pack_valid(struct packed_git *); void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *); unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, size_t *sizep); @@ -479,6 +321,8 @@ extern int do_check_packed_object_crc; */ int packed_object_info(struct packed_git *pack, off_t offset, struct object_info *); +int packed_object_info_with_index_pos(struct packed_git *p, off_t obj_offset, + uint32_t *maybe_index_pos, struct object_info *oi); void mark_bad_packed_object(struct packed_git *, const struct object_id *); const struct packed_git *has_packed_and_bad(struct repository *, const struct object_id *); diff --git a/repack-geometry.c b/repack-geometry.c index 2064683dcf..15b3412950 100644 --- a/repack-geometry.c +++ b/repack-geometry.c @@ -32,7 +32,8 @@ void pack_geometry_init(struct pack_geometry *geometry, { struct packed_git *p; struct strbuf buf = STRBUF_INIT; - struct multi_pack_index *m = get_multi_pack_index(existing->source); + struct odb_source_files *files = odb_source_files_downcast(existing->source); + struct multi_pack_index *m = get_multi_pack_index(files->packed); repo_for_each_pack(existing->repo, p) { if (geometry->midx_layer_threshold_set && m && diff --git a/repack-midx.c b/repack-midx.c index b6b1de7180..7c7c3620e5 100644 --- a/repack-midx.c +++ b/repack-midx.c @@ -557,13 +557,14 @@ static void repack_make_midx_append_plan(struct repack_write_midx_opts *opts, struct midx_compaction_step **steps_p, size_t *steps_nr_p) { + struct odb_source_files *files = odb_source_files_downcast(opts->existing->source); struct multi_pack_index *m; struct midx_compaction_step *steps = NULL; struct midx_compaction_step *step; size_t steps_nr = 0, steps_alloc = 0; odb_reprepare(opts->existing->repo->objects); - m = get_multi_pack_index(opts->existing->source); + m = get_multi_pack_index(files->packed); if (opts->names->nr) { struct strbuf buf = STRBUF_INIT; @@ -606,6 +607,7 @@ static int repack_make_midx_compaction_plan(struct repack_write_midx_opts *opts, struct midx_compaction_step **steps_p, size_t *steps_nr_p) { + struct odb_source_files *files = odb_source_files_downcast(opts->existing->source); struct multi_pack_index *m; struct midx_compaction_step *steps = NULL; struct midx_compaction_step step = { 0 }; @@ -618,7 +620,7 @@ static int repack_make_midx_compaction_plan(struct repack_write_midx_opts *opts, opts->existing->repo); odb_reprepare(opts->existing->repo->objects); - m = get_multi_pack_index(opts->existing->source); + m = get_multi_pack_index(files->packed); for (i = 0; m && i < m->num_packs + m->num_packs_in_base; i++) { if (prepare_midx_pack(m, i)) { @@ -938,6 +940,7 @@ out: static int write_midx_incremental(struct repack_write_midx_opts *opts) { + struct odb_source_files *files = odb_source_files_downcast(opts->existing->source); struct midx_compaction_step *steps = NULL; struct strbuf lock_name = STRBUF_INIT; struct lock_file lf; @@ -946,7 +949,7 @@ static int write_midx_incremental(struct repack_write_midx_opts *opts) size_t i; int ret = 0; - get_midx_chain_filename(opts->existing->source, &lock_name); + get_midx_chain_filename(files->packed, &lock_name); if (safe_create_leading_directories(opts->existing->repo, lock_name.buf)) die_errno(_("unable to create leading directories of %s"), diff --git a/repack.c b/repack.c index 571dabb665..d2aa58e134 100644 --- a/repack.c +++ b/repack.c @@ -59,10 +59,10 @@ void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, bool wrote_incremental_midx) { struct strbuf buf = STRBUF_INIT; - struct odb_source *source = repo->objects->sources; - struct multi_pack_index *m = get_multi_pack_index(source); + struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); + struct multi_pack_index *m = get_multi_pack_index(files->packed); strbuf_addf(&buf, "%s.pack", base_name); - if (m && source->local && midx_contains_pack(m, buf.buf)) { + if (m && files->base.local && midx_contains_pack(m, buf.buf)) { clear_midx_file(repo); if (!wrote_incremental_midx) clear_incremental_midx_files(repo, NULL); diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index 790000fb26..fb16ec0176 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -13,13 +13,16 @@ static struct multi_pack_index *setup_midx(const char *object_dir) { + struct odb_source_files *files; struct odb_source *source; setup_git_directory(the_repository); source = odb_find_source(the_repository->objects, object_dir); if (!source) source = odb_add_to_alternates_memory(the_repository->objects, object_dir); - return load_multi_pack_index(source); + files = odb_source_files_downcast(source); + + return load_multi_pack_index(files->packed); } static int read_midx_file(const char *object_dir, const char *checksum, @@ -70,7 +73,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_packs; i++) printf("%s\n", m->pack_names[i]); - printf("object-dir: %s\n", m->source->path); + printf("object-dir: %s\n", m->source->base.path); if (show_objects) { struct object_id oid;