From 5e44529d206936336e568725aec4fb5fe64138b7 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 11 Jul 2022 14:11:15 -0400 Subject: pack: support SHA256 in packfiles --- include/git2/odb_backend.h | 66 +++++++--- src/libgit2/commit_graph.c | 5 +- src/libgit2/indexer.c | 3 +- src/libgit2/midx.c | 5 +- src/libgit2/mwindow.c | 7 +- src/libgit2/mwindow.h | 5 +- src/libgit2/odb.c | 13 +- src/libgit2/odb_pack.c | 88 ++++++++++---- src/libgit2/pack.c | 134 ++++++++++++--------- src/libgit2/pack.h | 30 +++-- tests/libgit2/odb/foreach.c | 10 +- tests/libgit2/odb/loose.c | 12 -- tests/libgit2/odb/pack_data_one_sha256.h | 10 ++ tests/libgit2/odb/packed_one.c | 8 +- tests/libgit2/odb/packed_one_sha256.c | 75 ++++++++++++ tests/libgit2/odb/sorting.c | 6 +- ...d823776d752efc71661592db317474a0cf292915f31.idx | Bin 0 -> 1376 bytes ...823776d752efc71661592db317474a0cf292915f31.pack | Bin 0 -> 1406 bytes 18 files changed, 348 insertions(+), 129 deletions(-) create mode 100644 tests/libgit2/odb/pack_data_one_sha256.h create mode 100644 tests/libgit2/odb/packed_one_sha256.c create mode 100644 tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.idx create mode 100644 tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.pack diff --git a/include/git2/odb_backend.h b/include/git2/odb_backend.h index 092eb91ae..f68dc4e4b 100644 --- a/include/git2/odb_backend.h +++ b/include/git2/odb_backend.h @@ -24,6 +24,26 @@ GIT_BEGIN_DECL * Constructors for in-box ODB backends. */ +/** Options for configuring a packfile object backend. */ +typedef struct { + unsigned int version; /**< version for the struct */ + + /** + * Type of object IDs to use for this object database, or + * 0 for default (currently SHA1). + */ + git_oid_t oid_type; +} git_odb_backend_pack_options; + +/* The current version of the diff options structure */ +#define GIT_ODB_BACKEND_PACK_OPTIONS_VERSION 1 + +/* Stack initializer for odb pack backend options. Alternatively use + * `git_odb_backend_pack_options_init` programmatic initialization. + */ +#define GIT_ODB_BACKEND_PACK_OPTIONS_INIT \ + { GIT_ODB_BACKEND_PACK_OPTIONS_VERSION } + /** * Create a backend for the packfiles. * @@ -32,7 +52,38 @@ GIT_BEGIN_DECL * * @return 0 or an error code */ -GIT_EXTERN(int) git_odb_backend_pack(git_odb_backend **out, const char *objects_dir); +#ifdef GIT_EXPERIMENTAL_SHA256 +GIT_EXTERN(int) git_odb_backend_pack( + git_odb_backend **out, + const char *objects_dir, + const git_odb_backend_pack_options *opts); +#else +GIT_EXTERN(int) git_odb_backend_pack( + git_odb_backend **out, + const char *objects_dir); +#endif + +/** + * Create a backend out of a single packfile + * + * This can be useful for inspecting the contents of a single + * packfile. + * + * @param out location to store the odb backend pointer + * @param index_file path to the packfile's .idx file + * + * @return 0 or an error code + */ +#ifdef GIT_EXPERIMENTAL_SHA256 +GIT_EXTERN(int) git_odb_backend_one_pack( + git_odb_backend **out, + const char *index_file, + const git_odb_backend_pack_options *opts); +#else +GIT_EXTERN(int) git_odb_backend_one_pack( + git_odb_backend **out, + const char *index_file); +#endif typedef enum { GIT_ODB_BACKEND_LOOSE_FSYNC = (1 << 0) @@ -90,19 +141,6 @@ GIT_EXTERN(int) git_odb_backend_loose( const char *objects_dir, git_odb_backend_loose_options *opts); -/** - * Create a backend out of a single packfile - * - * This can be useful for inspecting the contents of a single - * packfile. - * - * @param out location to store the odb backend pointer - * @param index_file path to the packfile's .idx file - * - * @return 0 or an error code - */ -GIT_EXTERN(int) git_odb_backend_one_pack(git_odb_backend **out, const char *index_file); - /** Streaming mode */ typedef enum { GIT_STREAM_RDONLY = (1 << 1), diff --git a/src/libgit2/commit_graph.c b/src/libgit2/commit_graph.c index d4cb93eb4..eafcab9a8 100644 --- a/src/libgit2/commit_graph.c +++ b/src/libgit2/commit_graph.c @@ -524,7 +524,7 @@ int git_commit_graph_entry_find( hi = ntohl(file->oid_fanout[(int)short_oid->id[0]]); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(file->oid_fanout[(int)short_oid->id[0] - 1])); - pos = git_pack__lookup_id(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id); + pos = git_pack__lookup_id(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1); if (pos >= 0) { /* An object matching exactly the oid was found */ @@ -712,7 +712,8 @@ int git_commit_graph_writer_add_index_file( if (error < 0) goto cleanup; - error = git_mwindow_get_pack(&p, idx_path); + /* TODO: SHA256 */ + error = git_mwindow_get_pack(&p, idx_path, 0); if (error < 0) goto cleanup; diff --git a/src/libgit2/indexer.c b/src/libgit2/indexer.c index 60625b6c4..089dd34e8 100644 --- a/src/libgit2/indexer.c +++ b/src/libgit2/indexer.c @@ -179,7 +179,8 @@ int git_indexer_new( if (fd < 0) goto cleanup; - error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path)); + /* TODO: SHA256 */ + error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path), 0); git_str_dispose(&tmp_path); if (error < 0) diff --git a/src/libgit2/midx.c b/src/libgit2/midx.c index 37de58ccd..1a0bdd252 100644 --- a/src/libgit2/midx.c +++ b/src/libgit2/midx.c @@ -392,7 +392,7 @@ int git_midx_entry_find( hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1])); - pos = git_pack__lookup_id(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id); + pos = git_pack__lookup_id(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1); if (pos >= 0) { /* An object matching exactly the oid was found */ @@ -549,7 +549,8 @@ int git_midx_writer_add( if (error < 0) return error; - error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf)); + /* TODO: SHA256 */ + error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf), 0); git_str_dispose(&idx_path_buf); if (error < 0) return error; diff --git a/src/libgit2/mwindow.c b/src/libgit2/mwindow.c index ad649490a..b8295d9d1 100644 --- a/src/libgit2/mwindow.c +++ b/src/libgit2/mwindow.c @@ -61,7 +61,10 @@ int git_mwindow_global_init(void) return git_runtime_shutdown_register(git_mwindow_global_shutdown); } -int git_mwindow_get_pack(struct git_pack_file **out, const char *path) +int git_mwindow_get_pack( + struct git_pack_file **out, + const char *path, + git_oid_t oid_type) { struct git_pack_file *pack; char *packname; @@ -86,7 +89,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path) } /* If we didn't find it, we need to create it */ - if ((error = git_packfile_alloc(&pack, path)) < 0) { + if ((error = git_packfile_alloc(&pack, path, oid_type)) < 0) { git_mutex_unlock(&git__mwindow_mutex); return error; } diff --git a/src/libgit2/mwindow.h b/src/libgit2/mwindow.h index e32ab99d4..8e6df2613 100644 --- a/src/libgit2/mwindow.h +++ b/src/libgit2/mwindow.h @@ -48,7 +48,10 @@ void git_mwindow_close(git_mwindow **w_cursor); extern int git_mwindow_global_init(void); struct git_pack_file; /* just declaration to avoid cyclical includes */ -int git_mwindow_get_pack(struct git_pack_file **out, const char *path); +int git_mwindow_get_pack( + struct git_pack_file **out, + const char *path, + git_oid_t oid_type); int git_mwindow_put_pack(struct git_pack_file *pack); #endif diff --git a/src/libgit2/odb.c b/src/libgit2/odb.c index 00d68d6f4..a91732948 100644 --- a/src/libgit2/odb.c +++ b/src/libgit2/odb.c @@ -632,6 +632,7 @@ int git_odb__add_default_backends( ino_t inode; git_odb_backend *loose, *packed; git_odb_backend_loose_options loose_opts = GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT; + git_odb_backend_pack_options pack_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT; /* TODO: inodes are not really relevant on Win32, so we need to find * a cross-platform workaround for this */ @@ -670,6 +671,7 @@ int git_odb__add_default_backends( loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC; loose_opts.oid_type = db->options.oid_type; + pack_opts.oid_type = db->options.oid_type; /* add the loose object backend */ if (git_odb_backend_loose(&loose, objects_dir, &loose_opts) < 0 || @@ -677,8 +679,15 @@ int git_odb__add_default_backends( return -1; /* add the packed file backend */ - if (git_odb_backend_pack(&packed, objects_dir) < 0 || - add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0) +#ifdef GIT_EXPERIMENTAL_SHA256 + if (git_odb_backend_pack(&packed, objects_dir, &pack_opts) < 0) + return -1; +#else + if (git_odb_backend_pack(&packed, objects_dir) < 0) + return -1; +#endif + + if (add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0) return -1; if (git_mutex_lock(&db->lock) < 0) { diff --git a/src/libgit2/odb_pack.c b/src/libgit2/odb_pack.c index d6243cd93..0f55c229b 100644 --- a/src/libgit2/odb_pack.c +++ b/src/libgit2/odb_pack.c @@ -26,6 +26,7 @@ struct pack_backend { git_odb_backend parent; + git_odb_backend_pack_options opts; git_midx_file *midx; git_vector midx_packs; git_vector packs; @@ -251,7 +252,7 @@ static int packfile_load__cb(void *data, git_str *path) if (git_vector_search2(NULL, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0) return 0; - error = git_mwindow_get_pack(&pack, path->ptr); + error = git_mwindow_get_pack(&pack, path->ptr, backend->opts.oid_type); /* ignore missing .pack file as git does */ if (error == GIT_ENOTFOUND) { @@ -270,10 +271,11 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen { struct git_pack_file *last_found = backend->last_found, *p; git_midx_entry midx_entry; + size_t oid_hexsize = git_oid_hexsize(backend->opts.oid_type); size_t i; if (backend->midx && - git_midx_entry_find(&midx_entry, backend->midx, oid, GIT_OID_SHA1_HEXSIZE) == 0 && + git_midx_entry_find(&midx_entry, backend->midx, oid, oid_hexsize) == 0 && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) { e->offset = midx_entry.offset; git_oid_cpy(&e->id, &midx_entry.sha1); @@ -282,21 +284,21 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen } if (last_found && - git_pack_entry_find(e, last_found, oid, GIT_OID_SHA1_HEXSIZE) == 0) + git_pack_entry_find(e, last_found, oid, oid_hexsize) == 0) return 0; git_vector_foreach(&backend->packs, i, p) { if (p == last_found) continue; - if (git_pack_entry_find(e, p, oid, GIT_OID_SHA1_HEXSIZE) == 0) { + if (git_pack_entry_find(e, p, oid, oid_hexsize) == 0) { backend->last_found = p; return 0; } } return git_odb__error_notfound( - "failed to find pack entry", oid, GIT_OID_SHA1_HEXSIZE); + "failed to find pack entry", oid, oid_hexsize); } static int pack_entry_find_prefix( @@ -307,7 +309,7 @@ static int pack_entry_find_prefix( { int error; size_t i; - git_oid found_full_oid = GIT_OID_SHA1_ZERO; + git_oid found_full_oid = { 0 }; bool found = false; struct git_pack_file *last_found = backend->last_found, *p; git_midx_entry midx_entry; @@ -425,7 +427,10 @@ static int process_multi_pack_index_pack( } /* Pack was not found. Allocate a new one. */ - error = git_mwindow_get_pack(&pack, git_str_cstr(&pack_path)); + error = git_mwindow_get_pack( + &pack, + git_str_cstr(&pack_path), + backend->opts.oid_type); git_str_dispose(&pack_path); if (error < 0) return error; @@ -596,27 +601,28 @@ static int pack_backend__read_prefix( void **buffer_p, size_t *len_p, git_object_t *type_p, - git_odb_backend *backend, + git_odb_backend *_backend, const git_oid *short_oid, size_t len) { + struct pack_backend *backend = (struct pack_backend *)_backend; int error = 0; if (len < GIT_OID_MINPREFIXLEN) error = git_odb__error_ambiguous("prefix length too short"); - else if (len >= GIT_OID_SHA1_HEXSIZE) { + else if (len >= git_oid_hexsize(backend->opts.oid_type)) { /* We can fall back to regular read method */ - error = pack_backend__read(buffer_p, len_p, type_p, backend, short_oid); + error = pack_backend__read(buffer_p, len_p, type_p, _backend, short_oid); if (!error) git_oid_cpy(out_oid, short_oid); } else { struct git_pack_entry e; git_rawobj raw = {NULL}; - if ((error = pack_entry_find_prefix( - &e, (struct pack_backend *)backend, short_oid, len)) == 0 && - (error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0) + if ((error = pack_entry_find_prefix(&e, + backend, short_oid, len)) == 0 && + (error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0) { *buffer_p = raw.data; *len_p = raw.len; @@ -840,7 +846,10 @@ static void pack_backend__free(git_odb_backend *_backend) git__free(backend); } -static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) +static int pack_backend__alloc( + struct pack_backend **out, + size_t initial_size, + const git_odb_backend_pack_options *opts) { struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend)); GIT_ERROR_CHECK_ALLOC(backend); @@ -849,12 +858,19 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) git__free(backend); return -1; } + if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) { git_vector_free(&backend->midx_packs); git__free(backend); return -1; } + if (opts) + memcpy(&backend->opts, opts, sizeof(git_odb_backend_pack_options)); + + if (!backend->opts.oid_type) + backend->opts.oid_type = GIT_OID_DEFAULT; + backend->parent.version = GIT_ODB_BACKEND_VERSION; backend->parent.read = &pack_backend__read; @@ -873,17 +889,31 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) return 0; } -int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx) +#ifdef GIT_EXPERIMENTAL_SHA256 +int git_odb_backend_one_pack( + git_odb_backend **backend_out, + const char *idx, + const git_odb_backend_pack_options *opts) +#else +int git_odb_backend_one_pack( + git_odb_backend **backend_out, + const char *idx) +#endif { struct pack_backend *backend = NULL; struct git_pack_file *packfile = NULL; - if (pack_backend__alloc(&backend, 1) < 0) +#ifndef GIT_EXPERIMENTAL_SHA256 + git_odb_backend_pack_options *opts = NULL; +#endif + + git_oid_t oid_type = opts ? opts->oid_type : 0; + + if (pack_backend__alloc(&backend, 1, opts) < 0) return -1; - if (git_mwindow_get_pack(&packfile, idx) < 0 || - git_vector_insert(&backend->packs, packfile) < 0) - { + if (git_mwindow_get_pack(&packfile, idx, oid_type) < 0 || + git_vector_insert(&backend->packs, packfile) < 0) { pack_backend__free((git_odb_backend *)backend); return -1; } @@ -892,18 +922,30 @@ int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx) return 0; } -int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) +#ifdef GIT_EXPERIMENTAL_SHA256 +int git_odb_backend_pack( + git_odb_backend **backend_out, + const char *objects_dir, + const git_odb_backend_pack_options *opts) +#else +int git_odb_backend_pack( + git_odb_backend **backend_out, + const char *objects_dir) +#endif { int error = 0; struct pack_backend *backend = NULL; git_str path = GIT_STR_INIT; - if (pack_backend__alloc(&backend, 8) < 0) +#ifndef GIT_EXPERIMENTAL_SHA256 + git_odb_backend_pack_options *opts = NULL; +#endif + + if (pack_backend__alloc(&backend, 8, opts) < 0) return -1; if (!(error = git_str_joinpath(&path, objects_dir, "pack")) && - git_fs_path_isdir(git_str_cstr(&path))) - { + git_fs_path_isdir(git_str_cstr(&path))) { backend->pack_folder = git_str_detach(&path); error = pack_backend__refresh((git_odb_backend *)backend); } diff --git a/src/libgit2/pack.c b/src/libgit2/pack.c index 4ea6b5909..513f31c03 100644 --- a/src/libgit2/pack.c +++ b/src/libgit2/pack.c @@ -32,7 +32,7 @@ static int packfile_unpack_compressed( * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid * is ambiguous within the pack. * This method assumes that len is between - * GIT_OID_MINPREFIXLEN and GIT_OID_SHA1_HEXSIZE. + * GIT_OID_MINPREFIXLEN and the oid type's hexsize. */ static int pack_entry_find_offset( off64_t *offset_out, @@ -205,6 +205,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) size_t idx_size; struct stat st; int error; + /* TODO: properly open the file without access time using O_NOATIME */ git_file fd = git_futils_open_ro(path); if (fd < 0) @@ -218,8 +219,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) if (!S_ISREG(st.st_mode) || !git__is_sizet(st.st_size) || - (idx_size = (size_t)st.st_size) < 4 * 256 + 20 + 20) - { + (idx_size = (size_t)st.st_size) < (4 * 256) + (p->oid_size * 2)) { p_close(fd); git_error_set(GIT_ERROR_ODB, "invalid pack index '%s'", path); return -1; @@ -242,8 +242,9 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) return packfile_error("unsupported index version"); } - } else + } else { version = 1; + } nr = 0; index = idx_map; @@ -264,11 +265,11 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) /* * Total size: * - 256 index entries 4 bytes each - * - 24-byte entries * nr (20-byte sha1 + 4-byte offset) - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum + * - 24/36-byte entries * nr (20/32 byte SHA + 4-byte offset) + * - 20/32-byte SHA of the packfile + * - 20/32-byte SHA file checksum */ - if (idx_size != 4*256 + nr * 24 + 20 + 20) { + if (idx_size != (4 * 256 + (nr * (p->oid_size + 4)) + (p->oid_size * 2))) { git_futils_mmap_free(&p->index_map); return packfile_error("index is corrupted"); } @@ -277,16 +278,16 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) * Minimum size: * - 8 bytes of header * - 256 index entries 4 bytes each - * - 20-byte sha1 entry * nr + * - 20/32-byte SHA entry * nr * - 4-byte crc entry * nr * - 4-byte offset entry * nr - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum + * - 20/32-byte SHA of the packfile + * - 20/32-byte SHA file checksum * And after the 4-byte offset table might be a * variable sized table containing 8-byte entries * for offsets larger than 2^31. */ - unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20; + unsigned long min_size = 8 + (4 * 256) + (nr * (p->oid_size + 4 + 4)) + (p->oid_size * 2); unsigned long max_size = min_size; if (nr) @@ -365,12 +366,12 @@ static unsigned char *pack_window_open( * Don't allow a negative offset, as that means we've wrapped * around. */ - if (offset > (p->mwf.size - 20)) + if (offset > (p->mwf.size - p->oid_size)) goto cleanup; if (offset < 0) goto cleanup; - pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, 20, left); + pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, p->oid_size, left); cleanup: git_mutex_unlock(&p->mwf.lock); @@ -473,13 +474,13 @@ int git_packfile_unpack_header( return error; } - /* pack_window_open() assures us we have [base, base + 20) available - * as a range that we can look at at. (Its actually the hash - * size that is assured.) With our object header encoding - * the maximum deflated object size is 2^137, which is just - * insane, so we know won't exceed what we have been given. + /* pack_window_open() assures us we have [base, base + oid_size) + * available as a range that we can look at at. (It's actually + * the hash size that is assured.) With our object header + * encoding the maximum deflated object size is 2^137, which is + * just insane, so we know won't exceed what we have been given. */ - base = git_mwindow_open(&p->mwf, w_curs, *curpos, 20, &left); + base = git_mwindow_open(&p->mwf, w_curs, *curpos, p->oid_size, &left); git_mutex_unlock(&p->lock); git_mutex_unlock(&p->mwf.lock); if (base == NULL) @@ -977,11 +978,12 @@ int get_delta_base( /* Assumption: the only reason this would fail is because the file is too small */ if (base_info == NULL) return GIT_EBUFS; - /* pack_window_open() assured us we have [base_info, base_info + 20) - * as a range that we can look at without walking off the - * end of the mapped window. Its actually the hash size - * that is assured. An OFS_DELTA longer than the hash size - * is stupid, as then a REF_DELTA would be smaller to store. + /* pack_window_open() assured us we have + * [base_info, base_info + oid_size) as a range that we can look + * at without walking off the end of the mapped window. Its + * actually the hash size that is assured. An OFS_DELTA longer + * than the hash size is stupid, as then a REF_DELTA would be + * smaller to store. */ if (type == GIT_OBJECT_OFS_DELTA) { unsigned used = 0; @@ -1002,7 +1004,7 @@ int get_delta_base( *curpos += used; } else if (type == GIT_OBJECT_REF_DELTA) { git_oid base_oid; - git_oid_fromraw(&base_oid, base_info, GIT_OID_SHA1); + git_oid_fromraw(&base_oid, base_info, p->oid_type); /* If we have the cooperative cache, search in it first */ if (p->has_cache) { @@ -1012,7 +1014,7 @@ int get_delta_base( if (entry->offset == 0) return packfile_error("delta offset is zero"); - *curpos += 20; + *curpos += p->oid_size; *delta_base_out = entry->offset; return 0; } else { @@ -1025,9 +1027,9 @@ int get_delta_base( } /* The base entry _must_ be in the same pack */ - if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, GIT_OID_SHA1_HEXSIZE) < 0) + if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, p->oid_hexsize) < 0) return packfile_error("base entry delta is not in the same pack"); - *curpos += 20; + *curpos += p->oid_size; } else return packfile_error("unknown object type"); @@ -1083,8 +1085,8 @@ static int packfile_open_locked(struct git_pack_file *p) { struct stat st; struct git_pack_header hdr; - unsigned char sha1[GIT_OID_SHA1_SIZE]; - unsigned char *idx_sha1; + unsigned char checksum[GIT_OID_MAX_SIZE]; + unsigned char *idx_checksum; if (pack_index_open_locked(p) < 0) return git_odb__error_notfound("failed to open packfile", NULL, 0); @@ -1131,12 +1133,13 @@ static int packfile_open_locked(struct git_pack_file *p) /* Verify the pack matches its index. */ if (p->num_objects != ntohl(hdr.hdr_entries) || - p_pread(p->mwf.fd, sha1, GIT_OID_SHA1_SIZE, p->mwf.size - GIT_OID_SHA1_SIZE) < 0) + p_pread(p->mwf.fd, checksum, p->oid_size, p->mwf.size - p->oid_size) < 0) goto cleanup; - idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40; + idx_checksum = ((unsigned char *)p->index_map.data) + + p->index_map.len - (p->oid_size * 2); - if (git_oid_raw_cmp(sha1, idx_sha1, GIT_OID_SHA1_SIZE) != 0) + if (git_oid_raw_cmp(checksum, idx_checksum, p->oid_size) != 0) goto cleanup; if (git_mwindow_file_register(&p->mwf) < 0) @@ -1171,7 +1174,10 @@ int git_packfile__name(char **out, const char *path) return 0; } -int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) +int git_packfile_alloc( + struct git_pack_file **pack_out, + const char *path, + git_oid_t oid_type) { struct stat st; struct git_pack_file *p; @@ -1219,6 +1225,9 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) p->pack_local = 1; p->mtime = (git_time_t)st.st_mtime; p->index_version = -1; + p->oid_type = oid_type ? oid_type : GIT_OID_DEFAULT; + p->oid_size = git_oid_size(p->oid_type); + p->oid_hexsize = git_oid_hexsize(p->oid_type); if (git_mutex_init(&p->lock) < 0) { git_error_set(GIT_ERROR_OS, "failed to initialize packfile mutex"); @@ -1260,9 +1269,9 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t end = index + p->index_map.len; index += 4 * 256; if (p->index_version == 1) - return ntohl(*((uint32_t *)(index + 24 * n))); + return ntohl(*((uint32_t *)(index + (p->oid_size + 4) * n))); - index += 8 + p->num_objects * (20 + 4); + index += 8 + p->num_objects * (p->oid_size + 4); off32 = ntohl(*((uint32_t *)(index + 4 * n))); if (!(off32 & 0x80000000)) return off32; @@ -1273,7 +1282,7 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t return -1; return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) | - ntohl(*((uint32_t *)(index + 4))); + ntohl(*((uint32_t *)(index + 4))); } static int git__memcmp4(const void *a, const void *b) { @@ -1326,15 +1335,18 @@ int git_pack_foreach_entry( } if (p->index_version > 1) { - const unsigned char *off = index + 24 * p->num_objects; + const unsigned char *off = index + + (p->oid_size + 4) * p->num_objects; + for (i = 0; i < p->num_objects; i++) git_vector_insert(&offsets, (void*)&off[4 * i]); + git_vector_sort(&offsets); git_vector_foreach(&offsets, i, current) git_vector_insert(&oids, (void*)&index[5 * (current - off)]); } else { for (i = 0; i < p->num_objects; i++) - git_vector_insert(&offsets, (void*)&index[24 * i]); + git_vector_insert(&offsets, (void*)&index[(p->oid_size + 4) * i]); git_vector_sort(&offsets); git_vector_foreach(&offsets, i, current) git_vector_insert(&oids, (void*)¤t[4]); @@ -1362,7 +1374,7 @@ int git_pack_foreach_entry( git_array_clear(oids); GIT_ERROR_CHECK_ALLOC(oid); } - git_oid_fromraw(oid, p->ids[i], GIT_OID_SHA1); + git_oid_fromraw(oid, p->ids[i], p->oid_type); } git_mutex_unlock(&p->lock); @@ -1412,10 +1424,13 @@ int git_pack_foreach_entry_offset( /* all offsets should have been validated by pack_index_check_locked */ if (p->index_version > 1) { - const unsigned char *offsets = index + 24 * p->num_objects; + const unsigned char *offsets = index + + (p->oid_size + 4) * p->num_objects; const unsigned char *large_offset_ptr; - const unsigned char *large_offsets = index + 28 * p->num_objects; - const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - 20; + const unsigned char *large_offsets = index + + (p->oid_size + 8) * p->num_objects; + const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - p->oid_size; + for (i = 0; i < p->num_objects; i++) { current_offset = ntohl(*(const uint32_t *)(offsets + 4 * i)); if (current_offset & 0x80000000) { @@ -1428,7 +1443,7 @@ int git_pack_foreach_entry_offset( ntohl(*((uint32_t *)(large_offset_ptr + 4))); } - git_oid_fromraw(¤t_oid, (index + 20 * i), GIT_OID_SHA1); + git_oid_fromraw(¤t_oid, (index + p->oid_size * i), p->oid_type); if ((error = cb(¤t_oid, current_offset, data)) != 0) { error = git_error_set_after_callback(error); goto cleanup; @@ -1436,8 +1451,8 @@ int git_pack_foreach_entry_offset( } } else { for (i = 0; i < p->num_objects; i++) { - current_offset = ntohl(*(const uint32_t *)(index + 24 * i)); - git_oid_fromraw(¤t_oid, (index + 24 * i + 4), GIT_OID_SHA1); + current_offset = ntohl(*(const uint32_t *)(index + (p->oid_size + 4) * i)); + git_oid_fromraw(¤t_oid, (index + (p->oid_size + 4) * i + 4), p->oid_type); if ((error = cb(¤t_oid, current_offset, data)) != 0) { error = git_error_set_after_callback(error); goto cleanup; @@ -1455,13 +1470,15 @@ int git_pack__lookup_id( size_t stride, unsigned lo, unsigned hi, - const unsigned char *oid_prefix) + const unsigned char *oid_prefix, + const git_oid_t oid_type) { const unsigned char *base = oid_lookup_table; + size_t oid_size = git_oid_size(oid_type); while (lo < hi) { unsigned mi = (lo + hi) / 2; - int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, GIT_OID_SHA1_SIZE); + int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, oid_size); if (!cmp) return mi; @@ -1516,9 +1533,9 @@ static int pack_entry_find_offset( lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1])); if (p->index_version > 1) { - stride = 20; + stride = p->oid_size; } else { - stride = 24; + stride = p->oid_size + 4; index += 4; } @@ -1527,7 +1544,8 @@ static int pack_entry_find_offset( short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects); #endif - pos = git_pack__lookup_id(index, stride, lo, hi, short_oid->id); + pos = git_pack__lookup_id(index, stride, lo, hi, + short_oid->id, p->oid_type); if (pos >= 0) { /* An object matching exactly the oid was found */ @@ -1545,7 +1563,9 @@ static int pack_entry_find_offset( } } - if (found && len != GIT_OID_SHA1_HEXSIZE && pos + 1 < (int)p->num_objects) { + if (found && + len != p->oid_hexsize && + pos + 1 < (int)p->num_objects) { /* Check for ambiguousity */ const unsigned char *next = current + stride; @@ -1570,13 +1590,13 @@ static int pack_entry_find_offset( } *offset_out = offset; - git_oid_fromraw(found_oid, current, GIT_OID_SHA1); + git_oid_fromraw(found_oid, current, p->oid_type); #ifdef INDEX_DEBUG_LOOKUP { - unsigned char hex_sha1[GIT_OID_SHA1_HEXSIZE + 1]; + char hex_sha1[p->oid_hexsize + 1]; git_oid_fmt(hex_sha1, found_oid); - hex_sha1[GIT_OID_SHA1_HEXSIZE] = '\0'; + hex_sha1[p->oid_hexsize] = '\0'; printf("found lo=%d %s\n", lo, hex_sha1); } #endif @@ -1598,7 +1618,7 @@ int git_pack_entry_find( GIT_ASSERT_ARG(p); - if (len == GIT_OID_SHA1_HEXSIZE && p->num_bad_objects) { + if (len == p->oid_hexsize && p->num_bad_objects) { unsigned i; for (i = 0; i < p->num_bad_objects; i++) if (git_oid__cmp(short_oid, &p->bad_object_ids[i]) == 0) diff --git a/src/libgit2/pack.h b/src/libgit2/pack.h index 6c9dbf20b..1a9eb14b2 100644 --- a/src/libgit2/pack.h +++ b/src/libgit2/pack.h @@ -101,9 +101,15 @@ struct git_pack_file { uint32_t num_bad_objects; git_oid *bad_object_ids; /* array of git_oid */ + git_oid_t oid_type; + unsigned oid_hexsize:7, + oid_size:6, + pack_local:1, + pack_keep:1, + has_cache:1; + int index_version; git_time_t mtime; - unsigned pack_local:1, pack_keep:1, has_cache:1; git_oidmap *idx_cache; unsigned char **ids; @@ -116,21 +122,22 @@ struct git_pack_file { }; /** - * Return the position where an OID (or a prefix) would be inserted within the - * OID Lookup Table of an .idx file. This performs binary search between the lo - * and hi indices. + * Return the position where an OID (or a prefix) would be inserted within + * the OID Lookup Table of an .idx file. This performs binary search + * between the lo and hi indices. * - * The stride parameter is provided because .idx files version 1 store the OIDs - * interleaved with the 4-byte file offsets of the objects within the .pack - * file (stride = 24), whereas files with version 2 store them in a contiguous - * flat array (stride = 20). + * The stride parameter is provided because .idx files version 1 store the + * OIDs interleaved with the 4-byte file offsets of the objects within the + * .pack file (stride = oid_size + 4), whereas files with version 2 store + * them in a contiguous flat array (stride = oid_size). */ int git_pack__lookup_id( const void *id_lookup_table, size_t stride, unsigned lo, unsigned hi, - const unsigned char *id_prefix); + const unsigned char *id_prefix, + const git_oid_t oid_type); struct git_pack_entry { off64_t offset; @@ -178,7 +185,10 @@ int get_delta_base( off64_t delta_obj_offset); void git_packfile_free(struct git_pack_file *p, bool unlink_packfile); -int git_packfile_alloc(struct git_pack_file **pack_out, const char *path); +int git_packfile_alloc( + struct git_pack_file **pack_out, + const char *path, + git_oid_t oid_type); int git_pack_entry_find( struct git_pack_entry *e, diff --git a/tests/libgit2/odb/foreach.c b/tests/libgit2/odb/foreach.c index 98b5d80b7..5e4dae41b 100644 --- a/tests/libgit2/odb/foreach.c +++ b/tests/libgit2/odb/foreach.c @@ -52,7 +52,15 @@ void test_odb_foreach__one_pack(void) int nobj = 0; cl_git_pass(git_odb_new(&_odb, NULL)); - cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"))); + cl_git_pass(git_odb_backend_one_pack( + &backend, + cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx") + +#ifdef GIT_EXPERIMENTAL_SHA256 +, NULL +#endif + + )); cl_git_pass(git_odb_add_backend(_odb, backend, 1)); _repo = NULL; diff --git a/tests/libgit2/odb/loose.c b/tests/libgit2/odb/loose.c index 93895deb5..9651ebab6 100644 --- a/tests/libgit2/odb/loose.c +++ b/tests/libgit2/odb/loose.c @@ -188,18 +188,6 @@ void test_odb_loose__exists_sha256(void) #endif } -void test_odb_loose__zzz(void) -{ - write_object_files(&one_sha256); - write_object_files(&commit_sha256); - write_object_files(&tree_sha256); - write_object_files(&tag_sha256); - write_object_files(&zero_sha256); - write_object_files(&two_sha256); - write_object_files(&some_sha256); - abort(); -} - void test_odb_loose__simple_reads_sha1(void) { test_read_object(&commit); diff --git a/tests/libgit2/odb/pack_data_one_sha256.h b/tests/libgit2/odb/pack_data_one_sha256.h new file mode 100644 index 000000000..4b9282a40 --- /dev/null +++ b/tests/libgit2/odb/pack_data_one_sha256.h @@ -0,0 +1,10 @@ +/* Just a few to make sure it's working, the rest is tested already */ +static const char *packed_objects_one_sha256[] = { + "083c2b8b445640d171e7aa667b0b32007006f786711032ebb82931cca69cc15b", + "34a4854b5426f92460b4ae35e6d79746be66c38bffd6ef3bc4f053ed78a36ba4", + "473a0f4c3be8a93681a267e3b1e9a7dcda1185436fe141f7749120a303721813", + "4c0d52d180c61d01ce1a91dec5ee58f0cbe65fd59433aea803ab927965493fd7", + "a2a430fb63b294f868af4ef6ccc9c3e8256e370859ce578a23837ac85337f562", + "f535d7595d5d0e5e530b5deb34542c96491fea300a1318036b605306548cb225", + "f8625e43f9e04f24291f77cdbe4c71b3c2a3b0003f60419b3ed06a058d766c8b" +}; diff --git a/tests/libgit2/odb/packed_one.c b/tests/libgit2/odb/packed_one.c index 777b54b98..8822585c6 100644 --- a/tests/libgit2/odb/packed_one.c +++ b/tests/libgit2/odb/packed_one.c @@ -11,7 +11,13 @@ void test_odb_packed_one__initialize(void) git_odb_backend *backend = NULL; cl_git_pass(git_odb_new(&_odb, NULL)); - cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx"))); + cl_git_pass(git_odb_backend_one_pack( + &backend, + cl_fixture("testrepo.git/objects/pack/pack-a81e489679b7d3418f9ab594bda8ceb37dd4c695.idx") +#ifdef GIT_EXPERIMENTAL_SHA256 + , NULL +#endif + )); cl_git_pass(git_odb_add_backend(_odb, backend, 1)); } diff --git a/tests/libgit2/odb/packed_one_sha256.c b/tests/libgit2/odb/packed_one_sha256.c new file mode 100644 index 000000000..6cebc321b --- /dev/null +++ b/tests/libgit2/odb/packed_one_sha256.c @@ -0,0 +1,75 @@ +#include "clar_libgit2.h" +#include "git2/odb_backend.h" + +#include "pack_data_one_sha256.h" +#include "pack.h" + +#ifdef GIT_EXPERIMENTAL_SHA256 +static git_odb *_odb; +#endif + +void test_odb_packed_one_sha256__initialize(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + git_odb_backend *backend = NULL; + git_odb_options odb_opts = GIT_ODB_OPTIONS_INIT; + git_odb_backend_pack_options backend_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT; + + odb_opts.oid_type = GIT_OID_SHA256; + backend_opts.oid_type = GIT_OID_SHA256; + + cl_git_pass(git_odb_new(&_odb, &odb_opts)); + cl_git_pass(git_odb_backend_one_pack(&backend, cl_fixture("packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.idx"), &backend_opts)); + cl_git_pass(git_odb_add_backend(_odb, backend, 1)); +#endif +} + +void test_odb_packed_one_sha256__cleanup(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + git_odb_free(_odb); + _odb = NULL; +#endif +} + +void test_odb_packed_one_sha256__mass_read(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(packed_objects_one_sha256); ++i) { + git_oid id; + git_odb_object *obj; + + cl_git_pass(git_oid_fromstr(&id, packed_objects_one_sha256[i], GIT_OID_SHA256)); + cl_assert(git_odb_exists(_odb, &id) == 1); + cl_git_pass(git_odb_read(&obj, _odb, &id)); + + git_odb_object_free(obj); + } +#endif +} + +void test_odb_packed_one_sha256__read_header_0(void) +{ +#ifdef GIT_EXPERIMENTAL_SHA256 + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(packed_objects_one_sha256); ++i) { + git_oid id; + git_odb_object *obj; + size_t len; + git_object_t type; + + cl_git_pass(git_oid_fromstr(&id, packed_objects_one_sha256[i], GIT_OID_SHA256)); + + cl_git_pass(git_odb_read(&obj, _odb, &id)); + cl_git_pass(git_odb_read_header(&len, &type, _odb, &id)); + + cl_assert(obj->cached.size == len); + cl_assert(obj->cached.type == type); + + git_odb_object_free(obj); + } +#endif +} diff --git a/tests/libgit2/odb/sorting.c b/tests/libgit2/odb/sorting.c index 33ce289b5..dc4d615c7 100644 --- a/tests/libgit2/odb/sorting.c +++ b/tests/libgit2/odb/sorting.c @@ -82,7 +82,11 @@ void test_odb_sorting__override_default_backend_priority(void) cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_LOOSE_PRIORITY, 5)); cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ODB_PACKED_PRIORITY, 3)); - git_odb_backend_pack(&packed, "./testrepo.git/objects"); + git_odb_backend_pack(&packed, "./testrepo.git/objects" +#ifdef GIT_EXPERIMENTAL_SHA256 + , NULL +#endif + ); git_odb_backend_loose(&loose, "./testrepo.git/objects", NULL); cl_git_pass(git_odb_open(&new_odb, cl_fixture("testrepo.git/objects"), NULL)); diff --git a/tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.idx b/tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.idx new file mode 100644 index 000000000..7aa472cea Binary files /dev/null and b/tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.idx differ diff --git a/tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.pack b/tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.pack new file mode 100644 index 000000000..adbe70ad3 Binary files /dev/null and b/tests/resources/packfile-sha256/pack-b4a043c0ec5e079e8ac67d823776d752efc71661592db317474a0cf292915f31.pack differ -- cgit v1.2.1