diff options
author | Yuang Li <yuangli88@hotmail.com> | 2022-08-11 10:29:57 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-11 10:29:57 +0100 |
commit | 7c2b1f45ff02ba12544b3f81d433ee698b44127b (patch) | |
tree | 967b7e0e30de827ceba74a8ef9b707723d7a1c59 | |
parent | a491917f8e37108b6180590736d833095626b1ec (diff) | |
parent | 8b521f018b734f58db2b9aec184e0f96422f140e (diff) | |
download | libgit2-7c2b1f45ff02ba12544b3f81d433ee698b44127b.tar.gz |
Merge pull request #6 from lya001/shallow-clone-network
Shallow clone network
-rw-r--r-- | include/git2/remote.h | 16 | ||||
-rw-r--r-- | include/git2/repository.h | 1 | ||||
-rw-r--r-- | include/git2/sys/transport.h | 26 | ||||
-rw-r--r-- | src/libgit2/clone.c | 5 | ||||
-rw-r--r-- | src/libgit2/commit.c | 2 | ||||
-rw-r--r-- | src/libgit2/fetch.c | 33 | ||||
-rw-r--r-- | src/libgit2/grafts.c | 10 | ||||
-rw-r--r-- | src/libgit2/grafts.h | 2 | ||||
-rw-r--r-- | src/libgit2/object.c | 21 | ||||
-rw-r--r-- | src/libgit2/object.h | 6 | ||||
-rw-r--r-- | src/libgit2/remote.c | 6 | ||||
-rw-r--r-- | src/libgit2/remote.h | 1 | ||||
-rw-r--r-- | src/libgit2/repository.c | 53 | ||||
-rw-r--r-- | src/libgit2/repository.h | 3 | ||||
-rw-r--r-- | src/libgit2/transports/local.c | 6 | ||||
-rw-r--r-- | src/libgit2/transports/smart.c | 46 | ||||
-rw-r--r-- | src/libgit2/transports/smart.h | 23 | ||||
-rw-r--r-- | src/libgit2/transports/smart_pkt.c | 94 | ||||
-rw-r--r-- | src/libgit2/transports/smart_protocol.c | 64 | ||||
-rw-r--r-- | src/util/array.h | 4 | ||||
-rw-r--r-- | tests/libgit2/clone/shallow.c | 174 | ||||
-rw-r--r-- | tests/libgit2/transports/smart/shallowarray.c | 52 |
22 files changed, 608 insertions, 40 deletions
diff --git a/include/git2/remote.h b/include/git2/remote.h index 8c9c26f3f..f1cee17aa 100644 --- a/include/git2/remote.h +++ b/include/git2/remote.h @@ -744,6 +744,20 @@ typedef struct { git_proxy_options proxy_opts; /** + * Depth of the fetch to perform, has to be a positive integer. + * + * The default is -1, which will fetch the full history. + */ + int depth; + + /** + * Convert a shallow repository to a full repository. + * + * The default is 0, which means the flag is off. + */ + int unshallow; + + /** * Whether to allow off-site redirects. If this is not * specified, the `http.followRedirects` configuration setting * will be consulted. @@ -758,7 +772,7 @@ typedef struct { #define GIT_FETCH_OPTIONS_VERSION 1 #define GIT_FETCH_OPTIONS_INIT { GIT_FETCH_OPTIONS_VERSION, GIT_REMOTE_CALLBACKS_INIT, GIT_FETCH_PRUNE_UNSPECIFIED, 1, \ - GIT_REMOTE_DOWNLOAD_TAGS_UNSPECIFIED, GIT_PROXY_OPTIONS_INIT } + GIT_REMOTE_DOWNLOAD_TAGS_UNSPECIFIED, GIT_PROXY_OPTIONS_INIT, -1, 0 } /** * Initialize git_fetch_options structure diff --git a/include/git2/repository.h b/include/git2/repository.h index c87f3c962..258b9ac00 100644 --- a/include/git2/repository.h +++ b/include/git2/repository.h @@ -11,6 +11,7 @@ #include "types.h" #include "oid.h" #include "buffer.h" +#include "oidarray.h" /** * @file git2/repository.h diff --git a/include/git2/sys/transport.h b/include/git2/sys/transport.h index 06ae7079f..391765b77 100644 --- a/include/git2/sys/transport.h +++ b/include/git2/sys/transport.h @@ -25,6 +25,24 @@ GIT_BEGIN_DECL +/** + * Flags to pass to transport + * + * Currently unused. + */ +typedef enum { + GIT_TRANSPORTFLAGS_NONE = 0, +} git_transport_flags_t; + +typedef struct git_shallowarray git_shallowarray; + +typedef struct { + const git_remote_head * const *refs; + size_t count; + git_shallowarray *shallow_roots; + int depth; +} git_fetch_negotiation; + struct git_transport { unsigned int version; /**< The struct version */ @@ -84,8 +102,7 @@ struct git_transport { int GIT_CALLBACK(negotiate_fetch)( git_transport *transport, git_repository *repo, - const git_remote_head * const *refs, - size_t count); + const git_fetch_negotiation *fetch_data); /** * Start downloading the packfile from the remote repository. @@ -430,6 +447,11 @@ GIT_EXTERN(int) git_smart_subtransport_ssh( git_transport *owner, void *param); +GIT_EXTERN(size_t) git_shallowarray_count(git_shallowarray *array); +GIT_EXTERN(const git_oid *) git_shallowarray_get(git_shallowarray *array, size_t idx); +GIT_EXTERN(int) git_shallowarray_add(git_shallowarray *array, git_oid *oid); +GIT_EXTERN(int) git_shallowarray_remove(git_shallowarray *array, git_oid *oid); + /** @} */ GIT_END_DECL #endif diff --git a/src/libgit2/clone.c b/src/libgit2/clone.c index 1843875f8..6f34cb7ca 100644 --- a/src/libgit2/clone.c +++ b/src/libgit2/clone.c @@ -409,7 +409,10 @@ static int clone_into(git_repository *repo, git_remote *_remote, const git_fetch memcpy(&fetch_opts, opts, sizeof(git_fetch_options)); fetch_opts.update_fetchhead = 0; - fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL; + + if (opts->depth <= 0) + fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL; + git_str_printf(&reflog_message, "clone: from %s", git_remote_url(remote)); if ((error = git_remote_fetch(remote, NULL, &fetch_opts, git_str_cstr(&reflog_message))) != 0) diff --git a/src/libgit2/commit.c b/src/libgit2/commit.c index 75cc8837c..528d8beb7 100644 --- a/src/libgit2/commit.c +++ b/src/libgit2/commit.c @@ -422,7 +422,7 @@ static int commit_parse(git_commit *commit, const char *data, size_t size, unsig buffer += tree_len; } - while (git_oid__parse(&parent_id, &buffer, buffer_end, "parent ") == 0) { + while (git_object__parse_oid_header(&parent_id, &buffer, buffer_end, "parent ", GIT_OID_SHA1) == 0) { git_oid *new_id = git_array_alloc(commit->parent_ids); GIT_ERROR_CHECK_ALLOC(new_id); diff --git a/src/libgit2/fetch.c b/src/libgit2/fetch.c index 5c2fee617..3216b261c 100644 --- a/src/libgit2/fetch.c +++ b/src/libgit2/fetch.c @@ -20,6 +20,7 @@ #include "netops.h" #include "repository.h" #include "refs.h" +#include "transports/smart.h" static int maybe_want(git_remote *remote, git_remote_head *head, git_refspec *tagspec, git_remote_autotag_option_t tagopt) { @@ -59,8 +60,10 @@ static int mark_local(git_remote *remote) return -1; git_vector_foreach(&remote->refs, i, head) { - /* If we have the object, mark it so we don't ask for it */ - if (git_odb_exists(odb, &head->oid)) + /* If we have the object, mark it so we don't ask for it. + However if we are unshallowing, we need to ask for it + even though the head exists locally. */ + if (remote->nego.depth != INT_MAX && git_odb_exists(odb, &head->oid)) head->local = 1; else remote->need_pack = 1; @@ -172,6 +175,7 @@ int git_fetch_negotiate(git_remote *remote, const git_fetch_options *opts) git_transport *t = remote->transport; remote->need_pack = 0; + remote->nego.depth = opts->unshallow ? INT_MAX : opts->depth; if (filter_wants(remote, opts) < 0) return -1; @@ -180,24 +184,43 @@ int git_fetch_negotiate(git_remote *remote, const git_fetch_options *opts) if (!remote->need_pack) return 0; + if (opts->unshallow && opts->depth > 0) { + git_error_set(GIT_ERROR_INVALID, "options '--depth' and '--unshallow' cannot be used together"); + return -1; + } + /* * Now we have everything set up so we can start tell the * server what we want and what we have. */ + remote->nego.refs = (const git_remote_head * const *)remote->refs.contents; + remote->nego.count = remote->refs.length; + remote->nego.shallow_roots = git__malloc(sizeof(git_shallowarray)); + + git_array_init(remote->nego.shallow_roots->array); + + git_repository__shallow_roots(&remote->nego.shallow_roots->array, remote->repo); + return t->negotiate_fetch(t, remote->repo, - (const git_remote_head * const *)remote->refs.contents, - remote->refs.length); + &remote->nego); } int git_fetch_download_pack(git_remote *remote) { git_transport *t = remote->transport; + int error; if (!remote->need_pack) return 0; - return t->download_pack(t, remote->repo, &remote->stats); + if ((error = t->download_pack(t, remote->repo, &remote->stats)) < 0) + return error; + + if ((error = git_repository__shallow_roots_write(remote->repo, remote->nego.shallow_roots->array)) < 0) + return error; + + return 0; } int git_fetch_options_init(git_fetch_options *opts, unsigned int version) diff --git a/src/libgit2/grafts.c b/src/libgit2/grafts.c index 82be2a680..dd1be3434 100644 --- a/src/libgit2/grafts.c +++ b/src/libgit2/grafts.c @@ -43,6 +43,9 @@ int git_grafts_from_file(git_grafts **out, const char *path) git_grafts *grafts = NULL; int error; + if (*out) + return git_grafts_refresh(*out); + if ((error = git_grafts_new(&grafts)) < 0) goto error; @@ -220,9 +223,8 @@ int git_grafts_get(git_commit_graft **out, git_grafts *grafts, const git_oid *oi return 0; } -int git_grafts_get_oids(git_oidarray *out, git_grafts *grafts) +int git_grafts_get_oids(git_array_oid_t *out, git_grafts *grafts) { - git_array_oid_t oids = GIT_ARRAY_INIT; const git_oid *oid; size_t i = 0; int error; @@ -230,13 +232,11 @@ int git_grafts_get_oids(git_oidarray *out, git_grafts *grafts) assert(out && grafts); while ((error = git_oidmap_iterate(NULL, grafts->commits, &i, &oid)) == 0) { - git_oid *cpy = git_array_alloc(oids); + git_oid *cpy = git_array_alloc(*out); GIT_ERROR_CHECK_ALLOC(cpy); git_oid_cpy(cpy, oid); } - git_oidarray__from_array(out, &oids); - return 0; } diff --git a/src/libgit2/grafts.h b/src/libgit2/grafts.h index fd9ef6736..4139438bb 100644 --- a/src/libgit2/grafts.h +++ b/src/libgit2/grafts.h @@ -31,7 +31,7 @@ int git_grafts_parse(git_grafts *grafts, const char *content, size_t contentlen) int git_grafts_add(git_grafts *grafts, const git_oid *oid, git_array_oid_t parents); int git_grafts_remove(git_grafts *grafts, const git_oid *oid); int git_grafts_get(git_commit_graft **out, git_grafts *grafts, const git_oid *oid); -int git_grafts_get_oids(git_oidarray *out, git_grafts *grafts); +int git_grafts_get_oids(git_array_oid_t *out, git_grafts *grafts); size_t git_grafts_size(git_grafts *grafts); #endif diff --git a/src/libgit2/object.c b/src/libgit2/object.c index d45465678..5ce4f1142 100644 --- a/src/libgit2/object.c +++ b/src/libgit2/object.c @@ -104,15 +104,13 @@ int git_object__from_raw( return 0; } -int git_object__from_odb_object( +int git_object__init_from_odb_object( git_object **object_out, git_repository *repo, git_odb_object *odb_obj, git_object_t type) { - int error; size_t object_size; - git_object_def *def; git_object *object = NULL; GIT_ASSERT_ARG(object_out); @@ -139,6 +137,23 @@ int git_object__from_odb_object( object->cached.size = odb_obj->cached.size; object->repo = repo; + *object_out = object; + return 0; +} + +int git_object__from_odb_object( + git_object **object_out, + git_repository *repo, + git_odb_object *odb_obj, + git_object_t type) +{ + int error; + git_object_def *def; + git_object *object = NULL; + + if ((error = git_object__init_from_odb_object(&object, repo, odb_obj, type)) < 0) + return error; + /* Parse raw object data */ def = &git_objects_table[odb_obj->cached.type]; GIT_ASSERT(def->free && def->parse); diff --git a/src/libgit2/object.h b/src/libgit2/object.h index 980e8627e..9099f8607 100644 --- a/src/libgit2/object.h +++ b/src/libgit2/object.h @@ -35,6 +35,12 @@ int git_object__from_raw( size_t size, git_object_t type); +int git_object__init_from_odb_object( + git_object **object_out, + git_repository *repo, + git_odb_object *odb_obj, + git_object_t type); + int git_object__from_odb_object( git_object **object_out, git_repository *repo, diff --git a/src/libgit2/remote.c b/src/libgit2/remote.c index 02d271d7d..63346e941 100644 --- a/src/libgit2/remote.c +++ b/src/libgit2/remote.c @@ -22,6 +22,7 @@ #include "git2/types.h" #include "git2/oid.h" #include "git2/net.h" +#include "transports/smart.h" #define CONFIG_URL_FMT "remote.%s.url" #define CONFIG_PUSHURL_FMT "remote.%s.pushurl" @@ -2149,6 +2150,11 @@ void git_remote_free(git_remote *remote) remote->transport = NULL; } + if (remote->nego.shallow_roots) { + git_array_clear(remote->nego.shallow_roots->array); + git__free(remote->nego.shallow_roots); + } + git_vector_free(&remote->refs); free_refspecs(&remote->refspecs); diff --git a/src/libgit2/remote.h b/src/libgit2/remote.h index 41ee58e0f..df3aea29d 100644 --- a/src/libgit2/remote.h +++ b/src/libgit2/remote.h @@ -37,6 +37,7 @@ struct git_remote { git_remote_autotag_option_t download_tags; int prune_refs; int passed_refspecs; + git_fetch_negotiation nego; }; int git_remote__urlfordirection(git_str *url_out, struct git_remote *remote, int direction, const git_remote_callbacks *callbacks); diff --git a/src/libgit2/repository.c b/src/libgit2/repository.c index 761772d55..6b77007aa 100644 --- a/src/libgit2/repository.c +++ b/src/libgit2/repository.c @@ -3340,6 +3340,59 @@ int git_repository_state_cleanup(git_repository *repo) return git_repository__cleanup_files(repo, state_files, ARRAY_SIZE(state_files)); } +int git_repository__shallow_roots(git_array_oid_t *out, git_repository *repo) { + int error = 0; + + if (!repo->shallow_grafts && (error = load_grafts(repo)) < 0) + return error; + + if ((error = git_grafts_refresh(repo->shallow_grafts)) < 0) + return error; + + if ((error = git_grafts_get_oids(out, repo->shallow_grafts)) < 0) + return error; + + return 0; +} + +int git_repository__shallow_roots_write(git_repository *repo, git_array_oid_t roots) +{ + git_filebuf file = GIT_FILEBUF_INIT; + git_str path = GIT_STR_INIT; + int error = 0; + size_t idx; + git_oid *oid; + + assert(repo); + + if ((error = git_str_joinpath(&path, repo->gitdir, "shallow")) < 0) + goto on_error; + + if ((error = git_filebuf_open(&file, git_str_cstr(&path), GIT_FILEBUF_HASH_CONTENTS, 0666)) < 0) + goto on_error; + + git_array_foreach(roots, idx, oid) { + git_filebuf_write(&file, git_oid_tostr_s(oid), GIT_OID_HEXSZ); + git_filebuf_write(&file, "\n", 1); + } + + git_filebuf_commit(&file); + + if ((error = load_grafts(repo)) < 0) { + error = -1; + goto on_error; + } + + if (git_array_size(roots) == 0) { + remove(path.ptr); + } + +on_error: + git_str_dispose(&path); + + return error; +} + int git_repository_is_shallow(git_repository *repo) { git_str path = GIT_STR_INIT; diff --git a/src/libgit2/repository.h b/src/libgit2/repository.h index 3cca53b3e..c7966bcd2 100644 --- a/src/libgit2/repository.h +++ b/src/libgit2/repository.h @@ -244,6 +244,9 @@ extern size_t git_repository__reserved_names_posix_len; bool git_repository__reserved_names( git_str **out, size_t *outlen, git_repository *repo, bool include_ntfs); +int git_repository__shallow_roots(git_array_oid_t *out, git_repository *repo); +int git_repository__shallow_roots_write(git_repository *repo, git_array_oid_t roots); + /* * The default branch for the repository; the `init.defaultBranch` * configuration option, if set, or `master` if it is not. diff --git a/src/libgit2/transports/local.c b/src/libgit2/transports/local.c index 6c754a034..24f49cc65 100644 --- a/src/libgit2/transports/local.c +++ b/src/libgit2/transports/local.c @@ -284,15 +284,13 @@ static int local_ls(const git_remote_head ***out, size_t *size, git_transport *t static int local_negotiate_fetch( git_transport *transport, git_repository *repo, - const git_remote_head * const *refs, - size_t count) + const git_fetch_negotiation *wants) { transport_local *t = (transport_local*)transport; git_remote_head *rhead; unsigned int i; - GIT_UNUSED(refs); - GIT_UNUSED(count); + GIT_UNUSED(wants); /* Fill in the loids */ git_vector_foreach(&t->refs, i, rhead) { diff --git a/src/libgit2/transports/smart.c b/src/libgit2/transports/smart.c index 7f57dba2a..b0925c8bb 100644 --- a/src/libgit2/transports/smart.c +++ b/src/libgit2/transports/smart.c @@ -482,3 +482,49 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param) *out = (git_transport *) t; return 0; } + +size_t git_shallowarray_count(git_shallowarray *array) +{ + return git_array_size(array->array); +} + +const git_oid * git_shallowarray_get(git_shallowarray *array, size_t idx) +{ + return git_array_get(array->array, idx); +} + +int git_shallowarray_add(git_shallowarray *array, git_oid *oid) +{ + size_t oid_index; + + if (git_array_search(&oid_index, array->array, (git_array_compare_cb)git_oid_cmp, &oid) < 0) { + git_oid *tmp = git_array_alloc(array->array); + GIT_ERROR_CHECK_ALLOC(tmp); + + git_oid_cpy(tmp, oid); + } + + return 0; +} + +int git_shallowarray_remove(git_shallowarray *array, git_oid *oid) +{ + git_array_oid_t new_array = GIT_ARRAY_INIT; + git_oid *element; + git_oid *tmp; + size_t i; + + git_array_foreach(array->array, i, element) { + if (git_oid_cmp(oid, element)) { + tmp = git_array_alloc(new_array); + GIT_ERROR_CHECK_ALLOC(tmp); + + git_oid_cpy(tmp, element); + } + } + + git_array_clear(array->array); + array->array = new_array; + + return 0; +} diff --git a/src/libgit2/transports/smart.h b/src/libgit2/transports/smart.h index 9323d6c44..bc072d2fe 100644 --- a/src/libgit2/transports/smart.h +++ b/src/libgit2/transports/smart.h @@ -14,6 +14,7 @@ #include "netops.h" #include "push.h" #include "str.h" +#include "oidarray.h" #include "git2/sys/transport.h" #define GIT_SIDE_BAND_DATA 1 @@ -32,6 +33,7 @@ #define GIT_CAP_SYMREF "symref" #define GIT_CAP_WANT_TIP_SHA1 "allow-tip-sha1-in-want" #define GIT_CAP_WANT_REACHABLE_SHA1 "allow-reachable-sha1-in-want" +#define GIT_CAP_SHALLOW "shallow" extern bool git_smart__ofs_delta_enabled; @@ -48,7 +50,9 @@ typedef enum { GIT_PKT_PROGRESS, GIT_PKT_OK, GIT_PKT_NG, - GIT_PKT_UNPACK + GIT_PKT_UNPACK, + GIT_PKT_SHALLOW, + GIT_PKT_UNSHALLOW, } git_pkt_type; /* Used for multi_ack and multi_ack_detailed */ @@ -120,6 +124,11 @@ typedef struct { int unpack_ok; } git_pkt_unpack; +typedef struct { + git_pkt_type type; + git_oid oid; +} git_pkt_shallow; + typedef struct transport_smart_caps { unsigned int common:1, ofs_delta:1, @@ -132,7 +141,8 @@ typedef struct transport_smart_caps { report_status:1, thin_pack:1, want_tip_sha1:1, - want_reachable_sha1:1; + want_reachable_sha1:1, + shallow:1; } transport_smart_caps; typedef int (*packetsize_cb)(size_t received, void *payload); @@ -167,8 +177,7 @@ int git_smart__push(git_transport *transport, git_push *push); int git_smart__negotiate_fetch( git_transport *transport, git_repository *repo, - const git_remote_head * const *refs, - size_t count); + const git_fetch_negotiation *wants); int git_smart__download_pack( git_transport *transport, @@ -186,8 +195,12 @@ int git_pkt_parse_line(git_pkt **head, const char **endptr, const char *line, si int git_pkt_buffer_flush(git_str *buf); int git_pkt_send_flush(GIT_SOCKET s); int git_pkt_buffer_done(git_str *buf); -int git_pkt_buffer_wants(const git_remote_head * const *refs, size_t count, transport_smart_caps *caps, git_str *buf); +int git_pkt_buffer_wants(const git_fetch_negotiation *wants, transport_smart_caps *caps, git_str *buf); int git_pkt_buffer_have(git_oid *oid, git_str *buf); void git_pkt_free(git_pkt *pkt); +struct git_shallowarray { + git_array_oid_t array; +}; + #endif diff --git a/src/libgit2/transports/smart_pkt.c b/src/libgit2/transports/smart_pkt.c index e679819fa..832da450c 100644 --- a/src/libgit2/transports/smart_pkt.c +++ b/src/libgit2/transports/smart_pkt.c @@ -366,6 +366,50 @@ static int unpack_pkt(git_pkt **out, const char *line, size_t len) return 0; } +static int shallow_pkt(git_pkt **out, const char *line, size_t len) +{ + git_pkt_shallow *pkt; + + pkt = git__calloc(1, sizeof(git_pkt_shallow)); + GIT_ERROR_CHECK_ALLOC(pkt); + + pkt->type = GIT_PKT_SHALLOW; + line += 7; + len -= 7; + + if (len >= GIT_OID_HEXSZ) { + git_oid_fromstr(&pkt->oid, line + 1); + line += GIT_OID_HEXSZ + 1; + len -= GIT_OID_HEXSZ + 1; + } + + *out = (git_pkt *) pkt; + + return 0; +} + +static int unshallow_pkt(git_pkt **out, const char *line, size_t len) +{ + git_pkt_shallow *pkt; + + pkt = git__calloc(1, sizeof(git_pkt_shallow)); + GIT_ERROR_CHECK_ALLOC(pkt); + + pkt->type = GIT_PKT_UNSHALLOW; + line += 9; + len -= 9; + + if (len >= GIT_OID_HEXSZ) { + git_oid_fromstr(&pkt->oid, line + 1); + line += GIT_OID_HEXSZ + 1; + len -= GIT_OID_HEXSZ + 1; + } + + *out = (git_pkt *) pkt; + + return 0; +} + static int parse_len(size_t *out, const char *line, size_t linelen) { char num[PKT_LEN_SIZE + 1]; @@ -492,6 +536,10 @@ int git_pkt_parse_line( error = ng_pkt(pkt, line, len); else if (!git__prefixncmp(line, len, "unpack")) error = unpack_pkt(pkt, line, len); + else if (!git__prefixcmp(line, "shallow")) + error = shallow_pkt(pkt, line, len); + else if (!git__prefixcmp(line, "unshallow")) + error = unshallow_pkt(pkt, line, len); else error = ref_pkt(pkt, line, len); @@ -557,6 +605,9 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca if (caps->ofs_delta) git_str_puts(&str, GIT_CAP_OFS_DELTA " "); + if (caps->shallow) + git_str_puts(&str, GIT_CAP_SHALLOW " "); + if (git_str_oom(&str)) return -1; @@ -586,8 +637,7 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca */ int git_pkt_buffer_wants( - const git_remote_head * const *refs, - size_t count, + const git_fetch_negotiation *wants, transport_smart_caps *caps, git_str *buf) { @@ -595,22 +645,22 @@ int git_pkt_buffer_wants( const git_remote_head *head; if (caps->common) { - for (; i < count; ++i) { - head = refs[i]; + for (; i < wants->count; ++i) { + head = wants->refs[i]; if (!head->local) break; } - if (buffer_want_with_caps(refs[i], caps, buf) < 0) + if (buffer_want_with_caps(wants->refs[i], caps, buf) < 0) return -1; i++; } - for (; i < count; ++i) { + for (; i < wants->count; ++i) { char oid[GIT_OID_SHA1_HEXSIZE]; - head = refs[i]; + head = wants->refs[i]; if (head->local) continue; @@ -622,6 +672,36 @@ int git_pkt_buffer_wants( return -1; } + /* Tell the server about our shallow objects */ + for (i = 0; i < git_shallowarray_count(wants->shallow_roots); i++) { + char oid[GIT_OID_HEXSZ]; + git_str shallow_buf = GIT_STR_INIT; + + git_oid_fmt(oid, git_shallowarray_get(wants->shallow_roots, i)); + git_str_puts(&shallow_buf, "shallow "); + git_str_put(&shallow_buf, oid, GIT_OID_HEXSZ); + git_str_putc(&shallow_buf, '\n'); + + git_str_printf(buf, "%04x%s", (unsigned int)git_str_len(&shallow_buf) + 4, git_str_cstr(&shallow_buf)); + + git_str_dispose(&shallow_buf); + + if (git_str_oom(buf)) + return -1; + } + + if (wants->depth > 0) { + git_str deepen_buf = GIT_STR_INIT; + + git_str_printf(&deepen_buf, "deepen %d\n", wants->depth); + git_str_printf(buf,"%04x%s", (unsigned int)git_str_len(&deepen_buf) + 4, git_str_cstr(&deepen_buf)); + + git_str_dispose(&deepen_buf); + + if (git_str_oom(buf)) + return -1; + } + return git_pkt_buffer_flush(buf); } diff --git a/src/libgit2/transports/smart_protocol.c b/src/libgit2/transports/smart_protocol.c index 09778b335..a44d0c853 100644 --- a/src/libgit2/transports/smart_protocol.c +++ b/src/libgit2/transports/smart_protocol.c @@ -214,6 +214,11 @@ int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vec if (!git__prefixcmp(ptr, GIT_CAP_WANT_REACHABLE_SHA1)) { caps->common = caps->want_reachable_sha1 = 1; ptr += strlen(GIT_CAP_DELETE_REFS); + } + + if (!git__prefixcmp(ptr, GIT_CAP_SHALLOW)) { + caps->common = caps->shallow = 1; + ptr += strlen(GIT_CAP_SHALLOW); continue; } @@ -317,7 +322,26 @@ static int wait_while_ack(gitno_buffer *buf) return 0; } -int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *wants, size_t count) +static int cap_not_sup_err(const char *cap_name) +{ + git_error_set(GIT_ERROR_NET, "server doesn't support %s", cap_name); + return GIT_EINVALID; +} + +/* Disables server capabilities we're not interested in */ +static int setup_caps(transport_smart_caps *caps, const git_fetch_negotiation *wants) +{ + if (wants->depth) { + if (!caps->shallow) + return cap_not_sup_err(GIT_CAP_SHALLOW); + } else { + caps->shallow = 0; + } + + return 0; +} + +int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_fetch_negotiation *wants) { transport_smart *t = (transport_smart *)transport; git_revwalk__push_options opts = GIT_REVWALK__PUSH_OPTIONS_INIT; @@ -329,7 +353,10 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c unsigned int i; git_oid oid; - if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0) + if ((error = setup_caps(&t->caps, wants)) < 0) + return error; + + if ((error = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0) return error; if ((error = git_revwalk_new(&walk, repo)) < 0) @@ -339,6 +366,35 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c if ((error = git_revwalk__push_glob(walk, "refs/*", &opts)) < 0) goto on_error; + if (wants->depth > 0) { + git_pkt_shallow *pkt; + + if ((error = git_smart__negotiation_step(&t->parent, data.ptr, data.size)) < 0) + goto on_error; + + while ((error = recv_pkt((git_pkt **)&pkt, NULL, buf)) == 0) { + + if (pkt->type == GIT_PKT_SHALLOW) { + git_shallowarray_add(wants->shallow_roots, &pkt->oid); + } else if (pkt->type == GIT_PKT_UNSHALLOW) { + git_shallowarray_remove(wants->shallow_roots, &pkt->oid); + } else if (pkt->type == GIT_PKT_FLUSH) { + /* Server is done, stop processing shallow oids */ + break; + } else { + git_error_set(GIT_ERROR_NET, "Unexpected pkt type"); + goto on_error; + } + + git_pkt_free((git_pkt *) pkt); + } + + git_pkt_free((git_pkt *) pkt); + + if (error < 0) { + goto on_error; + } + } /* * Our support for ACK extensions is simply to parse them. On * the first ACK we will accept that as enough common @@ -401,7 +457,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c git_pkt_ack *pkt; unsigned int j; - if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0) + if ((error = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0) goto on_error; git_vector_foreach(&t->common, j, pkt) { @@ -421,7 +477,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c git_pkt_ack *pkt; unsigned int j; - if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0) + if ((error = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0) goto on_error; git_vector_foreach(&t->common, j, pkt) { diff --git a/src/util/array.h b/src/util/array.h index cbab52ad1..bf66e1c5a 100644 --- a/src/util/array.h +++ b/src/util/array.h @@ -85,12 +85,14 @@ on_oom: #define git_array_foreach(a, i, element) \ for ((i) = 0; (i) < (a).size && ((element) = &(a).ptr[(i)]); (i)++) +typedef int (*git_array_compare_cb)(const void *, const void *); + GIT_INLINE(int) git_array__search( size_t *out, void *array_ptr, size_t item_size, size_t array_len, - int (*compare)(const void *, const void *), + git_array_compare_cb compare, const void *key) { size_t lim; diff --git a/tests/libgit2/clone/shallow.c b/tests/libgit2/clone/shallow.c new file mode 100644 index 000000000..2a88d5d05 --- /dev/null +++ b/tests/libgit2/clone/shallow.c @@ -0,0 +1,174 @@ +#include "clar_libgit2.h" +#include "futils.h" +#include "repository.h" + +void test_clone_shallow__initialize(void) +{ + cl_git_pass(git_libgit2_opts(GIT_OPT_ENABLE_SHALLOW, 1)); +} + +void test_clone_shallow__cleanup(void) +{ + git_libgit2_opts(GIT_OPT_ENABLE_SHALLOW, 0); + cl_git_sandbox_cleanup(); +} + +static int remote_single_branch(git_remote **out, git_repository *repo, const char *name, const char *url, void *payload) +{ + GIT_UNUSED(payload); + + cl_git_pass(git_remote_create_with_fetchspec(out, repo, name, url, "+refs/heads/master:refs/remotes/origin/master")); + + return 0; +} + +void test_clone_shallow__clone_depth_zero(void) +{ + git_str path = GIT_STR_INIT; + git_repository *repo; + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + git_array_oid_t roots = GIT_ARRAY_INIT; + + clone_opts.fetch_opts.depth = 0; + clone_opts.remote_cb = remote_single_branch; + + git_str_joinpath(&path, clar_sandbox_path(), "shallowclone_0"); + + cl_git_pass(git_clone(&repo, "https://github.com/libgit2/TestGitRepository", git_str_cstr(&path), &clone_opts)); + + /* cloning with depth 0 results in a full clone. */ + cl_assert_equal_b(false, git_repository_is_shallow(repo)); + + /* full clones do not have shallow roots. */ + cl_git_pass(git_repository__shallow_roots(&roots, repo)); + cl_assert_equal_i(0, roots.size); + + git_array_clear(roots); + git_str_dispose(&path); + git_repository_free(repo); +} + +void test_clone_shallow__clone_depth_one(void) +{ + git_str path = GIT_STR_INIT; + git_repository *repo; + git_revwalk *walk; + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + git_oid oid; + git_array_oid_t roots = GIT_ARRAY_INIT; + size_t num_commits = 0; + int error = 0; + + clone_opts.fetch_opts.depth = 1; + clone_opts.remote_cb = remote_single_branch; + + git_str_joinpath(&path, clar_sandbox_path(), "shallowclone_1"); + + cl_git_pass(git_clone(&repo, "https://github.com/libgit2/TestGitRepository", git_str_cstr(&path), &clone_opts)); + + cl_assert_equal_b(true, git_repository_is_shallow(repo)); + + cl_git_pass(git_repository__shallow_roots(&roots, repo)); + cl_assert_equal_i(1, roots.size); + cl_assert_equal_s("49322bb17d3acc9146f98c97d078513228bbf3c0", git_oid_tostr_s(&roots.ptr[0])); + + git_revwalk_new(&walk, repo); + + git_revwalk_push_head(walk); + + while ((error = git_revwalk_next(&oid, walk)) == GIT_OK) { + num_commits++; + } + + cl_assert_equal_i(num_commits, 1); + cl_assert_equal_i(error, GIT_ITEROVER); + + git_array_clear(roots); + git_str_dispose(&path); + git_revwalk_free(walk); + git_repository_free(repo); +} + +void test_clone_shallow__clone_depth_five(void) +{ + git_str path = GIT_STR_INIT; + git_repository *repo; + git_revwalk *walk; + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + git_oid oid; + git_array_oid_t roots = GIT_ARRAY_INIT; + size_t num_commits = 0; + int error = 0; + + clone_opts.fetch_opts.depth = 5; + clone_opts.remote_cb = remote_single_branch; + + git_str_joinpath(&path, clar_sandbox_path(), "shallowclone_5"); + + cl_git_pass(git_clone(&repo, "https://github.com/libgit2/TestGitRepository", git_str_cstr(&path), &clone_opts)); + + cl_assert_equal_b(true, git_repository_is_shallow(repo)); + + cl_git_pass(git_repository__shallow_roots(&roots, repo)); + cl_assert_equal_i(3, roots.size); + cl_assert_equal_s("c070ad8c08840c8116da865b2d65593a6bb9cd2a", git_oid_tostr_s(&roots.ptr[0])); + cl_assert_equal_s("0966a434eb1a025db6b71485ab63a3bfbea520b6", git_oid_tostr_s(&roots.ptr[1])); + cl_assert_equal_s("83834a7afdaa1a1260568567f6ad90020389f664", git_oid_tostr_s(&roots.ptr[2])); + + git_revwalk_new(&walk, repo); + + git_revwalk_push_head(walk); + + while ((error = git_revwalk_next(&oid, walk)) == GIT_OK) { + num_commits++; + } + + cl_assert_equal_i(num_commits, 13); + cl_assert_equal_i(error, GIT_ITEROVER); + + git_array_clear(roots); + git_str_dispose(&path); + git_revwalk_free(walk); + git_repository_free(repo); +} + +void test_clone_shallow__unshallow(void) +{ + git_str path = GIT_STR_INIT; + git_repository *repo; + git_revwalk *walk; + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; + git_remote *origin = NULL; + git_oid oid; + size_t num_commits = 0; + int error = 0; + + clone_opts.fetch_opts.depth = 5; + clone_opts.remote_cb = remote_single_branch; + + git_str_joinpath(&path, clar_sandbox_path(), "unshallow"); + cl_git_pass(git_clone(&repo, "https://github.com/libgit2/TestGitRepository", git_str_cstr(&path), &clone_opts)); + cl_assert_equal_b(true, git_repository_is_shallow(repo)); + + fetch_opts.unshallow = 1; + cl_git_pass(git_remote_lookup(&origin, repo, "origin")); + + cl_git_pass(git_remote_fetch(origin, NULL, &fetch_opts, NULL)); + cl_assert_equal_b(false, git_repository_is_shallow(repo)); + + git_revwalk_new(&walk, repo); + git_revwalk_push_head(walk); + + while ((error = git_revwalk_next(&oid, walk)) == GIT_OK) { + num_commits++; + } + + cl_assert_equal_i(num_commits, 21); + cl_assert_equal_i(error, GIT_ITEROVER); + + git_remote_free(origin); + git_str_dispose(&path); + git_revwalk_free(walk); + git_repository_free(repo); +} diff --git a/tests/libgit2/transports/smart/shallowarray.c b/tests/libgit2/transports/smart/shallowarray.c new file mode 100644 index 000000000..c51e62713 --- /dev/null +++ b/tests/libgit2/transports/smart/shallowarray.c @@ -0,0 +1,52 @@ +#include "clar_libgit2.h" + +#include "git2/oid.h" +#include "git2/transport.h" + +#include "common.h" +#include "transports/smart.h" +#include "oid.h" + +#include <assert.h> + +#define oid_0 "c070ad8c08840c8116da865b2d65593a6bb9cd2a" +#define oid_1 "0966a434eb1a025db6b71485ab63a3bfbea520b6" +#define oid_2 "83834a7afdaa1a1260568567f6ad90020389f664" + +void test_transports_smart_shallowarray__add_and_remove_oid_from_shallowarray(void) +{ + git_oid oid_0_obj, oid_1_obj, oid_2_obj; + git_shallowarray *shallow_roots = git__malloc(sizeof(git_shallowarray)); + git_array_init(shallow_roots->array); + + git_oid_fromstr(&oid_0_obj, oid_0); + git_oid_fromstr(&oid_1_obj, oid_1); + git_oid_fromstr(&oid_2_obj, oid_2); + + git_shallowarray_add(shallow_roots, &oid_0_obj); + git_shallowarray_add(shallow_roots, &oid_1_obj); + git_shallowarray_add(shallow_roots, &oid_2_obj); + + cl_assert_equal_i(3, shallow_roots->array.size); + cl_assert_equal_s("c070ad8c08840c8116da865b2d65593a6bb9cd2a", git_oid_tostr_s(&shallow_roots->array.ptr[0])); + cl_assert_equal_s("0966a434eb1a025db6b71485ab63a3bfbea520b6", git_oid_tostr_s(&shallow_roots->array.ptr[1])); + cl_assert_equal_s("83834a7afdaa1a1260568567f6ad90020389f664", git_oid_tostr_s(&shallow_roots->array.ptr[2])); + + git_shallowarray_remove(shallow_roots, &oid_2_obj); + + cl_assert_equal_i(2, shallow_roots->array.size); + cl_assert_equal_s("c070ad8c08840c8116da865b2d65593a6bb9cd2a", git_oid_tostr_s(&shallow_roots->array.ptr[0])); + cl_assert_equal_s("0966a434eb1a025db6b71485ab63a3bfbea520b6", git_oid_tostr_s(&shallow_roots->array.ptr[1])); + + git_shallowarray_remove(shallow_roots, &oid_1_obj); + + cl_assert_equal_i(1, shallow_roots->array.size); + cl_assert_equal_s("c070ad8c08840c8116da865b2d65593a6bb9cd2a", git_oid_tostr_s(&shallow_roots->array.ptr[0])); + + git_shallowarray_remove(shallow_roots, &oid_0_obj); + + cl_assert_equal_i(0, shallow_roots->array.size); + + git_array_clear(shallow_roots->array); + git__free(shallow_roots); +} |