During negotiation, fetch-pack eventually reports as "have" lines all commits reachable from all refs. Allow the user to restrict the commits sent in this way by providing a whitelist of tips; only the tips themselves and their ancestors will be sent. This feature is only supported for protocols that support connect or stateless-connect (such as HTTP with protocol v2). This will speed up negotiation when the repository has multiple relatively independent branches (for example, when a repository interacts with multiple repositories, such as with linux-next [1] and torvalds/linux [2]), and the user knows which local branch is likely to have commits in common with the upstream branch they are fetching. [1] https://kernel.googlesource.com/pub/scm/linux/kernel/git/next/linux-next/ [2] https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/ Signed-off-by: Jonathan Tan <jonathantanmy@xxxxxxxxxx> --- v2 is exactly the same as the original, except with user-facing documentation in Documentation/fetch-options.txt. > What's the plan to expose this "feature" to end-users? There is no > end-user facing documentation added by this patch, and in-code > comments only talk about what (mechanical) effect the option has, > but not when a user may want to use the feature, or how the user > would best decide the set of commits to pass to this new option. Jonathan Nieder also mentioned this. Lack of documentation was an oversight, sorry. I've added it in this version. > Would something like this > > git fetch $(git for-each-ref \ > --format=--nego-tip="%(objectname)" \ > refs/remotes/linux-next/) \ > linux-next > > be an expected typical way to pull from one remote, exposing only > the tips of refs we got from that remote and not the ones we > obtained from other places? Yes, that is one way. Alternatively, if the user is only fetching one branch, they may also want to specify a single branch. --- Documentation/fetch-options.txt | 12 +++++++ builtin/fetch.c | 21 +++++++++++++ fetch-pack.c | 19 ++++++++++-- fetch-pack.h | 7 +++++ t/t5510-fetch.sh | 55 +++++++++++++++++++++++++++++++++ transport-helper.c | 3 ++ transport.c | 1 + transport.h | 10 ++++++ 8 files changed, 126 insertions(+), 2 deletions(-) diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt index 97d3217df9..80c4c94595 100644 --- a/Documentation/fetch-options.txt +++ b/Documentation/fetch-options.txt @@ -42,6 +42,18 @@ the current repository has the same history as the source repository. .git/shallow. This option updates .git/shallow and accept such refs. +--negotiation-tip:: + By default, Git will report, to the server, commits reachable + from all local refs to find common commits in an attempt to + reduce the size of the to-be-received packfile. If specified, + Git will only report commits reachable from the given commit. + This is useful to speed up fetches when the user knows which + local ref is likely to have commits in common with the + upstream ref being fetched. ++ +This option may be specified more than once; if so, Git will report +commits reachable from any of the given commits. + ifndef::git-pull[] --dry-run:: Show what would be done, without making any changes. diff --git a/builtin/fetch.c b/builtin/fetch.c index ea5b9669ad..12daec0f3b 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -63,6 +63,7 @@ static int shown_url = 0; static struct refspec refmap = REFSPEC_INIT_FETCH; static struct list_objects_filter_options filter_options; static struct string_list server_options = STRING_LIST_INIT_DUP; +static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP; static int git_fetch_config(const char *k, const char *v, void *cb) { @@ -174,6 +175,8 @@ static struct option builtin_fetch_options[] = { TRANSPORT_FAMILY_IPV4), OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"), TRANSPORT_FAMILY_IPV6), + OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"), + N_("report that we have only objects reachable from this object")), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), OPT_END() }; @@ -1075,6 +1078,24 @@ static struct transport *prepare_transport(struct remote *remote, int deepen) filter_options.filter_spec); set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); } + if (negotiation_tip.nr) { + struct oid_array *oids; + if (transport->smart_options) { + int i; + oids = xcalloc(1, sizeof(*oids)); + for (i = 0; i < negotiation_tip.nr; i++) { + struct object_id oid; + if (get_oid(negotiation_tip.items[i].string, + &oid)) + die("%s is not a valid object", + negotiation_tip.items[i].string); + oid_array_append(oids, &oid); + } + transport->smart_options->negotiation_tips = oids; + } else { + warning("Ignoring --negotiation-tip because the protocol does not support it."); + } + } return transport; } diff --git a/fetch-pack.c b/fetch-pack.c index ba12085c4a..c66bd49bd1 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -213,6 +213,21 @@ static int next_flush(int stateless_rpc, int count) return count; } +static void mark_tips(struct fetch_negotiator *negotiator, + const struct oid_array *negotiation_tips) +{ + int i; + if (!negotiation_tips) { + for_each_ref(rev_list_insert_ref_oid, negotiator); + return; + } + + for (i = 0; i < negotiation_tips->nr; i++) + rev_list_insert_ref(negotiator, NULL, + &negotiation_tips->oid[i]); + return; +} + static int find_common(struct fetch_negotiator *negotiator, struct fetch_pack_args *args, int fd[2], struct object_id *result_oid, @@ -230,7 +245,7 @@ static int find_common(struct fetch_negotiator *negotiator, if (args->stateless_rpc && multi_ack == 1) die(_("--stateless-rpc requires multi_ack_detailed")); - for_each_ref(rev_list_insert_ref_oid, negotiator); + mark_tips(negotiator, args->negotiation_tips); for_each_cached_alternate(negotiator, insert_one_alternate_object); fetching = 0; @@ -1295,7 +1310,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, else state = FETCH_SEND_REQUEST; - for_each_ref(rev_list_insert_ref_oid, &negotiator); + mark_tips(&negotiator, args->negotiation_tips); for_each_cached_alternate(&negotiator, insert_one_alternate_object); break; diff --git a/fetch-pack.h b/fetch-pack.h index bb45a366a8..1859ee9275 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -16,6 +16,13 @@ struct fetch_pack_args { const struct string_list *deepen_not; struct list_objects_filter_options filter_options; const struct string_list *server_options; + + /* + * If not NULL, during packfile negotiation, fetch-pack will send "have" + * lines only with these tips and their ancestors. + */ + const struct oid_array *negotiation_tips; + unsigned deepen_relative:1; unsigned quiet:1; unsigned keep_pack:1; diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index e402aee6a2..ea1b5e53c1 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -865,4 +865,59 @@ test_expect_success C_LOCALE_OUTPUT 'fetch compact output' ' test_cmp expect actual ' +negotiator_tip () { + SERVER="$1" + URL="$2" + USE_PROTOCOL_V2="$3" + + rm -rf "$SERVER" client && + git init "$SERVER" && + test_commit -C "$SERVER" alpha_1 && + test_commit -C "$SERVER" alpha_2 && + git -C "$SERVER" checkout --orphan beta && + test_commit -C "$SERVER" beta_1 && + test_commit -C "$SERVER" beta_2 && + + git clone "$URL" client && + + if [ "$USE_PROTOCOL_V2" -eq 1 ] + then + git -C "$SERVER" config protocol.version 2 + git -C client config protocol.version 2 + fi && + + test_commit -C "$SERVER" beta_s && + git -C "$SERVER" checkout master && + test_commit -C "$SERVER" alpha_s && + git -C "$SERVER" tag -d alpha_1 alpha_2 beta_1 beta_2 && + + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 --negotiation-tip=beta_1 \ + origin alpha_s beta_s && + + # Ensure that {alpha,beta}_1 are sent as "have", but not {alpha_beta}_2 + ALPHA_1=$(git -C client rev-parse alpha_1) && + grep "fetch> have $ALPHA_1" trace && + BETA_1=$(git -C client rev-parse beta_1) && + grep "fetch> have $BETA_1" trace && + ALPHA_2=$(git -C client rev-parse alpha_2) && + ! grep "fetch> have $ALPHA_2" trace && + BETA_2=$(git -C client rev-parse beta_2) && + ! grep "fetch> have $BETA_2" trace +} + +test_expect_success '--negotiator-tip limits "have" lines sent' ' + negotiator_tip server server 0 +' + +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success 'fetch with --filter=blob:limit=0 and HTTP' ' + negotiator_tip "$HTTPD_DOCUMENT_ROOT_PATH/server" \ + "$HTTPD_URL/smart/server" 1 +' + +stop_httpd + test_done diff --git a/transport-helper.c b/transport-helper.c index 1f8ff7e942..ad8f7c7726 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -684,6 +684,9 @@ static int fetch(struct transport *transport, transport, "filter", data->transport_options.filter_options.filter_spec); + if (data->transport_options.negotiation_tips) + warning("Ignoring --negotiation-tip because the protocol does not support it."); + if (data->fetch) return fetch_with_fetch(transport, nr_heads, to_fetch); diff --git a/transport.c b/transport.c index a32da30dee..9f10f8ad9f 100644 --- a/transport.c +++ b/transport.c @@ -318,6 +318,7 @@ static int fetch_refs_via_pack(struct transport *transport, args.filter_options = data->options.filter_options; args.stateless_rpc = transport->stateless_rpc; args.server_options = transport->server_options; + args.negotiation_tips = data->options.negotiation_tips; if (!data->got_remote_heads) refs_tmp = get_refs_via_connect(transport, 0, NULL); diff --git a/transport.h b/transport.h index 7792b08582..d31be5be63 100644 --- a/transport.h +++ b/transport.h @@ -25,6 +25,16 @@ struct git_transport_options { const char *receivepack; struct push_cas_option *cas; struct list_objects_filter_options filter_options; + + /* + * This is only used during fetch. See the documentation of + * negotiation_tips in struct fetch_pack_args. + * + * This field is only supported by transports that support connect or + * stateless_connect. Set this field directly instead of using + * transport_set_option(). + */ + struct oid_array *negotiation_tips; }; enum transport_family { -- 2.18.0.rc2.346.g013aa6912e-goog
