Changes since v2 [1] - PATCH v2 03/25, transport-helper.c: do not send null option to remote, is dropped. It's just a hack. Even if we want to set NULL to a transport option to say "reset to defaults", we need to tell remote helper about it, not silently ignore it. 15/25 solves it without the old patch. - get_shallow_commits_by_rev_list() avoids going through all in-core objects in 13/25 - split check_unreachable()'s double usage, 23/25 (new) and 24/25. 23/25 actually makes interdiff a bit messy to read. - move --not arguments before revs in 19/25 - some more small changes for other comments, see interdiff. [1] http://thread.gmane.org/gmane.comp.version-control.git/285414 Interdiff diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 67a13af..8265348 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -50,6 +50,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) struct child_process *conn; struct fetch_pack_args args; struct sha1_array shallow = SHA1_ARRAY_INIT; + struct string_list deepen_not = STRING_LIST_INIT_DUP; packet_trace_identity("fetch-pack"); @@ -58,14 +59,13 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) for (i = 1; i < argc && *argv[i] == '-'; i++) { const char *arg = argv[i]; - const char *value; - if (skip_prefix(arg, "--upload-pack=", &value)) { - args.uploadpack = value; + if (skip_prefix(arg, "--upload-pack=", &arg)) { + args.uploadpack = arg; continue; } - if (skip_prefix(arg, "--exec=", &value)) { - args.uploadpack = value; + if (skip_prefix(arg, "--exec=", &arg)) { + args.uploadpack = arg; continue; } if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) { @@ -101,22 +101,16 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) args.verbose = 1; continue; } - if (skip_prefix(arg, "--depth=", &value)) { - args.depth = strtol(value, NULL, 0); + if (skip_prefix(arg, "--depth=", &arg)) { + args.depth = strtol(arg, NULL, 0); continue; } - if (skip_prefix(arg, "--shallow-since=", &value)) { - args.deepen_since = xstrdup(value); + if (skip_prefix(arg, "--shallow-since=", &arg)) { + args.deepen_since = xstrdup(arg); continue; } - if (skip_prefix(arg, "--shallow-exclude=", &value)) { - static struct string_list *deepen_not; - if (!deepen_not) { - deepen_not = xmalloc(sizeof(*deepen_not)); - string_list_init(deepen_not, 1); - args.deepen_not = deepen_not; - } - string_list_append(deepen_not, value); + if (skip_prefix(arg, "--shallow-exclude=", &arg)) { + string_list_append(&deepen_not, arg); continue; } if (!strcmp(arg, "--deepen-relative")) { @@ -150,6 +144,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) } usage(fetch_pack_usage); } + if (deepen_not.nr) + args.deepen_not = &deepen_not; if (i < argc) dest = argv[i++]; diff --git a/builtin/fetch.c b/builtin/fetch.c index c5f0e9e..808c11b 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -875,7 +875,7 @@ static void set_option(struct transport *transport, const char *name, const char name, transport->url); } -static struct transport *prepare_transport(struct remote *remote) +static struct transport *prepare_transport(struct remote *remote, int deepen) { struct transport *transport; transport = transport_get(remote, NULL); @@ -886,9 +886,9 @@ static struct transport *prepare_transport(struct remote *remote) set_option(transport, TRANS_OPT_KEEP, "yes"); if (depth) set_option(transport, TRANS_OPT_DEPTH, depth); - if (deepen_since) + if (deepen && deepen_since) set_option(transport, TRANS_OPT_DEEPEN_SINCE, deepen_since); - if (deepen_not.nr) + if (deepen && deepen_not.nr) set_option(transport, TRANS_OPT_DEEPEN_NOT, (const char *)&deepen_not); if (deepen_relative) @@ -900,15 +900,24 @@ static struct transport *prepare_transport(struct remote *remote) static void backfill_tags(struct transport *transport, struct ref *ref_map) { - if (transport->cannot_reuse) { - gsecondary = prepare_transport(transport->remote); + int cannot_reuse; + + /* + * Once we have set TRANS_OPT_DEEPEN_SINCE, we can't unset it + * when remote helper is used (setting it to an empty string + * is not unsetting). We could extend the remote helper + * protocol for that, but for now, just force a new connection + * without deepen-since. Similar story for deepen-not. + */ + cannot_reuse = transport->cannot_reuse || + deepen_since || deepen_not.nr; + if (cannot_reuse) { + gsecondary = prepare_transport(transport->remote, 0); transport = gsecondary; } transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL); transport_set_option(transport, TRANS_OPT_DEPTH, "0"); - transport_set_option(transport, TRANS_OPT_DEEPEN_SINCE, NULL); - transport_set_option(transport, TRANS_OPT_DEEPEN_NOT, NULL); transport_set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, NULL); fetch_refs(transport, ref_map); @@ -1121,7 +1130,7 @@ static int fetch_one(struct remote *remote, int argc, const char **argv) die(_("No remote repository specified. Please, specify either a URL or a\n" "remote name from which new revisions should be fetched.")); - gtransport = prepare_transport(remote); + gtransport = prepare_transport(remote, 1); if (prune < 0) { /* no command line request */ diff --git a/fetch-pack.c b/fetch-pack.c index 69a99e5..ace9217 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -857,7 +857,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, allow_unadvertised_object_request |= ALLOW_TIP_SHA1; } if (server_supports("allow-reachable-sha1-in-want")) { - print_verbose(args, "Server supports allow-reachable-sha1-in-want\n"); + print_verbose(args, "Server supports allow-reachable-sha1-in-want"); allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; } if (!server_supports("thin-pack")) diff --git a/remote-curl.c b/remote-curl.c index ea06c5d..3f1a8f5 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -750,8 +750,7 @@ static int fetch_git(struct discovery *heads, struct argv_array args = ARGV_ARRAY_INIT; argv_array_pushl(&args, "fetch-pack", "--stateless-rpc", - "--stdin", "--lock-pack", - NULL); + "--stdin", "--lock-pack", NULL); if (options.followtags) argv_array_push(&args, "--include-tag"); if (options.thin) diff --git a/shallow.c b/shallow.c index f5d5c1d..5dca743 100644 --- a/shallow.c +++ b/shallow.c @@ -141,7 +141,7 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, static void show_commit(struct commit *commit, void *data) { - commit->object.flags |= *(int *)data; + commit_list_insert(commit, data); } /* @@ -155,20 +155,15 @@ struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av, int not_shallow_flag) { struct commit_list *result = NULL, *p; + struct commit_list *not_shallow_list = NULL; struct rev_info revs; - unsigned int i, nr; + int both_flags = shallow_flag | not_shallow_flag; /* * SHALLOW (excluded) and NOT_SHALLOW (included) should not be * set at this point. But better be safe than sorry. */ - nr = get_max_object_index(); - for (i = 0; i < nr; i++) { - struct object *o = get_indexed_object(i); - if (!o || o->type != OBJ_COMMIT) - continue; - o->flags &= ~(shallow_flag | not_shallow_flag); - } + clear_object_flags(both_flags); is_repository_shallow(); /* make sure shallows are read */ @@ -176,10 +171,13 @@ struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av, save_commit_buffer = 0; setup_revisions(ac, av, &revs, NULL); - /* Mark all reachable commits as NOT_SHALLOW */ if (prepare_revision_walk(&revs)) die("revision walk setup failed"); - traverse_commit_list(&revs, show_commit, NULL, ¬_shallow_flag); + traverse_commit_list(&revs, show_commit, NULL, ¬_shallow_list); + + /* Mark all reachable commits as NOT_SHALLOW */ + for (p = not_shallow_list; p; p = p->next) + p->item->object.flags |= not_shallow_flag; /* * mark border commits SHALLOW + NOT_SHALLOW. @@ -188,14 +186,8 @@ struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av, * A, later. If NOT_SHALLOW on A is cleared at step 1, B * itself is considered border at step 2, which is incorrect. */ - nr = get_max_object_index(); - for (i = 0; i < nr; i++) { - struct object *o = get_indexed_object(i); - struct commit *c = (struct commit *)o; - - if (!o || o->type != OBJ_COMMIT || - !(o->flags & not_shallow_flag)) - continue; + for (p = not_shallow_list; p; p = p->next) { + struct commit *c = p->item; if (parse_commit(c)) die("unable to parse commit %s", @@ -203,21 +195,21 @@ struct commit_list *get_shallow_commits_by_rev_list(int ac, const char **av, for (p = c->parents; p; p = p->next) if (!(p->item->object.flags & not_shallow_flag)) { - o->flags |= shallow_flag; + c->object.flags |= shallow_flag; commit_list_insert(c, &result); break; } } + free_commit_list(not_shallow_list); /* * Now we can clean up NOT_SHALLOW on border commits. Having * both flags set can confuse the caller. */ for (p = result; p; p = p->next) { - struct object *ro = &p->item->object; - if ((ro->flags & not_shallow_flag) && - (ro->flags & shallow_flag)) - ro->flags &= ~not_shallow_flag; + struct object *o = &p->item->object; + if ((o->flags & both_flags) == both_flags) + o->flags &= ~not_shallow_flag; } return result; } diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index ed8d18c..f512098 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -654,8 +654,10 @@ test_expect_success 'clone shallow since ...' ' test_expect_success 'fetch shallow since ...' ' git -C shallow11 fetch --shallow-since "200000000 +0700" origin && git -C shallow11 log --pretty=tformat:%s origin/master >actual && - echo three >expected && - echo two >>expected && + cat >expected <<-\EOF && + three + two + EOF test_cmp expected actual ' diff --git a/t/t5539-fetch-http-shallow.sh b/t/t5539-fetch-http-shallow.sh index 06389eb..25f8968 100755 --- a/t/t5539-fetch-http-shallow.sh +++ b/t/t5539-fetch-http-shallow.sh @@ -91,8 +91,10 @@ test_expect_success 'clone shallow since ...' ' test_expect_success 'fetch shallow since ...' ' git -C shallow11 fetch --shallow-since "200000000 +0700" origin && git -C shallow11 log --pretty=tformat:%s origin/master >actual && - echo three >expected && - echo two >>expected && + cat >expected <<-\EOF && + three + two + EOF test_cmp expected actual ' diff --git a/transport-helper.c b/transport-helper.c index 3c1ae6f..b894b60 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -290,9 +290,6 @@ static int string_list_set_helper_option(struct helper_data *data, struct strbuf buf = STRBUF_INIT; int i, ret = 0; - if (!list) - return 0; - for (i = 0; i < list->nr; i++) { strbuf_addf(&buf, "option %s ", name); quote_c_style(list->items[i].string, &buf, NULL, 0); @@ -337,12 +334,8 @@ static int set_helper_option(struct transport *transport, strbuf_addf(&buf, "option %s ", name); if (is_bool) strbuf_addstr(&buf, value ? "true" : "false"); - else if (value) + else quote_c_style(value, &buf, NULL, 0); - else { - strbuf_release(&buf); - return 0; - } strbuf_addch(&buf, '\n'); ret = strbuf_set_helper_option(data, &buf); diff --git a/transport.c b/transport.c index 2f3823a..f9c9e67 100644 --- a/transport.c +++ b/transport.c @@ -482,6 +482,7 @@ static int set_git_option(struct git_transport_options *opts, return 0; } else if (!strcmp(name, TRANS_OPT_DEEPEN_NOT)) { opts->deepen_not = (const struct string_list *)value; + return 0; } else if (!strcmp(name, TRANS_OPT_DEEPEN_RELATIVE)) { opts->deepen_relative = !!value; return 0; diff --git a/upload-pack.c b/upload-pack.c index f05971e..18b914a 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -452,33 +452,26 @@ static int is_our_ref(struct object *o) (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)); return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF); } -/* - * If reachable is NULL, return 1 if there is no unreachable object, - * zero otherwise. - * - * If reachable is not NULL, it's filled with reachable objects. - * Return value is irrelevant. The caller has to compare reachable and - * src to find out if there's any unreachable object. - */ -static int check_unreachable(struct object_array *reachable, - struct object_array *src) + +static int do_reachable_revlist(struct child_process *cmd, + struct object_array *src, + struct object_array *reachable) { static const char *argv[] = { "rev-list", "--stdin", NULL, }; - static struct child_process cmd = CHILD_PROCESS_INIT; struct object *o; char namebuf[42]; /* ^ + SHA-1 + LF */ int i; - cmd.argv = argv; - cmd.git_cmd = 1; - cmd.no_stderr = 1; - cmd.in = -1; - cmd.out = -1; + cmd->argv = argv; + cmd->git_cmd = 1; + cmd->no_stderr = 1; + cmd->in = -1; + cmd->out = -1; - if (start_command(&cmd)) - return 0; + if (start_command(cmd)) + return -1; /* * If rev-list --stdin encounters an unknown commit, it @@ -498,8 +491,8 @@ static int check_unreachable(struct object_array *reachable, if (!is_our_ref(o)) continue; memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); - if (write_in_full(cmd.in, namebuf, 42) < 0) - return 0; + if (write_in_full(cmd->in, namebuf, 42) < 0) + return -1; } namebuf[40] = '\n'; for (i = 0; i < src->nr; i++) { @@ -512,22 +505,26 @@ static int check_unreachable(struct object_array *reachable, if (reachable && o->type == OBJ_COMMIT) o->flags |= TMP_MARK; memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); - if (write_in_full(cmd.in, namebuf, 41) < 0) - return 0; + if (write_in_full(cmd->in, namebuf, 41) < 0) + return -1; } - close(cmd.in); + close(cmd->in); sigchain_pop(SIGPIPE); - - /* - * The commits out of the rev-list are not ancestors of - * our ref. - */ - if (!reachable) { - i = read_in_full(cmd.out, namebuf, 1); - if (i) return 0; - } else { +} + +static int get_reachable_list(struct object_array *src, + struct object_array *reachable) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + int i, ret = do_reachable_revlist(&cmd, src, reachable); + struct object *o; + char namebuf[42]; /* ^ + SHA-1 + LF */ + + if (ret < 0) + return -1; + while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) { struct object_id sha1; @@ -539,15 +536,38 @@ static int check_unreachable(struct object_array *reachable, o->flags &= ~TMP_MARK; } } - for (i = get_max_object_index(); 0 < i; ) { - o = get_indexed_object(--i); + for (i = get_max_object_index(); 0 < i; i--) { + o = get_indexed_object(i - 1); if (o && o->type == OBJ_COMMIT && (o->flags & TMP_MARK)) { add_object_array(o, NULL, reachable); o->flags &= ~TMP_MARK; } } - } + close(cmd.out); + + if (finish_command(&cmd)) + return -1; + + return 0; +} + +static int check_unreachable(struct object_array *src) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + int i, ret = do_reachable_revlist(&cmd, src, NULL); + char buf[1]; + + if (ret < 0) + return 0; + + /* + * The commits out of the rev-list are not ancestors of + * our ref. + */ + i = read_in_full(cmd.out, buf, 1); + if (i) + return 0; close(cmd.out); /* @@ -573,7 +593,7 @@ static void check_non_tip(void) */ if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1)) ; /* error */ - else if (check_unreachable(NULL, &want_obj)) + else if (check_unreachable(&want_obj)) return; /* Pick one of them (we know there at least is one) */ @@ -645,7 +665,7 @@ static void deepen(int depth, int deepen_relative, } else if (deepen_relative) { struct object_array reachable_shallows = OBJECT_ARRAY_INIT; - (void)check_unreachable(&reachable_shallows, shallows); + get_reachable_list(shallows, &reachable_shallows); result = get_shallow_commits(&reachable_shallows, depth + 1, SHALLOW, NOT_SHALLOW); @@ -729,7 +749,7 @@ static void receive_needs(void) char *ref = NULL; unsigned char sha1[20]; if (expand_ref(arg, strlen(arg), sha1, &ref) != 1) - die("Ambiguous deepen-not: %s", line); + die("git upload-pack: ambiguous deepen-not: %s", line); string_list_append(&deepen_not, ref); free(ref); deepen_rev_list = 1; @@ -791,7 +811,7 @@ static void receive_needs(void) if (depth == 0 && !deepen_rev_list && shallows.nr == 0) return; if (depth > 0 && deepen_rev_list) - die("--depth and --shallow-since (or --shallow-exclude) cannot be used together"); + die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); if (depth > 0) deepen(depth, deepen_relative, &shallows); else if (deepen_rev_list) { @@ -801,16 +821,17 @@ static void receive_needs(void) argv_array_push(&av, "rev-list"); if (deepen_since) argv_array_pushf(&av, "--max-age=%lu", deepen_since); - for (i = 0; i < want_obj.nr; i++) { - struct object *o = want_obj.objects[i].item; - argv_array_push(&av, oid_to_hex(&o->oid)); - } if (deepen_not.nr) { argv_array_push(&av, "--not"); for (i = 0; i < deepen_not.nr; i++) { struct string_list_item *s = deepen_not.items + i; argv_array_push(&av, s->string); } + argv_array_push(&av, "--not"); + } + for (i = 0; i < want_obj.nr; i++) { + struct object *o = want_obj.objects[i].item; + argv_array_push(&av, oid_to_hex(&o->oid)); } deepen_by_rev_list(av.argc, av.argv, &shallows); argv_array_clear(&av); Nguyễn Thái Ngọc Duy (25): remote-curl.c: convert fetch_git() to use argv_array transport-helper.c: refactor set_helper_option() upload-pack: move shallow deepen code out of receive_needs() upload-pack: move "shallow" sending code out of deepen() upload-pack: remove unused variable "backup" upload-pack: move "unshallow" sending code out of deepen() upload-pack: use skip_prefix() instead of starts_with() upload-pack: tighten number parsing at "deepen" lines upload-pack: move rev-list code out of check_non_tip() fetch-pack: use skip_prefix() instead of starts_with() fetch-pack: use a common function for verbose printing fetch-pack: use a separate flag for fetch in deepening mode shallow.c: implement a generic shallow boundary finder based on rev-list upload-pack: add deepen-since to cut shallow repos based on time fetch: define shallow boundary with --shallow-since clone: define shallow clone boundary based on time with --shallow-since t5500, t5539: tests for shallow depth since a specific date refs: add expand_ref() upload-pack: support define shallow boundary by excluding revisions fetch: define shallow boundary with --shallow-exclude clone: define shallow clone boundary with --shallow-exclude t5500, t5539: tests for shallow depth excluding a ref upload-pack: split check_unreachable() in two, prep for get_reachable_list() upload-pack: add get_reachable_list() fetch, upload-pack: --deepen=N extends shallow boundary by N commits Documentation/fetch-options.txt | 14 + Documentation/git-clone.txt | 8 + Documentation/git-fetch-pack.txt | 14 + Documentation/gitremote-helpers.txt | 11 + Documentation/technical/pack-protocol.txt | 4 +- Documentation/technical/protocol-capabilities.txt | 25 ++ builtin/clone.c | 32 +- builtin/fetch-pack.c | 27 +- builtin/fetch.c | 58 +++- commit.h | 2 + fetch-pack.c | 128 ++++---- fetch-pack.h | 4 + object.h | 2 +- refs.c | 8 +- refs.h | 1 + remote-curl.c | 74 +++-- shallow.c | 77 +++++ t/t5500-fetch-pack.sh | 69 +++++ t/t5539-fetch-http-shallow.sh | 74 +++++ transport-helper.c | 62 +++- transport.c | 12 + transport.h | 14 + upload-pack.c | 338 ++++++++++++++++------ 23 files changed, 858 insertions(+), 200 deletions(-) -- 2.7.1.532.gd9e3aaa -- To unsubscribe from this list: send the line "unsubscribe git" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html