On Mon, Sep 9, 2024 at 3:17 PM Shubham Kanodia via GitGitGadget <gitgitgadget@xxxxxxxxx> wrote: > > From: Shubham Kanodia <shubham.kanodia10@xxxxxxxxx> > > This commit introduces a new configuration option, > remote.<name>.prefetchref, which allows users to specify specific > ref patterns to be prefetched during a git fetch --prefetch > operation. > > The new option accepts a space-separated list of ref patterns. > When the --prefetch option is used with git fetch, only the refs > matching these patterns will be prefetched, instead of the > default behavior of prefetching all fetchable refs. > > Example usage in .git/config: > [remote "origin"] > prefetchref = "refs/heads/main refs/heads/feature/*" > > This change allows users to optimize their prefetch operations, potentially > reducing network traffic and improving performance for large repositories > with many refs. > > Signed-off-by: Shubham Kanodia <shubham.kanodia10@xxxxxxxxx> > --- > remote: introduce config to set prefetch refs > > Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-1782%2Fpastelsky%2Fsk%2Fremote-prefetchref-v1 > Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-1782/pastelsky/sk/remote-prefetchref-v1 > Pull-Request: https://github.com/gitgitgadget/git/pull/1782 > > Documentation/config/remote.txt | 6 +++ > builtin/fetch.c | 29 +++++++++++++- > remote.c | 8 ++++ > remote.h | 3 ++ > t/t7900-maintenance.sh | 70 +++++++++++++++++++++++++++++++++ > 5 files changed, 115 insertions(+), 1 deletion(-) > > diff --git a/Documentation/config/remote.txt b/Documentation/config/remote.txt > index 8efc53e836d..b25d76dd3b1 100644 > --- a/Documentation/config/remote.txt > +++ b/Documentation/config/remote.txt > @@ -33,6 +33,12 @@ remote.<name>.fetch:: > The default set of "refspec" for linkgit:git-fetch[1]. See > linkgit:git-fetch[1]. > > +remote.<name>.prefetchref:: > + Specify the refs to be prefetched when fetching from this remote. > + The value is a space-separated list of ref patterns (e.g., "refs/heads/master refs/heads/develop*"). > + These patterns are used as the source part of the refspecs for prefetching. > + This can be used to optimize fetch operations by specifying exactly which refs should be prefetched. > + > remote.<name>.push:: > The default set of "refspec" for linkgit:git-push[1]. See > linkgit:git-push[1]. > diff --git a/builtin/fetch.c b/builtin/fetch.c > index b2b5aee5bf2..6e584fa2ebb 100644 > --- a/builtin/fetch.c > +++ b/builtin/fetch.c > @@ -434,6 +434,30 @@ static void find_non_local_tags(const struct ref *refs, > oidset_clear(&fetch_oids); > } > > +static void apply_prefetch_refspec(struct remote *remote, struct refspec *rs) > +{ > + if (remote->prefetch_refs.nr > 0) { > + int i; > + for (i = 0; i < remote->prefetch_refs.nr; i++) { > + const char *src = remote->prefetch_refs.items[i].string; > + struct strbuf dst = STRBUF_INIT; > + > + strbuf_addf(&dst, "refs/prefetch/%s/", remote->name); > + if (starts_with(src, "refs/heads/")) { > + strbuf_addstr(&dst, src + 11); > + } else if (starts_with(src, "refs/")) { > + strbuf_addstr(&dst, src + 5); > + } else { > + strbuf_addstr(&dst, src); > + } > + > + refspec_appendf(rs, "%s:%s", src, dst.buf); > + strbuf_release(&dst); > + } > + } > +} > + > + > static void filter_prefetch_refspec(struct refspec *rs) > { > int i; > @@ -502,8 +526,11 @@ static struct ref *get_ref_map(struct remote *remote, > int existing_refs_populated = 0; > > filter_prefetch_refspec(rs); > - if (remote) > + if (remote) { > filter_prefetch_refspec(&remote->fetch); > + if (prefetch) > + apply_prefetch_refspec(remote, rs); > + } > > if (rs->nr) { > struct refspec *fetch_refspec; > diff --git a/remote.c b/remote.c > index 8f3dee13186..b46d62b2c47 100644 > --- a/remote.c > +++ b/remote.c > @@ -141,6 +141,7 @@ static struct remote *make_remote(struct remote_state *remote_state, > ret->prune = -1; /* unspecified */ > ret->prune_tags = -1; /* unspecified */ > ret->name = xstrndup(name, len); > + string_list_init_dup(&ret->prefetch_refs); > refspec_init(&ret->push, REFSPEC_PUSH); > refspec_init(&ret->fetch, REFSPEC_FETCH); > > @@ -166,6 +167,7 @@ static void remote_clear(struct remote *remote) > free((char *)remote->uploadpack); > FREE_AND_NULL(remote->http_proxy); > FREE_AND_NULL(remote->http_proxy_authmethod); > + string_list_clear(&remote->prefetch_refs, 0); > } > > static void add_merge(struct branch *branch, const char *name) > @@ -456,6 +458,12 @@ static int handle_config(const char *key, const char *value, > remote->prune = git_config_bool(key, value); > else if (!strcmp(subkey, "prunetags")) > remote->prune_tags = git_config_bool(key, value); > + else if (!strcmp(subkey, "prefetchref")) { > + if (!value) > + return config_error_nonbool(key); > + string_list_split(&remote->prefetch_refs, value, ' ', -1); > + return 0; > + } > else if (!strcmp(subkey, "url")) { > if (!value) > return config_error_nonbool(key); > diff --git a/remote.h b/remote.h > index b901b56746d..c18e68e0d8d 100644 > --- a/remote.h > +++ b/remote.h > @@ -5,6 +5,7 @@ > #include "hashmap.h" > #include "refspec.h" > #include "strvec.h" > +#include "string-list.h" > > struct option; > struct transport_ls_refs_options; > @@ -77,6 +78,8 @@ struct remote { > > struct refspec fetch; > > + struct string_list prefetch_refs; > + > /* > * The setting for whether to fetch tags (as a separate rule from the > * configured refspecs); > diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh > index abae7a97546..2ad5b922d83 100755 > --- a/t/t7900-maintenance.sh > +++ b/t/t7900-maintenance.sh > @@ -245,6 +245,76 @@ test_expect_success 'prefetch multiple remotes' ' > test_subcommand git fetch remote2 $fetchargs <skip-remote1.txt > ' > > +test_expect_success 'prefetch only acts on remote.<name>.prefetchref refs if present' ' > + test_create_repo prefetch-test-mixed-patterns && > + ( > + cd prefetch-test-mixed-patterns && > + test_commit initial && > + git clone . clone1 && > + git clone . clone2 && > + > + git remote add remote1 "file://$(pwd)/clone1" && > + git remote add remote2 "file://$(pwd)/clone2" && > + > + # Set single prefetchref pattern for remote1 and multiple for remote2 > + git config remote.remote1.prefetchref "refs/heads/foo" && > + git config remote.remote2.prefetchref "refs/heads/feature/* refs/heads/topic" && > + > + # Create branches in clone1 and push > + ( > + cd clone1 && > + git checkout -b foo && > + test_commit foo-commit && > + git checkout -b feature/a && > + test_commit feature-a-commit && > + git checkout -b other && > + test_commit other-commit && > + git push origin foo feature/a other > + ) && > + > + # Create branches in clone2 and push > + ( > + cd clone2 && > + git checkout -b topic && > + test_commit master-commit && > + git checkout -b feature/x && > + test_commit feature-x-commit && > + git checkout -b feature/y && > + test_commit feature-y-commit && > + git checkout -b dev && > + test_commit dev-commit && > + git push origin topic feature/x feature/y dev > + ) && > + > + # Run maintenance prefetch task > + GIT_TRACE2_EVENT="$(pwd)/prefetch.txt" git maintenance run --task=prefetch 2>/dev/null && > + > + # Check that only specified refs were prefetched > + fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" && > + test_subcommand git fetch remote1 $fetchargs <prefetch.txt && > + test_subcommand git fetch remote2 $fetchargs <prefetch.txt && > + ls -R .git/refs/prefetch && > + > + # Verify that only specified refs are in the prefetch refs for remote1 > + git rev-parse refs/prefetch/remotes/remote1/foo && > + test_must_fail git rev-parse refs/prefetch/remotes/remote1/feature/a && > + test_must_fail git rev-parse refs/prefetch/remotes/remote1/other && > + > + # Verify that only specified refs are in the prefetch refs for remote2 > + git rev-parse refs/prefetch/remotes/remote2/feature/x && > + git rev-parse refs/prefetch/remotes/remote2/feature/y && > + git rev-parse refs/prefetch/remotes/remote2/topic && > + test_must_fail git rev-parse refs/prefetch/remotes/remote2/dev && > + > + # Fetch all refs and compare > + git fetch --all && > + test_cmp_rev refs/remotes/remote1/foo refs/prefetch/remotes/remote1/foo && > + test_cmp_rev refs/remotes/remote2/feature/x refs/prefetch/remotes/remote2/feature/x && > + test_cmp_rev refs/remotes/remote2/feature/y refs/prefetch/remotes/remote2/feature/y && > + test_cmp_rev refs/remotes/remote2/topic refs/prefetch/remotes/remote2/topic > + ) > +' > + > test_expect_success 'loose-objects task' ' > # Repack everything so we know the state of the object dir > git repack -adk && > > base-commit: 2e7b89e038c0c888acf61f1b4ee5a43d4dd5e94c > -- > gitgitgadget This is a continuation of my work, as we deemed that adding a `remote.<remote-name>.prefetch` was unnecessary given that there were already ways to stop fetching refs from a given remote using `skipFetchAll`. Looking at getting early feedback on the direction & implementation here since this isn't as straightforward as my last contribution. I would appreciate any thoughts! The config's name is tentatively `prefetchref,` but I'm open to suggestions.
