From: Shubham Kanodia <shubham.kanodia10@xxxxxxxxx>
This commit introduces a new configuration option,
remote.<name>.prefetchref, which allows users to specify specific
ref patterns to be prefetched during a git fetch --prefetch
operation.
The new option accepts a space-separated list of ref patterns.
When the --prefetch option is used with git fetch, only the refs
matching these patterns will be prefetched, instead of the
default behavior of prefetching all fetchable refs.
Example usage in .git/config:
[remote "origin"]
prefetchref = "refs/heads/main refs/heads/feature/*"
This change allows users to optimize their prefetch operations, potentially
reducing network traffic and improving performance for large repositories
with many refs.
Signed-off-by: Shubham Kanodia <shubham.kanodia10@xxxxxxxxx>
---
remote: introduce config to set prefetch refs
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-1782%2Fpastelsky%2Fsk%2Fremote-prefetchref-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-1782/pastelsky/sk/remote-prefetchref-v2
Pull-Request: https://github.com/gitgitgadget/git/pull/1782
Range-diff vs v1:
1: 3113d8b8635 ! 1: f9f9e637bfa remote: introduce config to set prefetch refs
@@ Documentation/config/remote.txt: remote.<name>.fetch::
linkgit:git-push[1].
## builtin/fetch.c ##
-@@ builtin/fetch.c: static void find_non_local_tags(const struct ref *refs,
- oidset_clear(&fetch_oids);
+@@
+ #include "trace.h"
+ #include "trace2.h"
+ #include "bundle-uri.h"
++#include "wildmatch.h"
+
+ #define FORCED_UPDATES_DELAY_WARNING_IN_MS (10 * 1000)
+
+@@ builtin/fetch.c: static void filter_prefetch_refspec(struct refspec *rs)
+ }
}
-+static void apply_prefetch_refspec(struct remote *remote, struct refspec *rs)
++static int matches_prefetch_refs(const char *refname, const struct string_list *prefetch_refs)
+{
-+ if (remote->prefetch_refs.nr > 0) {
-+ int i;
-+ for (i = 0; i < remote->prefetch_refs.nr; i++) {
-+ const char *src = remote->prefetch_refs.items[i].string;
-+ struct strbuf dst = STRBUF_INIT;
-+
-+ strbuf_addf(&dst, "refs/prefetch/%s/", remote->name);
-+ if (starts_with(src, "refs/heads/")) {
-+ strbuf_addstr(&dst, src + 11);
-+ } else if (starts_with(src, "refs/")) {
-+ strbuf_addstr(&dst, src + 5);
-+ } else {
-+ strbuf_addstr(&dst, src);
-+ }
++ int i;
++ int has_positive = 0;
++ int matched_positive = 0;
++ int matched_negative = 0;
++
++ for (i = 0; i < prefetch_refs->nr; i++) {
++ const char *pattern = prefetch_refs->items[i].string;
++ int is_negative = (*pattern == '!');
++
++ if (is_negative)
++ pattern++;
++ else
++ has_positive = 1;
+
-+ refspec_appendf(rs, "%s:%s", src, dst.buf);
-+ strbuf_release(&dst);
++ if (wildmatch(pattern, refname, 0) == 0) {
++ if (is_negative)
++ matched_negative = 1;
++ else
++ matched_positive = 1;
+ }
+ }
++
++ if (!has_positive)
++ return !matched_negative;
++
++ return matched_positive && !matched_negative;
+}
+
+
- static void filter_prefetch_refspec(struct refspec *rs)
- {
- int i;
++static void ref_remove(struct ref **head, struct ref *to_remove)
++{
++ struct ref **pp, *p;
++
++ for (pp = head; (p = *pp) != NULL; pp = &p->next) {
++ if (p == to_remove) {
++ *pp = p->next;
++ return;
++ }
++ }
++}
++
+ static struct ref *get_ref_map(struct remote *remote,
+ const struct ref *remote_refs,
+ struct refspec *rs,
@@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
int existing_refs_populated = 0;
filter_prefetch_refspec(rs);
-- if (remote)
-+ if (remote) {
++
+ if (remote)
filter_prefetch_refspec(&remote->fetch);
-+ if (prefetch)
-+ apply_prefetch_refspec(remote, rs);
+
+@@ builtin/fetch.c: static struct ref *get_ref_map(struct remote *remote,
+ else
+ ref_map = apply_negative_refspecs(ref_map, &remote->fetch);
+
++ /**
++ * Filter out advertised refs that we don't want to fetch during
++ * prefetch if a prefetchref config is set
++ */
++ if (prefetch && remote->prefetch_refs.nr) {
++ struct ref *ref, *next;
++ for (ref = ref_map; ref; ref = next) {
++ next = ref->next;
++
++ if (!matches_prefetch_refs(ref->name, &remote->prefetch_refs)) {
++ ref_remove(&ref_map, ref);
++ free_one_ref(ref);
++ }
++ }
+ }
++
+ ref_map = ref_remove_duplicates(ref_map);
- if (rs->nr) {
- struct refspec *fetch_refspec;
+ for (rm = ref_map; rm; rm = rm->next) {
## remote.c ##
@@ remote.c: static struct remote *make_remote(struct remote_state *remote_state,
@@ t/t7900-maintenance.sh: test_expect_success 'prefetch multiple remotes' '
test_subcommand git fetch remote2 $fetchargs <skip-remote1.txt
'
-+test_expect_success 'prefetch only acts on remote.<name>.prefetchref refs if present' '
-+ test_create_repo prefetch-test-mixed-patterns &&
++test_expect_success 'prefetch with positive prefetch ref patterns' '
++ test_create_repo filter-prefetch-positive &&
+ (
-+ cd prefetch-test-mixed-patterns &&
++ cd filter-prefetch-positive &&
+ test_commit initial &&
-+ git clone . clone1 &&
+ git clone . clone2 &&
-+
-+ git remote add remote1 "file://$(pwd)/clone1" &&
+ git remote add remote2 "file://$(pwd)/clone2" &&
+
-+ # Set single prefetchref pattern for remote1 and multiple for remote2
-+ git config remote.remote1.prefetchref "refs/heads/foo" &&
-+ git config remote.remote2.prefetchref "refs/heads/feature/* refs/heads/topic" &&
-+
-+ # Create branches in clone1 and push
-+ (
-+ cd clone1 &&
-+ git checkout -b foo &&
-+ test_commit foo-commit &&
-+ git checkout -b feature/a &&
-+ test_commit feature-a-commit &&
-+ git checkout -b other &&
-+ test_commit other-commit &&
-+ git push origin foo feature/a other
-+ ) &&
-+
-+ # Create branches in clone2 and push
-+ (
-+ cd clone2 &&
-+ git checkout -b topic &&
-+ test_commit master-commit &&
-+ git checkout -b feature/x &&
-+ test_commit feature-x-commit &&
-+ git checkout -b feature/y &&
-+ test_commit feature-y-commit &&
-+ git checkout -b dev &&
-+ test_commit dev-commit &&
-+ git push origin topic feature/x feature/y dev
-+ ) &&
-+
-+ # Run maintenance prefetch task
-+ GIT_TRACE2_EVENT="$(pwd)/prefetch.txt" git maintenance run --task=prefetch 2>/dev/null &&
-+
-+ # Check that only specified refs were prefetched
++ cd clone2 &&
++ git checkout -b feature && test_commit feature-commit-2 &&
++ git checkout -b wip/test && test_commit wip-test-commit-2 &&
++ git checkout -b topic/x && test_commit topic-x-commit-2 &&
++ git push -f origin feature wip/test topic/x&&
++ cd .. &&
++
++ git config remote.remote2.prefetchref "refs/heads/feature" &&
+ fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" &&
-+ test_subcommand git fetch remote1 $fetchargs <prefetch.txt &&
-+ test_subcommand git fetch remote2 $fetchargs <prefetch.txt &&
-+ ls -R .git/refs/prefetch &&
-+
-+ # Verify that only specified refs are in the prefetch refs for remote1
-+ git rev-parse refs/prefetch/remotes/remote1/foo &&
-+ test_must_fail git rev-parse refs/prefetch/remotes/remote1/feature/a &&
-+ test_must_fail git rev-parse refs/prefetch/remotes/remote1/other &&
-+
-+ # Verify that only specified refs are in the prefetch refs for remote2
-+ git rev-parse refs/prefetch/remotes/remote2/feature/x &&
-+ git rev-parse refs/prefetch/remotes/remote2/feature/y &&
-+ git rev-parse refs/prefetch/remotes/remote2/topic &&
-+ test_must_fail git rev-parse refs/prefetch/remotes/remote2/dev &&
-+
-+ # Fetch all refs and compare
-+ git fetch --all &&
-+ test_cmp_rev refs/remotes/remote1/foo refs/prefetch/remotes/remote1/foo &&
-+ test_cmp_rev refs/remotes/remote2/feature/x refs/prefetch/remotes/remote2/feature/x &&
-+ test_cmp_rev refs/remotes/remote2/feature/y refs/prefetch/remotes/remote2/feature/y &&
-+ test_cmp_rev refs/remotes/remote2/topic refs/prefetch/remotes/remote2/topic
++ GIT_TRACE2_EVENT="$(pwd)/prefetch-positive.txt" git maintenance run --task=prefetch 2>/dev/null &&
++ test_subcommand git fetch remote2 $fetchargs <prefetch-positive.txt &&
++
++ git rev-parse refs/prefetch/remotes/remote2/feature &&
++ test_must_fail git rev-parse refs/prefetch/remotes/remote2/wip/test &&
++ test_must_fail git rev-parse refs/prefetch/remotes/remote2/topic/x
++ )
++'
++
++test_expect_success 'prefetch with negative prefetch ref patterns' '
++ test_create_repo filter-prefetch-negative &&
++ (
++ cd filter-prefetch-negative &&
++ test_commit initial &&
++ git clone . clone3 &&
++ git remote add remote3 "file://$(pwd)/clone3" &&
++ cat .git/config &&
++
++ cd clone3 &&
++ git checkout -b feature && test_commit feature-commit-3 &&
++ git checkout -b wip/test && test_commit wip-test-commit-3 &&
++ git checkout -b topic/x && test_commit topic-x-commit-3 &&
++ git push -f origin feature wip/test topic/x &&
++ cd .. &&
++
++ git config remote.remote3.prefetchref "!refs/heads/wip/*" &&
++ fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" &&
++ GIT_TRACE2_EVENT="$(pwd)/prefetch-negative.txt" git maintenance run --task=prefetch 2>/dev/null &&
++ test_subcommand git fetch remote3 $fetchargs <prefetch-negative.txt &&
++ git rev-parse refs/prefetch/remotes/remote3/feature &&
++ git rev-parse refs/prefetch/remotes/remote3/topic/x &&
++ test_must_fail git rev-parse refs/prefetch/remotes/remote3/wip/test
++ )
++'
++
++test_expect_success 'prefetch with positive & negative prefetch ref patterns' '
++ test_create_repo filter-prefetch-mixed &&
++ (
++ cd filter-prefetch-mixed &&
++ test_commit initial &&
++ git clone . clone4 &&
++ git remote add remote4 "file://$(pwd)/clone4" &&
++
++ cd clone4 &&
++ git checkout -b feature && test_commit feature-commit-4 &&
++ git checkout -b topic/x && test_commit topic-x-commit-4 &&
++ git checkout -b topic/y && test_commit topic-y-commit-4 &&
++ git push -f origin feature topic/x topic/y &&
++ cd .. &&
++
++ git config remote.remote4.prefetchref "refs/heads/topic/* !refs/heads/topic/y" &&
++ # git config --add remote.remote4.prefetchref "!refs/topic/y" &&
++ cat .git/config &&
++ fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" &&
++ GIT_TRACE2_EVENT="$(pwd)/prefetch-mixed.txt" git maintenance run --task=prefetch 2>/dev/null &&
++ test_subcommand git fetch remote4 $fetchargs <prefetch-mixed.txt &&
++
++ test_must_fail git rev-parse refs/prefetch/remotes/remote4/feature &&
++ test_must_fail git rev-parse refs/prefetch/remotes/remote4/topic/y &&
++ git rev-parse refs/prefetch/remotes/remote4/topic/x
+ )
+'
+
Documentation/config/remote.txt | 6 +++
builtin/fetch.c | 61 +++++++++++++++++++++++++
remote.c | 8 ++++
remote.h | 3 ++
t/t7900-maintenance.sh | 80 +++++++++++++++++++++++++++++++++
5 files changed, 158 insertions(+)
diff --git a/Documentation/config/remote.txt b/Documentation/config/remote.txt
index 8efc53e836d..b25d76dd3b1 100644
--- a/Documentation/config/remote.txt
+++ b/Documentation/config/remote.txt
@@ -33,6 +33,12 @@ remote.<name>.fetch::
The default set of "refspec" for linkgit:git-fetch[1]. See
linkgit:git-fetch[1].
+remote.<name>.prefetchref::
+ Specify the refs to be prefetched when fetching from this remote.
+ The value is a space-separated list of ref patterns (e.g., "refs/heads/master refs/heads/develop*").
+ These patterns are used as the source part of the refspecs for prefetching.
+ This can be used to optimize fetch operations by specifying exactly which refs should be prefetched.
+
remote.<name>.push::
The default set of "refspec" for linkgit:git-push[1]. See
linkgit:git-push[1].
diff --git a/builtin/fetch.c b/builtin/fetch.c
index b2b5aee5bf2..16c8a31c2e1 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -38,6 +38,7 @@
#include "trace.h"
#include "trace2.h"
#include "bundle-uri.h"
+#include "wildmatch.h"
#define FORCED_UPDATES_DELAY_WARNING_IN_MS (10 * 1000)
@@ -485,6 +486,49 @@ static void filter_prefetch_refspec(struct refspec *rs)
}
}
+static int matches_prefetch_refs(const char *refname, const struct string_list *prefetch_refs)
+{
+ int i;
+ int has_positive = 0;
+ int matched_positive = 0;
+ int matched_negative = 0;
+
+ for (i = 0; i < prefetch_refs->nr; i++) {
+ const char *pattern = prefetch_refs->items[i].string;
+ int is_negative = (*pattern == '!');
+
+ if (is_negative)
+ pattern++;
+ else
+ has_positive = 1;
+
+ if (wildmatch(pattern, refname, 0) == 0) {
+ if (is_negative)
+ matched_negative = 1;
+ else
+ matched_positive = 1;
+ }
+ }
+
+ if (!has_positive)
+ return !matched_negative;
+
+ return matched_positive && !matched_negative;
+}
+
+
+static void ref_remove(struct ref **head, struct ref *to_remove)
+{
+ struct ref **pp, *p;
+
+ for (pp = head; (p = *pp) != NULL; pp = &p->next) {
+ if (p == to_remove) {
+ *pp = p->next;
+ return;
+ }
+ }
+}
+
static struct ref *get_ref_map(struct remote *remote,
const struct ref *remote_refs,
struct refspec *rs,
@@ -502,6 +546,7 @@ static struct ref *get_ref_map(struct remote *remote,
int existing_refs_populated = 0;
filter_prefetch_refspec(rs);
+
if (remote)
filter_prefetch_refspec(&remote->fetch);
@@ -610,6 +655,22 @@ static struct ref *get_ref_map(struct remote *remote,
else
ref_map = apply_negative_refspecs(ref_map, &remote->fetch);
+ /**
+ * Filter out advertised refs that we don't want to fetch during
+ * prefetch if a prefetchref config is set
+ */
+ if (prefetch && remote->prefetch_refs.nr) {
+ struct ref *ref, *next;
+ for (ref = ref_map; ref; ref = next) {
+ next = ref->next;
+
+ if (!matches_prefetch_refs(ref->name, &remote->prefetch_refs)) {
+ ref_remove(&ref_map, ref);
+ free_one_ref(ref);
+ }
+ }
+ }
+
ref_map = ref_remove_duplicates(ref_map);
for (rm = ref_map; rm; rm = rm->next) {
diff --git a/remote.c b/remote.c
index 8f3dee13186..b46d62b2c47 100644
--- a/remote.c
+++ b/remote.c
@@ -141,6 +141,7 @@ static struct remote *make_remote(struct remote_state *remote_state,
ret->prune = -1; /* unspecified */
ret->prune_tags = -1; /* unspecified */
ret->name = xstrndup(name, len);
+ string_list_init_dup(&ret->prefetch_refs);
refspec_init(&ret->push, REFSPEC_PUSH);
refspec_init(&ret->fetch, REFSPEC_FETCH);
@@ -166,6 +167,7 @@ static void remote_clear(struct remote *remote)
free((char *)remote->uploadpack);
FREE_AND_NULL(remote->http_proxy);
FREE_AND_NULL(remote->http_proxy_authmethod);
+ string_list_clear(&remote->prefetch_refs, 0);
}
static void add_merge(struct branch *branch, const char *name)
@@ -456,6 +458,12 @@ static int handle_config(const char *key, const char *value,
remote->prune = git_config_bool(key, value);
else if (!strcmp(subkey, "prunetags"))
remote->prune_tags = git_config_bool(key, value);
+ else if (!strcmp(subkey, "prefetchref")) {
+ if (!value)
+ return config_error_nonbool(key);
+ string_list_split(&remote->prefetch_refs, value, ' ', -1);
+ return 0;
+ }
else if (!strcmp(subkey, "url")) {
if (!value)
return config_error_nonbool(key);
diff --git a/remote.h b/remote.h
index b901b56746d..c18e68e0d8d 100644
--- a/remote.h
+++ b/remote.h
@@ -5,6 +5,7 @@
#include "hashmap.h"
#include "refspec.h"
#include "strvec.h"
+#include "string-list.h"
struct option;
struct transport_ls_refs_options;
@@ -77,6 +78,8 @@ struct remote {
struct refspec fetch;
+ struct string_list prefetch_refs;
+
/*
* The setting for whether to fetch tags (as a separate rule from the
* configured refspecs);
diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh
index abae7a97546..5b64257eb7d 100755
--- a/t/t7900-maintenance.sh
+++ b/t/t7900-maintenance.sh
@@ -245,6 +245,86 @@ test_expect_success 'prefetch multiple remotes' '
test_subcommand git fetch remote2 $fetchargs <skip-remote1.txt
'
+test_expect_success 'prefetch with positive prefetch ref patterns' '
+ test_create_repo filter-prefetch-positive &&
+ (
+ cd filter-prefetch-positive &&
+ test_commit initial &&
+ git clone . clone2 &&
+ git remote add remote2 "file://$(pwd)/clone2" &&
+
+ cd clone2 &&
+ git checkout -b feature && test_commit feature-commit-2 &&
+ git checkout -b wip/test && test_commit wip-test-commit-2 &&
+ git checkout -b topic/x && test_commit topic-x-commit-2 &&
+ git push -f origin feature wip/test topic/x&&
+ cd .. &&
+
+ git config remote.remote2.prefetchref "refs/heads/feature" &&
+ fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" &&
+ GIT_TRACE2_EVENT="$(pwd)/prefetch-positive.txt" git maintenance run --task=prefetch 2>/dev/null &&
+ test_subcommand git fetch remote2 $fetchargs <prefetch-positive.txt &&
+
+ git rev-parse refs/prefetch/remotes/remote2/feature &&
+ test_must_fail git rev-parse refs/prefetch/remotes/remote2/wip/test &&
+ test_must_fail git rev-parse refs/prefetch/remotes/remote2/topic/x
+ )
+'
+
+test_expect_success 'prefetch with negative prefetch ref patterns' '
+ test_create_repo filter-prefetch-negative &&
+ (
+ cd filter-prefetch-negative &&
+ test_commit initial &&
+ git clone . clone3 &&
+ git remote add remote3 "file://$(pwd)/clone3" &&
+ cat .git/config &&
+
+ cd clone3 &&
+ git checkout -b feature && test_commit feature-commit-3 &&
+ git checkout -b wip/test && test_commit wip-test-commit-3 &&
+ git checkout -b topic/x && test_commit topic-x-commit-3 &&
+ git push -f origin feature wip/test topic/x &&
+ cd .. &&
+
+ git config remote.remote3.prefetchref "!refs/heads/wip/*" &&
+ fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" &&
+ GIT_TRACE2_EVENT="$(pwd)/prefetch-negative.txt" git maintenance run --task=prefetch 2>/dev/null &&
+ test_subcommand git fetch remote3 $fetchargs <prefetch-negative.txt &&
+ git rev-parse refs/prefetch/remotes/remote3/feature &&
+ git rev-parse refs/prefetch/remotes/remote3/topic/x &&
+ test_must_fail git rev-parse refs/prefetch/remotes/remote3/wip/test
+ )
+'
+
+test_expect_success 'prefetch with positive & negative prefetch ref patterns' '
+ test_create_repo filter-prefetch-mixed &&
+ (
+ cd filter-prefetch-mixed &&
+ test_commit initial &&
+ git clone . clone4 &&
+ git remote add remote4 "file://$(pwd)/clone4" &&
+
+ cd clone4 &&
+ git checkout -b feature && test_commit feature-commit-4 &&
+ git checkout -b topic/x && test_commit topic-x-commit-4 &&
+ git checkout -b topic/y && test_commit topic-y-commit-4 &&
+ git push -f origin feature topic/x topic/y &&
+ cd .. &&
+
+ git config remote.remote4.prefetchref "refs/heads/topic/* !refs/heads/topic/y" &&
+ # git config --add remote.remote4.prefetchref "!refs/topic/y" &&
+ cat .git/config &&
+ fetchargs="--prefetch --prune --no-tags --no-write-fetch-head --recurse-submodules=no --quiet" &&
+ GIT_TRACE2_EVENT="$(pwd)/prefetch-mixed.txt" git maintenance run --task=prefetch 2>/dev/null &&
+ test_subcommand git fetch remote4 $fetchargs <prefetch-mixed.txt &&
+
+ test_must_fail git rev-parse refs/prefetch/remotes/remote4/feature &&
+ test_must_fail git rev-parse refs/prefetch/remotes/remote4/topic/y &&
+ git rev-parse refs/prefetch/remotes/remote4/topic/x
+ )
+'
+
test_expect_success 'loose-objects task' '
# Repack everything so we know the state of the object dir
git repack -adk &&
base-commit: 2e7b89e038c0c888acf61f1b4ee5a43d4dd5e94c
--
gitgitgadget
