If configured and selected, the mirrors are tried in turn until one
succeeds, re-writing the refs to a refs/mirrors/<remote>/<hostname>/ space.
No refs from the mirrors are ever written to the real refs/heads or
refs/tags spaces, but their being available locally will speed up fetching
from the real remote if they are more up to date than the local version.
Signed-off-by: Sam Vilain <sam@xxxxxxxxxx>
---
builtin-fetch.c | 161 ++++++++++++++++++++++++++++++++++++++++++++--
remote.c | 14 ++++-
remote.h | 1 +
t/t5560-mirror-fetch.sh | 46 +++++++++++++
transport.c | 41 ++++++++++++
transport.h | 5 ++
6 files changed, 260 insertions(+), 8 deletions(-)
create mode 100644 t/t5560-mirror-fetch.sh
diff --git a/builtin-fetch.c b/builtin-fetch.c
index 209f502..b3b8766 100644
--- a/builtin-fetch.c
+++ b/builtin-fetch.c
@@ -45,8 +45,8 @@ static struct option builtin_fetch_options[] = {
OPT_BOOLEAN('k', "keep", &keep, "keep downloaded pack"),
OPT_BOOLEAN('u', "update-head-ok", &update_head_ok,
"allow updating of HEAD ref"),
- OPT_BOOLEAN('M', "mirror", &use_mirror,
- "use mirror if available"),
+ OPT_SET_INT('M', "use-mirror", &use_mirror,
+ "use mirror if available", 1),
OPT_STRING(0, "depth", &depth, "DEPTH",
"deepen history of shallow clone"),
OPT_END()
@@ -109,6 +109,109 @@ static void find_non_local_tags(struct transport *transport,
struct ref **head,
struct ref ***tail);
+char* get_url_hostname(const char *url)
+{
+ char *scratch = xstrdup(url);
+ char *host = strstr(url, "://");
+ char c;
+ char *end, *rh;
+ if (host) {
+ host += 3;
+ c = '/';
+ }
+ else {
+ host = scratch;
+ c = ':';
+ }
+
+ if (host[0] == '[') {
+ end = strchr(host + 1, ']');
+ if (end) {
+ *end = 0;
+ host++;
+ }
+ }
+ else {
+ end = strchr(host, c);
+ if (end && !has_dos_drive_prefix(url) ) {
+ *end = 0;
+ }
+ else {
+ host = "localhost";
+ }
+ }
+ rh = xstrdup(host);
+ free(scratch);
+ return rh;
+}
+
+const char *mirror_ref(const char* remote_name, const char* mirror_hostname,
+ const char* refname)
+{
+ int has_refs, new_sz;
+ char *rv, *dst;
+
+ // *rs[i] = *refspec[i]; ?
+ has_refs = ( strstr(refname, "refs/") == refname );
+ /* "refs/"(0 or 5) "mirrors/"(8) remote "/"(1) hostname "/"(1) */
+ new_sz = (has_refs ? 0 : 5) + 8
+ + strlen(remote_name) + 1
+ + strlen(mirror_hostname) + 1
+ + strlen(refname) + 1;
+ rv = xmalloc( new_sz );
+ strcpy(rv, "refs/mirrors/");
+ dst = rv + 13;
+ strcpy(dst, remote_name);
+ dst += strlen(remote_name);
+ *dst++ = '/';
+ strcpy(dst, mirror_hostname);
+ dst += strlen(mirror_hostname);
+ *dst++ = '/';
+ strcpy(dst, refname+(has_refs?5:0));
+ return rv;
+}
+
+struct ref *mirror_refmap(struct transport* transport,
+ struct ref* ref_map)
+{
+ struct ref *rm, *mirror_refmap, *last, *rv, *peer_ref;
+
+ const char* remote_name = transport->remote->name;
+ const char* mirror_hostname = get_url_hostname(transport->url);
+ int c = 0;
+
+ last = NULL;
+ rv = NULL;
+ for (rm = ref_map; rm; rm = rm->next) {
+ const char *new_dst;
+
+ // skip refs we already have locally, to avoid ref churn
+ if (has_sha1_file(rm->old_sha1))
+ continue;
+
+ mirror_refmap = alloc_ref(rm->name);
+ mirror_refmap->remote_status = rm->remote_status;
+ hashcpy(mirror_refmap->old_sha1, rm->old_sha1);
+ hashcpy(mirror_refmap->new_sha1, rm->new_sha1);
+
+ if (last)
+ last->next = mirror_refmap;
+ else
+ rv = mirror_refmap;
+ c++;
+
+ new_dst = mirror_ref(remote_name, mirror_hostname, rm->name);
+
+ peer_ref = alloc_ref(new_dst);
+ mirror_refmap->peer_ref = peer_ref;
+ peer_ref->force = 1;
+ last = mirror_refmap;
+ }
+
+ return rv;
+}
+
+
static struct ref *get_ref_map(struct transport *transport,
struct refspec *refs, int ref_count, int tags,
int *autotags)
@@ -165,10 +268,14 @@ static struct ref *get_ref_map(struct transport *transport,
if (tags == TAGS_DEFAULT && *autotags)
find_non_local_tags(transport, &ref_map, &tail);
ref_remove_duplicates(ref_map);
+ if (strcmp(transport->url, transport->remote->url[0]) != 0) {
+ return mirror_refmap(transport, ref_map);
+ }
return ref_map;
}
+
#define STORE_REF_ERROR_OTHER 1
#define STORE_REF_ERROR_DF_CONFLICT 2
@@ -638,6 +745,7 @@ static int do_fetch(struct transport *transport,
}
ref_map = get_ref_map(transport, refs, ref_count, tags, &autotags);
+
if (!update_head_ok)
check_not_current_branch(ref_map);
@@ -688,13 +796,18 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
int i;
static const char **refs = NULL;
int ref_nr = 0;
- int exit_code;
+ int exit_code = 0;
+ int urls_remaining = 1;
+ struct transport *real_transport = NULL;
+ const char *mirror = NULL;
+ struct refspec *refspec;
/* Record the command line for the reflog */
strbuf_addstr(&default_rla, "fetch");
for (i = 1; i < argc; i++)
strbuf_addf(&default_rla, " %s", argv[i]);
+ use_mirror = -1;
argc = parse_options(argc, argv, prefix,
builtin_fetch_options, builtin_fetch_usage, 0);
@@ -706,6 +819,10 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
if (!remote)
die("Where do you want to fetch from today?");
+ if (use_mirror == -1) {
+ use_mirror = remote->use_mirror ? 1 : 0;
+ }
+
transport = transport_get(remote, remote->url[0]);
if (verbosity >= 2)
transport->verbose = 1;
@@ -742,9 +859,39 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
sigchain_push_common(unlock_pack_on_signal);
atexit(unlock_pack);
- exit_code = do_fetch(transport,
- parse_fetch_refspec(ref_nr, refs), ref_nr);
- transport_disconnect(transport);
- transport = NULL;
+ refspec = parse_fetch_refspec(ref_nr, refs);
+ if (use_mirror) {
+ real_transport = transport;
+ urls_remaining = remote->mirror_url_nr + 1;
+ }
+ while (urls_remaining) {
+ if (use_mirror && (urls_remaining > 1) ) {
+ transport = transport_next_mirror(real_transport, mirror);
+ mirror = transport->url;
+ warning("trying mirror: %s", mirror);
+ // real_transport may not have these options - re-set them.
+ if (upload_pack)
+ set_option(TRANS_OPT_UPLOADPACK, upload_pack);
+ if (keep)
+ set_option(TRANS_OPT_KEEP, "yes");
+ if (depth)
+ set_option(TRANS_OPT_DEPTH, depth);
+
+ }
+ exit_code = do_fetch(transport, refspec, ref_nr);
+ transport_disconnect(transport);
+ transport = NULL;
+ urls_remaining--;
+ if (use_mirror) {
+ if (!exit_code && urls_remaining >= 1) {
+ warning("successful fetch from mirror");
+ urls_remaining = 1;
+ }
+ if (urls_remaining == 1) {
+ transport = real_transport;
+ warning("trying master: %s", transport->url);
+ }
+ }
+ }
return exit_code;
}
diff --git a/remote.c b/remote.c
index 65df03d..5f08e10 100644
--- a/remote.c
+++ b/remote.c
@@ -139,8 +139,9 @@ static struct remote *make_remote(const char *name, int len)
for (i = 0; i < remotes_nr; i++) {
if (len ? (!strncmp(name, remotes[i]->name, len) &&
!remotes[i]->name[len]) :
- !strcmp(name, remotes[i]->name))
+ !strcmp(name, remotes[i]->name)) {
return remotes[i];
+ }
}
ret = xcalloc(1, sizeof(struct remote));
@@ -683,6 +684,7 @@ static struct refspec *parse_push_refspec(int nr_refspec, const char **refspec)
return parse_refspec_internal(nr_refspec, refspec, 0, 0);
}
+
static int valid_remote_nick(const char *name)
{
if (!name[0] || is_dot_or_dotdot(name))
@@ -786,6 +788,16 @@ int remote_has_url(struct remote *remote, const char *url)
return 0;
}
+int remote_mirror_idx(struct remote *remote, const char *mirror_url)
+{
+ int i;
+ for (i = 0; i < remote->mirror_url_nr; i++) {
+ if (!strcmp(remote->mirror_url[i], mirror_url))
+ return i;
+ }
+ return -1;
+}
+
static int match_name_with_pattern(const char *key, const char *name,
const char *value, char **result)
{
diff --git a/remote.h b/remote.h
index c720b9a..da208ff 100644
--- a/remote.h
+++ b/remote.h
@@ -61,6 +61,7 @@ typedef int each_remote_fn(struct remote *remote, void *priv);
int for_each_remote(each_remote_fn fn, void *priv);
int remote_has_url(struct remote *remote, const char *url);
+int remote_mirror_idx(struct remote *remote, const char *mirror_url);
struct refspec {
unsigned force : 1;
diff --git a/t/t5560-mirror-fetch.sh b/t/t5560-mirror-fetch.sh
new file mode 100644
index 0000000..940dc0e
--- /dev/null
+++ b/t/t5560-mirror-fetch.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# Copyright (c) 2009 Sam Vilain
+#
+
+test_description='mirror fetch test'
+
+. ./test-lib.sh
+
+test_expect_success setup '
+ echo >file master initial &&
+ git add file &&
+ git commit -a -m "Master initial" &&
+ git clone . master &&
+ git clone master mirror &&
+ cd master &&
+ echo >file master update &&
+ git commit -a -m "Master update" &&
+ cd .. &&
+ mkdir clone &&
+ cd clone &&
+ git init &&
+ git remote add origin ../master &&
+ git config remote.origin.mirror-url ../mirror
+'
+
+# in later iterations we'll expect these mirror tracking refs to be
+# cleaned up once they are confirmed reachable from the master, but
+# for now they leave a sufficient breadcrumb of the operation
+
+test_expect_success 'fetch using mirror - explicit' '
+ git fetch --use-mirror origin refs/heads/*:refs/remotes/origin/* &&
+ git rev-parse refs/mirrors/origin/localhost/heads/master
+'
+
+test_expect_success 'fetch using mirror - default' '
+ cd .. &&
+ mkdir clone2 &&
+ cd clone2 &&
+ git init &&
+ git remote add origin ../master &&
+ git config remote.origin.mirror-url ../mirror
+ git fetch --use-mirror &&
+ git rev-parse refs/mirrors/origin/localhost/heads/master
+'
+test_done
diff --git a/transport.c b/transport.c
index 644a30a..0dc0185 100644
--- a/transport.c
+++ b/transport.c
@@ -859,6 +859,47 @@ struct transport *transport_get(struct remote *remote, const char *url)
return ret;
}
+struct transport *transport_next_mirror(struct transport *transport,
+ const char *last_mirror)
+{
+ struct transport *ret;
+ struct remote* remote = transport->remote;
+ int mirror_idx = -1;
+ const char* url;
+
+ if (!last_mirror) {
+ if (remote->preferred_mirror) {
+ mirror_idx = remote_mirror_idx(
+ remote,
+ remote->preferred_mirror
+ );
+ if (mirror_idx == -1) {
+ warning("preferred mirror '%s' not listed "
+ "in remote.%s.mirror-url",
+ remote->preferred_mirror,
+ remote->name);
+ }
+ }
+ else {
+ mirror_idx = 0;
+ }
+ }
+ else {
+ mirror_idx = remote_mirror_idx(remote, last_mirror) + 1;
+ // caller must check that we are not looping indefinitely
+ mirror_idx %= remote->mirror_url_nr;
+ }
+
+ url = remote->mirror_url[mirror_idx];
+ ret = transport_get(remote, url);
+
+ // copy settings - caller must re-set options
+ ret->verbose = transport->verbose;
+ ret->progress = transport->progress;
+
+ return ret;
+}
+
int transport_set_option(struct transport *transport,
const char *name, const char *value)
{
diff --git a/transport.h b/transport.h
index c14da6f..9890157 100644
--- a/transport.h
+++ b/transport.h
@@ -41,6 +41,9 @@ struct transport {
/* Returns a transport suitable for the url */
struct transport *transport_get(struct remote *, const char *);
+/* Returns a transport for a mirror */
+struct transport *transport_next_mirror(struct transport *transport, const char *last_mirror);
+
/* Transport options which apply to git:// and scp-style URLs */
/* The program to use on the remote side to send a pack */
@@ -78,6 +81,8 @@ int transport_fetch_refs(struct transport *transport, const struct ref *refs);
void transport_unlock_pack(struct transport *transport);
int transport_disconnect(struct transport *transport);
char *transport_anonymize_url(const char *url);
+struct refspec *mirror_refspec(struct transport* transport,
+ struct refspec *refspec, int refspec_nr);
/* Transport methods defined outside transport.c */
int transport_helper_init(struct transport *transport, const char *name);
--
1.6.3.3
--
To unsubscribe from this list: send the line "unsubscribe git" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html