On Tue, Jan 2, 2018 at 4:18 PM, Brandon Williams <bmwill@xxxxxxxxxx> wrote:
> When communicating with a v2 server, perform a fetch by requesting the
> 'fetch' command.
>
> Signed-off-by: Brandon Williams <bmwill@xxxxxxxxxx>
> ---
> builtin/fetch-pack.c | 2 +-
> fetch-pack.c | 267 ++++++++++++++++++++++++++++++++++++++++++++++++-
> fetch-pack.h | 4 +-
> t/t5701-protocol-v2.sh | 40 ++++++++
> transport.c | 8 +-
> 5 files changed, 314 insertions(+), 7 deletions(-)
>
> diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c
> index f492e8abd..867dd3cc7 100644
> --- a/builtin/fetch-pack.c
> +++ b/builtin/fetch-pack.c
> @@ -213,7 +213,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
> }
>
> ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought,
> - &shallow, pack_lockfile_ptr);
> + &shallow, pack_lockfile_ptr, protocol_v0);
> if (pack_lockfile) {
> printf("lock %s\n", pack_lockfile);
> fflush(stdout);
> diff --git a/fetch-pack.c b/fetch-pack.c
> index 9f6b07ad9..c26fdc539 100644
> --- a/fetch-pack.c
> +++ b/fetch-pack.c
> @@ -1008,6 +1008,262 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
> return ref;
> }
>
> +static void add_wants(const struct ref *wants, struct strbuf *req_buf)
> +{
> + for ( ; wants ; wants = wants->next) {
> + const struct object_id *remote = &wants->old_oid;
> + const char *remote_hex;
> + struct object *o;
> +
> + /*
> + * If that object is complete (i.e. it is an ancestor of a
> + * local ref), we tell them we have it but do not have to
> + * tell them about its ancestors, which they already know
> + * about.
> + *
> + * We use lookup_object here because we are only
> + * interested in the case we *know* the object is
> + * reachable and we have already scanned it.
> + */
> + if (((o = lookup_object(remote->hash)) != NULL) &&
> + (o->flags & COMPLETE)) {
> + continue;
> + }
> +
> + remote_hex = oid_to_hex(remote);
> + packet_buf_write(req_buf, "want %s\n", remote_hex);
> + }
> +}
> +
> +static int add_haves(struct strbuf *req_buf, int *in_vain)
> +{
> + int ret = 0;
> + int haves_added = 0;
> + const struct object_id *oid;
> +
> + while ((oid = get_rev())) {
> + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
> + if (++haves_added >= INITIAL_FLUSH)
> + break;
> + };
> +
> + *in_vain += haves_added;
> + if (!haves_added || *in_vain >= MAX_IN_VAIN) {
> + /* Send Done */
> + packet_buf_write(req_buf, "done\n");
> + ret = 1;
> + }
> +
> + return ret;
> +}
> +
> +static int send_haves(int fd_out, int *in_vain)
> +{
> + int ret = 0;
> + struct strbuf req_buf = STRBUF_INIT;
> +
> + ret = add_haves(&req_buf, in_vain);
> +
> + /* Send request */
> + packet_buf_flush(&req_buf);
> + write_or_die(fd_out, req_buf.buf, req_buf.len);
> +
> + strbuf_release(&req_buf);
> + return ret;
> +}
> +
> +static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
> + const struct ref *wants, struct oidset *common,
> + int *in_vain)
> +{
> + int ret = 0;
> + struct strbuf req_buf = STRBUF_INIT;
> +
> + packet_buf_write(&req_buf, "command=fetch");
> + packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
> + if (args->stateless_rpc)
> + packet_buf_write(&req_buf, "stateless-rpc=true");
> +
> + packet_buf_delim(&req_buf);
> + if (args->use_thin_pack)
> + packet_buf_write(&req_buf, "thin-pack");
> + if (args->no_progress)
> + packet_buf_write(&req_buf, "no-progress");
> + if (args->include_tag)
> + packet_buf_write(&req_buf, "include-tag");
> + if (prefer_ofs_delta)
> + packet_buf_write(&req_buf, "ofs-delta");
> +
> + /* add wants */
> + add_wants(wants, &req_buf);
> +
> + /*
> + * If we are running stateless-rpc we need to add all the common
> + * commits we've found in previous rounds
> + */
> + if (args->stateless_rpc) {
> + struct oidset_iter iter;
> + const struct object_id *oid;
> + oidset_iter_init(common, &iter);
> +
> + while ((oid = oidset_iter_next(&iter))) {
> + packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
> + }
> + }
> +
> + /* Add initial haves */
> + ret = add_haves(&req_buf, in_vain);
> +
> + /* Send request */
> + packet_buf_flush(&req_buf);
> + write_or_die(fd_out, req_buf.buf, req_buf.len);
> +
> + strbuf_release(&req_buf);
> + return ret;
> +}
> +
> +static enum ack_type process_ack(const char *line, struct object_id *oid)
> +{
> + const char *arg;
> +
> + if (!strcmp(line, "NAK"))
> + return NAK;
> + if (skip_prefix(line, "ACK ", &arg)) {
> + if (!parse_oid_hex(arg, oid, &arg)) {
> + if (strstr(arg, "continue"))
> + return ACK_continue;
> + if (strstr(arg, "common"))
> + return ACK_common;
> + if (strstr(arg, "ready"))
> + return ACK_ready;
> + return ACK;
> + }
> + }
> + if (skip_prefix(line, "ERR ", &arg))
> + die(_("remote error: %s"), arg);
> + die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
> +}
> +
> +static int process_acks(struct packet_reader *reader, struct oidset *common)
> +{
> + int got_ready = 0;
> + int got_common = 0;
> + while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
> + struct object_id oid;
> + struct commit *commit;
> + enum ack_type ack = process_ack(reader->line, &oid);
> +
> + switch (ack) {
> + case ACK_ready:
> + clear_prio_queue(&rev_list);
> + got_ready = 1;
> + /* fallthrough */
> + case ACK_common:
> + oidset_insert(common, &oid);
> + commit = lookup_commit(&oid);
> + mark_common(commit, 0, 1);
> + got_common = 1;
> + break;
> + case NAK:
> + break;
> + case ACK:
> + case ACK_continue:
> + die("ACK/ACK_continue not supported");
> + }
> + }
> +
> + if (reader->status != PACKET_READ_FLUSH &&
> + reader->status != PACKET_READ_DELIM)
> + die("Error during processing acks: %d", reader->status);
> +
> + /* return 0 if no common, 1 if there are common, or 2 if ready */
> + return got_ready + got_common;
> +}
> +
> +#define FETCH_CHECK_LOCAL 0
> +#define FETCH_SEND_REQUEST 1
> +#define FETCH_PROCESS_ACKS 2
> +#define FETCH_SEND_HAVES 3
> +#define FETCH_GET_PACK 4
> +#define FETCH_DONE 5
> +
> +static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
> + int fd[2],
> + const struct ref *orig_ref,
> + struct ref **sought, int nr_sought,
> + char **pack_lockfile)
> +{
> + struct ref *ref = copy_ref_list(orig_ref);
> + int state = FETCH_CHECK_LOCAL;
Is there any reason to use #defines over an enum here?
> + struct oidset common = OIDSET_INIT;
> + struct packet_reader reader;
> + int in_vain = 0;
> + packet_reader_init(&reader, fd[0], NULL, 0,
> + PACKET_READ_CHOMP_NEWLINE);
> +
> + while (state != FETCH_DONE) {
> + switch (state) {
> + case FETCH_CHECK_LOCAL:
> + sort_ref_list(&ref, ref_compare_name);
> + QSORT(sought, nr_sought, cmp_ref_by_name);
> +
> + /* v2 supports these by default */
> + allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
> + use_sideband = 2;
> +
> + /* Filter 'ref' by 'sought' and those that aren't local */
> + if (everything_local(args, &ref, sought, nr_sought))
> + state = FETCH_DONE;
> + else
> + state = FETCH_SEND_REQUEST;
> + break;
> + case FETCH_SEND_REQUEST:
> + if (send_fetch_request(fd[1], args, ref, &common, &in_vain))
> + state = FETCH_GET_PACK;
> + else
> + state = FETCH_PROCESS_ACKS;
> + break;
> + case FETCH_PROCESS_ACKS:
> + /* Process ACKs/NAKs */
> + switch (process_acks(&reader, &common)) {
> + case 2:
> + state = FETCH_GET_PACK;
> + break;
> + case 1:
> + in_vain = 0;
> + /* fallthrough */
> + default:
> + if (args->stateless_rpc)
> + state = FETCH_SEND_REQUEST;
> + else
> + state = FETCH_SEND_HAVES;
> + break;
> + }
> + break;
> + case FETCH_SEND_HAVES:
> + if (send_haves(fd[1], &in_vain))
> + state = FETCH_GET_PACK;
> + else
> + state = FETCH_PROCESS_ACKS;
> + break;
> + case FETCH_GET_PACK:
> + /* get the pack */
> + if (get_pack(args, fd, pack_lockfile))
> + die(_("git fetch-pack: fetch failed."));
> +
> + state = FETCH_DONE;
> + break;
> + case FETCH_DONE:
> + break;
> + default:
> + die("invalid state");
> + }
> + }
> +
> + oidset_clear(&common);
> + return ref;
> +}
> +
> static void fetch_pack_config(void)
> {
> git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
> @@ -1153,7 +1409,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
> const char *dest,
> struct ref **sought, int nr_sought,
> struct oid_array *shallow,
> - char **pack_lockfile)
> + char **pack_lockfile,
> + enum protocol_version version)
> {
> struct ref *ref_cpy;
> struct shallow_info si;
> @@ -1167,8 +1424,12 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
> die(_("no matching remote head"));
> }
> prepare_shallow_info(&si, shallow);
> - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
> - &si, pack_lockfile);
> + if (version == protocol_v2)
> + ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
> + pack_lockfile);
> + else
> + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
> + &si, pack_lockfile);
> reprepare_packed_git();
> update_shallow(args, sought, nr_sought, &si);
> clear_shallow_info(&si);
> diff --git a/fetch-pack.h b/fetch-pack.h
> index b6aeb43a8..7afca7305 100644
> --- a/fetch-pack.h
> +++ b/fetch-pack.h
> @@ -3,6 +3,7 @@
>
> #include "string-list.h"
> #include "run-command.h"
> +#include "protocol.h"
>
> struct oid_array;
>
> @@ -43,7 +44,8 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
> struct ref **sought,
> int nr_sought,
> struct oid_array *shallow,
> - char **pack_lockfile);
> + char **pack_lockfile,
> + enum protocol_version version);
>
> /*
> * Print an appropriate error message for each sought ref that wasn't
> diff --git a/t/t5701-protocol-v2.sh b/t/t5701-protocol-v2.sh
> index 7d8aeb766..3e411e178 100755
> --- a/t/t5701-protocol-v2.sh
> +++ b/t/t5701-protocol-v2.sh
> @@ -33,4 +33,44 @@ test_expect_success 'ref advertisment is filtered with ls-remote using protocol
> ! grep "refs/tags/" log
> '
>
> +test_expect_success 'clone with file:// using protocol v2' '
> + GIT_TRACE_PACKET=1 git -c protocol.version=2 \
> + clone "file://$(pwd)/file_parent" file_child 2>log &&
> +
> + git -C file_child log -1 --format=%s >actual &&
> + git -C file_parent log -1 --format=%s >expect &&
> + test_cmp expect actual &&
> +
> + # Server responded using protocol v1
> + grep "clone< version 2" log
> +'
> +
> +test_expect_success 'fetch with file:// using protocol v2' '
> + test_commit -C file_parent two &&
> +
> + GIT_TRACE_PACKET=1 git -C file_child -c protocol.version=2 \
> + fetch origin 2>log &&
> +
> + git -C file_child log -1 --format=%s origin/master >actual &&
> + git -C file_parent log -1 --format=%s >expect &&
> + test_cmp expect actual &&
> +
> + # Server responded using protocol v1
> + grep "fetch< version 2" log
> +'
> +
> +test_expect_success 'ref advertisment is filtered during fetch using protocol v2' '
> + test_commit -C file_parent three &&
> +
> + GIT_TRACE_PACKET=1 git -C file_child -c protocol.version=2 \
> + fetch origin master 2>log &&
> +
> + git -C file_child log -1 --format=%s origin/master >actual &&
> + git -C file_parent log -1 --format=%s >expect &&
> + test_cmp expect actual &&
> +
> + grep "ref-pattern master" log &&
> + ! grep "refs/tags/" log
> +'
> +
> test_done
> diff --git a/transport.c b/transport.c
> index 6ea3905e3..4fdbd9adc 100644
> --- a/transport.c
> +++ b/transport.c
> @@ -256,14 +256,18 @@ static int fetch_refs_via_pack(struct transport *transport,
>
> switch (data->version) {
> case protocol_v2:
> - die("support for protocol v2 not implemented yet");
> + refs = fetch_pack(&args, data->fd, data->conn,
> + refs_tmp ? refs_tmp : transport->remote_refs,
> + dest, to_fetch, nr_heads, &data->shallow,
> + &transport->pack_lockfile, data->version);
> + packet_flush(data->fd[1]);
> break;
> case protocol_v1:
> case protocol_v0:
> refs = fetch_pack(&args, data->fd, data->conn,
> refs_tmp ? refs_tmp : transport->remote_refs,
> dest, to_fetch, nr_heads, &data->shallow,
> - &transport->pack_lockfile);
> + &transport->pack_lockfile, data->version);
> break;
> case protocol_unknown_version:
> BUG("unknown protocol version");
> --
> 2.15.1.620.gb9897f4670-goog
>
