Merge branch 'jt/connectivity-check-after-unshallow'
authorJunio C Hamano <gitster@pobox.com>
Tue, 24 Jul 2018 21:50:44 +0000 (14:50 -0700)
committerJunio C Hamano <gitster@pobox.com>
Tue, 24 Jul 2018 21:50:44 +0000 (14:50 -0700)
"git fetch" failed to correctly validate the set of objects it
received when making a shallow history deeper, which has been
corrected.

* jt/connectivity-check-after-unshallow:
  fetch-pack: write shallow, then check connectivity
  fetch-pack: implement ref-in-want
  fetch-pack: put shallow info in output parameter
  fetch: refactor to make function args narrower
  fetch: refactor fetch_refs into two functions
  fetch: refactor the population of peer ref OIDs
  upload-pack: test negotiation with changing repository
  upload-pack: implement ref-in-want
  test-pkt-line: add unpack-sideband subcommand

22 files changed:
Documentation/config.txt
Documentation/technical/protocol-v2.txt
builtin/clone.c
builtin/fetch.c
connected.c
connected.h
fetch-object.c
fetch-pack.c
fetch-pack.h
remote.c
remote.h
t/helper/test-pkt-line.c
t/lib-httpd.sh
t/lib-httpd/apache.conf
t/lib-httpd/apply-one-time-sed.sh [new file with mode: 0644]
t/t5537-fetch-shallow.sh
t/t5703-upload-pack-ref-in-want.sh [new file with mode: 0755]
transport-helper.c
transport-internal.h
transport.c
transport.h
upload-pack.c

index a32172a..43b2de7 100644 (file)
@@ -3489,6 +3489,13 @@ Note that this configuration variable is ignored if it is seen in the
 repository-level config (this is a safety measure against fetching from
 untrusted repositories).
 
+uploadpack.allowRefInWant::
+       If this option is set, `upload-pack` will support the `ref-in-want`
+       feature of the protocol version 2 `fetch` command.  This feature
+       is intended for the benefit of load-balanced servers which may
+       not have the same view of what OIDs their refs point to due to
+       replication delay.
+
 url.<base>.insteadOf::
        Any URL that starts with this value will be rewritten to
        start, instead, with <base>. In cases where some site serves a
index f58f24b..09e4e02 100644 (file)
@@ -298,12 +298,21 @@ included in the client's request:
        for use with partial clone and partial fetch operations. See
        `rev-list` for possible "filter-spec" values.
 
+If the 'ref-in-want' feature is advertised, the following argument can
+be included in the client's request as well as the potential addition of
+the 'wanted-refs' section in the server's response as explained below.
+
+    want-ref <ref>
+       Indicates to the server that the client wants to retrieve a
+       particular ref, where <ref> is the full name of a ref on the
+       server.
+
 The response of `fetch` is broken into a number of sections separated by
 delimiter packets (0001), with each section beginning with its section
 header.
 
     output = *section
-    section = (acknowledgments | shallow-info | packfile)
+    section = (acknowledgments | shallow-info | wanted-refs | packfile)
              (flush-pkt | delim-pkt)
 
     acknowledgments = PKT-LINE("acknowledgments" LF)
@@ -318,6 +327,10 @@ header.
     shallow = "shallow" SP obj-id
     unshallow = "unshallow" SP obj-id
 
+    wanted-refs = PKT-LINE("wanted-refs" LF)
+                 *PKT-LINE(wanted-ref LF)
+    wanted-ref = obj-id SP refname
+
     packfile = PKT-LINE("packfile" LF)
               *PKT-LINE(%x01-03 *%x00-ff)
 
@@ -378,6 +391,19 @@ header.
        * This section is only included if a packfile section is also
          included in the response.
 
+    wanted-refs section
+       * This section is only included if the client has requested a
+         ref using a 'want-ref' line and if a packfile section is also
+         included in the response.
+
+       * Always begins with the section header "wanted-refs".
+
+       * The server will send a ref listing ("<oid> <refname>") for
+         each reference requested using 'want-ref' lines.
+
+       * The server MUST NOT send any refs which were not requested
+         using 'want-ref' lines.
+
     packfile section
        * This section is only included if the client has sent 'want'
          lines in its request and either requested that no more
index 1d939af..2f84717 100644 (file)
@@ -1156,7 +1156,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
                        }
 
                if (!is_local && !complete_refs_before_fetch)
-                       transport_fetch_refs(transport, mapped_refs);
+                       transport_fetch_refs(transport, mapped_refs, NULL);
 
                remote_head = find_ref_by_name(refs, "HEAD");
                remote_head_points_at =
@@ -1198,7 +1198,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
        if (is_local)
                clone_local(path, git_dir);
        else if (refs && complete_refs_before_fetch)
-               transport_fetch_refs(transport, mapped_refs);
+               transport_fetch_refs(transport, mapped_refs, NULL);
 
        update_remote_refs(refs, mapped_refs, remote_head_points_at,
                           branch_top.buf, reflog_msg.buf, transport,
index d9bb72b..ac06f6a 100644 (file)
@@ -242,9 +242,9 @@ static int will_fetch(struct ref **head, const unsigned char *sha1)
        return 0;
 }
 
-static void find_non_local_tags(struct transport *transport,
-                       struct ref **head,
-                       struct ref ***tail)
+static void find_non_local_tags(const struct ref *refs,
+                               struct ref **head,
+                               struct ref ***tail)
 {
        struct string_list existing_refs = STRING_LIST_INIT_DUP;
        struct string_list remote_refs = STRING_LIST_INIT_NODUP;
@@ -252,7 +252,7 @@ static void find_non_local_tags(struct transport *transport,
        struct string_list_item *item = NULL;
 
        for_each_ref(add_existing, &existing_refs);
-       for (ref = transport_get_remote_refs(transport, NULL); ref; ref = ref->next) {
+       for (ref = refs; ref; ref = ref->next) {
                if (!starts_with(ref->name, "refs/tags/"))
                        continue;
 
@@ -326,7 +326,8 @@ static void find_non_local_tags(struct transport *transport,
        string_list_clear(&remote_refs, 0);
 }
 
-static struct ref *get_ref_map(struct transport *transport,
+static struct ref *get_ref_map(struct remote *remote,
+                              const struct ref *remote_refs,
                               struct refspec *rs,
                               int tags, int *autotags)
 {
@@ -334,26 +335,11 @@ static struct ref *get_ref_map(struct transport *transport,
        struct ref *rm;
        struct ref *ref_map = NULL;
        struct ref **tail = &ref_map;
-       struct argv_array ref_prefixes = ARGV_ARRAY_INIT;
 
        /* opportunistically-updated references: */
        struct ref *orefs = NULL, **oref_tail = &orefs;
 
-       const struct ref *remote_refs;
-
-       if (rs->nr)
-               refspec_ref_prefixes(rs, &ref_prefixes);
-       else if (transport->remote && transport->remote->fetch.nr)
-               refspec_ref_prefixes(&transport->remote->fetch, &ref_prefixes);
-
-       if (ref_prefixes.argc &&
-           (tags == TAGS_SET || (tags == TAGS_DEFAULT && !rs->nr))) {
-               argv_array_push(&ref_prefixes, "refs/tags/");
-       }
-
-       remote_refs = transport_get_remote_refs(transport, &ref_prefixes);
-
-       argv_array_clear(&ref_prefixes);
+       struct string_list existing_refs = STRING_LIST_INIT_DUP;
 
        if (rs->nr) {
                struct refspec *fetch_refspec;
@@ -390,7 +376,7 @@ static struct ref *get_ref_map(struct transport *transport,
                if (refmap.nr)
                        fetch_refspec = &refmap;
                else
-                       fetch_refspec = &transport->remote->fetch;
+                       fetch_refspec = &remote->fetch;
 
                for (i = 0; i < fetch_refspec->nr; i++)
                        get_fetch_map(ref_map, &fetch_refspec->items[i], &oref_tail, 1);
@@ -398,7 +384,6 @@ static struct ref *get_ref_map(struct transport *transport,
                die("--refmap option is only meaningful with command-line refspec(s).");
        } else {
                /* Use the defaults */
-               struct remote *remote = transport->remote;
                struct branch *branch = branch_get(NULL);
                int has_merge = branch_has_merge_config(branch);
                if (remote &&
@@ -437,7 +422,7 @@ static struct ref *get_ref_map(struct transport *transport,
                /* also fetch all tags */
                get_fetch_map(remote_refs, tag_refspec, &tail, 0);
        else if (tags == TAGS_DEFAULT && *autotags)
-               find_non_local_tags(transport, &ref_map, &tail);
+               find_non_local_tags(remote_refs, &ref_map, &tail);
 
        /* Now append any refs to be updated opportunistically: */
        *tail = orefs;
@@ -446,7 +431,23 @@ static struct ref *get_ref_map(struct transport *transport,
                tail = &rm->next;
        }
 
-       return ref_remove_duplicates(ref_map);
+       ref_map = ref_remove_duplicates(ref_map);
+
+       for_each_ref(add_existing, &existing_refs);
+       for (rm = ref_map; rm; rm = rm->next) {
+               if (rm->peer_ref) {
+                       struct string_list_item *peer_item =
+                               string_list_lookup(&existing_refs,
+                                                  rm->peer_ref->name);
+                       if (peer_item) {
+                               struct object_id *old_oid = peer_item->util;
+                               oidcpy(&rm->peer_ref->old_oid, old_oid);
+                       }
+               }
+       }
+       string_list_clear(&existing_refs, 1);
+
+       return ref_map;
 }
 
 #define STORE_REF_ERROR_OTHER 1
@@ -756,7 +757,7 @@ static int iterate_ref_map(void *cb_data, struct object_id *oid)
 }
 
 static int store_updated_refs(const char *raw_url, const char *remote_name,
-               struct ref *ref_map)
+                             int connectivity_checked, struct ref *ref_map)
 {
        FILE *fp;
        struct commit *commit;
@@ -778,10 +779,12 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
        else
                url = xstrdup("foreign");
 
-       rm = ref_map;
-       if (check_connected(iterate_ref_map, &rm, NULL)) {
-               rc = error(_("%s did not send all necessary objects\n"), url);
-               goto abort;
+       if (!connectivity_checked) {
+               rm = ref_map;
+               if (check_connected(iterate_ref_map, &rm, NULL)) {
+                       rc = error(_("%s did not send all necessary objects\n"), url);
+                       goto abort;
+               }
        }
 
        prepare_format_display(ref_map);
@@ -933,15 +936,32 @@ static int quickfetch(struct ref *ref_map)
        return check_connected(iterate_ref_map, &rm, &opt);
 }
 
-static int fetch_refs(struct transport *transport, struct ref *ref_map)
+static int fetch_refs(struct transport *transport, struct ref *ref_map,
+                     struct ref **updated_remote_refs)
 {
        int ret = quickfetch(ref_map);
        if (ret)
-               ret = transport_fetch_refs(transport, ref_map);
+               ret = transport_fetch_refs(transport, ref_map,
+                                          updated_remote_refs);
        if (!ret)
-               ret |= store_updated_refs(transport->url,
-                               transport->remote->name,
-                               ref_map);
+               /*
+                * Keep the new pack's ".keep" file around to allow the caller
+                * time to update refs to reference the new objects.
+                */
+               return 0;
+       transport_unlock_pack(transport);
+       return ret;
+}
+
+/* Update local refs based on the ref values fetched from a remote */
+static int consume_refs(struct transport *transport, struct ref *ref_map)
+{
+       int connectivity_checked = transport->smart_options
+               ? transport->smart_options->connectivity_checked : 0;
+       int ret = store_updated_refs(transport->url,
+                                    transport->remote->name,
+                                    connectivity_checked,
+                                    ref_map);
        transport_unlock_pack(transport);
        return ret;
 }
@@ -1087,7 +1107,8 @@ static void backfill_tags(struct transport *transport, struct ref *ref_map)
        transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, NULL);
        transport_set_option(transport, TRANS_OPT_DEPTH, "0");
        transport_set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, NULL);
-       fetch_refs(transport, ref_map);
+       if (!fetch_refs(transport, ref_map, NULL))
+               consume_refs(transport, ref_map);
 
        if (gsecondary) {
                transport_disconnect(gsecondary);
@@ -1098,13 +1119,12 @@ static void backfill_tags(struct transport *transport, struct ref *ref_map)
 static int do_fetch(struct transport *transport,
                    struct refspec *rs)
 {
-       struct string_list existing_refs = STRING_LIST_INIT_DUP;
        struct ref *ref_map;
-       struct ref *rm;
        int autotags = (transport->remote->fetch_tags == 1);
        int retcode = 0;
-
-       for_each_ref(add_existing, &existing_refs);
+       const struct ref *remote_refs;
+       struct ref *updated_remote_refs = NULL;
+       struct argv_array ref_prefixes = ARGV_ARRAY_INIT;
 
        if (tags == TAGS_DEFAULT) {
                if (transport->remote->fetch_tags == 2)
@@ -1120,22 +1140,24 @@ static int do_fetch(struct transport *transport,
                        goto cleanup;
        }
 
-       ref_map = get_ref_map(transport, rs, tags, &autotags);
-       if (!update_head_ok)
-               check_not_current_branch(ref_map);
+       if (rs->nr)
+               refspec_ref_prefixes(rs, &ref_prefixes);
+       else if (transport->remote && transport->remote->fetch.nr)
+               refspec_ref_prefixes(&transport->remote->fetch, &ref_prefixes);
 
-       for (rm = ref_map; rm; rm = rm->next) {
-               if (rm->peer_ref) {
-                       struct string_list_item *peer_item =
-                               string_list_lookup(&existing_refs,
-                                                  rm->peer_ref->name);
-                       if (peer_item) {
-                               struct object_id *old_oid = peer_item->util;
-                               oidcpy(&rm->peer_ref->old_oid, old_oid);
-                       }
-               }
+       if (ref_prefixes.argc &&
+           (tags == TAGS_SET || (tags == TAGS_DEFAULT && !rs->nr))) {
+               argv_array_push(&ref_prefixes, "refs/tags/");
        }
 
+       remote_refs = transport_get_remote_refs(transport, &ref_prefixes);
+       argv_array_clear(&ref_prefixes);
+
+       ref_map = get_ref_map(transport->remote, remote_refs, rs,
+                             tags, &autotags);
+       if (!update_head_ok)
+               check_not_current_branch(ref_map);
+
        if (tags == TAGS_DEFAULT && autotags)
                transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, "1");
        if (prune) {
@@ -1152,7 +1174,24 @@ static int do_fetch(struct transport *transport,
                                   transport->url);
                }
        }
-       if (fetch_refs(transport, ref_map)) {
+
+       if (fetch_refs(transport, ref_map, &updated_remote_refs)) {
+               free_refs(ref_map);
+               retcode = 1;
+               goto cleanup;
+       }
+       if (updated_remote_refs) {
+               /*
+                * Regenerate ref_map using the updated remote refs.  This is
+                * to account for additional information which may be provided
+                * by the transport (e.g. shallow info).
+                */
+               free_refs(ref_map);
+               ref_map = get_ref_map(transport->remote, updated_remote_refs, rs,
+                                     tags, &autotags);
+               free_refs(updated_remote_refs);
+       }
+       if (consume_refs(transport, ref_map)) {
                free_refs(ref_map);
                retcode = 1;
                goto cleanup;
@@ -1164,14 +1203,13 @@ static int do_fetch(struct transport *transport,
        if (tags == TAGS_DEFAULT && autotags) {
                struct ref **tail = &ref_map;
                ref_map = NULL;
-               find_non_local_tags(transport, &ref_map, &tail);
+               find_non_local_tags(remote_refs, &ref_map, &tail);
                if (ref_map)
                        backfill_tags(transport, ref_map);
                free_refs(ref_map);
        }
 
  cleanup:
-       string_list_clear(&existing_refs, 1);
        return retcode;
 }
 
index 91feb78..1bba888 100644 (file)
@@ -58,8 +58,10 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
        argv_array_push(&rev_list.args, "--stdin");
        if (repository_format_partial_clone)
                argv_array_push(&rev_list.args, "--exclude-promisor-objects");
-       argv_array_push(&rev_list.args, "--not");
-       argv_array_push(&rev_list.args, "--all");
+       if (!opt->is_deepening_fetch) {
+               argv_array_push(&rev_list.args, "--not");
+               argv_array_push(&rev_list.args, "--all");
+       }
        argv_array_push(&rev_list.args, "--quiet");
        if (opt->progress)
                argv_array_pushf(&rev_list.args, "--progress=%s",
index a53f03a..322dc76 100644 (file)
@@ -38,6 +38,13 @@ struct check_connected_options {
         * Insert these variables into the environment of the child process.
         */
        const char **env;
+
+       /*
+        * If non-zero, check the ancestry chain completely, not stopping at
+        * any existing ref. This is necessary when deepening existing refs
+        * during a fetch.
+        */
+       unsigned is_deepening_fetch : 1;
 };
 
 #define CHECK_CONNECTED_INIT { 0 }
index 853624f..48fe63d 100644 (file)
@@ -19,7 +19,7 @@ static void fetch_refs(const char *remote_name, struct ref *ref)
 
        transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
        transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");
-       transport_fetch_refs(transport, ref);
+       transport_fetch_refs(transport, ref, NULL);
        fetch_if_missing = original_fetch_if_missing;
 }
 
index dbd879a..7ccb9c0 100644 (file)
@@ -20,6 +20,7 @@
 #include "oidset.h"
 #include "packfile.h"
 #include "object-store.h"
+#include "connected.h"
 
 static int transfer_unpack_limit = -1;
 static int fetch_unpack_limit = -1;
@@ -1103,9 +1104,10 @@ static void add_shallow_requests(struct strbuf *req_buf,
 
 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
 {
+       int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
+
        for ( ; wants ; wants = wants->next) {
                const struct object_id *remote = &wants->old_oid;
-               const char *remote_hex;
                struct object *o;
 
                /*
@@ -1123,8 +1125,10 @@ static void add_wants(const struct ref *wants, struct strbuf *req_buf)
                        continue;
                }
 
-               remote_hex = oid_to_hex(remote);
-               packet_buf_write(req_buf, "want %s\n", remote_hex);
+               if (!use_ref_in_want || wants->exact_oid)
+                       packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
+               else
+                       packet_buf_write(req_buf, "want-ref %s\n", wants->name);
        }
 }
 
@@ -1335,6 +1339,32 @@ static void receive_shallow_info(struct fetch_pack_args *args,
        args->deepen = 1;
 }
 
+static void receive_wanted_refs(struct packet_reader *reader, struct ref *refs)
+{
+       process_section_header(reader, "wanted-refs", 0);
+       while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
+               struct object_id oid;
+               const char *end;
+               struct ref *r = NULL;
+
+               if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
+                       die("expected wanted-ref, got '%s'", reader->line);
+
+               for (r = refs; r; r = r->next) {
+                       if (!strcmp(end, r->name)) {
+                               oidcpy(&r->old_oid, &oid);
+                               break;
+                       }
+               }
+
+               if (!r)
+                       die("unexpected wanted-ref: '%s'", reader->line);
+       }
+
+       if (reader->status != PACKET_READ_DELIM)
+               die("error processing wanted refs: %d", reader->status);
+}
+
 enum fetch_state {
        FETCH_CHECK_LOCAL = 0,
        FETCH_SEND_REQUEST,
@@ -1409,6 +1439,9 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
                        if (process_section_header(&reader, "shallow-info", 1))
                                receive_shallow_info(args, &reader);
 
+                       if (process_section_header(&reader, "wanted-refs", 1))
+                               receive_wanted_refs(&reader, ref);
+
                        /* get the pack */
                        process_section_header(&reader, "packfile", 0);
                        if (get_pack(args, fd, pack_lockfile))
@@ -1471,12 +1504,13 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr)
 }
 
 static void update_shallow(struct fetch_pack_args *args,
-                          struct ref **sought, int nr_sought,
+                          struct ref *refs,
                           struct shallow_info *si)
 {
        struct oid_array ref = OID_ARRAY_INIT;
        int *status;
        int i;
+       struct ref *r;
 
        if (args->deepen && alternate_shallow_file) {
                if (*alternate_shallow_file == '\0') { /* --unshallow */
@@ -1518,8 +1552,8 @@ static void update_shallow(struct fetch_pack_args *args,
        remove_nonexistent_theirs_shallow(si);
        if (!si->nr_ours && !si->nr_theirs)
                return;
-       for (i = 0; i < nr_sought; i++)
-               oid_array_append(&ref, &sought[i]->old_oid);
+       for (r = refs; r; r = r->next)
+               oid_array_append(&ref, &r->old_oid);
        si->ref = &ref;
 
        if (args->update_shallow) {
@@ -1553,17 +1587,29 @@ static void update_shallow(struct fetch_pack_args *args,
         * remote is also shallow, check what ref is safe to update
         * without updating .git/shallow
         */
-       status = xcalloc(nr_sought, sizeof(*status));
+       status = xcalloc(ref.nr, sizeof(*status));
        assign_shallow_commits_to_refs(si, NULL, status);
        if (si->nr_ours || si->nr_theirs) {
-               for (i = 0; i < nr_sought; i++)
+               for (r = refs, i = 0; r; r = r->next, i++)
                        if (status[i])
-                               sought[i]->status = REF_STATUS_REJECT_SHALLOW;
+                               r->status = REF_STATUS_REJECT_SHALLOW;
        }
        free(status);
        oid_array_clear(&ref);
 }
 
+static int iterate_ref_map(void *cb_data, struct object_id *oid)
+{
+       struct ref **rm = cb_data;
+       struct ref *ref = *rm;
+
+       if (!ref)
+               return -1; /* end of the list */
+       *rm = ref->next;
+       oidcpy(oid, &ref->old_oid);
+       return 0;
+}
+
 struct ref *fetch_pack(struct fetch_pack_args *args,
                       int fd[], struct child_process *conn,
                       const struct ref *ref,
@@ -1592,7 +1638,25 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
                                        &si, pack_lockfile);
        reprepare_packed_git(the_repository);
-       update_shallow(args, sought, nr_sought, &si);
+
+       if (!args->cloning && args->deepen) {
+               struct check_connected_options opt = CHECK_CONNECTED_INIT;
+               struct ref *iterator = ref_cpy;
+               opt.shallow_file = alternate_shallow_file;
+               if (args->deepen)
+                       opt.is_deepening_fetch = 1;
+               if (check_connected(iterate_ref_map, &iterator, &opt)) {
+                       error(_("remote did not send all necessary objects"));
+                       free_refs(ref_cpy);
+                       ref_cpy = NULL;
+                       rollback_lock_file(&shallow_lock);
+                       goto cleanup;
+               }
+               args->connectivity_checked = 1;
+       }
+
+       update_shallow(args, ref_cpy, &si);
+cleanup:
        clear_shallow_info(&si);
        return ref_cpy;
 }
index bb45a36..2160be9 100644 (file)
@@ -41,6 +41,21 @@ struct fetch_pack_args {
         * regardless of which object flags it uses (if any).
         */
        unsigned no_dependents:1;
+
+       /*
+        * Because fetch_pack() overwrites the shallow file upon a
+        * successful deepening non-clone fetch, if this struct
+        * specifies such a fetch, fetch_pack() needs to perform a
+        * connectivity check before deciding if a fetch is successful
+        * (and overwriting the shallow file). fetch_pack() sets this
+        * field to 1 if such a connectivity check was performed.
+        *
+        * This is different from check_self_contained_and_connected
+        * in that the former allows existing objects in the
+        * repository to satisfy connectivity needs, whereas the
+        * latter doesn't.
+        */
+       unsigned connectivity_checked:1;
 };
 
 /*
index 539285f..3fd4345 100644 (file)
--- a/remote.c
+++ b/remote.c
@@ -1736,6 +1736,7 @@ int get_fetch_map(const struct ref *remote_refs,
                if (refspec->exact_sha1) {
                        ref_map = alloc_ref(name);
                        get_oid_hex(name, &ref_map->old_oid);
+                       ref_map->exact_oid = 1;
                } else {
                        ref_map = get_remote_ref(remote_refs, name);
                }
index 45ecc6c..9762921 100644 (file)
--- a/remote.h
+++ b/remote.h
@@ -73,6 +73,7 @@ struct ref {
                force:1,
                forced_update:1,
                expect_old_sha1:1,
+               exact_oid:1,
                deletion:1;
 
        enum {
index 0f19e53..30775f9 100644 (file)
@@ -1,3 +1,4 @@
+#include "cache.h"
 #include "pkt-line.h"
 
 static void pack_line(const char *line)
@@ -48,6 +49,36 @@ static void unpack(void)
        }
 }
 
+static void unpack_sideband(void)
+{
+       struct packet_reader reader;
+       packet_reader_init(&reader, 0, NULL, 0,
+                          PACKET_READ_GENTLE_ON_EOF |
+                          PACKET_READ_CHOMP_NEWLINE);
+
+       while (packet_reader_read(&reader) != PACKET_READ_EOF) {
+               int band;
+               int fd;
+
+               switch (reader.status) {
+               case PACKET_READ_EOF:
+                       break;
+               case PACKET_READ_NORMAL:
+                       band = reader.line[0] & 0xff;
+                       if (band < 1 || band > 2)
+                               die("unexpected side band %d", band);
+                       fd = band;
+
+                       write_or_die(fd, reader.line + 1, reader.pktlen - 1);
+                       break;
+               case PACKET_READ_FLUSH:
+                       return;
+               case PACKET_READ_DELIM:
+                       break;
+               }
+       }
+}
+
 int cmd_main(int argc, const char **argv)
 {
        if (argc < 2)
@@ -57,6 +88,8 @@ int cmd_main(int argc, const char **argv)
                pack(argc - 2, argv + 2);
        else if (!strcmp(argv[1], "unpack"))
                unpack();
+       else if (!strcmp(argv[1], "unpack-sideband"))
+               unpack_sideband();
        else
                die("invalid argument '%s'", argv[1]);
 
index 435a374..ed41b15 100644 (file)
@@ -132,6 +132,7 @@ prepare_httpd() {
        cp "$TEST_PATH"/passwd "$HTTPD_ROOT_PATH"
        install_script broken-smart-http.sh
        install_script error.sh
+       install_script apply-one-time-sed.sh
 
        ln -s "$LIB_HTTPD_MODULE_PATH" "$HTTPD_ROOT_PATH/modules"
 
index 724d9ae..581c010 100644 (file)
@@ -111,9 +111,14 @@ Alias /auth/dumb/ www/auth/dumb/
        SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
        SetEnv GIT_HTTP_EXPORT_ALL
 </LocationMatch>
+<LocationMatch /one_time_sed/>
+       SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
+       SetEnv GIT_HTTP_EXPORT_ALL
+</LocationMatch>
 ScriptAliasMatch /smart_*[^/]*/(.*) ${GIT_EXEC_PATH}/git-http-backend/$1
 ScriptAlias /broken_smart/ broken-smart-http.sh/
 ScriptAlias /error/ error.sh/
+ScriptAliasMatch /one_time_sed/(.*) apply-one-time-sed.sh/$1
 <Directory ${GIT_EXEC_PATH}>
        Options FollowSymlinks
 </Directory>
@@ -123,6 +128,9 @@ ScriptAlias /error/ error.sh/
 <Files error.sh>
   Options ExecCGI
 </Files>
+<Files apply-one-time-sed.sh>
+       Options ExecCGI
+</Files>
 <Files ${GIT_EXEC_PATH}/git-http-backend>
        Options ExecCGI
 </Files>
diff --git a/t/lib-httpd/apply-one-time-sed.sh b/t/lib-httpd/apply-one-time-sed.sh
new file mode 100644 (file)
index 0000000..fcef728
--- /dev/null
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# If "one-time-sed" exists in $HTTPD_ROOT_PATH, run sed on the HTTP response,
+# using the contents of "one-time-sed" as the sed command to be run. If the
+# response was modified as a result, delete "one-time-sed" so that subsequent
+# HTTP responses are no longer modified.
+#
+# This can be used to simulate the effects of the repository changing in
+# between HTTP request-response pairs.
+if [ -e one-time-sed ]; then
+       "$GIT_EXEC_PATH/git-http-backend" >out
+       sed "$(cat one-time-sed)" <out >out_modified
+
+       if diff out out_modified >/dev/null; then
+               cat out
+       else
+               cat out_modified
+               rm one-time-sed
+       fi
+else
+       "$GIT_EXEC_PATH/git-http-backend"
+fi
index 943231a..7045685 100755 (executable)
@@ -186,4 +186,47 @@ EOF
        test_cmp expect actual
 '
 
+. "$TEST_DIRECTORY"/lib-httpd.sh
+start_httpd
+
+REPO="$HTTPD_DOCUMENT_ROOT_PATH/repo"
+
+test_expect_success 'shallow fetches check connectivity before writing shallow file' '
+       rm -rf "$REPO" client &&
+
+       git init "$REPO" &&
+       test_commit -C "$REPO" one &&
+       test_commit -C "$REPO" two &&
+       test_commit -C "$REPO" three &&
+
+       git init client &&
+
+       # Use protocol v2 to ensure that shallow information is sent exactly
+       # once by the server, since we are planning to manipulate it.
+       git -C "$REPO" config protocol.version 2 &&
+       git -C client config protocol.version 2 &&
+
+       git -C client fetch --depth=2 "$HTTPD_URL/one_time_sed/repo" master:a_branch &&
+
+       # Craft a situation in which the server sends back an unshallow request
+       # with an empty packfile. This is done by refetching with a shorter
+       # depth (to ensure that the packfile is empty), and overwriting the
+       # shallow line in the response with the unshallow line we want.
+       printf "s/0034shallow %s/0036unshallow %s/" \
+              "$(git -C "$REPO" rev-parse HEAD)" \
+              "$(git -C "$REPO" rev-parse HEAD^)" \
+              >"$HTTPD_ROOT_PATH/one-time-sed" &&
+       test_must_fail git -C client fetch --depth=1 "$HTTPD_URL/one_time_sed/repo" \
+               master:a_branch &&
+
+       # Ensure that the one-time-sed script was used.
+       ! test -e "$HTTPD_ROOT_PATH/one-time-sed" &&
+
+       # Ensure that the resulting repo is consistent, despite our failure to
+       # fetch.
+       git -C client fsck
+'
+
+stop_httpd
+
 test_done
diff --git a/t/t5703-upload-pack-ref-in-want.sh b/t/t5703-upload-pack-ref-in-want.sh
new file mode 100755 (executable)
index 0000000..a73c55a
--- /dev/null
@@ -0,0 +1,377 @@
+#!/bin/sh
+
+test_description='upload-pack ref-in-want'
+
+. ./test-lib.sh
+
+get_actual_refs () {
+       sed -n -e '/wanted-refs/,/0001/{
+               /wanted-refs/d
+               /0001/d
+               p
+               }' <out | test-pkt-line unpack >actual_refs
+}
+
+get_actual_commits () {
+       sed -n -e '/packfile/,/0000/{
+               /packfile/d
+               p
+               }' <out | test-pkt-line unpack-sideband >o.pack &&
+       git index-pack o.pack &&
+       git verify-pack -v o.idx | grep commit | cut -c-40 | sort >actual_commits
+}
+
+check_output () {
+       get_actual_refs &&
+       test_cmp expected_refs actual_refs &&
+       get_actual_commits &&
+       test_cmp expected_commits actual_commits
+}
+
+# c(o/foo) d(o/bar)
+#        \ /
+#         b   e(baz)  f(master)
+#          \__  |  __/
+#             \ | /
+#               a
+test_expect_success 'setup repository' '
+       test_commit a &&
+       git checkout -b o/foo &&
+       test_commit b &&
+       test_commit c &&
+       git checkout -b o/bar b &&
+       test_commit d &&
+       git checkout -b baz a &&
+       test_commit e &&
+       git checkout master &&
+       test_commit f
+'
+
+test_expect_success 'config controls ref-in-want advertisement' '
+       git serve --advertise-capabilities >out &&
+       ! grep -a ref-in-want out &&
+
+       git config uploadpack.allowRefInWant false &&
+       git serve --advertise-capabilities >out &&
+       ! grep -a ref-in-want out &&
+
+       git config uploadpack.allowRefInWant true &&
+       git serve --advertise-capabilities >out &&
+       grep -a ref-in-want out
+'
+
+test_expect_success 'invalid want-ref line' '
+       test-pkt-line pack >in <<-EOF &&
+       command=fetch
+       0001
+       no-progress
+       want-ref refs/heads/non-existent
+       done
+       0000
+       EOF
+
+       test_must_fail git serve --stateless-rpc 2>out <in &&
+       grep "unknown ref" out
+'
+
+test_expect_success 'basic want-ref' '
+       cat >expected_refs <<-EOF &&
+       $(git rev-parse f) refs/heads/master
+       EOF
+       git rev-parse f | sort >expected_commits &&
+
+       test-pkt-line pack >in <<-EOF &&
+       command=fetch
+       0001
+       no-progress
+       want-ref refs/heads/master
+       have $(git rev-parse a)
+       done
+       0000
+       EOF
+
+       git serve --stateless-rpc >out <in &&
+       check_output
+'
+
+test_expect_success 'multiple want-ref lines' '
+       cat >expected_refs <<-EOF &&
+       $(git rev-parse c) refs/heads/o/foo
+       $(git rev-parse d) refs/heads/o/bar
+       EOF
+       git rev-parse c d | sort >expected_commits &&
+
+       test-pkt-line pack >in <<-EOF &&
+       command=fetch
+       0001
+       no-progress
+       want-ref refs/heads/o/foo
+       want-ref refs/heads/o/bar
+       have $(git rev-parse b)
+       done
+       0000
+       EOF
+
+       git serve --stateless-rpc >out <in &&
+       check_output
+'
+
+test_expect_success 'mix want and want-ref' '
+       cat >expected_refs <<-EOF &&
+       $(git rev-parse f) refs/heads/master
+       EOF
+       git rev-parse e f | sort >expected_commits &&
+
+       test-pkt-line pack >in <<-EOF &&
+       command=fetch
+       0001
+       no-progress
+       want-ref refs/heads/master
+       want $(git rev-parse e)
+       have $(git rev-parse a)
+       done
+       0000
+       EOF
+
+       git serve --stateless-rpc >out <in &&
+       check_output
+'
+
+test_expect_success 'want-ref with ref we already have commit for' '
+       cat >expected_refs <<-EOF &&
+       $(git rev-parse c) refs/heads/o/foo
+       EOF
+       >expected_commits &&
+
+       test-pkt-line pack >in <<-EOF &&
+       command=fetch
+       0001
+       no-progress
+       want-ref refs/heads/o/foo
+       have $(git rev-parse c)
+       done
+       0000
+       EOF
+
+       git serve --stateless-rpc >out <in &&
+       check_output
+'
+
+. "$TEST_DIRECTORY"/lib-httpd.sh
+start_httpd
+
+REPO="$HTTPD_DOCUMENT_ROOT_PATH/repo"
+LOCAL_PRISTINE="$(pwd)/local_pristine"
+
+test_expect_success 'setup repos for change-while-negotiating test' '
+       (
+               git init "$REPO" &&
+               cd "$REPO" &&
+               >.git/git-daemon-export-ok &&
+               test_commit m1 &&
+               git tag -d m1 &&
+
+               # Local repo with many commits (so that negotiation will take
+               # more than 1 request/response pair)
+               git clone "http://127.0.0.1:$LIB_HTTPD_PORT/smart/repo" "$LOCAL_PRISTINE" &&
+               cd "$LOCAL_PRISTINE" &&
+               git checkout -b side &&
+               for i in $(seq 1 33); do test_commit s$i; done &&
+
+               # Add novel commits to upstream
+               git checkout master &&
+               cd "$REPO" &&
+               test_commit m2 &&
+               test_commit m3 &&
+               git tag -d m2 m3
+       ) &&
+       git -C "$LOCAL_PRISTINE" remote set-url origin "http://127.0.0.1:$LIB_HTTPD_PORT/one_time_sed/repo" &&
+       git -C "$LOCAL_PRISTINE" config protocol.version 2
+'
+
+inconsistency () {
+       # Simulate that the server initially reports $2 as the ref
+       # corresponding to $1, and after that, $1 as the ref corresponding to
+       # $1. This corresponds to the real-life situation where the server's
+       # repository appears to change during negotiation, for example, when
+       # different servers in a load-balancing arrangement serve (stateless)
+       # RPCs during a single negotiation.
+       printf "s/%s/%s/" \
+              $(git -C "$REPO" rev-parse $1 | tr -d "\n") \
+              $(git -C "$REPO" rev-parse $2 | tr -d "\n") \
+              >"$HTTPD_ROOT_PATH/one-time-sed"
+}
+
+test_expect_success 'server is initially ahead - no ref in want' '
+       git -C "$REPO" config uploadpack.allowRefInWant false &&
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       inconsistency master 1234567890123456789012345678901234567890 &&
+       test_must_fail git -C local fetch 2>err &&
+       grep "ERR upload-pack: not our ref" err
+'
+
+test_expect_success 'server is initially ahead - ref in want' '
+       git -C "$REPO" config uploadpack.allowRefInWant true &&
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       inconsistency master 1234567890123456789012345678901234567890 &&
+       git -C local fetch &&
+
+       git -C "$REPO" rev-parse --verify master >expected &&
+       git -C local rev-parse --verify refs/remotes/origin/master >actual &&
+       test_cmp expected actual
+'
+
+test_expect_success 'server is initially behind - no ref in want' '
+       git -C "$REPO" config uploadpack.allowRefInWant false &&
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       inconsistency master "master^" &&
+       git -C local fetch &&
+
+       git -C "$REPO" rev-parse --verify "master^" >expected &&
+       git -C local rev-parse --verify refs/remotes/origin/master >actual &&
+       test_cmp expected actual
+'
+
+test_expect_success 'server is initially behind - ref in want' '
+       git -C "$REPO" config uploadpack.allowRefInWant true &&
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       inconsistency master "master^" &&
+       git -C local fetch &&
+
+       git -C "$REPO" rev-parse --verify "master" >expected &&
+       git -C local rev-parse --verify refs/remotes/origin/master >actual &&
+       test_cmp expected actual
+'
+
+test_expect_success 'server loses a ref - ref in want' '
+       git -C "$REPO" config uploadpack.allowRefInWant true &&
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       echo "s/master/raster/" >"$HTTPD_ROOT_PATH/one-time-sed" &&
+       test_must_fail git -C local fetch 2>err &&
+
+       grep "ERR unknown ref refs/heads/raster" err
+'
+
+stop_httpd
+
+REPO="$(pwd)/repo"
+LOCAL_PRISTINE="$(pwd)/local_pristine"
+
+# $REPO
+# c(o/foo) d(o/bar)
+#        \ /
+#         b   e(baz)  f(master)
+#          \__  |  __/
+#             \ | /
+#               a
+#
+# $LOCAL_PRISTINE
+#              s32(side)
+#              |
+#              .
+#              .
+#              |
+#              a(master)
+test_expect_success 'setup repos for fetching with ref-in-want tests' '
+       (
+               git init "$REPO" &&
+               cd "$REPO" &&
+               test_commit a &&
+
+               # Local repo with many commits (so that negotiation will take
+               # more than 1 request/response pair)
+               rm -rf "$LOCAL_PRISTINE" &&
+               git clone "file://$REPO" "$LOCAL_PRISTINE" &&
+               cd "$LOCAL_PRISTINE" &&
+               git checkout -b side &&
+               for i in $(seq 1 33); do test_commit s$i; done &&
+
+               # Add novel commits to upstream
+               git checkout master &&
+               cd "$REPO" &&
+               git checkout -b o/foo &&
+               test_commit b &&
+               test_commit c &&
+               git checkout -b o/bar b &&
+               test_commit d &&
+               git checkout -b baz a &&
+               test_commit e &&
+               git checkout master &&
+               test_commit f
+       ) &&
+       git -C "$REPO" config uploadpack.allowRefInWant true &&
+       git -C "$LOCAL_PRISTINE" config protocol.version 2
+'
+
+test_expect_success 'fetching with exact OID' '
+       test_when_finished "rm -f log" &&
+
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       GIT_TRACE_PACKET="$(pwd)/log" git -C local fetch origin \
+               $(git -C "$REPO" rev-parse d):refs/heads/actual &&
+
+       git -C "$REPO" rev-parse "d" >expected &&
+       git -C local rev-parse refs/heads/actual >actual &&
+       test_cmp expected actual &&
+       grep "want $(git -C "$REPO" rev-parse d)" log
+'
+
+test_expect_success 'fetching multiple refs' '
+       test_when_finished "rm -f log" &&
+
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       GIT_TRACE_PACKET="$(pwd)/log" git -C local fetch origin master baz &&
+
+       git -C "$REPO" rev-parse "master" "baz" >expected &&
+       git -C local rev-parse refs/remotes/origin/master refs/remotes/origin/baz >actual &&
+       test_cmp expected actual &&
+       grep "want-ref refs/heads/master" log &&
+       grep "want-ref refs/heads/baz" log
+'
+
+test_expect_success 'fetching ref and exact OID' '
+       test_when_finished "rm -f log" &&
+
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       GIT_TRACE_PACKET="$(pwd)/log" git -C local fetch origin \
+               master $(git -C "$REPO" rev-parse b):refs/heads/actual &&
+
+       git -C "$REPO" rev-parse "master" "b" >expected &&
+       git -C local rev-parse refs/remotes/origin/master refs/heads/actual >actual &&
+       test_cmp expected actual &&
+       grep "want $(git -C "$REPO" rev-parse b)" log &&
+       grep "want-ref refs/heads/master" log
+'
+
+test_expect_success 'fetching with wildcard that does not match any refs' '
+       test_when_finished "rm -f log" &&
+
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       git -C local fetch origin refs/heads/none*:refs/heads/* >out &&
+       test_must_be_empty out
+'
+
+test_expect_success 'fetching with wildcard that matches multiple refs' '
+       test_when_finished "rm -f log" &&
+
+       rm -rf local &&
+       cp -r "$LOCAL_PRISTINE" local &&
+       GIT_TRACE_PACKET="$(pwd)/log" git -C local fetch origin refs/heads/o*:refs/heads/o* &&
+
+       git -C "$REPO" rev-parse "o/foo" "o/bar" >expected &&
+       git -C local rev-parse "o/foo" "o/bar" >actual &&
+       test_cmp expected actual &&
+       grep "want-ref refs/heads/o/foo" log &&
+       grep "want-ref refs/heads/o/bar" log
+'
+
+test_done
index 1f8ff7e..8b5abca 100644 (file)
@@ -651,14 +651,16 @@ static int connect_helper(struct transport *transport, const char *name,
 }
 
 static int fetch(struct transport *transport,
-                int nr_heads, struct ref **to_fetch)
+                int nr_heads, struct ref **to_fetch,
+                struct ref **fetched_refs)
 {
        struct helper_data *data = transport->data;
        int i, count;
 
        if (process_connect(transport, 0)) {
                do_take_over(transport);
-               return transport->vtable->fetch(transport, nr_heads, to_fetch);
+               return transport->vtable->fetch(transport, nr_heads, to_fetch,
+                                               fetched_refs);
        }
 
        count = 0;
index 1cde625..eeb6c34 100644 (file)
@@ -36,11 +36,18 @@ struct transport_vtable {
         * Fetch the objects for the given refs. Note that this gets
         * an array, and should ignore the list structure.
         *
+        * The transport *may* provide, in fetched_refs, the list of refs that
+        * it fetched.  If the transport knows anything about the fetched refs
+        * that the caller does not know (for example, shallow status), it
+        * should provide that list of refs and include that information in the
+        * list.
+        *
         * If the transport did not get hashes for refs in
         * get_refs_list(), it should set the old_sha1 fields in the
         * provided refs now.
         **/
-       int (*fetch)(struct transport *transport, int refs_nr, struct ref **refs);
+       int (*fetch)(struct transport *transport, int refs_nr, struct ref **refs,
+                    struct ref **fetched_refs);
 
        /**
         * Push the objects and refs. Send the necessary objects, and
index a32da30..fdd813f 100644 (file)
@@ -151,7 +151,8 @@ static struct ref *get_refs_from_bundle(struct transport *transport,
 }
 
 static int fetch_refs_from_bundle(struct transport *transport,
-                              int nr_heads, struct ref **to_fetch)
+                              int nr_heads, struct ref **to_fetch,
+                              struct ref **fetched_refs)
 {
        struct bundle_transport_data *data = transport->data;
        return unbundle(&data->header, data->fd,
@@ -287,7 +288,8 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus
 }
 
 static int fetch_refs_via_pack(struct transport *transport,
-                              int nr_heads, struct ref **to_fetch)
+                              int nr_heads, struct ref **to_fetch,
+                              struct ref **fetched_refs)
 {
        int ret = 0;
        struct git_transport_data *data = transport->data;
@@ -348,14 +350,19 @@ static int fetch_refs_via_pack(struct transport *transport,
        data->got_remote_heads = 0;
        data->options.self_contained_and_connected =
                args.self_contained_and_connected;
+       data->options.connectivity_checked = args.connectivity_checked;
 
        if (refs == NULL)
                ret = -1;
        if (report_unmatched_refs(to_fetch, nr_heads))
                ret = -1;
 
+       if (fetched_refs)
+               *fetched_refs = refs;
+       else
+               free_refs(refs);
+
        free_refs(refs_tmp);
-       free_refs(refs);
        free(dest);
        return ret;
 }
@@ -1215,19 +1222,31 @@ const struct ref *transport_get_remote_refs(struct transport *transport,
        return transport->remote_refs;
 }
 
-int transport_fetch_refs(struct transport *transport, struct ref *refs)
+int transport_fetch_refs(struct transport *transport, struct ref *refs,
+                        struct ref **fetched_refs)
 {
        int rc;
        int nr_heads = 0, nr_alloc = 0, nr_refs = 0;
        struct ref **heads = NULL;
+       struct ref *nop_head = NULL, **nop_tail = &nop_head;
        struct ref *rm;
 
        for (rm = refs; rm; rm = rm->next) {
                nr_refs++;
                if (rm->peer_ref &&
                    !is_null_oid(&rm->old_oid) &&
-                   !oidcmp(&rm->peer_ref->old_oid, &rm->old_oid))
+                   !oidcmp(&rm->peer_ref->old_oid, &rm->old_oid)) {
+                       /*
+                        * These need to be reported as fetched, but we don't
+                        * actually need to fetch them.
+                        */
+                       if (fetched_refs) {
+                               struct ref *nop_ref = copy_ref(rm);
+                               *nop_tail = nop_ref;
+                               nop_tail = &nop_ref->next;
+                       }
                        continue;
+               }
                ALLOC_GROW(heads, nr_heads + 1, nr_alloc);
                heads[nr_heads++] = rm;
        }
@@ -1245,7 +1264,11 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs)
                        heads[nr_heads++] = rm;
        }
 
-       rc = transport->vtable->fetch(transport, nr_heads, heads);
+       rc = transport->vtable->fetch(transport, nr_heads, heads, fetched_refs);
+       if (fetched_refs && nop_head) {
+               *nop_tail = *fetched_refs;
+               *fetched_refs = nop_head;
+       }
 
        free(heads);
        return rc;
index 7792b08..7a9a7fc 100644 (file)
@@ -18,6 +18,17 @@ struct git_transport_options {
        unsigned deepen_relative : 1;
        unsigned from_promisor : 1;
        unsigned no_dependents : 1;
+
+       /*
+        * If this transport supports connect or stateless-connect,
+        * the corresponding field in struct fetch_pack_args is copied
+        * here after fetching.
+        *
+        * See the definition of connectivity_checked in struct
+        * fetch_pack_args for more information.
+        */
+       unsigned connectivity_checked:1;
+
        int depth;
        const char *deepen_since;
        const struct string_list *deepen_not;
@@ -218,7 +229,8 @@ int transport_push(struct transport *connection,
 const struct ref *transport_get_remote_refs(struct transport *transport,
                                            const struct argv_array *ref_prefixes);
 
-int transport_fetch_refs(struct transport *transport, struct ref *refs);
+int transport_fetch_refs(struct transport *transport, struct ref *refs,
+                        struct ref **fetched_refs);
 void transport_unlock_pack(struct transport *transport);
 int transport_disconnect(struct transport *transport);
 char *transport_anonymize_url(const char *url);
index 936acab..de6106a 100644 (file)
@@ -65,6 +65,7 @@ static const char *pack_objects_hook;
 
 static int filter_capability_requested;
 static int allow_filter;
+static int allow_ref_in_want;
 static struct list_objects_filter_options filter_options;
 
 static void reset_timeout(void)
@@ -1077,6 +1078,8 @@ static int upload_pack_config(const char *var, const char *value, void *unused)
                        return git_config_string(&pack_objects_hook, var, value);
        } else if (!strcmp("uploadpack.allowfilter", var)) {
                allow_filter = git_config_bool(var, value);
+       } else if (!strcmp("uploadpack.allowrefinwant", var)) {
+               allow_ref_in_want = git_config_bool(var, value);
        }
        return parse_hide_refs_config(var, value, "uploadpack");
 }
@@ -1116,6 +1119,7 @@ void upload_pack(struct upload_pack_options *options)
 
 struct upload_pack_data {
        struct object_array wants;
+       struct string_list wanted_refs;
        struct oid_array haves;
 
        struct object_array shallows;
@@ -1137,12 +1141,14 @@ struct upload_pack_data {
 static void upload_pack_data_init(struct upload_pack_data *data)
 {
        struct object_array wants = OBJECT_ARRAY_INIT;
+       struct string_list wanted_refs = STRING_LIST_INIT_DUP;
        struct oid_array haves = OID_ARRAY_INIT;
        struct object_array shallows = OBJECT_ARRAY_INIT;
        struct string_list deepen_not = STRING_LIST_INIT_DUP;
 
        memset(data, 0, sizeof(*data));
        data->wants = wants;
+       data->wanted_refs = wanted_refs;
        data->haves = haves;
        data->shallows = shallows;
        data->deepen_not = deepen_not;
@@ -1151,6 +1157,7 @@ static void upload_pack_data_init(struct upload_pack_data *data)
 static void upload_pack_data_clear(struct upload_pack_data *data)
 {
        object_array_clear(&data->wants);
+       string_list_clear(&data->wanted_refs, 1);
        oid_array_clear(&data->haves);
        object_array_clear(&data->shallows);
        string_list_clear(&data->deepen_not, 0);
@@ -1187,6 +1194,34 @@ static int parse_want(const char *line)
        return 0;
 }
 
+static int parse_want_ref(const char *line, struct string_list *wanted_refs)
+{
+       const char *arg;
+       if (skip_prefix(line, "want-ref ", &arg)) {
+               struct object_id oid;
+               struct string_list_item *item;
+               struct object *o;
+
+               if (read_ref(arg, &oid)) {
+                       packet_write_fmt(1, "ERR unknown ref %s", arg);
+                       die("unknown ref %s", arg);
+               }
+
+               item = string_list_append(wanted_refs, arg);
+               item->util = oiddup(&oid);
+
+               o = parse_object_or_die(&oid, arg);
+               if (!(o->flags & WANTED)) {
+                       o->flags |= WANTED;
+                       add_object_array(o, NULL, &want_obj);
+               }
+
+               return 1;
+       }
+
+       return 0;
+}
+
 static int parse_have(const char *line, struct oid_array *haves)
 {
        const char *arg;
@@ -1212,6 +1247,8 @@ static void process_args(struct packet_reader *request,
                /* process want */
                if (parse_want(arg))
                        continue;
+               if (allow_ref_in_want && parse_want_ref(arg, &data->wanted_refs))
+                       continue;
                /* process have line */
                if (parse_have(arg, &data->haves))
                        continue;
@@ -1354,6 +1391,24 @@ static int process_haves_and_send_acks(struct upload_pack_data *data)
        return ret;
 }
 
+static void send_wanted_ref_info(struct upload_pack_data *data)
+{
+       const struct string_list_item *item;
+
+       if (!data->wanted_refs.nr)
+               return;
+
+       packet_write_fmt(1, "wanted-refs\n");
+
+       for_each_string_list_item(item, &data->wanted_refs) {
+               packet_write_fmt(1, "%s %s\n",
+                                oid_to_hex(item->util),
+                                item->string);
+       }
+
+       packet_delim(1);
+}
+
 static void send_shallow_info(struct upload_pack_data *data)
 {
        /* No shallow info needs to be sent */
@@ -1421,6 +1476,7 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys,
                                state = FETCH_DONE;
                        break;
                case FETCH_SEND_PACK:
+                       send_wanted_ref_info(&data);
                        send_shallow_info(&data);
 
                        packet_write_fmt(1, "packfile\n");
@@ -1441,12 +1497,22 @@ int upload_pack_advertise(struct repository *r,
 {
        if (value) {
                int allow_filter_value;
+               int allow_ref_in_want;
+
                strbuf_addstr(value, "shallow");
+
                if (!repo_config_get_bool(the_repository,
                                         "uploadpack.allowfilter",
                                         &allow_filter_value) &&
                    allow_filter_value)
                        strbuf_addstr(value, " filter");
+
+               if (!repo_config_get_bool(the_repository,
+                                        "uploadpack.allowrefinwant",
+                                        &allow_ref_in_want) &&
+                   allow_ref_in_want)
+                       strbuf_addstr(value, " ref-in-want");
        }
+
        return 1;
 }