Merge branch 'jt/fetch-negotiator-skipping'
[git/git.git] / fetch-pack.c
index 786e2ee..5714bcb 100644 (file)
@@ -18,6 +18,8 @@
 #include "sha1-array.h"
 #include "oidset.h"
 #include "packfile.h"
+#include "object-store.h"
+#include "connected.h"
 #include "fetch-negotiator.h"
 
 static int transfer_unpack_limit = -1;
@@ -77,7 +79,7 @@ static void cache_one_alternate(const char *refname,
                                void *vcache)
 {
        struct alternate_object_cache *cache = vcache;
-       struct object *obj = parse_object(oid);
+       struct object *obj = parse_object(the_repository, oid);
 
        if (!obj || (obj->flags & ALTERNATE))
                return;
@@ -108,7 +110,9 @@ static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
                               const char *refname,
                               const struct object_id *oid)
 {
-       struct object *o = deref_tag(parse_object(oid), refname, 0);
+       struct object *o = deref_tag(the_repository,
+                                    parse_object(the_repository, oid),
+                                    refname, 0);
 
        if (o && o->type == OBJ_COMMIT)
                negotiator->add_tip(negotiator, (struct commit *)o);
@@ -214,6 +218,22 @@ static int next_flush(int stateless_rpc, int count)
        return count;
 }
 
+static void mark_tips(struct fetch_negotiator *negotiator,
+                     const struct oid_array *negotiation_tips)
+{
+       int i;
+
+       if (!negotiation_tips) {
+               for_each_ref(rev_list_insert_ref_oid, negotiator);
+               return;
+       }
+
+       for (i = 0; i < negotiation_tips->nr; i++)
+               rev_list_insert_ref(negotiator, NULL,
+                                   &negotiation_tips->oid[i]);
+       return;
+}
+
 static int find_common(struct fetch_negotiator *negotiator,
                       struct fetch_pack_args *args,
                       int fd[2], struct object_id *result_oid,
@@ -231,7 +251,7 @@ static int find_common(struct fetch_negotiator *negotiator,
        if (args->stateless_rpc && multi_ack == 1)
                die(_("--stateless-rpc requires multi_ack_detailed"));
 
-       for_each_ref(rev_list_insert_ref_oid, negotiator);
+       mark_tips(negotiator, args->negotiation_tips);
        for_each_cached_alternate(negotiator, insert_one_alternate_object);
 
        fetching = 0;
@@ -250,7 +270,7 @@ static int find_common(struct fetch_negotiator *negotiator,
                 * interested in the case we *know* the object is
                 * reachable and we have already scanned it.
                 */
-               if (((o = lookup_object(remote->hash)) != NULL) &&
+               if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
                                (o->flags & COMPLETE)) {
                        continue;
                }
@@ -287,7 +307,7 @@ static int find_common(struct fetch_negotiator *negotiator,
                return 1;
        }
 
-       if (is_repository_shallow())
+       if (is_repository_shallow(the_repository))
                write_shallow_commits(&req_buf, 1, NULL);
        if (args->depth > 0)
                packet_buf_write(&req_buf, "deepen %d", args->depth);
@@ -318,16 +338,16 @@ static int find_common(struct fetch_negotiator *negotiator,
                        if (skip_prefix(line, "shallow ", &arg)) {
                                if (get_oid_hex(arg, &oid))
                                        die(_("invalid shallow line: %s"), line);
-                               register_shallow(&oid);
+                               register_shallow(the_repository, &oid);
                                continue;
                        }
                        if (skip_prefix(line, "unshallow ", &arg)) {
                                if (get_oid_hex(arg, &oid))
                                        die(_("invalid unshallow line: %s"), line);
-                               if (!lookup_object(oid.hash))
+                               if (!lookup_object(the_repository, oid.hash))
                                        die(_("object not found: %s"), line);
                                /* make sure that it is parsed as shallow */
-                               if (!parse_object(&oid))
+                               if (!parse_object(the_repository, &oid))
                                        die(_("error in object: %s"), line);
                                if (unregister_shallow(&oid))
                                        die(_("no shallow found: %s"), line);
@@ -386,8 +406,10 @@ static int find_common(struct fetch_negotiator *negotiator,
                                case ACK_ready:
                                case ACK_continue: {
                                        struct commit *commit =
-                                               lookup_commit(result_oid);
+                                               lookup_commit(the_repository,
+                                                             result_oid);
                                        int was_common;
+
                                        if (!commit)
                                                die(_("invalid commit %s"), oid_to_hex(result_oid));
                                        was_common = negotiator->ack(negotiator, commit);
@@ -461,14 +483,14 @@ static struct commit_list *complete;
 
 static int mark_complete(const struct object_id *oid)
 {
-       struct object *o = parse_object(oid);
+       struct object *o = parse_object(the_repository, oid);
 
        while (o && o->type == OBJ_TAG) {
                struct tag *t = (struct tag *) o;
                if (!t->tagged)
                        break; /* broken repository */
                o->flags |= COMPLETE;
-               o = parse_object(&t->tagged->oid);
+               o = parse_object(the_repository, &t->tagged->oid);
        }
        if (o && o->type == OBJ_COMMIT) {
                struct commit *commit = (struct commit *)o;
@@ -549,11 +571,11 @@ static void filter_refs(struct fetch_pack_args *args,
                                }
                                i++;
                        }
-               }
 
-               if (!keep && args->fetch_all &&
-                   (!args->deepen || !starts_with(ref->name, "refs/tags/")))
-                       keep = 1;
+                       if (!keep && args->fetch_all &&
+                           (!args->deepen || !starts_with(ref->name, "refs/tags/")))
+                               keep = 1;
+               }
 
                if (keep) {
                        *newtail = ref;
@@ -669,7 +691,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
 
                if (!has_object_file_with_flags(&ref->old_oid, flags))
                        continue;
-               o = parse_object(&ref->old_oid);
+               o = parse_object(the_repository, &ref->old_oid);
                if (!o)
                        continue;
 
@@ -700,7 +722,9 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
                 * Don't mark them common yet; the server has to be told so first.
                 */
                for (ref = *refs; ref; ref = ref->next) {
-                       struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
+                       struct object *o = deref_tag(the_repository,
+                                                    lookup_object(the_repository,
+                                                    ref->old_oid.hash),
                                                     NULL, 0);
 
                        if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
@@ -728,7 +752,7 @@ static int everything_local(struct fetch_pack_args *args,
                const struct object_id *remote = &ref->old_oid;
                struct object *o;
 
-               o = lookup_object(remote->hash);
+               o = lookup_object(the_repository, remote->hash);
                if (!o || !(o->flags & COMPLETE)) {
                        retval = 0;
                        print_verbose(args, "want %s (%s)", oid_to_hex(remote),
@@ -895,7 +919,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
        sort_ref_list(&ref, ref_compare_name);
        QSORT(sought, nr_sought, cmp_ref_by_name);
 
-       if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
+       if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
                die(_("Server does not support shallow clients"));
        if (args->depth > 0 || args->deepen_since || args->deepen_not)
                args->deepen = 1;
@@ -996,7 +1020,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 static void add_shallow_requests(struct strbuf *req_buf,
                                 const struct fetch_pack_args *args)
 {
-       if (is_repository_shallow())
+       if (is_repository_shallow(the_repository))
                write_shallow_commits(req_buf, 1, NULL);
        if (args->depth > 0)
                packet_buf_write(req_buf, "deepen %d", args->depth);
@@ -1015,9 +1039,10 @@ static void add_shallow_requests(struct strbuf *req_buf,
 
 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
 {
+       int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
+
        for ( ; wants ; wants = wants->next) {
                const struct object_id *remote = &wants->old_oid;
-               const char *remote_hex;
                struct object *o;
 
                /*
@@ -1030,13 +1055,15 @@ static void add_wants(const struct ref *wants, struct strbuf *req_buf)
                 * interested in the case we *know* the object is
                 * reachable and we have already scanned it.
                 */
-               if (((o = lookup_object(remote->hash)) != NULL) &&
+               if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
                    (o->flags & COMPLETE)) {
                        continue;
                }
 
-               remote_hex = oid_to_hex(remote);
-               packet_buf_write(req_buf, "want %s\n", remote_hex);
+               if (!use_ref_in_want || wants->exact_oid)
+                       packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
+               else
+                       packet_buf_write(req_buf, "want-ref %s\n", wants->name);
        }
 }
 
@@ -1111,7 +1138,7 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
        /* Add shallow-info and deepen request */
        if (server_supports_feature("fetch", "shallow", 0))
                add_shallow_requests(&req_buf, args);
-       else if (is_repository_shallow() || args->deepen)
+       else if (is_repository_shallow(the_repository) || args->deepen)
                die(_("Server does not support shallow requests"));
 
        /* Add filter */
@@ -1192,7 +1219,7 @@ static int process_acks(struct fetch_negotiator *negotiator,
                        if (!get_oid_hex(arg, &oid)) {
                                struct commit *commit;
                                oidset_insert(common, &oid);
-                               commit = lookup_commit(&oid);
+                               commit = lookup_commit(the_repository, &oid);
                                negotiator->ack(negotiator, commit);
                        }
                        continue;
@@ -1225,16 +1252,16 @@ static void receive_shallow_info(struct fetch_pack_args *args,
                if (skip_prefix(reader->line, "shallow ", &arg)) {
                        if (get_oid_hex(arg, &oid))
                                die(_("invalid shallow line: %s"), reader->line);
-                       register_shallow(&oid);
+                       register_shallow(the_repository, &oid);
                        continue;
                }
                if (skip_prefix(reader->line, "unshallow ", &arg)) {
                        if (get_oid_hex(arg, &oid))
                                die(_("invalid unshallow line: %s"), reader->line);
-                       if (!lookup_object(oid.hash))
+                       if (!lookup_object(the_repository, oid.hash))
                                die(_("object not found: %s"), reader->line);
                        /* make sure that it is parsed as shallow */
-                       if (!parse_object(&oid))
+                       if (!parse_object(the_repository, &oid))
                                die(_("error in object: %s"), reader->line);
                        if (unregister_shallow(&oid))
                                die(_("no shallow found: %s"), reader->line);
@@ -1251,6 +1278,32 @@ static void receive_shallow_info(struct fetch_pack_args *args,
        args->deepen = 1;
 }
 
+static void receive_wanted_refs(struct packet_reader *reader, struct ref *refs)
+{
+       process_section_header(reader, "wanted-refs", 0);
+       while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
+               struct object_id oid;
+               const char *end;
+               struct ref *r = NULL;
+
+               if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
+                       die("expected wanted-ref, got '%s'", reader->line);
+
+               for (r = refs; r; r = r->next) {
+                       if (!strcmp(end, r->name)) {
+                               oidcpy(&r->old_oid, &oid);
+                               break;
+                       }
+               }
+
+               if (!r)
+                       die("unexpected wanted-ref: '%s'", reader->line);
+       }
+
+       if (reader->status != PACKET_READ_DELIM)
+               die("error processing wanted refs: %d", reader->status);
+}
+
 enum fetch_state {
        FETCH_CHECK_LOCAL = 0,
        FETCH_SEND_REQUEST,
@@ -1296,7 +1349,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
                        else
                                state = FETCH_SEND_REQUEST;
 
-                       for_each_ref(rev_list_insert_ref_oid, &negotiator);
+                       mark_tips(&negotiator, args->negotiation_tips);
                        for_each_cached_alternate(&negotiator,
                                                  insert_one_alternate_object);
                        break;
@@ -1327,6 +1380,9 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
                        if (process_section_header(&reader, "shallow-info", 1))
                                receive_shallow_info(args, &reader);
 
+                       if (process_section_header(&reader, "wanted-refs", 1))
+                               receive_wanted_refs(&reader, ref);
+
                        /* get the pack */
                        process_section_header(&reader, "packfile", 0);
                        if (get_pack(args, fd, pack_lockfile))
@@ -1392,16 +1448,17 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr)
 }
 
 static void update_shallow(struct fetch_pack_args *args,
-                          struct ref **sought, int nr_sought,
+                          struct ref *refs,
                           struct shallow_info *si)
 {
        struct oid_array ref = OID_ARRAY_INIT;
        int *status;
        int i;
+       struct ref *r;
 
        if (args->deepen && alternate_shallow_file) {
                if (*alternate_shallow_file == '\0') { /* --unshallow */
-                       unlink_or_warn(git_path_shallow());
+                       unlink_or_warn(git_path_shallow(the_repository));
                        rollback_lock_file(&shallow_lock);
                } else
                        commit_lock_file(&shallow_lock);
@@ -1439,8 +1496,8 @@ static void update_shallow(struct fetch_pack_args *args,
        remove_nonexistent_theirs_shallow(si);
        if (!si->nr_ours && !si->nr_theirs)
                return;
-       for (i = 0; i < nr_sought; i++)
-               oid_array_append(&ref, &sought[i]->old_oid);
+       for (r = refs; r; r = r->next)
+               oid_array_append(&ref, &r->old_oid);
        si->ref = &ref;
 
        if (args->update_shallow) {
@@ -1474,17 +1531,29 @@ static void update_shallow(struct fetch_pack_args *args,
         * remote is also shallow, check what ref is safe to update
         * without updating .git/shallow
         */
-       status = xcalloc(nr_sought, sizeof(*status));
+       status = xcalloc(ref.nr, sizeof(*status));
        assign_shallow_commits_to_refs(si, NULL, status);
        if (si->nr_ours || si->nr_theirs) {
-               for (i = 0; i < nr_sought; i++)
+               for (r = refs, i = 0; r; r = r->next, i++)
                        if (status[i])
-                               sought[i]->status = REF_STATUS_REJECT_SHALLOW;
+                               r->status = REF_STATUS_REJECT_SHALLOW;
        }
        free(status);
        oid_array_clear(&ref);
 }
 
+static int iterate_ref_map(void *cb_data, struct object_id *oid)
+{
+       struct ref **rm = cb_data;
+       struct ref *ref = *rm;
+
+       if (!ref)
+               return -1; /* end of the list */
+       *rm = ref->next;
+       oidcpy(oid, &ref->old_oid);
+       return 0;
+}
+
 struct ref *fetch_pack(struct fetch_pack_args *args,
                       int fd[], struct child_process *conn,
                       const struct ref *ref,
@@ -1513,7 +1582,25 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
                                        &si, pack_lockfile);
        reprepare_packed_git(the_repository);
-       update_shallow(args, sought, nr_sought, &si);
+
+       if (!args->cloning && args->deepen) {
+               struct check_connected_options opt = CHECK_CONNECTED_INIT;
+               struct ref *iterator = ref_cpy;
+               opt.shallow_file = alternate_shallow_file;
+               if (args->deepen)
+                       opt.is_deepening_fetch = 1;
+               if (check_connected(iterate_ref_map, &iterator, &opt)) {
+                       error(_("remote did not send all necessary objects"));
+                       free_refs(ref_cpy);
+                       ref_cpy = NULL;
+                       rollback_lock_file(&shallow_lock);
+                       goto cleanup;
+               }
+               args->connectivity_checked = 1;
+       }
+
+       update_shallow(args, ref_cpy, &si);
+cleanup:
        clear_shallow_info(&si);
        return ref_cpy;
 }