Merge branch 'sb/object-store-lookup'
authorJunio C Hamano <gitster@pobox.com>
Thu, 2 Aug 2018 22:30:42 +0000 (15:30 -0700)
committerJunio C Hamano <gitster@pobox.com>
Thu, 2 Aug 2018 22:30:42 +0000 (15:30 -0700)
lookup_commit_reference() and friends have been updated to find
in-core object for a specific in-core repository instance.

* sb/object-store-lookup: (32 commits)
  commit.c: allow lookup_commit_reference to handle arbitrary repositories
  commit.c: allow lookup_commit_reference_gently to handle arbitrary repositories
  tag.c: allow deref_tag to handle arbitrary repositories
  object.c: allow parse_object to handle arbitrary repositories
  object.c: allow parse_object_buffer to handle arbitrary repositories
  commit.c: allow get_cached_commit_buffer to handle arbitrary repositories
  commit.c: allow set_commit_buffer to handle arbitrary repositories
  commit.c: migrate the commit buffer to the parsed object store
  commit-slabs: remove realloc counter outside of slab struct
  commit.c: allow parse_commit_buffer to handle arbitrary repositories
  tag: allow parse_tag_buffer to handle arbitrary repositories
  tag: allow lookup_tag to handle arbitrary repositories
  commit: allow lookup_commit to handle arbitrary repositories
  tree: allow lookup_tree to handle arbitrary repositories
  blob: allow lookup_blob to handle arbitrary repositories
  object: allow lookup_object to handle arbitrary repositories
  object: allow object_as_type to handle arbitrary repositories
  tag: add repository argument to deref_tag
  tag: add repository argument to parse_tag_buffer
  tag: add repository argument to lookup_tag
  ...

35 files changed:
1  2 
builtin/branch.c
builtin/checkout.c
builtin/clone.c
builtin/fetch.c
builtin/fmt-merge-msg.c
builtin/fsck.c
builtin/grep.c
builtin/log.c
builtin/pack-objects.c
builtin/receive-pack.c
builtin/rev-list.c
commit-graph.c
commit.c
commit.h
fast-import.c
fetch-pack.c
fsck.c
line-log.c
list-objects.c
log-tree.c
merge-recursive.c
negotiator/default.c
object.h
pretty.c
ref-filter.c
refs.c
refs/files-backend.c
remote.c
revision.c
sequencer.c
sha1-file.c
sha1-name.c
submodule.c
upload-pack.c
wt-status.c

Simple merge
Simple merge
diff --cc builtin/clone.c
Simple merge
diff --cc builtin/fetch.c
Simple merge
@@@ -344,7 -344,9 +345,9 @@@ static void shortlog(const char *name
        const struct object_id *oid = &origin_data->oid;
        int limit = opts->shortlog_len;
  
-       branch = deref_tag(parse_object(oid), oid_to_hex(oid), the_hash_algo->hexsz);
+       branch = deref_tag(the_repository, parse_object(the_repository, oid),
+                          oid_to_hex(oid),
 -                         GIT_SHA1_HEXSZ);
++                         the_hash_algo->hexsz);
        if (!branch || branch->type != OBJ_COMMIT)
                return;
  
diff --cc builtin/fsck.c
Simple merge
diff --cc builtin/grep.c
Simple merge
diff --cc builtin/log.c
Simple merge
Simple merge
Simple merge
Simple merge
diff --cc commit-graph.c
@@@ -244,12 -241,8 +244,12 @@@ static struct commit_list **insert_pare
  {
        struct commit *c;
        struct object_id oid;
 +
 +      if (pos >= g->num_commits)
 +              die("invalid parent position %"PRIu64, pos);
 +
        hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
-       c = lookup_commit(&oid);
+       c = lookup_commit(the_repository, &oid);
        if (!c)
                die("could not find commit %s", oid_to_hex(&oid));
        c->graph_pos = pos;
@@@ -740,11 -698,10 +740,11 @@@ void write_commit_graph(const char *obj
                        struct object_id oid;
                        struct commit *result;
  
 -                      if (commit_hex[i] && parse_oid_hex(commit_hex[i], &oid, &end))
 +                      if (commit_hex->items[i].string &&
 +                          parse_oid_hex(commit_hex->items[i].string, &oid, &end))
                                continue;
  
-                       result = lookup_commit_reference_gently(&oid, 1);
+                       result = lookup_commit_reference_gently(the_repository, &oid, 1);
  
                        if (result) {
                                ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
        oids.alloc = 0;
        oids.nr = 0;
  }
-               graph_commit = lookup_commit(&cur_oid);
 +
 +#define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
 +static int verify_commit_graph_error;
 +
 +static void graph_report(const char *fmt, ...)
 +{
 +      va_list ap;
 +
 +      verify_commit_graph_error = 1;
 +      va_start(ap, fmt);
 +      vfprintf(stderr, fmt, ap);
 +      fprintf(stderr, "\n");
 +      va_end(ap);
 +}
 +
 +#define GENERATION_ZERO_EXISTS 1
 +#define GENERATION_NUMBER_EXISTS 2
 +
 +int verify_commit_graph(struct repository *r, struct commit_graph *g)
 +{
 +      uint32_t i, cur_fanout_pos = 0;
 +      struct object_id prev_oid, cur_oid, checksum;
 +      int generation_zero = 0;
 +      struct hashfile *f;
 +      int devnull;
 +
 +      if (!g) {
 +              graph_report("no commit-graph file loaded");
 +              return 1;
 +      }
 +
 +      verify_commit_graph_error = 0;
 +
 +      if (!g->chunk_oid_fanout)
 +              graph_report("commit-graph is missing the OID Fanout chunk");
 +      if (!g->chunk_oid_lookup)
 +              graph_report("commit-graph is missing the OID Lookup chunk");
 +      if (!g->chunk_commit_data)
 +              graph_report("commit-graph is missing the Commit Data chunk");
 +
 +      if (verify_commit_graph_error)
 +              return verify_commit_graph_error;
 +
 +      devnull = open("/dev/null", O_WRONLY);
 +      f = hashfd(devnull, NULL);
 +      hashwrite(f, g->data, g->data_len - g->hash_len);
 +      finalize_hashfile(f, checksum.hash, CSUM_CLOSE);
 +      if (hashcmp(checksum.hash, g->data + g->data_len - g->hash_len)) {
 +              graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
 +              verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
 +      }
 +
 +      for (i = 0; i < g->num_commits; i++) {
 +              struct commit *graph_commit;
 +
 +              hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
 +
 +              if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
 +                      graph_report("commit-graph has incorrect OID order: %s then %s",
 +                                   oid_to_hex(&prev_oid),
 +                                   oid_to_hex(&cur_oid));
 +
 +              oidcpy(&prev_oid, &cur_oid);
 +
 +              while (cur_oid.hash[0] > cur_fanout_pos) {
 +                      uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
 +
 +                      if (i != fanout_value)
 +                              graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
 +                                           cur_fanout_pos, fanout_value, i);
 +                      cur_fanout_pos++;
 +              }
 +
-               graph_commit = lookup_commit(&cur_oid);
++              graph_commit = lookup_commit(r, &cur_oid);
 +              if (!parse_commit_in_graph_one(g, graph_commit))
 +                      graph_report("failed to parse %s from commit-graph",
 +                                   oid_to_hex(&cur_oid));
 +      }
 +
 +      while (cur_fanout_pos < 256) {
 +              uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
 +
 +              if (g->num_commits != fanout_value)
 +                      graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
 +                                   cur_fanout_pos, fanout_value, i);
 +
 +              cur_fanout_pos++;
 +      }
 +
 +      if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
 +              return verify_commit_graph_error;
 +
 +      for (i = 0; i < g->num_commits; i++) {
 +              struct commit *graph_commit, *odb_commit;
 +              struct commit_list *graph_parents, *odb_parents;
 +              uint32_t max_generation = 0;
 +
 +              hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
 +
++              graph_commit = lookup_commit(r, &cur_oid);
 +              odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
 +              if (parse_commit_internal(odb_commit, 0, 0)) {
 +                      graph_report("failed to parse %s from object database",
 +                                   oid_to_hex(&cur_oid));
 +                      continue;
 +              }
 +
 +              if (oidcmp(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid,
 +                         get_commit_tree_oid(odb_commit)))
 +                      graph_report("root tree OID for commit %s in commit-graph is %s != %s",
 +                                   oid_to_hex(&cur_oid),
 +                                   oid_to_hex(get_commit_tree_oid(graph_commit)),
 +                                   oid_to_hex(get_commit_tree_oid(odb_commit)));
 +
 +              graph_parents = graph_commit->parents;
 +              odb_parents = odb_commit->parents;
 +
 +              while (graph_parents) {
 +                      if (odb_parents == NULL) {
 +                              graph_report("commit-graph parent list for commit %s is too long",
 +                                           oid_to_hex(&cur_oid));
 +                              break;
 +                      }
 +
 +                      if (oidcmp(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
 +                              graph_report("commit-graph parent for %s is %s != %s",
 +                                           oid_to_hex(&cur_oid),
 +                                           oid_to_hex(&graph_parents->item->object.oid),
 +                                           oid_to_hex(&odb_parents->item->object.oid));
 +
 +                      if (graph_parents->item->generation > max_generation)
 +                              max_generation = graph_parents->item->generation;
 +
 +                      graph_parents = graph_parents->next;
 +                      odb_parents = odb_parents->next;
 +              }
 +
 +              if (odb_parents != NULL)
 +                      graph_report("commit-graph parent list for commit %s terminates early",
 +                                   oid_to_hex(&cur_oid));
 +
 +              if (!graph_commit->generation) {
 +                      if (generation_zero == GENERATION_NUMBER_EXISTS)
 +                              graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere",
 +                                           oid_to_hex(&cur_oid));
 +                      generation_zero = GENERATION_ZERO_EXISTS;
 +              } else if (generation_zero == GENERATION_ZERO_EXISTS)
 +                      graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere",
 +                                   oid_to_hex(&cur_oid));
 +
 +              if (generation_zero == GENERATION_ZERO_EXISTS)
 +                      continue;
 +
 +              /*
 +               * If one of our parents has generation GENERATION_NUMBER_MAX, then
 +               * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid
 +               * extra logic in the following condition.
 +               */
 +              if (max_generation == GENERATION_NUMBER_MAX)
 +                      max_generation--;
 +
 +              if (graph_commit->generation != max_generation + 1)
 +                      graph_report("commit-graph generation for commit %s is %u != %u",
 +                                   oid_to_hex(&cur_oid),
 +                                   graph_commit->generation,
 +                                   max_generation + 1);
 +
 +              if (graph_commit->date != odb_commit->date)
 +                      graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime,
 +                                   oid_to_hex(&cur_oid),
 +                                   graph_commit->date,
 +                                   odb_commit->date);
 +      }
 +
 +      return verify_commit_graph_error;
 +}
diff --cc commit.c
+++ b/commit.c
@@@ -446,10 -466,9 +466,10 @@@ int parse_commit_internal(struct commi
                return error("Object %s not a commit",
                             oid_to_hex(&item->object.oid));
        }
-       ret = parse_commit_buffer(item, buffer, size, 0);
 +
+       ret = parse_commit_buffer(the_repository, item, buffer, size, 0);
        if (save_commit_buffer && !ret) {
-               set_commit_buffer(item, buffer, size);
+               set_commit_buffer(the_repository, item, buffer, size);
                return 0;
        }
        free(buffer);
diff --cc commit.h
+++ b/commit.h
@@@ -76,8 -78,7 +78,8 @@@ struct commit *lookup_commit_reference_
   */
  struct commit *lookup_commit_or_die(const struct object_id *oid, const char *ref_name);
  
- int parse_commit_buffer(struct commit *item, const void *buffer, unsigned long size, int check_graph);
+ int parse_commit_buffer(struct repository *r, struct commit *item, const void *buffer, unsigned long size, int check_graph);
 +int parse_commit_internal(struct commit *item, int quiet_on_missing, int use_commit_graph);
  int parse_commit_gently(struct commit *item, int quiet_on_missing);
  static inline int parse_commit(struct commit *item)
  {
diff --cc fast-import.c
Simple merge
diff --cc fetch-pack.c
@@@ -102,17 -106,32 +102,19 @@@ static void for_each_cached_alternate(s
        }
  
        for (i = 0; i < cache.nr; i++)
 -              cb(cache.items[i]);
 +              cb(negotiator, cache.items[i]);
  }
  
 -static void rev_list_push(struct commit *commit, int mark)
 -{
 -      if (!(commit->object.flags & mark)) {
 -              commit->object.flags |= mark;
 -
 -              if (parse_commit(commit))
 -                      return;
 -
 -              prio_queue_put(&rev_list, commit);
 -
 -              if (!(commit->object.flags & COMMON))
 -                      non_common_revs++;
 -      }
 -}
 -
 -static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 +static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
 +                             const char *refname,
 +                             const struct object_id *oid)
  {
-       struct object *o = deref_tag(parse_object(oid), refname, 0);
+       struct object *o = deref_tag(the_repository,
+                                    parse_object(the_repository, oid),
+                                    refname, 0);
  
        if (o && o->type == OBJ_COMMIT)
 -              rev_list_push((struct commit *)o, SEEN);
 +              negotiator->add_tip(negotiator, (struct commit *)o);
  
        return 0;
  }
@@@ -387,14 -500,13 +389,16 @@@ static int find_common(struct fetch_neg
                                case ACK_ready:
                                case ACK_continue: {
                                        struct commit *commit =
-                                               lookup_commit(result_oid);
+                                               lookup_commit(the_repository,
+                                                             result_oid);
 +                                      int was_common;
++
                                        if (!commit)
                                                die(_("invalid commit %s"), oid_to_hex(result_oid));
 +                                      was_common = negotiator->ack(negotiator, commit);
                                        if (args->stateless_rpc
                                         && ack == ACK_common
 -                                       && !(commit->object.flags & COMMON)) {
 +                                       && !was_common) {
                                                /* We need to replay the have for this object
                                                 * on the next RPC request so the peer knows
                                                 * it is in common with us.
@@@ -1196,8 -1282,8 +1202,8 @@@ static int process_acks(struct fetch_ne
                        if (!get_oid_hex(arg, &oid)) {
                                struct commit *commit;
                                oidset_insert(common, &oid);
-                               commit = lookup_commit(&oid);
+                               commit = lookup_commit(the_repository, &oid);
 -                              mark_common(commit, 0, 1);
 +                              negotiator->ack(negotiator, commit);
                        }
                        continue;
                }
diff --cc fsck.c
Simple merge
diff --cc line-log.c
Simple merge
diff --cc list-objects.c
Simple merge
diff --cc log-tree.c
Simple merge
Simple merge
index 382fc77,0000000..4b78f6b
mode 100644,000000..100644
--- /dev/null
@@@ -1,176 -1,0 +1,176 @@@
-       struct object *o = deref_tag(parse_object(oid), refname, 0);
 +#include "cache.h"
 +#include "default.h"
 +#include "../commit.h"
 +#include "../fetch-negotiator.h"
 +#include "../prio-queue.h"
 +#include "../refs.h"
 +#include "../tag.h"
 +
 +/* Remember to update object flag allocation in object.h */
 +#define COMMON                (1U << 2)
 +#define COMMON_REF    (1U << 3)
 +#define SEEN          (1U << 4)
 +#define POPPED                (1U << 5)
 +
 +static int marked;
 +
 +struct negotiation_state {
 +      struct prio_queue rev_list;
 +      int non_common_revs;
 +};
 +
 +static void rev_list_push(struct negotiation_state *ns,
 +                        struct commit *commit, int mark)
 +{
 +      if (!(commit->object.flags & mark)) {
 +              commit->object.flags |= mark;
 +
 +              if (parse_commit(commit))
 +                      return;
 +
 +              prio_queue_put(&ns->rev_list, commit);
 +
 +              if (!(commit->object.flags & COMMON))
 +                      ns->non_common_revs++;
 +      }
 +}
 +
 +static int clear_marks(const char *refname, const struct object_id *oid,
 +                     int flag, void *cb_data)
 +{
++      struct object *o = deref_tag(the_repository, parse_object(the_repository, oid), refname, 0);
 +
 +      if (o && o->type == OBJ_COMMIT)
 +              clear_commit_marks((struct commit *)o,
 +                                 COMMON | COMMON_REF | SEEN | POPPED);
 +      return 0;
 +}
 +
 +/*
 + * This function marks a rev and its ancestors as common.
 + * In some cases, it is desirable to mark only the ancestors (for example
 + * when only the server does not yet know that they are common).
 + */
 +static void mark_common(struct negotiation_state *ns, struct commit *commit,
 +              int ancestors_only, int dont_parse)
 +{
 +      if (commit != NULL && !(commit->object.flags & COMMON)) {
 +              struct object *o = (struct object *)commit;
 +
 +              if (!ancestors_only)
 +                      o->flags |= COMMON;
 +
 +              if (!(o->flags & SEEN))
 +                      rev_list_push(ns, commit, SEEN);
 +              else {
 +                      struct commit_list *parents;
 +
 +                      if (!ancestors_only && !(o->flags & POPPED))
 +                              ns->non_common_revs--;
 +                      if (!o->parsed && !dont_parse)
 +                              if (parse_commit(commit))
 +                                      return;
 +
 +                      for (parents = commit->parents;
 +                                      parents;
 +                                      parents = parents->next)
 +                              mark_common(ns, parents->item, 0,
 +                                          dont_parse);
 +              }
 +      }
 +}
 +
 +/*
 + * Get the next rev to send, ignoring the common.
 + */
 +static const struct object_id *get_rev(struct negotiation_state *ns)
 +{
 +      struct commit *commit = NULL;
 +
 +      while (commit == NULL) {
 +              unsigned int mark;
 +              struct commit_list *parents;
 +
 +              if (ns->rev_list.nr == 0 || ns->non_common_revs == 0)
 +                      return NULL;
 +
 +              commit = prio_queue_get(&ns->rev_list);
 +              parse_commit(commit);
 +              parents = commit->parents;
 +
 +              commit->object.flags |= POPPED;
 +              if (!(commit->object.flags & COMMON))
 +                      ns->non_common_revs--;
 +
 +              if (commit->object.flags & COMMON) {
 +                      /* do not send "have", and ignore ancestors */
 +                      commit = NULL;
 +                      mark = COMMON | SEEN;
 +              } else if (commit->object.flags & COMMON_REF)
 +                      /* send "have", and ignore ancestors */
 +                      mark = COMMON | SEEN;
 +              else
 +                      /* send "have", also for its ancestors */
 +                      mark = SEEN;
 +
 +              while (parents) {
 +                      if (!(parents->item->object.flags & SEEN))
 +                              rev_list_push(ns, parents->item, mark);
 +                      if (mark & COMMON)
 +                              mark_common(ns, parents->item, 1, 0);
 +                      parents = parents->next;
 +              }
 +      }
 +
 +      return &commit->object.oid;
 +}
 +
 +static void known_common(struct fetch_negotiator *n, struct commit *c)
 +{
 +      if (!(c->object.flags & SEEN)) {
 +              rev_list_push(n->data, c, COMMON_REF | SEEN);
 +              mark_common(n->data, c, 1, 1);
 +      }
 +}
 +
 +static void add_tip(struct fetch_negotiator *n, struct commit *c)
 +{
 +      n->known_common = NULL;
 +      rev_list_push(n->data, c, SEEN);
 +}
 +
 +static const struct object_id *next(struct fetch_negotiator *n)
 +{
 +      n->known_common = NULL;
 +      n->add_tip = NULL;
 +      return get_rev(n->data);
 +}
 +
 +static int ack(struct fetch_negotiator *n, struct commit *c)
 +{
 +      int known_to_be_common = !!(c->object.flags & COMMON);
 +      mark_common(n->data, c, 0, 1);
 +      return known_to_be_common;
 +}
 +
 +static void release(struct fetch_negotiator *n)
 +{
 +      clear_prio_queue(&((struct negotiation_state *)n->data)->rev_list);
 +      FREE_AND_NULL(n->data);
 +}
 +
 +void default_negotiator_init(struct fetch_negotiator *negotiator)
 +{
 +      struct negotiation_state *ns;
 +      negotiator->known_common = known_common;
 +      negotiator->add_tip = add_tip;
 +      negotiator->next = next;
 +      negotiator->ack = ack;
 +      negotiator->release = release;
 +      negotiator->data = ns = xcalloc(1, sizeof(*ns));
 +      ns->rev_list.compare = compare_commits_by_commit_date;
 +
 +      if (marked)
 +              for_each_ref(clear_marks, NULL);
 +      marked = 1;
 +}
diff --cc object.h
Simple merge
diff --cc pretty.c
Simple merge
diff --cc ref-filter.c
Simple merge
diff --cc refs.c
Simple merge
Simple merge
diff --cc remote.c
Simple merge
diff --cc revision.c
Simple merge
diff --cc sequencer.c
Simple merge
diff --cc sha1-file.c
Simple merge
diff --cc sha1-name.c
Simple merge
diff --cc submodule.c
Simple merge
diff --cc upload-pack.c
Simple merge
diff --cc wt-status.c
@@@ -2340,17 -2340,7 +2340,17 @@@ int has_uncommitted_changes(int ignore_
        if (ignore_submodules)
                rev_info.diffopt.flags.ignore_submodules = 1;
        rev_info.diffopt.flags.quick = 1;
 +
        add_head_to_pending(&rev_info);
-               struct tree *tree = lookup_tree(the_hash_algo->empty_tree);
 +      if (!rev_info.pending.nr) {
 +              /*
 +               * We have no head (or it's corrupt); use the empty tree,
 +               * which will complain if the index is non-empty.
 +               */
++              struct tree *tree = lookup_tree(the_repository, the_hash_algo->empty_tree);
 +              add_pending_object(&rev_info, &tree->object, "");
 +      }
 +
        diff_setup_done(&rev_info.diffopt);
        result = run_diff_index(&rev_info, 1);
        return diff_result_code(&rev_info.diffopt, result);