Merge branch 'ds/midx-too-many-packs'
authorJunio C Hamano <gitster@pobox.com>
Sun, 19 May 2019 07:45:30 +0000 (16:45 +0900)
committerJunio C Hamano <gitster@pobox.com>
Sun, 19 May 2019 07:45:30 +0000 (16:45 +0900)
The code to generate the multi-pack idx file was not prepared to
see too many packfiles and ran out of open file descriptor, which
has been corrected.

* ds/midx-too-many-packs:
  midx: add packs to packed_git linked list
  midx: pass a repository pointer

1  2 
builtin/pack-objects.c
object-store.h
packfile.c
sha1-name.c

diff --combined builtin/pack-objects.c
@@@ -97,7 -97,7 +97,7 @@@ static off_t reuse_packfile_offset
  static int use_bitmap_index_default = 1;
  static int use_bitmap_index = -1;
  static int write_bitmap_index;
 -static uint16_t write_bitmap_options;
 +static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE;
  
  static int exclude_promisor_objects;
  
@@@ -964,8 -964,6 +964,8 @@@ static void write_pack_file(void
        if (written != nr_result)
                die(_("wrote %"PRIu32" objects while expecting %"PRIu32),
                    written, nr_result);
 +      trace2_data_intmax("pack-objects", the_repository,
 +                         "write_pack_file/wrote", nr_result);
  }
  
  static int no_try_delta(const char *path)
@@@ -1080,7 -1078,7 +1080,7 @@@ static int want_object_in_pack(const st
  
        for (m = get_multi_pack_index(the_repository); m; m = m->next) {
                struct pack_entry e;
-               if (fill_midx_entry(oid, &e, m)) {
+               if (fill_midx_entry(the_repository, oid, &e, m)) {
                        struct packed_git *p = e.p;
                        off_t offset;
  
diff --combined object-store.h
@@@ -76,7 -76,8 +76,8 @@@ struct packed_git 
                 pack_keep_in_core:1,
                 freshened:1,
                 do_not_close:1,
-                pack_promisor:1;
+                pack_promisor:1,
+                multi_pack_index:1;
        unsigned char hash[GIT_MAX_RAWSZ];
        struct revindex_entry *revindex;
        /* something like ".git/objects/pack/xxxxx.pack" */
@@@ -128,12 -129,6 +129,6 @@@ struct raw_object_store 
        /* A most-recently-used ordered version of the packed_git list. */
        struct list_head packed_git_mru;
  
-       /*
-        * A linked list containing all packfiles, starting with those
-        * contained in the multi_pack_index.
-        */
-       struct packed_git *all_packs;
        /*
         * A fast, rough count of the number of objects in the repository.
         * These two fields are not meant for direct access. Use
@@@ -162,10 -157,10 +157,10 @@@ const char *loose_object_path(struct re
  void *map_loose_object(struct repository *r, const struct object_id *oid,
                       unsigned long *size);
  
 -extern void *read_object_file_extended(struct repository *r,
 -                                     const struct object_id *oid,
 -                                     enum object_type *type,
 -                                     unsigned long *size, int lookup_replace);
 +void *read_object_file_extended(struct repository *r,
 +                              const struct object_id *oid,
 +                              enum object_type *type,
 +                              unsigned long *size, int lookup_replace);
  static inline void *repo_read_object_file(struct repository *r,
                                          const struct object_id *oid,
                                          enum object_type *type,
  /* Read and unpack an object file into memory, write memory to an object file */
  int oid_object_info(struct repository *r, const struct object_id *, unsigned long *);
  
 -extern int hash_object_file(const void *buf, unsigned long len,
 -                          const char *type, struct object_id *oid);
 +int hash_object_file(const void *buf, unsigned long len,
 +                   const char *type, struct object_id *oid);
  
 -extern int write_object_file(const void *buf, unsigned long len,
 -                           const char *type, struct object_id *oid);
 +int write_object_file(const void *buf, unsigned long len,
 +                    const char *type, struct object_id *oid);
  
 -extern int hash_object_file_literally(const void *buf, unsigned long len,
 -                                    const char *type, struct object_id *oid,
 -                                    unsigned flags);
 +int hash_object_file_literally(const void *buf, unsigned long len,
 +                             const char *type, struct object_id *oid,
 +                             unsigned flags);
  
 -extern int pretend_object_file(void *, unsigned long, enum object_type,
 -                             struct object_id *oid);
 +int pretend_object_file(void *, unsigned long, enum object_type,
 +                      struct object_id *oid);
  
 -extern int force_object_loose(const struct object_id *oid, time_t mtime);
 +int force_object_loose(const struct object_id *oid, time_t mtime);
  
  /*
   * Open the loose object at path, check its hash, and return the contents,
@@@ -227,9 -222,9 +222,9 @@@ int repo_has_object_file_with_flags(str
   * with the specified name.  This function does not respect replace
   * references.
   */
 -extern int has_loose_object_nonlocal(const struct object_id *);
 +int has_loose_object_nonlocal(const struct object_id *);
  
 -extern void assert_oid_type(const struct object_id *oid, enum object_type expect);
 +void assert_oid_type(const struct object_id *oid, enum object_type expect);
  
  struct object_info {
        /* Request */
diff --combined packfile.c
@@@ -903,25 -903,25 +903,25 @@@ static void prepare_packed_git(struct r
   * all unreachable objects about to be pruned, in which case they're not really
   * interesting as a measure of repo size in the first place.
   */
 -unsigned long approximate_object_count(void)
 +unsigned long repo_approximate_object_count(struct repository *r)
  {
 -      if (!the_repository->objects->approximate_object_count_valid) {
 +      if (!r->objects->approximate_object_count_valid) {
                unsigned long count;
                struct multi_pack_index *m;
                struct packed_git *p;
  
 -              prepare_packed_git(the_repository);
 +              prepare_packed_git(r);
                count = 0;
 -              for (m = get_multi_pack_index(the_repository); m; m = m->next)
 +              for (m = get_multi_pack_index(r); m; m = m->next)
                        count += m->num_objects;
 -              for (p = the_repository->objects->packed_git; p; p = p->next) {
 +              for (p = r->objects->packed_git; p; p = p->next) {
                        if (open_pack_index(p))
                                continue;
                        count += p->num_objects;
                }
 -              the_repository->objects->approximate_object_count = count;
 +              r->objects->approximate_object_count = count;
        }
 -      return the_repository->objects->approximate_object_count;
 +      return r->objects->approximate_object_count;
  }
  
  static void *get_next_packed_git(const void *p)
@@@ -994,8 -994,6 +994,6 @@@ static void prepare_packed_git(struct r
        }
        rearrange_packed_git(r);
  
-       r->objects->all_packs = NULL;
        prepare_packed_git_mru(r);
        r->objects->packed_git_initialized = 1;
  }
@@@ -1026,26 -1024,16 +1024,16 @@@ struct multi_pack_index *get_multi_pack
  
  struct packed_git *get_all_packs(struct repository *r)
  {
-       prepare_packed_git(r);
-       if (!r->objects->all_packs) {
-               struct packed_git *p = r->objects->packed_git;
-               struct multi_pack_index *m;
-               for (m = r->objects->multi_pack_index; m; m = m->next) {
-                       uint32_t i;
-                       for (i = 0; i < m->num_packs; i++) {
-                               if (!prepare_midx_pack(m, i)) {
-                                       m->packs[i]->next = p;
-                                       p = m->packs[i];
-                               }
-                       }
-               }
+       struct multi_pack_index *m;
  
-               r->objects->all_packs = p;
+       prepare_packed_git(r);
+       for (m = r->objects->multi_pack_index; m; m = m->next) {
+               uint32_t i;
+               for (i = 0; i < m->num_packs; i++)
+                       prepare_midx_pack(r, m, i);
        }
  
-       return r->objects->all_packs;
+       return r->objects->packed_git;
  }
  
  struct list_head *get_packed_git_mru(struct repository *r)
@@@ -1998,13 -1986,13 +1986,13 @@@ int find_pack_entry(struct repository *
                return 0;
  
        for (m = r->objects->multi_pack_index; m; m = m->next) {
-               if (fill_midx_entry(oid, e, m))
+               if (fill_midx_entry(r, oid, e, m))
                        return 1;
        }
  
        list_for_each(pos, &r->objects->packed_git_mru) {
                struct packed_git *p = list_entry(pos, struct packed_git, mru);
-               if (fill_pack_entry(oid, e, p)) {
+               if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) {
                        list_move(&p->mru, &r->objects->packed_git_mru);
                        return 1;
                }
diff --combined sha1-name.c
  #include "packfile.h"
  #include "object-store.h"
  #include "repository.h"
 +#include "submodule.h"
  #include "midx.h"
  #include "commit-reach.h"
  
 -static int get_oid_oneline(const char *, struct object_id *, struct commit_list *);
 +static int get_oid_oneline(struct repository *r, const char *, struct object_id *, struct commit_list *);
  
 -typedef int (*disambiguate_hint_fn)(const struct object_id *, void *);
 +typedef int (*disambiguate_hint_fn)(struct repository *, const struct object_id *, void *);
  
  struct disambiguate_state {
        int len; /* length of prefix in hex chars */
        char hex_pfx[GIT_MAX_HEXSZ + 1];
        struct object_id bin_pfx;
  
 +      struct repository *repo;
        disambiguate_hint_fn fn;
        void *cb_data;
        struct object_id candidate;
@@@ -40,7 -38,7 +40,7 @@@
  static void update_candidates(struct disambiguate_state *ds, const struct object_id *current)
  {
        if (ds->always_call_fn) {
 -              ds->ambiguous = ds->fn(current, ds->cb_data) ? 1 : 0;
 +              ds->ambiguous = ds->fn(ds->repo, current, ds->cb_data) ? 1 : 0;
                return;
        }
        if (!ds->candidate_exists) {
@@@ -60,7 -58,7 +60,7 @@@
        }
  
        if (!ds->candidate_checked) {
 -              ds->candidate_ok = ds->fn(&ds->candidate, ds->cb_data);
 +              ds->candidate_ok = ds->fn(ds->repo, &ds->candidate, ds->cb_data);
                ds->disambiguate_fn_used = 1;
                ds->candidate_checked = 1;
        }
@@@ -73,7 -71,7 +73,7 @@@
        }
  
        /* if we reach this point, we know ds->candidate satisfies fn */
 -      if (ds->fn(current, ds->cb_data)) {
 +      if (ds->fn(ds->repo, current, ds->cb_data)) {
                /*
                 * if both current and candidate satisfy fn, we cannot
                 * disambiguate.
@@@ -91,7 -89,9 +91,7 @@@ static void find_short_object_filename(
  {
        struct object_directory *odb;
  
 -      for (odb = the_repository->objects->odb;
 -           odb && !ds->ambiguous;
 -           odb = odb->next) {
 +      for (odb = ds->repo->objects->odb; odb && !ds->ambiguous; odb = odb->next) {
                int pos;
                struct oid_array *loose_objects;
  
@@@ -157,6 -157,9 +157,9 @@@ static void unique_in_pack(struct packe
        uint32_t num, i, first = 0;
        const struct object_id *current = NULL;
  
+       if (p->multi_pack_index)
+               return;
        if (open_pack_index(p) || !p->num_objects)
                return;
  
@@@ -182,10 -185,10 +185,10 @@@ static void find_short_packed_object(st
        struct multi_pack_index *m;
        struct packed_git *p;
  
 -      for (m = get_multi_pack_index(the_repository); m && !ds->ambiguous;
 +      for (m = get_multi_pack_index(ds->repo); m && !ds->ambiguous;
             m = m->next)
                unique_in_midx(m, ds);
 -      for (p = get_packed_git(the_repository); p && !ds->ambiguous;
 +      for (p = get_packed_git(ds->repo); p && !ds->ambiguous;
             p = p->next)
                unique_in_pack(p, ds);
  }
@@@ -215,7 -218,7 +218,7 @@@ static int finish_object_disambiguation
                 * same repository!
                 */
                ds->candidate_ok = (!ds->disambiguate_fn_used ||
 -                                  ds->fn(&ds->candidate, ds->cb_data));
 +                                  ds->fn(ds->repo, &ds->candidate, ds->cb_data));
  
        if (!ds->candidate_ok)
                return SHORT_NAME_AMBIGUOUS;
        return 0;
  }
  
 -static int disambiguate_commit_only(const struct object_id *oid, void *cb_data_unused)
 +static int disambiguate_commit_only(struct repository *r,
 +                                  const struct object_id *oid,
 +                                  void *cb_data_unused)
  {
 -      int kind = oid_object_info(the_repository, oid, NULL);
 +      int kind = oid_object_info(r, oid, NULL);
        return kind == OBJ_COMMIT;
  }
  
 -static int disambiguate_committish_only(const struct object_id *oid, void *cb_data_unused)
 +static int disambiguate_committish_only(struct repository *r,
 +                                      const struct object_id *oid,
 +                                      void *cb_data_unused)
  {
        struct object *obj;
        int kind;
  
 -      kind = oid_object_info(the_repository, oid, NULL);
 +      kind = oid_object_info(r, oid, NULL);
        if (kind == OBJ_COMMIT)
                return 1;
        if (kind != OBJ_TAG)
                return 0;
  
        /* We need to do this the hard way... */
 -      obj = deref_tag(the_repository, parse_object(the_repository, oid),
 -                      NULL, 0);
 +      obj = deref_tag(r, parse_object(r, oid), NULL, 0);
        if (obj && obj->type == OBJ_COMMIT)
                return 1;
        return 0;
  }
  
 -static int disambiguate_tree_only(const struct object_id *oid, void *cb_data_unused)
 +static int disambiguate_tree_only(struct repository *r,
 +                                const struct object_id *oid,
 +                                void *cb_data_unused)
  {
 -      int kind = oid_object_info(the_repository, oid, NULL);
 +      int kind = oid_object_info(r, oid, NULL);
        return kind == OBJ_TREE;
  }
  
 -static int disambiguate_treeish_only(const struct object_id *oid, void *cb_data_unused)
 +static int disambiguate_treeish_only(struct repository *r,
 +                                   const struct object_id *oid,
 +                                   void *cb_data_unused)
  {
        struct object *obj;
        int kind;
  
 -      kind = oid_object_info(the_repository, oid, NULL);
 +      kind = oid_object_info(r, oid, NULL);
        if (kind == OBJ_TREE || kind == OBJ_COMMIT)
                return 1;
        if (kind != OBJ_TAG)
                return 0;
  
        /* We need to do this the hard way... */
 -      obj = deref_tag(the_repository, parse_object(the_repository, oid),
 -                      NULL, 0);
 +      obj = deref_tag(r, parse_object(r, oid), NULL, 0);
        if (obj && (obj->type == OBJ_TREE || obj->type == OBJ_COMMIT))
                return 1;
        return 0;
  }
  
 -static int disambiguate_blob_only(const struct object_id *oid, void *cb_data_unused)
 +static int disambiguate_blob_only(struct repository *r,
 +                                const struct object_id *oid,
 +                                void *cb_data_unused)
  {
 -      int kind = oid_object_info(the_repository, oid, NULL);
 +      int kind = oid_object_info(r, oid, NULL);
        return kind == OBJ_BLOB;
  }
  
@@@ -318,8 -313,7 +321,8 @@@ int set_disambiguate_hint_config(const 
        return error("unknown hint type for '%s': %s", var, value);
  }
  
 -static int init_object_disambiguation(const char *name, int len,
 +static int init_object_disambiguation(struct repository *r,
 +                                    const char *name, int len,
                                      struct disambiguate_state *ds)
  {
        int i;
  
        ds->len = len;
        ds->hex_pfx[len] = '\0';
 -      prepare_alt_odb(the_repository);
 +      ds->repo = r;
 +      prepare_alt_odb(r);
        return 0;
  }
  
@@@ -361,25 -354,25 +364,25 @@@ static int show_ambiguous_object(const 
        struct strbuf desc = STRBUF_INIT;
        int type;
  
 -      if (ds->fn && !ds->fn(oid, ds->cb_data))
 +      if (ds->fn && !ds->fn(ds->repo, oid, ds->cb_data))
                return 0;
  
 -      type = oid_object_info(the_repository, oid, NULL);
 +      type = oid_object_info(ds->repo, oid, NULL);
        if (type == OBJ_COMMIT) {
 -              struct commit *commit = lookup_commit(the_repository, oid);
 +              struct commit *commit = lookup_commit(ds->repo, oid);
                if (commit) {
                        struct pretty_print_context pp = {0};
                        pp.date_mode.type = DATE_SHORT;
                        format_commit_message(commit, " %ad - %s", &desc, &pp);
                }
        } else if (type == OBJ_TAG) {
 -              struct tag *tag = lookup_tag(the_repository, oid);
 +              struct tag *tag = lookup_tag(ds->repo, oid);
                if (!parse_tag(tag) && tag->tag)
                        strbuf_addf(&desc, " %s", tag->tag);
        }
  
        advise("  %s %s%s",
 -             find_unique_abbrev(oid, DEFAULT_ABBREV),
 +             repo_find_unique_abbrev(ds->repo, oid, DEFAULT_ABBREV),
               type_name(type) ? type_name(type) : "unknown type",
               desc.buf);
  
@@@ -393,18 -386,10 +396,18 @@@ static int collect_ambiguous(const stru
        return 0;
  }
  
 +static int repo_collect_ambiguous(struct repository *r,
 +                                const struct object_id *oid,
 +                                void *data)
 +{
 +      return collect_ambiguous(oid, data);
 +}
 +
 +static struct repository *sort_ambiguous_repo;
  static int sort_ambiguous(const void *a, const void *b)
  {
 -      int a_type = oid_object_info(the_repository, a, NULL);
 -      int b_type = oid_object_info(the_repository, b, NULL);
 +      int a_type = oid_object_info(sort_ambiguous_repo, a, NULL);
 +      int b_type = oid_object_info(sort_ambiguous_repo, b, NULL);
        int a_type_sort;
        int b_type_sort;
  
        return a_type_sort > b_type_sort ? 1 : -1;
  }
  
 -static enum get_oid_result get_short_oid(const char *name, int len,
 +static void sort_ambiguous_oid_array(struct repository *r, struct oid_array *a)
 +{
 +      /* mutex will be needed if this code is to be made thread safe */
 +      sort_ambiguous_repo = r;
 +      QSORT(a->oid, a->nr, sort_ambiguous);
 +      sort_ambiguous_repo = NULL;
 +}
 +
 +static enum get_oid_result get_short_oid(struct repository *r,
 +                                       const char *name, int len,
                                         struct object_id *oid,
                                         unsigned flags)
  {
        struct disambiguate_state ds;
        int quietly = !!(flags & GET_OID_QUIETLY);
  
 -      if (init_object_disambiguation(name, len, &ds) < 0)
 +      if (init_object_disambiguation(r, name, len, &ds) < 0)
                return -1;
  
        if (HAS_MULTI_BITS(flags & GET_OID_DISAMBIGUATORS))
                        ds.fn = NULL;
  
                advise(_("The candidates are:"));
 -              for_each_abbrev(ds.hex_pfx, collect_ambiguous, &collect);
 -              QSORT(collect.oid, collect.nr, sort_ambiguous);
 +              repo_for_each_abbrev(r, ds.hex_pfx, collect_ambiguous, &collect);
 +              sort_ambiguous_oid_array(r, &collect);
  
                if (oid_array_for_each(&collect, show_ambiguous_object, &ds))
                        BUG("show_ambiguous_object shouldn't return non-zero");
        return status;
  }
  
 -int for_each_abbrev(const char *prefix, each_abbrev_fn fn, void *cb_data)
 +int repo_for_each_abbrev(struct repository *r, const char *prefix,
 +                       each_abbrev_fn fn, void *cb_data)
  {
        struct oid_array collect = OID_ARRAY_INIT;
        struct disambiguate_state ds;
        int ret;
  
 -      if (init_object_disambiguation(prefix, strlen(prefix), &ds) < 0)
 +      if (init_object_disambiguation(r, prefix, strlen(prefix), &ds) < 0)
                return -1;
  
        ds.always_call_fn = 1;
 -      ds.fn = collect_ambiguous;
 +      ds.fn = repo_collect_ambiguous;
        ds.cb_data = &collect;
        find_short_object_filename(&ds);
        find_short_packed_object(&ds);
@@@ -545,7 -520,6 +548,7 @@@ struct min_abbrev_data 
        unsigned int init_len;
        unsigned int cur_len;
        char *hex;
 +      struct repository *repo;
        const struct object_id *oid;
  };
  
@@@ -574,13 -548,6 +577,13 @@@ static int extend_abbrev_len(const stru
        return 0;
  }
  
 +static int repo_extend_abbrev_len(struct repository *r,
 +                                const struct object_id *oid,
 +                                void *cb_data)
 +{
 +      return extend_abbrev_len(oid, cb_data);
 +}
 +
  static void find_abbrev_len_for_midx(struct multi_pack_index *m,
                                     struct min_abbrev_data *mad)
  {
@@@ -625,6 -592,9 +628,9 @@@ static void find_abbrev_len_for_pack(st
        struct object_id oid;
        const struct object_id *mad_oid;
  
+       if (p->multi_pack_index)
+               return;
        if (open_pack_index(p) || !p->num_objects)
                return;
  
@@@ -658,22 -628,21 +664,22 @@@ static void find_abbrev_len_packed(stru
        struct multi_pack_index *m;
        struct packed_git *p;
  
 -      for (m = get_multi_pack_index(the_repository); m; m = m->next)
 +      for (m = get_multi_pack_index(mad->repo); m; m = m->next)
                find_abbrev_len_for_midx(m, mad);
 -      for (p = get_packed_git(the_repository); p; p = p->next)
 +      for (p = get_packed_git(mad->repo); p; p = p->next)
                find_abbrev_len_for_pack(p, mad);
  }
  
 -int find_unique_abbrev_r(char *hex, const struct object_id *oid, int len)
 +int repo_find_unique_abbrev_r(struct repository *r, char *hex,
 +                            const struct object_id *oid, int len)
  {
        struct disambiguate_state ds;
        struct min_abbrev_data mad;
        struct object_id oid_ret;
 -      const unsigned hexsz = the_hash_algo->hexsz;
 +      const unsigned hexsz = r->hash_algo->hexsz;
  
        if (len < 0) {
 -              unsigned long count = approximate_object_count();
 +              unsigned long count = repo_approximate_object_count(r);
                /*
                 * Add one because the MSB only tells us the highest bit set,
                 * not including the value of all the _other_ bits (so "15"
        if (len == hexsz || !len)
                return hexsz;
  
 +      mad.repo = r;
        mad.init_len = len;
        mad.cur_len = len;
        mad.hex = hex;
  
        find_abbrev_len_packed(&mad);
  
 -      if (init_object_disambiguation(hex, mad.cur_len, &ds) < 0)
 +      if (init_object_disambiguation(r, hex, mad.cur_len, &ds) < 0)
                return -1;
  
 -      ds.fn = extend_abbrev_len;
 +      ds.fn = repo_extend_abbrev_len;
        ds.always_call_fn = 1;
        ds.cb_data = (void *)&mad;
  
        return mad.cur_len;
  }
  
 -const char *find_unique_abbrev(const struct object_id *oid, int len)
 +const char *repo_find_unique_abbrev(struct repository *r,
 +                                  const struct object_id *oid,
 +                                  int len)
  {
        static int bufno;
        static char hexbuffer[4][GIT_MAX_HEXSZ + 1];
        char *hex = hexbuffer[bufno];
        bufno = (bufno + 1) % ARRAY_SIZE(hexbuffer);
 -      find_unique_abbrev_r(hex, oid, len);
 +      repo_find_unique_abbrev_r(r, hex, oid, len);
        return hex;
  }
  
@@@ -783,11 -749,11 +789,11 @@@ static inline int push_mark(const char 
        return at_mark(string, len, suffix, ARRAY_SIZE(suffix));
  }
  
 -static enum get_oid_result get_oid_1(const char *name, int len, struct object_id *oid, unsigned lookup_flags);
 -static int interpret_nth_prior_checkout(const char *name, int namelen, struct strbuf *buf);
 +static enum get_oid_result get_oid_1(struct repository *r, const char *name, int len, struct object_id *oid, unsigned lookup_flags);
 +static int interpret_nth_prior_checkout(struct repository *r, const char *name, int namelen, struct strbuf *buf);
  
 -static int get_oid_basic(const char *str, int len, struct object_id *oid,
 -                        unsigned int flags)
 +static int get_oid_basic(struct repository *r, const char *str, int len,
 +                       struct object_id *oid, unsigned int flags)
  {
        static const char *warn_msg = "refname '%.*s' is ambiguous.";
        static const char *object_name_msg = N_(
        int refs_found = 0;
        int at, reflog_len, nth_prior = 0;
  
 -      if (len == the_hash_algo->hexsz && !get_oid_hex(str, oid)) {
 +      if (len == r->hash_algo->hexsz && !get_oid_hex(str, oid)) {
                if (warn_ambiguous_refs && warn_on_object_refname_ambiguity) {
 -                      refs_found = dwim_ref(str, len, &tmp_oid, &real_ref);
 +                      refs_found = repo_dwim_ref(r, str, len, &tmp_oid, &real_ref);
                        if (refs_found > 0) {
                                warning(warn_msg, len, str);
                                if (advice_object_name_warning)
                struct strbuf buf = STRBUF_INIT;
                int detached;
  
 -              if (interpret_nth_prior_checkout(str, len, &buf) > 0) {
 -                      detached = (buf.len == the_hash_algo->hexsz && !get_oid_hex(buf.buf, oid));
 +              if (interpret_nth_prior_checkout(r, str, len, &buf) > 0) {
 +                      detached = (buf.len == r->hash_algo->hexsz && !get_oid_hex(buf.buf, oid));
                        strbuf_release(&buf);
                        if (detached)
                                return 0;
  
        if (!len && reflog_len)
                /* allow "@{...}" to mean the current branch reflog */
 -              refs_found = dwim_ref("HEAD", 4, oid, &real_ref);
 +              refs_found = repo_dwim_ref(r, "HEAD", 4, oid, &real_ref);
        else if (reflog_len)
 -              refs_found = dwim_log(str, len, oid, &real_ref);
 +              refs_found = repo_dwim_log(r, str, len, oid, &real_ref);
        else
 -              refs_found = dwim_ref(str, len, oid, &real_ref);
 +              refs_found = repo_dwim_ref(r, str, len, oid, &real_ref);
  
        if (!refs_found)
                return -1;
  
        if (warn_ambiguous_refs && !(flags & GET_OID_QUIETLY) &&
            (refs_found > 1 ||
 -           !get_short_oid(str, len, &tmp_oid, GET_OID_QUIETLY)))
 +           !get_short_oid(r, str, len, &tmp_oid, GET_OID_QUIETLY)))
                warning(warn_msg, len, str);
  
        if (reflog_len) {
                                return -1;
                        }
                }
 -              if (read_ref_at(real_ref, flags, at_time, nth, oid, NULL,
 +              if (read_ref_at(get_main_ref_store(r),
 +                              real_ref, flags, at_time, nth, oid, NULL,
                                &co_time, &co_tz, &co_cnt)) {
                        if (!len) {
                                if (starts_with(real_ref, "refs/heads/")) {
        return 0;
  }
  
 -static enum get_oid_result get_parent(const char *name, int len,
 +static enum get_oid_result get_parent(struct repository *r,
 +                                    const char *name, int len,
                                      struct object_id *result, int idx)
  {
        struct object_id oid;
 -      enum get_oid_result ret = get_oid_1(name, len, &oid,
 +      enum get_oid_result ret = get_oid_1(r, name, len, &oid,
                                            GET_OID_COMMITTISH);
        struct commit *commit;
        struct commit_list *p;
  
        if (ret)
                return ret;
 -      commit = lookup_commit_reference(the_repository, &oid);
 +      commit = lookup_commit_reference(r, &oid);
        if (parse_commit(commit))
                return MISSING_OBJECT;
        if (!idx) {
        return MISSING_OBJECT;
  }
  
 -static enum get_oid_result get_nth_ancestor(const char *name, int len,
 +static enum get_oid_result get_nth_ancestor(struct repository *r,
 +                                          const char *name, int len,
                                            struct object_id *result,
                                            int generation)
  {
        struct commit *commit;
        int ret;
  
 -      ret = get_oid_1(name, len, &oid, GET_OID_COMMITTISH);
 +      ret = get_oid_1(r, name, len, &oid, GET_OID_COMMITTISH);
        if (ret)
                return ret;
 -      commit = lookup_commit_reference(the_repository, &oid);
 +      commit = lookup_commit_reference(r, &oid);
        if (!commit)
                return MISSING_OBJECT;
  
        return FOUND;
  }
  
 -struct object *peel_to_type(const char *name, int namelen,
 -                          struct object *o, enum object_type expected_type)
 +struct object *repo_peel_to_type(struct repository *r, const char *name, int namelen,
 +                               struct object *o, enum object_type expected_type)
  {
        if (name && !namelen)
                namelen = strlen(name);
        while (1) {
 -              if (!o || (!o->parsed && !parse_object(the_repository, &o->oid)))
 +              if (!o || (!o->parsed && !parse_object(r, &o->oid)))
                        return NULL;
                if (expected_type == OBJ_ANY || o->type == expected_type)
                        return o;
                if (o->type == OBJ_TAG)
                        o = ((struct tag*) o)->tagged;
                else if (o->type == OBJ_COMMIT)
 -                      o = &(get_commit_tree(((struct commit *)o))->object);
 +                      o = &(repo_get_commit_tree(r, ((struct commit *)o))->object);
                else {
                        if (name)
                                error("%.*s: expected %s type, but the object "
        }
  }
  
 -static int peel_onion(const char *name, int len, struct object_id *oid,
 -                    unsigned lookup_flags)
 +static int peel_onion(struct repository *r, const char *name, int len,
 +                    struct object_id *oid, unsigned lookup_flags)
  {
        struct object_id outer;
        const char *sp;
        else if (expected_type == OBJ_TREE)
                lookup_flags |= GET_OID_TREEISH;
  
 -      if (get_oid_1(name, sp - name - 2, &outer, lookup_flags))
 +      if (get_oid_1(r, name, sp - name - 2, &outer, lookup_flags))
                return -1;
  
 -      o = parse_object(the_repository, &outer);
 +      o = parse_object(r, &outer);
        if (!o)
                return -1;
        if (!expected_type) {
 -              o = deref_tag(the_repository, o, name, sp - name - 2);
 -              if (!o || (!o->parsed && !parse_object(the_repository, &o->oid)))
 +              o = deref_tag(r, o, name, sp - name - 2);
 +              if (!o || (!o->parsed && !parse_object(r, &o->oid)))
                        return -1;
                oidcpy(oid, &o->oid);
                return 0;
         * if we do not get the needed object, we should
         * barf.
         */
 -      o = peel_to_type(name, len, o, expected_type);
 +      o = repo_peel_to_type(r, name, len, o, expected_type);
        if (!o)
                return -1;
  
  
                prefix = xstrndup(sp + 1, name + len - 1 - (sp + 1));
                commit_list_insert((struct commit *)o, &list);
 -              ret = get_oid_oneline(prefix, oid, list);
 +              ret = get_oid_oneline(r, prefix, oid, list);
                free(prefix);
                return ret;
        }
        return 0;
  }
  
 -static int get_describe_name(const char *name, int len, struct object_id *oid)
 +static int get_describe_name(struct repository *r,
 +                           const char *name, int len,
 +                           struct object_id *oid)
  {
        const char *cp;
        unsigned flags = GET_OID_QUIETLY | GET_OID_COMMIT;
                        if (ch == 'g' && cp[-1] == '-') {
                                cp++;
                                len -= cp - name;
 -                              return get_short_oid(cp, len, oid, flags);
 +                              return get_short_oid(r,
 +                                                   cp, len, oid, flags);
                        }
                }
        }
        return -1;
  }
  
 -static enum get_oid_result get_oid_1(const char *name, int len,
 +static enum get_oid_result get_oid_1(struct repository *r,
 +                                   const char *name, int len,
                                     struct object_id *oid,
                                     unsigned lookup_flags)
  {
                if (!num && len1 == len - 1)
                        num = 1;
                if (has_suffix == '^')
 -                      return get_parent(name, len1, oid, num);
 +                      return get_parent(r, name, len1, oid, num);
                /* else if (has_suffix == '~') -- goes without saying */
 -              return get_nth_ancestor(name, len1, oid, num);
 +              return get_nth_ancestor(r, name, len1, oid, num);
        }
  
 -      ret = peel_onion(name, len, oid, lookup_flags);
 +      ret = peel_onion(r, name, len, oid, lookup_flags);
        if (!ret)
                return FOUND;
  
 -      ret = get_oid_basic(name, len, oid, lookup_flags);
 +      ret = get_oid_basic(r, name, len, oid, lookup_flags);
        if (!ret)
                return FOUND;
  
        /* It could be describe output that is "SOMETHING-gXXXX" */
 -      ret = get_describe_name(name, len, oid);
 +      ret = get_describe_name(r, name, len, oid);
        if (!ret)
                return FOUND;
  
 -      return get_short_oid(name, len, oid, lookup_flags);
 +      return get_short_oid(r, name, len, oid, lookup_flags);
  }
  
  /*
  /* Remember to update object flag allocation in object.h */
  #define ONELINE_SEEN (1u<<20)
  
 +struct handle_one_ref_cb {
 +      struct repository *repo;
 +      struct commit_list **list;
 +};
 +
  static int handle_one_ref(const char *path, const struct object_id *oid,
                          int flag, void *cb_data)
  {
 -      struct commit_list **list = cb_data;
 -      struct object *object = parse_object(the_repository, oid);
 +      struct handle_one_ref_cb *cb = cb_data;
 +      struct commit_list **list = cb->list;
 +      struct object *object = parse_object(cb->repo, oid);
        if (!object)
                return 0;
        if (object->type == OBJ_TAG) {
 -              object = deref_tag(the_repository, object, path,
 +              object = deref_tag(cb->repo, object, path,
                                   strlen(path));
                if (!object)
                        return 0;
        return 0;
  }
  
 -static int get_oid_oneline(const char *prefix, struct object_id *oid,
 -                          struct commit_list *list)
 +static int get_oid_oneline(struct repository *r,
 +                         const char *prefix, struct object_id *oid,
 +                         struct commit_list *list)
  {
        struct commit_list *backup = NULL, *l;
        int found = 0;
                int matches;
  
                commit = pop_most_recent_commit(&list, ONELINE_SEEN);
 -              if (!parse_object(the_repository, &commit->object.oid))
 +              if (!parse_object(r, &commit->object.oid))
                        continue;
                buf = get_commit_buffer(commit, NULL);
                p = strstr(buf, "\n\n");
@@@ -1312,8 -1264,7 +1318,8 @@@ static int grab_nth_branch_switch(struc
   * Parse @{-N} syntax, return the number of characters parsed
   * if successful; otherwise signal an error with negative value.
   */
 -static int interpret_nth_prior_checkout(const char *name, int namelen,
 +static int interpret_nth_prior_checkout(struct repository *r,
 +                                      const char *name, int namelen,
                                        struct strbuf *buf)
  {
        long nth;
        cb.remaining = nth;
        strbuf_init(&cb.buf, 20);
  
 -      retval = 0;
 -      if (0 < for_each_reflog_ent_reverse("HEAD", grab_nth_branch_switch, &cb)) {
 +      retval = refs_for_each_reflog_ent_reverse(get_main_ref_store(r),
 +                      "HEAD", grab_nth_branch_switch, &cb);
 +      if (0 < retval) {
                strbuf_reset(buf);
                strbuf_addbuf(buf, &cb.buf);
                retval = brace - name + 1;
 -      }
 +      } else
 +              retval = 0;
  
        strbuf_release(&cb.buf);
        return retval;
  }
  
 -int get_oid_mb(const char *name, struct object_id *oid)
 +int repo_get_oid_mb(struct repository *r,
 +                  const char *name,
 +                  struct object_id *oid)
  {
        struct commit *one, *two;
        struct commit_list *mbs;
  
        dots = strstr(name, "...");
        if (!dots)
 -              return get_oid(name, oid);
 +              return repo_get_oid(r, name, oid);
        if (dots == name)
 -              st = get_oid("HEAD", &oid_tmp);
 +              st = repo_get_oid(r, "HEAD", &oid_tmp);
        else {
                struct strbuf sb;
                strbuf_init(&sb, dots - name);
                strbuf_add(&sb, name, dots - name);
 -              st = get_oid_committish(sb.buf, &oid_tmp);
 +              st = repo_get_oid_committish(r, sb.buf, &oid_tmp);
                strbuf_release(&sb);
        }
        if (st)
                return st;
 -      one = lookup_commit_reference_gently(the_repository, &oid_tmp, 0);
 +      one = lookup_commit_reference_gently(r, &oid_tmp, 0);
        if (!one)
                return -1;
  
 -      if (get_oid_committish(dots[3] ? (dots + 3) : "HEAD", &oid_tmp))
 +      if (repo_get_oid_committish(r, dots[3] ? (dots + 3) : "HEAD", &oid_tmp))
                return -1;
 -      two = lookup_commit_reference_gently(the_repository, &oid_tmp, 0);
 +      two = lookup_commit_reference_gently(r, &oid_tmp, 0);
        if (!two)
                return -1;
 +      if (r != the_repository)
 +              BUG("sorry get_merge_bases() can't take struct repository yet");
        mbs = get_merge_bases(one, two);
        if (!mbs || mbs->next)
                st = -1;
@@@ -1418,8 -1363,7 +1424,8 @@@ static int interpret_empty_at(const cha
        return 1;
  }
  
 -static int reinterpret(const char *name, int namelen, int len,
 +static int reinterpret(struct repository *r,
 +                     const char *name, int namelen, int len,
                       struct strbuf *buf, unsigned allowed)
  {
        /* we have extra data, which might need further processing */
        int ret;
  
        strbuf_add(buf, name + len, namelen - len);
 -      ret = interpret_branch_name(buf->buf, buf->len, &tmp, allowed);
 +      ret = repo_interpret_branch_name(r, buf->buf, buf->len, &tmp, allowed);
        /* that data was not interpreted, remove our cruft */
        if (ret < 0) {
                strbuf_setlen(buf, used);
        return ret - used + len;
  }
  
 -static void set_shortened_ref(struct strbuf *buf, const char *ref)
 +static void set_shortened_ref(struct repository *r, struct strbuf *buf, const char *ref)
  {
 -      char *s = shorten_unambiguous_ref(ref, 0);
 +      char *s = refs_shorten_unambiguous_ref(get_main_ref_store(r), ref, 0);
        strbuf_reset(buf);
        strbuf_addstr(buf, s);
        free(s);
@@@ -1464,8 -1408,7 +1470,8 @@@ static int branch_interpret_allowed(con
        return 0;
  }
  
 -static int interpret_branch_mark(const char *name, int namelen,
 +static int interpret_branch_mark(struct repository *r,
 +                               const char *name, int namelen,
                                 int at, struct strbuf *buf,
                                 int (*get_mark)(const char *, int),
                                 const char *(*get_data)(struct branch *,
        if (!branch_interpret_allowed(value, allowed))
                return -1;
  
 -      set_shortened_ref(buf, value);
 +      set_shortened_ref(r, buf, value);
        return len + at;
  }
  
 -int interpret_branch_name(const char *name, int namelen, struct strbuf *buf,
 -                        unsigned allowed)
 +int repo_interpret_branch_name(struct repository *r,
 +                             const char *name, int namelen,
 +                             struct strbuf *buf,
 +                             unsigned allowed)
  {
        char *at;
        const char *start;
                namelen = strlen(name);
  
        if (!allowed || (allowed & INTERPRET_BRANCH_LOCAL)) {
 -              len = interpret_nth_prior_checkout(name, namelen, buf);
 +              len = interpret_nth_prior_checkout(r, name, namelen, buf);
                if (!len) {
                        return len; /* syntax Ok, not enough switches */
                } else if (len > 0) {
                        if (len == namelen)
                                return len; /* consumed all */
                        else
 -                              return reinterpret(name, namelen, len, buf, allowed);
 +                              return reinterpret(r, name, namelen, len, buf, allowed);
                }
        }
  
                if (!allowed || (allowed & INTERPRET_BRANCH_HEAD)) {
                        len = interpret_empty_at(name, namelen, at - name, buf);
                        if (len > 0)
 -                              return reinterpret(name, namelen, len, buf,
 +                              return reinterpret(r, name, namelen, len, buf,
                                                   allowed);
                }
  
 -              len = interpret_branch_mark(name, namelen, at - name, buf,
 +              len = interpret_branch_mark(r, name, namelen, at - name, buf,
                                            upstream_mark, branch_get_upstream,
                                            allowed);
                if (len > 0)
                        return len;
  
 -              len = interpret_branch_mark(name, namelen, at - name, buf,
 +              len = interpret_branch_mark(r, name, namelen, at - name, buf,
                                            push_mark, branch_get_push,
                                            allowed);
                if (len > 0)
@@@ -1589,10 -1530,10 +1595,10 @@@ int strbuf_check_branch_ref(struct strb
   * This is like "get_oid_basic()", except it allows "object ID expressions",
   * notably "xyz^" for "parent of xyz"
   */
 -int get_oid(const char *name, struct object_id *oid)
 +int repo_get_oid(struct repository *r, const char *name, struct object_id *oid)
  {
        struct object_context unused;
 -      return get_oid_with_context(the_repository, name, 0, oid, &unused);
 +      return get_oid_with_context(r, name, 0, oid, &unused);
  }
  
  /*
@@@ -1625,48 -1566,43 +1631,48 @@@ int get_oidf(struct object_id *oid, con
   * commit-ish. It is merely to give a hint to the disambiguation
   * machinery.
   */
 -int get_oid_committish(const char *name, struct object_id *oid)
 +int repo_get_oid_committish(struct repository *r,
 +                          const char *name,
 +                          struct object_id *oid)
  {
        struct object_context unused;
 -      return get_oid_with_context(the_repository,
 -                                  name, GET_OID_COMMITTISH,
 +      return get_oid_with_context(r, name, GET_OID_COMMITTISH,
                                    oid, &unused);
  }
  
 -int get_oid_treeish(const char *name, struct object_id *oid)
 +int repo_get_oid_treeish(struct repository *r,
 +                       const char *name,
 +                       struct object_id *oid)
  {
        struct object_context unused;
 -      return get_oid_with_context(the_repository,
 -                                  name, GET_OID_TREEISH,
 +      return get_oid_with_context(r, name, GET_OID_TREEISH,
                                    oid, &unused);
  }
  
 -int get_oid_commit(const char *name, struct object_id *oid)
 +int repo_get_oid_commit(struct repository *r,
 +                      const char *name,
 +                      struct object_id *oid)
  {
        struct object_context unused;
 -      return get_oid_with_context(the_repository,
 -                                  name, GET_OID_COMMIT,
 +      return get_oid_with_context(r, name, GET_OID_COMMIT,
                                    oid, &unused);
  }
  
 -int get_oid_tree(const char *name, struct object_id *oid)
 +int repo_get_oid_tree(struct repository *r,
 +                    const char *name,
 +                    struct object_id *oid)
  {
        struct object_context unused;
 -      return get_oid_with_context(the_repository,
 -                                  name, GET_OID_TREE,
 +      return get_oid_with_context(r, name, GET_OID_TREE,
                                    oid, &unused);
  }
  
 -int get_oid_blob(const char *name, struct object_id *oid)
 +int repo_get_oid_blob(struct repository *r,
 +                    const char *name,
 +                    struct object_id *oid)
  {
        struct object_context unused;
 -      return get_oid_with_context(the_repository,
 -                                  name, GET_OID_BLOB,
 +      return get_oid_with_context(r, name, GET_OID_BLOB,
                                    oid, &unused);
  }
  
@@@ -1678,7 -1614,7 +1684,7 @@@ static void diagnose_invalid_oid_path(c
                                      int object_name_len)
  {
        struct object_id oid;
 -      unsigned mode;
 +      unsigned short mode;
  
        if (!prefix)
                prefix = "";
  }
  
  /* Must be called only when :stage:filename doesn't exist. */
 -static void diagnose_invalid_index_path(struct index_state *istate,
 +static void diagnose_invalid_index_path(struct repository *r,
                                        int stage,
                                        const char *prefix,
                                        const char *filename)
  {
 +      struct index_state *istate = r->index;
        const struct cache_entry *ce;
        int pos;
        unsigned namelen = strlen(filename);
                            ce_stage(ce), filename);
        }
  
 -      if (file_exists(filename))
 +      if (repo_file_exists(r, filename))
                die("Path '%s' exists on disk, but not in the index.", filename);
        if (is_missing_file_error(errno))
                die("Path '%s' does not exist (neither on disk nor in the index).",
  }
  
  
 -static char *resolve_relative_path(const char *rel)
 +static char *resolve_relative_path(struct repository *r, const char *rel)
  {
        if (!starts_with(rel, "./") && !starts_with(rel, "../"))
                return NULL;
  
 -      if (!is_inside_work_tree())
 +      if (r != the_repository || !is_inside_work_tree())
                die("relative path syntax can't be used outside working tree.");
  
        /* die() inside prefix_path() if resolved path is outside worktree */
@@@ -1792,7 -1727,7 +1798,7 @@@ static enum get_oid_result get_oid_with
        memset(oc, 0, sizeof(*oc));
        oc->mode = S_IFINVALID;
        strbuf_init(&oc->symlink_path, 0);
 -      ret = get_oid_1(name, namelen, oid, flags);
 +      ret = get_oid_1(repo, name, namelen, oid, flags);
        if (!ret)
                return ret;
        /*
                char *new_path = NULL;
                int pos;
                if (!only_to_die && namelen > 2 && name[1] == '/') {
 +                      struct handle_one_ref_cb cb;
                        struct commit_list *list = NULL;
  
 -                      for_each_ref(handle_one_ref, &list);
 -                      head_ref(handle_one_ref, &list);
 +                      cb.repo = repo;
 +                      cb.list = &list;
 +                      refs_for_each_ref(repo->refs, handle_one_ref, &cb);
 +                      refs_head_ref(repo->refs, handle_one_ref, &cb);
                        commit_list_sort_by_date(&list);
 -                      return get_oid_oneline(name + 2, oid, list);
 +                      return get_oid_oneline(repo, name + 2, oid, list);
                }
                if (namelen < 3 ||
                    name[2] != ':' ||
                        stage = name[1] - '0';
                        cp = name + 3;
                }
 -              new_path = resolve_relative_path(cp);
 +              new_path = resolve_relative_path(repo, cp);
                if (!new_path) {
                        namelen = namelen - (cp - name);
                } else {
                        oc->path = xstrdup(cp);
  
                if (!repo->index->cache)
 -                      repo_read_index(the_repository);
 +                      repo_read_index(repo);
                pos = index_name_pos(repo->index, cp, namelen);
                if (pos < 0)
                        pos = -pos - 1;
                        pos++;
                }
                if (only_to_die && name[1] && name[1] != '/')
 -                      diagnose_invalid_index_path(repo->index, stage, prefix, cp);
 +                      diagnose_invalid_index_path(repo, stage, prefix, cp);
                free(new_path);
                return -1;
        }
                sub_flags &= ~GET_OID_DISAMBIGUATORS;
                sub_flags |= GET_OID_TREEISH;
  
 -              if (!get_oid_1(name, len, &tree_oid, sub_flags)) {
 +              if (!get_oid_1(repo, name, len, &tree_oid, sub_flags)) {
                        const char *filename = cp+1;
                        char *new_filename = NULL;
  
 -                      new_filename = resolve_relative_path(filename);
 +                      new_filename = resolve_relative_path(repo, filename);
                        if (new_filename)
                                filename = new_filename;
 +                      /*
 +                       * NEEDSWORK: Eventually get_tree_entry*() should
 +                       * learn to take struct repository directly and we
 +                       * would not need to inject submodule odb to the
 +                       * in-core odb.
 +                       */
 +                      if (repo != the_repository)
 +                              add_to_alternates_memory(repo->objects->odb->path);
                        if (flags & GET_OID_FOLLOW_SYMLINKS) {
                                ret = get_tree_entry_follow_symlinks(&tree_oid,
                                        filename, oid, &oc->symlink_path,
   * exist in 'HEAD'" when given "HEAD:doc", or it may return in which case
   * you have a chance to diagnose the error further.
   */
 -void maybe_die_on_misspelt_object_name(const char *name, const char *prefix)
 +void maybe_die_on_misspelt_object_name(struct repository *r,
 +                                     const char *name,
 +                                     const char *prefix)
  {
        struct object_context oc;
        struct object_id oid;
 -      get_oid_with_context_1(the_repository, name, GET_OID_ONLY_TO_DIE,
 +      get_oid_with_context_1(r, name, GET_OID_ONLY_TO_DIE,
                               prefix, &oid, &oc);
  }