Merge branch 'ab/commit-graph-write-progress'
authorJunio C Hamano <gitster@pobox.com>
Tue, 5 Feb 2019 22:26:14 +0000 (14:26 -0800)
committerJunio C Hamano <gitster@pobox.com>
Tue, 5 Feb 2019 22:26:14 +0000 (14:26 -0800)
The codepath to show progress meter while writing out commit-graph
file has been improved.

* ab/commit-graph-write-progress:
  commit-graph write: emit a percentage for all progress
  commit-graph write: add itermediate progress
  commit-graph write: remove empty line for readability
  commit-graph write: add more descriptive progress output
  commit-graph write: show progress for object search
  commit-graph write: more descriptive "writing out" output
  commit-graph write: add "Writing out" progress output
  commit-graph: don't call write_graph_chunk_extra_edges() unnecessarily
  commit-graph: rename "large edges" to "extra edges"

1  2 
commit-graph.c
commit-graph.h
t/t5318-commit-graph.sh

diff --combined commit-graph.c
  #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
  #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
  #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
- #define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
+ #define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
  
 -#define GRAPH_DATA_WIDTH 36
 +#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
  
  #define GRAPH_VERSION_1 0x1
  #define GRAPH_VERSION GRAPH_VERSION_1
  
- #define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
 -#define GRAPH_OID_VERSION_SHA1 1
 -#define GRAPH_OID_LEN_SHA1 GIT_SHA1_RAWSZ
 -#define GRAPH_OID_VERSION GRAPH_OID_VERSION_SHA1
 -#define GRAPH_OID_LEN GRAPH_OID_LEN_SHA1
 -
+ #define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
 -#define GRAPH_PARENT_MISSING 0x7fffffff
  #define GRAPH_EDGE_LAST_MASK 0x7fffffff
  #define GRAPH_PARENT_NONE 0x70000000
  
  #define GRAPH_FANOUT_SIZE (4 * 256)
  #define GRAPH_CHUNKLOOKUP_WIDTH 12
  #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
 -                      + GRAPH_FANOUT_SIZE + GRAPH_OID_LEN)
 +                      + GRAPH_FANOUT_SIZE + the_hash_algo->rawsz)
  
  char *get_commit_graph_filename(const char *obj_dir)
  {
        return xstrfmt("%s/info/commit-graph", obj_dir);
  }
  
 +static uint8_t oid_version(void)
 +{
 +      return 1;
 +}
 +
  static struct commit_graph *alloc_commit_graph(void)
  {
        struct commit_graph *g = xcalloc(1, sizeof(*g));
@@@ -83,10 -84,16 +83,10 @@@ static int commit_graph_compatible(stru
  struct commit_graph *load_commit_graph_one(const char *graph_file)
  {
        void *graph_map;
 -      const unsigned char *data, *chunk_lookup;
        size_t graph_size;
        struct stat st;
 -      uint32_t i;
 -      struct commit_graph *graph;
 +      struct commit_graph *ret;
        int fd = git_open(graph_file);
 -      uint64_t last_chunk_offset;
 -      uint32_t last_chunk_id;
 -      uint32_t graph_signature;
 -      unsigned char graph_version, hash_version;
  
        if (fd < 0)
                return NULL;
                die(_("graph file %s is too small"), graph_file);
        }
        graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
 +      ret = parse_commit_graph(graph_map, fd, graph_size);
 +
 +      if (!ret) {
 +              munmap(graph_map, graph_size);
 +              close(fd);
 +              exit(1);
 +      }
 +
 +      return ret;
 +}
 +
 +struct commit_graph *parse_commit_graph(void *graph_map, int fd,
 +                                      size_t graph_size)
 +{
 +      const unsigned char *data, *chunk_lookup;
 +      uint32_t i;
 +      struct commit_graph *graph;
 +      uint64_t last_chunk_offset;
 +      uint32_t last_chunk_id;
 +      uint32_t graph_signature;
 +      unsigned char graph_version, hash_version;
 +
 +      if (!graph_map)
 +              return NULL;
 +
 +      if (graph_size < GRAPH_MIN_SIZE)
 +              return NULL;
 +
        data = (const unsigned char *)graph_map;
  
        graph_signature = get_be32(data);
        if (graph_signature != GRAPH_SIGNATURE) {
                error(_("graph signature %X does not match signature %X"),
                      graph_signature, GRAPH_SIGNATURE);
 -              goto cleanup_fail;
 +              return NULL;
        }
  
        graph_version = *(unsigned char*)(data + 4);
        if (graph_version != GRAPH_VERSION) {
                error(_("graph version %X does not match version %X"),
                      graph_version, GRAPH_VERSION);
 -              goto cleanup_fail;
 +              return NULL;
        }
  
        hash_version = *(unsigned char*)(data + 5);
 -      if (hash_version != GRAPH_OID_VERSION) {
 +      if (hash_version != oid_version()) {
                error(_("hash version %X does not match version %X"),
 -                    hash_version, GRAPH_OID_VERSION);
 -              goto cleanup_fail;
 +                    hash_version, oid_version());
 +              return NULL;
        }
  
        graph = alloc_commit_graph();
  
 -      graph->hash_len = GRAPH_OID_LEN;
 +      graph->hash_len = the_hash_algo->rawsz;
        graph->num_chunks = *(unsigned char*)(data + 6);
        graph->graph_fd = fd;
        graph->data = graph_map;
        last_chunk_offset = 8;
        chunk_lookup = data + 8;
        for (i = 0; i < graph->num_chunks; i++) {
 -              uint32_t chunk_id = get_be32(chunk_lookup + 0);
 -              uint64_t chunk_offset = get_be64(chunk_lookup + 4);
 +              uint32_t chunk_id;
 +              uint64_t chunk_offset;
                int chunk_repeated = 0;
  
 +              if (data + graph_size - chunk_lookup <
 +                  GRAPH_CHUNKLOOKUP_WIDTH) {
 +                      error(_("chunk lookup table entry missing; graph file may be incomplete"));
 +                      free(graph);
 +                      return NULL;
 +              }
 +
 +              chunk_id = get_be32(chunk_lookup + 0);
 +              chunk_offset = get_be64(chunk_lookup + 4);
 +
                chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH;
  
 -              if (chunk_offset > graph_size - GIT_MAX_RAWSZ) {
 +              if (chunk_offset > graph_size - the_hash_algo->rawsz) {
                        error(_("improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
                              (uint32_t)chunk_offset);
 -                      goto cleanup_fail;
 +                      free(graph);
 +                      return NULL;
                }
  
                switch (chunk_id) {
                                graph->chunk_commit_data = data + chunk_offset;
                        break;
  
-               case GRAPH_CHUNKID_LARGEEDGES:
-                       if (graph->chunk_large_edges)
+               case GRAPH_CHUNKID_EXTRAEDGES:
+                       if (graph->chunk_extra_edges)
                                chunk_repeated = 1;
                        else
-                               graph->chunk_large_edges = data + chunk_offset;
+                               graph->chunk_extra_edges = data + chunk_offset;
                        break;
                }
  
                if (chunk_repeated) {
                        error(_("chunk id %08x appears multiple times"), chunk_id);
 -                      goto cleanup_fail;
 +                      free(graph);
 +                      return NULL;
                }
  
                if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
        }
  
        return graph;
 -
 -cleanup_fail:
 -      munmap(graph_map, graph_size);
 -      close(fd);
 -      exit(1);
  }
  
  static void prepare_commit_graph_one(struct repository *r, const char *obj_dir)
@@@ -317,8 -289,7 +317,8 @@@ static int bsearch_graph(struct commit_
                            g->chunk_oid_lookup, g->hash_len, pos);
  }
  
 -static struct commit_list **insert_parent_or_die(struct commit_graph *g,
 +static struct commit_list **insert_parent_or_die(struct repository *r,
 +                                               struct commit_graph *g,
                                                 uint64_t pos,
                                                 struct commit_list **pptr)
  {
                die("invalid parent position %"PRIu64, pos);
  
        hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
 -      c = lookup_commit(the_repository, &oid);
 +      c = lookup_commit(r, &oid);
        if (!c)
                die(_("could not find commit %s"), oid_to_hex(&oid));
        c->graph_pos = pos;
@@@ -343,9 -314,7 +343,9 @@@ static void fill_commit_graph_info(stru
        item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
  }
  
 -static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t pos)
 +static int fill_commit_in_graph(struct repository *r,
 +                              struct commit *item,
 +                              struct commit_graph *g, uint32_t pos)
  {
        uint32_t edge_value;
        uint32_t *parent_data_ptr;
        edge_value = get_be32(commit_data + g->hash_len);
        if (edge_value == GRAPH_PARENT_NONE)
                return 1;
 -      pptr = insert_parent_or_die(g, edge_value, pptr);
 +      pptr = insert_parent_or_die(r, g, edge_value, pptr);
  
        edge_value = get_be32(commit_data + g->hash_len + 4);
        if (edge_value == GRAPH_PARENT_NONE)
                return 1;
-       if (!(edge_value & GRAPH_OCTOPUS_EDGES_NEEDED)) {
+       if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
 -              pptr = insert_parent_or_die(g, edge_value, pptr);
 +              pptr = insert_parent_or_die(r, g, edge_value, pptr);
                return 1;
        }
  
-       parent_data_ptr = (uint32_t*)(g->chunk_large_edges +
+       parent_data_ptr = (uint32_t*)(g->chunk_extra_edges +
                          4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
        do {
                edge_value = get_be32(parent_data_ptr);
 -              pptr = insert_parent_or_die(g,
 +              pptr = insert_parent_or_die(r, g,
                                            edge_value & GRAPH_EDGE_LAST_MASK,
                                            pptr);
                parent_data_ptr++;
@@@ -402,9 -371,7 +402,9 @@@ static int find_commit_in_graph(struct 
        }
  }
  
 -static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item)
 +static int parse_commit_in_graph_one(struct repository *r,
 +                                   struct commit_graph *g,
 +                                   struct commit *item)
  {
        uint32_t pos;
  
                return 1;
  
        if (find_commit_in_graph(item, g, &pos))
 -              return fill_commit_in_graph(item, g, pos);
 +              return fill_commit_in_graph(r, item, g, pos);
  
        return 0;
  }
@@@ -421,7 -388,7 +421,7 @@@ int parse_commit_in_graph(struct reposi
  {
        if (!prepare_commit_graph(r))
                return 0;
 -      return parse_commit_in_graph_one(r->objects->commit_graph, item);
 +      return parse_commit_in_graph_one(r, r->objects->commit_graph, item);
  }
  
  void load_commit_graph_info(struct repository *r, struct commit *item)
                fill_commit_graph_info(item, r->objects->commit_graph, pos);
  }
  
 -static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit *c)
 +static struct tree *load_tree_for_commit(struct repository *r,
 +                                       struct commit_graph *g,
 +                                       struct commit *c)
  {
        struct object_id oid;
        const unsigned char *commit_data = g->chunk_commit_data +
                                           GRAPH_DATA_WIDTH * (c->graph_pos);
  
        hashcpy(oid.hash, commit_data);
 -      c->maybe_tree = lookup_tree(the_repository, &oid);
 +      c->maybe_tree = lookup_tree(r, &oid);
  
        return c->maybe_tree;
  }
  
 -static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g,
 +static struct tree *get_commit_tree_in_graph_one(struct repository *r,
 +                                               struct commit_graph *g,
                                                 const struct commit *c)
  {
        if (c->maybe_tree)
        if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
                BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
  
 -      return load_tree_for_commit(g, (struct commit *)c);
 +      return load_tree_for_commit(r, g, (struct commit *)c);
  }
  
  struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c)
  {
 -      return get_commit_tree_in_graph_one(r->objects->commit_graph, c);
 +      return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c);
  }
  
  static void write_graph_chunk_fanout(struct hashfile *f,
                                     struct commit **commits,
-                                    int nr_commits)
+                                    int nr_commits,
+                                    struct progress *progress,
+                                    uint64_t *progress_cnt)
  {
        int i, count = 0;
        struct commit **list = commits;
                while (count < nr_commits) {
                        if ((*list)->object.oid.hash[0] != i)
                                break;
+                       display_progress(progress, ++*progress_cnt);
                        count++;
                        list++;
                }
  }
  
  static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
-                                  struct commit **commits, int nr_commits)
+                                  struct commit **commits, int nr_commits,
+                                  struct progress *progress,
+                                  uint64_t *progress_cnt)
  {
        struct commit **list = commits;
        int count;
-       for (count = 0; count < nr_commits; count++, list++)
+       for (count = 0; count < nr_commits; count++, list++) {
+               display_progress(progress, ++*progress_cnt);
                hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
+       }
  }
  
  static const unsigned char *commit_to_sha1(size_t index, void *table)
  }
  
  static void write_graph_chunk_data(struct hashfile *f, int hash_len,
-                                  struct commit **commits, int nr_commits)
+                                  struct commit **commits, int nr_commits,
+                                  struct progress *progress,
+                                  uint64_t *progress_cnt)
  {
        struct commit **list = commits;
        struct commit **last = commits + nr_commits;
                struct commit_list *parent;
                int edge_value;
                uint32_t packedDate[2];
+               display_progress(progress, ++*progress_cnt);
  
                parse_commit(*list);
                hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
                                              commit_to_sha1);
  
                        if (edge_value < 0)
 -                              edge_value = GRAPH_PARENT_MISSING;
 +                              BUG("missing parent %s for commit %s",
 +                                  oid_to_hex(&parent->item->object.oid),
 +                                  oid_to_hex(&(*list)->object.oid));
                }
  
                hashwrite_be32(f, edge_value);
                if (!parent)
                        edge_value = GRAPH_PARENT_NONE;
                else if (parent->next)
-                       edge_value = GRAPH_OCTOPUS_EDGES_NEEDED | num_extra_edges;
+                       edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
                else {
                        edge_value = sha1_pos(parent->item->object.oid.hash,
                                              commits,
                                              nr_commits,
                                              commit_to_sha1);
                        if (edge_value < 0)
 -                              edge_value = GRAPH_PARENT_MISSING;
 +                              BUG("missing parent %s for commit %s",
 +                                  oid_to_hex(&parent->item->object.oid),
 +                                  oid_to_hex(&(*list)->object.oid));
                }
  
                hashwrite_be32(f, edge_value);
  
-               if (edge_value & GRAPH_OCTOPUS_EDGES_NEEDED) {
+               if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
                        do {
                                num_extra_edges++;
                                parent = parent->next;
        }
  }
  
- static void write_graph_chunk_large_edges(struct hashfile *f,
+ static void write_graph_chunk_extra_edges(struct hashfile *f,
                                          struct commit **commits,
-                                         int nr_commits)
+                                         int nr_commits,
+                                         struct progress *progress,
+                                         uint64_t *progress_cnt)
  {
        struct commit **list = commits;
        struct commit **last = commits + nr_commits;
  
        while (list < last) {
                int num_parents = 0;
+               display_progress(progress, ++*progress_cnt);
                for (parent = (*list)->parents; num_parents < 3 && parent;
                     parent = parent->next)
                        num_parents++;
                                                  commit_to_sha1);
  
                        if (edge_value < 0)
 -                              edge_value = GRAPH_PARENT_MISSING;
 +                              BUG("missing parent %s for commit %s",
 +                                  oid_to_hex(&parent->item->object.oid),
 +                                  oid_to_hex(&(*list)->object.oid));
                        else if (!parent->next)
                                edge_value |= GRAPH_LAST_EDGE;
  
@@@ -680,15 -653,15 +695,15 @@@ static void add_missing_parents(struct 
  
  static void close_reachable(struct packed_oid_list *oids, int report_progress)
  {
-       int i, j;
+       int i;
        struct commit *commit;
        struct progress *progress = NULL;
  
        if (report_progress)
                progress = start_delayed_progress(
-                       _("Loading known commits in commit graph"), j = 0);
+                       _("Loading known commits in commit graph"), oids->nr);
        for (i = 0; i < oids->nr; i++) {
-               display_progress(progress, ++j);
+               display_progress(progress, i + 1);
                commit = lookup_commit(the_repository, &oids->list[i]);
                if (commit)
                        commit->object.flags |= UNINTERESTING;
         */
        if (report_progress)
                progress = start_delayed_progress(
-                       _("Expanding reachable commits in commit graph"), j = 0);
+                       _("Expanding reachable commits in commit graph"), oids->nr);
        for (i = 0; i < oids->nr; i++) {
-               display_progress(progress, ++j);
+               display_progress(progress, i + 1);
                commit = lookup_commit(the_repository, &oids->list[i]);
  
                if (commit && !parse_commit(commit))
  
        if (report_progress)
                progress = start_delayed_progress(
-                       _("Clearing commit marks in commit graph"), j = 0);
+                       _("Clearing commit marks in commit graph"), oids->nr);
        for (i = 0; i < oids->nr; i++) {
-               display_progress(progress, ++j);
+               display_progress(progress, i + 1);
                commit = lookup_commit(the_repository, &oids->list[i]);
  
                if (commit)
@@@ -810,13 -783,16 +825,17 @@@ void write_commit_graph(const char *obj
        int num_extra_edges;
        struct commit_list *parent;
        struct progress *progress = NULL;
 +      const unsigned hashsz = the_hash_algo->rawsz;
+       uint64_t progress_cnt = 0;
+       struct strbuf progress_title = STRBUF_INIT;
+       unsigned long approx_nr_objects;
  
        if (!commit_graph_compatible(the_repository))
                return;
  
        oids.nr = 0;
-       oids.alloc = approximate_object_count() / 32;
+       approx_nr_objects = approximate_object_count();
+       oids.alloc = approx_nr_objects / 32;
        oids.progress = NULL;
        oids.progress_done = 0;
  
                strbuf_addf(&packname, "%s/pack/", obj_dir);
                dirlen = packname.len;
                if (report_progress) {
-                       oids.progress = start_delayed_progress(
-                               _("Finding commits for commit graph"), 0);
+                       strbuf_addf(&progress_title,
+                                   Q_("Finding commits for commit graph in %d pack",
+                                      "Finding commits for commit graph in %d packs",
+                                      pack_indexes->nr),
+                                   pack_indexes->nr);
+                       oids.progress = start_delayed_progress(progress_title.buf, 0);
                        oids.progress_done = 0;
                }
                for (i = 0; i < pack_indexes->nr; i++) {
                        free(p);
                }
                stop_progress(&oids.progress);
+               strbuf_reset(&progress_title);
                strbuf_release(&packname);
        }
  
        if (commit_hex) {
-               if (report_progress)
-                       progress = start_delayed_progress(
-                               _("Finding commits for commit graph"),
-                               commit_hex->nr);
+               if (report_progress) {
+                       strbuf_addf(&progress_title,
+                                   Q_("Finding commits for commit graph from %d ref",
+                                      "Finding commits for commit graph from %d refs",
+                                      commit_hex->nr),
+                                   commit_hex->nr);
+                       progress = start_delayed_progress(progress_title.buf,
+                                                         commit_hex->nr);
+               }
                for (i = 0; i < commit_hex->nr; i++) {
                        const char *end;
                        struct object_id oid;
                        }
                }
                stop_progress(&progress);
+               strbuf_reset(&progress_title);
        }
  
        if (!pack_indexes && !commit_hex) {
                if (report_progress)
                        oids.progress = start_delayed_progress(
-                               _("Finding commits for commit graph"), 0);
+                               _("Finding commits for commit graph among packed objects"),
+                               approx_nr_objects);
                for_each_packed_object(add_packed_commits, &oids,
                                       FOR_EACH_OBJECT_PACK_ORDER);
+               if (oids.progress_done < approx_nr_objects)
+                       display_progress(oids.progress, approx_nr_objects);
                stop_progress(&oids.progress);
        }
  
        close_reachable(&oids, report_progress);
  
+       if (report_progress)
+               progress = start_delayed_progress(
+                       _("Counting distinct commits in commit graph"),
+                       oids.nr);
+       display_progress(progress, 0); /* TODO: Measure QSORT() progress */
        QSORT(oids.list, oids.nr, commit_compare);
        count_distinct = 1;
        for (i = 1; i < oids.nr; i++) {
+               display_progress(progress, i + 1);
                if (!oideq(&oids.list[i - 1], &oids.list[i]))
                        count_distinct++;
        }
+       stop_progress(&progress);
  
 -      if (count_distinct >= GRAPH_PARENT_MISSING)
 +      if (count_distinct >= GRAPH_EDGE_LAST_MASK)
                die(_("the commit graph format cannot write %d commits"), count_distinct);
  
        commits.nr = 0;
        ALLOC_ARRAY(commits.list, commits.alloc);
  
        num_extra_edges = 0;
+       if (report_progress)
+               progress = start_delayed_progress(
+                       _("Finding extra edges in commit graph"),
+                       oids.nr);
        for (i = 0; i < oids.nr; i++) {
                int num_parents = 0;
+               display_progress(progress, i + 1);
                if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
                        continue;
  
                commits.nr++;
        }
        num_chunks = num_extra_edges ? 4 : 3;
+       stop_progress(&progress);
  
 -      if (commits.nr >= GRAPH_PARENT_MISSING)
 +      if (commits.nr >= GRAPH_EDGE_LAST_MASK)
                die(_("too many commits to write graph"));
  
        compute_generation_numbers(&commits, report_progress);
        hashwrite_be32(f, GRAPH_SIGNATURE);
  
        hashwrite_u8(f, GRAPH_VERSION);
 -      hashwrite_u8(f, GRAPH_OID_VERSION);
 +      hashwrite_u8(f, oid_version());
        hashwrite_u8(f, num_chunks);
        hashwrite_u8(f, 0); /* unused padding byte */
  
        chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
        chunk_ids[2] = GRAPH_CHUNKID_DATA;
        if (num_extra_edges)
-               chunk_ids[3] = GRAPH_CHUNKID_LARGEEDGES;
+               chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES;
        else
                chunk_ids[3] = 0;
        chunk_ids[4] = 0;
  
        chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH;
        chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE;
 -      chunk_offsets[2] = chunk_offsets[1] + GRAPH_OID_LEN * commits.nr;
 -      chunk_offsets[3] = chunk_offsets[2] + (GRAPH_OID_LEN + 16) * commits.nr;
 +      chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr;
 +      chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr;
        chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges;
  
        for (i = 0; i <= num_chunks; i++) {
                hashwrite(f, chunk_write, 12);
        }
  
-       write_graph_chunk_fanout(f, commits.list, commits.nr);
-       write_graph_chunk_oids(f, hashsz, commits.list, commits.nr);
-       write_graph_chunk_data(f, hashsz, commits.list, commits.nr);
-       write_graph_chunk_large_edges(f, commits.list, commits.nr);
+       if (report_progress) {
+               strbuf_addf(&progress_title,
+                           Q_("Writing out commit graph in %d pass",
+                              "Writing out commit graph in %d passes",
+                              num_chunks),
+                           num_chunks);
+               progress = start_delayed_progress(
+                       progress_title.buf,
+                       num_chunks * commits.nr);
+       }
+       write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt);
 -      write_graph_chunk_oids(f, GRAPH_OID_LEN, commits.list, commits.nr, progress, &progress_cnt);
 -      write_graph_chunk_data(f, GRAPH_OID_LEN, commits.list, commits.nr, progress, &progress_cnt);
++      write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
++      write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
+       if (num_extra_edges)
+               write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt);
+       stop_progress(&progress);
+       strbuf_release(&progress_title);
  
        close_commit_graph(the_repository);
        finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
@@@ -1074,7 -1089,7 +1132,7 @@@ int verify_commit_graph(struct reposito
                }
  
                graph_commit = lookup_commit(r, &cur_oid);
 -              if (!parse_commit_in_graph_one(g, graph_commit))
 +              if (!parse_commit_in_graph_one(r, g, graph_commit))
                        graph_report("failed to parse %s from commit-graph",
                                     oid_to_hex(&cur_oid));
        }
                        continue;
                }
  
 -              if (!oideq(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid,
 +              if (!oideq(&get_commit_tree_in_graph_one(r, g, graph_commit)->object.oid,
                           get_commit_tree_oid(odb_commit)))
                        graph_report("root tree OID for commit %s in commit-graph is %s != %s",
                                     oid_to_hex(&cur_oid),
diff --combined commit-graph.h
@@@ -49,14 -49,11 +49,14 @@@ struct commit_graph 
        const uint32_t *chunk_oid_fanout;
        const unsigned char *chunk_oid_lookup;
        const unsigned char *chunk_commit_data;
-       const unsigned char *chunk_large_edges;
+       const unsigned char *chunk_extra_edges;
  };
  
  struct commit_graph *load_commit_graph_one(const char *graph_file);
  
 +struct commit_graph *parse_commit_graph(void *graph_map, int fd,
 +                                      size_t graph_size);
 +
  /*
   * Return 1 if and only if the repository has a commit-graph
   * file and generation numbers are computed in that file.
diff --combined t/t5318-commit-graph.sh
@@@ -122,7 -122,7 +122,7 @@@ test_expect_success 'write graph with m
        cd "$TRASH_DIRECTORY/full" &&
        git commit-graph write &&
        test_path_is_file $objdir/info/commit-graph &&
-       graph_read_expect "10" "large_edges"
+       graph_read_expect "10" "extra_edges"
  '
  
  graph_git_behavior 'merge 1 vs 2' full merge/1 merge/2
@@@ -157,7 -157,7 +157,7 @@@ test_expect_success 'write graph with n
        cd "$TRASH_DIRECTORY/full" &&
        git commit-graph write &&
        test_path_is_file $objdir/info/commit-graph &&
-       graph_read_expect "11" "large_edges"
+       graph_read_expect "11" "extra_edges"
  '
  
  graph_git_behavior 'full graph, commit 8 vs merge 1' full commits/8 merge/1
@@@ -167,7 -167,7 +167,7 @@@ test_expect_success 'write graph with n
        cd "$TRASH_DIRECTORY/full" &&
        git commit-graph write &&
        test_path_is_file $objdir/info/commit-graph &&
-       graph_read_expect "11" "large_edges"
+       graph_read_expect "11" "extra_edges"
  '
  
  graph_git_behavior 'cleared graph, commit 8 vs merge 1' full commits/8 merge/1
@@@ -177,7 -177,7 +177,7 @@@ test_expect_success 'build graph from l
        cd "$TRASH_DIRECTORY/full" &&
        cat new-idx | git commit-graph write --stdin-packs &&
        test_path_is_file $objdir/info/commit-graph &&
-       graph_read_expect "9" "large_edges"
+       graph_read_expect "9" "extra_edges"
  '
  
  graph_git_behavior 'graph from pack, commit 8 vs merge 1' full commits/8 merge/1
@@@ -200,7 -200,7 +200,7 @@@ test_expect_success 'build graph from c
        cd "$TRASH_DIRECTORY/full" &&
        git rev-parse merge/3 | git commit-graph write --stdin-commits --append &&
        test_path_is_file $objdir/info/commit-graph &&
-       graph_read_expect "10" "large_edges"
+       graph_read_expect "10" "extra_edges"
  '
  
  graph_git_behavior 'append graph, commit 8 vs merge 1' full commits/8 merge/1
@@@ -210,7 -210,7 +210,7 @@@ test_expect_success 'build graph using 
        cd "$TRASH_DIRECTORY/full" &&
        git commit-graph write --reachable &&
        test_path_is_file $objdir/info/commit-graph &&
-       graph_read_expect "11" "large_edges"
+       graph_read_expect "11" "extra_edges"
  '
  
  graph_git_behavior 'append graph, commit 8 vs merge 1' full commits/8 merge/1
@@@ -231,7 -231,7 +231,7 @@@ test_expect_success 'write graph in bar
        cd "$TRASH_DIRECTORY/bare" &&
        git commit-graph write &&
        test_path_is_file $baredir/info/commit-graph &&
-       graph_read_expect "11" "large_edges"
+       graph_read_expect "11" "extra_edges"
  '
  
  graph_git_behavior 'bare repo with graph, commit 8 vs merge 1' bare commits/8 merge/1
@@@ -366,10 -366,9 +366,10 @@@ GRAPH_OCTOPUS_DATA_OFFSET=$(($GRAPH_COM
  GRAPH_BYTE_OCTOPUS=$(($GRAPH_OCTOPUS_DATA_OFFSET + 4))
  GRAPH_BYTE_FOOTER=$(($GRAPH_OCTOPUS_DATA_OFFSET + 4 * $NUM_OCTOPUS_EDGES))
  
 -# usage: corrupt_graph_and_verify <position> <data> <string>
 +# usage: corrupt_graph_and_verify <position> <data> <string> [<zero_pos>]
  # Manipulates the commit-graph file at the position
 -# by inserting the data, then runs 'git commit-graph verify'
 +# by inserting the data, optionally zeroing the file
 +# starting at <zero_pos>, then runs 'git commit-graph verify'
  # and places the output in the file 'err'. Test 'err' for
  # the given string.
  corrupt_graph_and_verify() {
        data="${2:-\0}"
        grepstr=$3
        cd "$TRASH_DIRECTORY/full" &&
 +      orig_size=$(wc -c < $objdir/info/commit-graph) &&
 +      zero_pos=${4:-${orig_size}} &&
        test_when_finished mv commit-graph-backup $objdir/info/commit-graph &&
        cp $objdir/info/commit-graph commit-graph-backup &&
        printf "$data" | dd of="$objdir/info/commit-graph" bs=1 seek="$pos" conv=notrunc &&
 +      dd of="$objdir/info/commit-graph" bs=1 seek="$zero_pos" count=0 &&
 +      dd if=/dev/zero of="$objdir/info/commit-graph" bs=1 seek="$zero_pos" count=$(($orig_size - $zero_pos)) &&
        test_must_fail git commit-graph verify 2>test_err &&
 -      grep -v "^+" test_err >err
 +      grep -v "^+" test_err >err &&
        test_i18ngrep "$grepstr" err
  }
  
@@@ -489,11 -484,6 +489,11 @@@ test_expect_success 'detect invalid che
                "incorrect checksum"
  '
  
 +test_expect_success 'detect incorrect chunk count' '
 +      corrupt_graph_and_verify $GRAPH_BYTE_CHUNK_COUNT "\377" \
 +              "chunk lookup table entry missing" $GRAPH_CHUNK_LOOKUP_OFFSET
 +'
 +
  test_expect_success 'git fsck (checks commit-graph)' '
        cd "$TRASH_DIRECTORY/full" &&
        git fsck &&