commit-graph: not compatible with replace objects
[git/git.git] / commit-graph.c
1 #include "cache.h"
2 #include "config.h"
3 #include "dir.h"
4 #include "git-compat-util.h"
5 #include "lockfile.h"
6 #include "pack.h"
7 #include "packfile.h"
8 #include "commit.h"
9 #include "object.h"
10 #include "refs.h"
11 #include "revision.h"
12 #include "sha1-lookup.h"
13 #include "commit-graph.h"
14 #include "object-store.h"
15 #include "alloc.h"
16 #include "hashmap.h"
17 #include "replace-object.h"
18
19 #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
20 #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
21 #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
22 #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
23 #define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
24
25 #define GRAPH_DATA_WIDTH 36
26
27 #define GRAPH_VERSION_1 0x1
28 #define GRAPH_VERSION GRAPH_VERSION_1
29
30 #define GRAPH_OID_VERSION_SHA1 1
31 #define GRAPH_OID_LEN_SHA1 GIT_SHA1_RAWSZ
32 #define GRAPH_OID_VERSION GRAPH_OID_VERSION_SHA1
33 #define GRAPH_OID_LEN GRAPH_OID_LEN_SHA1
34
35 #define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
36 #define GRAPH_PARENT_MISSING 0x7fffffff
37 #define GRAPH_EDGE_LAST_MASK 0x7fffffff
38 #define GRAPH_PARENT_NONE 0x70000000
39
40 #define GRAPH_LAST_EDGE 0x80000000
41
42 #define GRAPH_HEADER_SIZE 8
43 #define GRAPH_FANOUT_SIZE (4 * 256)
44 #define GRAPH_CHUNKLOOKUP_WIDTH 12
45 #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
46 + GRAPH_FANOUT_SIZE + GRAPH_OID_LEN)
47
48 char *get_commit_graph_filename(const char *obj_dir)
49 {
50 return xstrfmt("%s/info/commit-graph", obj_dir);
51 }
52
53 static struct commit_graph *alloc_commit_graph(void)
54 {
55 struct commit_graph *g = xcalloc(1, sizeof(*g));
56 g->graph_fd = -1;
57
58 return g;
59 }
60
61 extern int read_replace_refs;
62
63 static int commit_graph_compatible(struct repository *r)
64 {
65 if (read_replace_refs) {
66 prepare_replace_object(r);
67 if (hashmap_get_size(&r->objects->replace_map->map))
68 return 0;
69 }
70
71 return 1;
72 }
73
74 struct commit_graph *load_commit_graph_one(const char *graph_file)
75 {
76 void *graph_map;
77 const unsigned char *data, *chunk_lookup;
78 size_t graph_size;
79 struct stat st;
80 uint32_t i;
81 struct commit_graph *graph;
82 int fd = git_open(graph_file);
83 uint64_t last_chunk_offset;
84 uint32_t last_chunk_id;
85 uint32_t graph_signature;
86 unsigned char graph_version, hash_version;
87
88 if (fd < 0)
89 return NULL;
90 if (fstat(fd, &st)) {
91 close(fd);
92 return NULL;
93 }
94 graph_size = xsize_t(st.st_size);
95
96 if (graph_size < GRAPH_MIN_SIZE) {
97 close(fd);
98 die("graph file %s is too small", graph_file);
99 }
100 graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
101 data = (const unsigned char *)graph_map;
102
103 graph_signature = get_be32(data);
104 if (graph_signature != GRAPH_SIGNATURE) {
105 error("graph signature %X does not match signature %X",
106 graph_signature, GRAPH_SIGNATURE);
107 goto cleanup_fail;
108 }
109
110 graph_version = *(unsigned char*)(data + 4);
111 if (graph_version != GRAPH_VERSION) {
112 error("graph version %X does not match version %X",
113 graph_version, GRAPH_VERSION);
114 goto cleanup_fail;
115 }
116
117 hash_version = *(unsigned char*)(data + 5);
118 if (hash_version != GRAPH_OID_VERSION) {
119 error("hash version %X does not match version %X",
120 hash_version, GRAPH_OID_VERSION);
121 goto cleanup_fail;
122 }
123
124 graph = alloc_commit_graph();
125
126 graph->hash_len = GRAPH_OID_LEN;
127 graph->num_chunks = *(unsigned char*)(data + 6);
128 graph->graph_fd = fd;
129 graph->data = graph_map;
130 graph->data_len = graph_size;
131
132 last_chunk_id = 0;
133 last_chunk_offset = 8;
134 chunk_lookup = data + 8;
135 for (i = 0; i < graph->num_chunks; i++) {
136 uint32_t chunk_id = get_be32(chunk_lookup + 0);
137 uint64_t chunk_offset = get_be64(chunk_lookup + 4);
138 int chunk_repeated = 0;
139
140 chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH;
141
142 if (chunk_offset > graph_size - GIT_MAX_RAWSZ) {
143 error("improper chunk offset %08x%08x", (uint32_t)(chunk_offset >> 32),
144 (uint32_t)chunk_offset);
145 goto cleanup_fail;
146 }
147
148 switch (chunk_id) {
149 case GRAPH_CHUNKID_OIDFANOUT:
150 if (graph->chunk_oid_fanout)
151 chunk_repeated = 1;
152 else
153 graph->chunk_oid_fanout = (uint32_t*)(data + chunk_offset);
154 break;
155
156 case GRAPH_CHUNKID_OIDLOOKUP:
157 if (graph->chunk_oid_lookup)
158 chunk_repeated = 1;
159 else
160 graph->chunk_oid_lookup = data + chunk_offset;
161 break;
162
163 case GRAPH_CHUNKID_DATA:
164 if (graph->chunk_commit_data)
165 chunk_repeated = 1;
166 else
167 graph->chunk_commit_data = data + chunk_offset;
168 break;
169
170 case GRAPH_CHUNKID_LARGEEDGES:
171 if (graph->chunk_large_edges)
172 chunk_repeated = 1;
173 else
174 graph->chunk_large_edges = data + chunk_offset;
175 break;
176 }
177
178 if (chunk_repeated) {
179 error("chunk id %08x appears multiple times", chunk_id);
180 goto cleanup_fail;
181 }
182
183 if (last_chunk_id == GRAPH_CHUNKID_OIDLOOKUP)
184 {
185 graph->num_commits = (chunk_offset - last_chunk_offset)
186 / graph->hash_len;
187 }
188
189 last_chunk_id = chunk_id;
190 last_chunk_offset = chunk_offset;
191 }
192
193 return graph;
194
195 cleanup_fail:
196 munmap(graph_map, graph_size);
197 close(fd);
198 exit(1);
199 }
200
201 static void prepare_commit_graph_one(struct repository *r, const char *obj_dir)
202 {
203 char *graph_name;
204
205 if (r->objects->commit_graph)
206 return;
207
208 graph_name = get_commit_graph_filename(obj_dir);
209 r->objects->commit_graph =
210 load_commit_graph_one(graph_name);
211
212 FREE_AND_NULL(graph_name);
213 }
214
215 /*
216 * Return 1 if commit_graph is non-NULL, and 0 otherwise.
217 *
218 * On the first invocation, this function attemps to load the commit
219 * graph if the_repository is configured to have one.
220 */
221 static int prepare_commit_graph(struct repository *r)
222 {
223 struct alternate_object_database *alt;
224 char *obj_dir;
225 int config_value;
226
227 if (r->objects->commit_graph_attempted)
228 return !!r->objects->commit_graph;
229 r->objects->commit_graph_attempted = 1;
230
231 if (repo_config_get_bool(r, "core.commitgraph", &config_value) ||
232 !config_value)
233 /*
234 * This repository is not configured to use commit graphs, so
235 * do not load one. (But report commit_graph_attempted anyway
236 * so that commit graph loading is not attempted again for this
237 * repository.)
238 */
239 return 0;
240
241 if (!commit_graph_compatible(r))
242 return 0;
243
244 obj_dir = r->objects->objectdir;
245 prepare_commit_graph_one(r, obj_dir);
246 prepare_alt_odb(r);
247 for (alt = r->objects->alt_odb_list;
248 !r->objects->commit_graph && alt;
249 alt = alt->next)
250 prepare_commit_graph_one(r, alt->path);
251 return !!r->objects->commit_graph;
252 }
253
254 static void close_commit_graph(void)
255 {
256 free_commit_graph(the_repository->objects->commit_graph);
257 the_repository->objects->commit_graph = NULL;
258 }
259
260 static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos)
261 {
262 return bsearch_hash(oid->hash, g->chunk_oid_fanout,
263 g->chunk_oid_lookup, g->hash_len, pos);
264 }
265
266 static struct commit_list **insert_parent_or_die(struct commit_graph *g,
267 uint64_t pos,
268 struct commit_list **pptr)
269 {
270 struct commit *c;
271 struct object_id oid;
272
273 if (pos >= g->num_commits)
274 die("invalid parent position %"PRIu64, pos);
275
276 hashcpy(oid.hash, g->chunk_oid_lookup + g->hash_len * pos);
277 c = lookup_commit(the_repository, &oid);
278 if (!c)
279 die("could not find commit %s", oid_to_hex(&oid));
280 c->graph_pos = pos;
281 return &commit_list_insert(c, pptr)->next;
282 }
283
284 static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, uint32_t pos)
285 {
286 const unsigned char *commit_data = g->chunk_commit_data + GRAPH_DATA_WIDTH * pos;
287 item->graph_pos = pos;
288 item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
289 }
290
291 static int fill_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t pos)
292 {
293 uint32_t edge_value;
294 uint32_t *parent_data_ptr;
295 uint64_t date_low, date_high;
296 struct commit_list **pptr;
297 const unsigned char *commit_data = g->chunk_commit_data + (g->hash_len + 16) * pos;
298
299 item->object.parsed = 1;
300 item->graph_pos = pos;
301
302 item->maybe_tree = NULL;
303
304 date_high = get_be32(commit_data + g->hash_len + 8) & 0x3;
305 date_low = get_be32(commit_data + g->hash_len + 12);
306 item->date = (timestamp_t)((date_high << 32) | date_low);
307
308 item->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
309
310 pptr = &item->parents;
311
312 edge_value = get_be32(commit_data + g->hash_len);
313 if (edge_value == GRAPH_PARENT_NONE)
314 return 1;
315 pptr = insert_parent_or_die(g, edge_value, pptr);
316
317 edge_value = get_be32(commit_data + g->hash_len + 4);
318 if (edge_value == GRAPH_PARENT_NONE)
319 return 1;
320 if (!(edge_value & GRAPH_OCTOPUS_EDGES_NEEDED)) {
321 pptr = insert_parent_or_die(g, edge_value, pptr);
322 return 1;
323 }
324
325 parent_data_ptr = (uint32_t*)(g->chunk_large_edges +
326 4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
327 do {
328 edge_value = get_be32(parent_data_ptr);
329 pptr = insert_parent_or_die(g,
330 edge_value & GRAPH_EDGE_LAST_MASK,
331 pptr);
332 parent_data_ptr++;
333 } while (!(edge_value & GRAPH_LAST_EDGE));
334
335 return 1;
336 }
337
338 static int find_commit_in_graph(struct commit *item, struct commit_graph *g, uint32_t *pos)
339 {
340 if (item->graph_pos != COMMIT_NOT_FROM_GRAPH) {
341 *pos = item->graph_pos;
342 return 1;
343 } else {
344 return bsearch_graph(g, &(item->object.oid), pos);
345 }
346 }
347
348 static int parse_commit_in_graph_one(struct commit_graph *g, struct commit *item)
349 {
350 uint32_t pos;
351
352 if (item->object.parsed)
353 return 1;
354
355 if (find_commit_in_graph(item, g, &pos))
356 return fill_commit_in_graph(item, g, pos);
357
358 return 0;
359 }
360
361 int parse_commit_in_graph(struct repository *r, struct commit *item)
362 {
363 if (!prepare_commit_graph(r))
364 return 0;
365 return parse_commit_in_graph_one(r->objects->commit_graph, item);
366 }
367
368 void load_commit_graph_info(struct repository *r, struct commit *item)
369 {
370 uint32_t pos;
371 if (!prepare_commit_graph(r))
372 return;
373 if (find_commit_in_graph(item, r->objects->commit_graph, &pos))
374 fill_commit_graph_info(item, r->objects->commit_graph, pos);
375 }
376
377 static struct tree *load_tree_for_commit(struct commit_graph *g, struct commit *c)
378 {
379 struct object_id oid;
380 const unsigned char *commit_data = g->chunk_commit_data +
381 GRAPH_DATA_WIDTH * (c->graph_pos);
382
383 hashcpy(oid.hash, commit_data);
384 c->maybe_tree = lookup_tree(the_repository, &oid);
385
386 return c->maybe_tree;
387 }
388
389 static struct tree *get_commit_tree_in_graph_one(struct commit_graph *g,
390 const struct commit *c)
391 {
392 if (c->maybe_tree)
393 return c->maybe_tree;
394 if (c->graph_pos == COMMIT_NOT_FROM_GRAPH)
395 BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
396
397 return load_tree_for_commit(g, (struct commit *)c);
398 }
399
400 struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c)
401 {
402 return get_commit_tree_in_graph_one(r->objects->commit_graph, c);
403 }
404
405 static void write_graph_chunk_fanout(struct hashfile *f,
406 struct commit **commits,
407 int nr_commits)
408 {
409 int i, count = 0;
410 struct commit **list = commits;
411
412 /*
413 * Write the first-level table (the list is sorted,
414 * but we use a 256-entry lookup to be able to avoid
415 * having to do eight extra binary search iterations).
416 */
417 for (i = 0; i < 256; i++) {
418 while (count < nr_commits) {
419 if ((*list)->object.oid.hash[0] != i)
420 break;
421 count++;
422 list++;
423 }
424
425 hashwrite_be32(f, count);
426 }
427 }
428
429 static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
430 struct commit **commits, int nr_commits)
431 {
432 struct commit **list = commits;
433 int count;
434 for (count = 0; count < nr_commits; count++, list++)
435 hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
436 }
437
438 static const unsigned char *commit_to_sha1(size_t index, void *table)
439 {
440 struct commit **commits = table;
441 return commits[index]->object.oid.hash;
442 }
443
444 static void write_graph_chunk_data(struct hashfile *f, int hash_len,
445 struct commit **commits, int nr_commits)
446 {
447 struct commit **list = commits;
448 struct commit **last = commits + nr_commits;
449 uint32_t num_extra_edges = 0;
450
451 while (list < last) {
452 struct commit_list *parent;
453 int edge_value;
454 uint32_t packedDate[2];
455
456 parse_commit(*list);
457 hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
458
459 parent = (*list)->parents;
460
461 if (!parent)
462 edge_value = GRAPH_PARENT_NONE;
463 else {
464 edge_value = sha1_pos(parent->item->object.oid.hash,
465 commits,
466 nr_commits,
467 commit_to_sha1);
468
469 if (edge_value < 0)
470 edge_value = GRAPH_PARENT_MISSING;
471 }
472
473 hashwrite_be32(f, edge_value);
474
475 if (parent)
476 parent = parent->next;
477
478 if (!parent)
479 edge_value = GRAPH_PARENT_NONE;
480 else if (parent->next)
481 edge_value = GRAPH_OCTOPUS_EDGES_NEEDED | num_extra_edges;
482 else {
483 edge_value = sha1_pos(parent->item->object.oid.hash,
484 commits,
485 nr_commits,
486 commit_to_sha1);
487 if (edge_value < 0)
488 edge_value = GRAPH_PARENT_MISSING;
489 }
490
491 hashwrite_be32(f, edge_value);
492
493 if (edge_value & GRAPH_OCTOPUS_EDGES_NEEDED) {
494 do {
495 num_extra_edges++;
496 parent = parent->next;
497 } while (parent);
498 }
499
500 if (sizeof((*list)->date) > 4)
501 packedDate[0] = htonl(((*list)->date >> 32) & 0x3);
502 else
503 packedDate[0] = 0;
504
505 packedDate[0] |= htonl((*list)->generation << 2);
506
507 packedDate[1] = htonl((*list)->date);
508 hashwrite(f, packedDate, 8);
509
510 list++;
511 }
512 }
513
514 static void write_graph_chunk_large_edges(struct hashfile *f,
515 struct commit **commits,
516 int nr_commits)
517 {
518 struct commit **list = commits;
519 struct commit **last = commits + nr_commits;
520 struct commit_list *parent;
521
522 while (list < last) {
523 int num_parents = 0;
524 for (parent = (*list)->parents; num_parents < 3 && parent;
525 parent = parent->next)
526 num_parents++;
527
528 if (num_parents <= 2) {
529 list++;
530 continue;
531 }
532
533 /* Since num_parents > 2, this initializer is safe. */
534 for (parent = (*list)->parents->next; parent; parent = parent->next) {
535 int edge_value = sha1_pos(parent->item->object.oid.hash,
536 commits,
537 nr_commits,
538 commit_to_sha1);
539
540 if (edge_value < 0)
541 edge_value = GRAPH_PARENT_MISSING;
542 else if (!parent->next)
543 edge_value |= GRAPH_LAST_EDGE;
544
545 hashwrite_be32(f, edge_value);
546 }
547
548 list++;
549 }
550 }
551
552 static int commit_compare(const void *_a, const void *_b)
553 {
554 const struct object_id *a = (const struct object_id *)_a;
555 const struct object_id *b = (const struct object_id *)_b;
556 return oidcmp(a, b);
557 }
558
559 struct packed_commit_list {
560 struct commit **list;
561 int nr;
562 int alloc;
563 };
564
565 struct packed_oid_list {
566 struct object_id *list;
567 int nr;
568 int alloc;
569 };
570
571 static int add_packed_commits(const struct object_id *oid,
572 struct packed_git *pack,
573 uint32_t pos,
574 void *data)
575 {
576 struct packed_oid_list *list = (struct packed_oid_list*)data;
577 enum object_type type;
578 off_t offset = nth_packed_object_offset(pack, pos);
579 struct object_info oi = OBJECT_INFO_INIT;
580
581 oi.typep = &type;
582 if (packed_object_info(the_repository, pack, offset, &oi) < 0)
583 die("unable to get type of object %s", oid_to_hex(oid));
584
585 if (type != OBJ_COMMIT)
586 return 0;
587
588 ALLOC_GROW(list->list, list->nr + 1, list->alloc);
589 oidcpy(&(list->list[list->nr]), oid);
590 list->nr++;
591
592 return 0;
593 }
594
595 static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit)
596 {
597 struct commit_list *parent;
598 for (parent = commit->parents; parent; parent = parent->next) {
599 if (!(parent->item->object.flags & UNINTERESTING)) {
600 ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc);
601 oidcpy(&oids->list[oids->nr], &(parent->item->object.oid));
602 oids->nr++;
603 parent->item->object.flags |= UNINTERESTING;
604 }
605 }
606 }
607
608 static void close_reachable(struct packed_oid_list *oids)
609 {
610 int i;
611 struct commit *commit;
612
613 for (i = 0; i < oids->nr; i++) {
614 commit = lookup_commit(the_repository, &oids->list[i]);
615 if (commit)
616 commit->object.flags |= UNINTERESTING;
617 }
618
619 /*
620 * As this loop runs, oids->nr may grow, but not more
621 * than the number of missing commits in the reachable
622 * closure.
623 */
624 for (i = 0; i < oids->nr; i++) {
625 commit = lookup_commit(the_repository, &oids->list[i]);
626
627 if (commit && !parse_commit(commit))
628 add_missing_parents(oids, commit);
629 }
630
631 for (i = 0; i < oids->nr; i++) {
632 commit = lookup_commit(the_repository, &oids->list[i]);
633
634 if (commit)
635 commit->object.flags &= ~UNINTERESTING;
636 }
637 }
638
639 static void compute_generation_numbers(struct packed_commit_list* commits)
640 {
641 int i;
642 struct commit_list *list = NULL;
643
644 for (i = 0; i < commits->nr; i++) {
645 if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY &&
646 commits->list[i]->generation != GENERATION_NUMBER_ZERO)
647 continue;
648
649 commit_list_insert(commits->list[i], &list);
650 while (list) {
651 struct commit *current = list->item;
652 struct commit_list *parent;
653 int all_parents_computed = 1;
654 uint32_t max_generation = 0;
655
656 for (parent = current->parents; parent; parent = parent->next) {
657 if (parent->item->generation == GENERATION_NUMBER_INFINITY ||
658 parent->item->generation == GENERATION_NUMBER_ZERO) {
659 all_parents_computed = 0;
660 commit_list_insert(parent->item, &list);
661 break;
662 } else if (parent->item->generation > max_generation) {
663 max_generation = parent->item->generation;
664 }
665 }
666
667 if (all_parents_computed) {
668 current->generation = max_generation + 1;
669 pop_commit(&list);
670
671 if (current->generation > GENERATION_NUMBER_MAX)
672 current->generation = GENERATION_NUMBER_MAX;
673 }
674 }
675 }
676 }
677
678 static int add_ref_to_list(const char *refname,
679 const struct object_id *oid,
680 int flags, void *cb_data)
681 {
682 struct string_list *list = (struct string_list *)cb_data;
683
684 string_list_append(list, oid_to_hex(oid));
685 return 0;
686 }
687
688 void write_commit_graph_reachable(const char *obj_dir, int append)
689 {
690 struct string_list list;
691
692 string_list_init(&list, 1);
693 for_each_ref(add_ref_to_list, &list);
694 write_commit_graph(obj_dir, NULL, &list, append);
695 }
696
697 void write_commit_graph(const char *obj_dir,
698 struct string_list *pack_indexes,
699 struct string_list *commit_hex,
700 int append)
701 {
702 struct packed_oid_list oids;
703 struct packed_commit_list commits;
704 struct hashfile *f;
705 uint32_t i, count_distinct = 0;
706 char *graph_name;
707 struct lock_file lk = LOCK_INIT;
708 uint32_t chunk_ids[5];
709 uint64_t chunk_offsets[5];
710 int num_chunks;
711 int num_extra_edges;
712 struct commit_list *parent;
713
714 if (!commit_graph_compatible(the_repository))
715 return;
716
717 oids.nr = 0;
718 oids.alloc = approximate_object_count() / 4;
719
720 if (append) {
721 prepare_commit_graph_one(the_repository, obj_dir);
722 if (the_repository->objects->commit_graph)
723 oids.alloc += the_repository->objects->commit_graph->num_commits;
724 }
725
726 if (oids.alloc < 1024)
727 oids.alloc = 1024;
728 ALLOC_ARRAY(oids.list, oids.alloc);
729
730 if (append && the_repository->objects->commit_graph) {
731 struct commit_graph *commit_graph =
732 the_repository->objects->commit_graph;
733 for (i = 0; i < commit_graph->num_commits; i++) {
734 const unsigned char *hash = commit_graph->chunk_oid_lookup +
735 commit_graph->hash_len * i;
736 hashcpy(oids.list[oids.nr++].hash, hash);
737 }
738 }
739
740 if (pack_indexes) {
741 struct strbuf packname = STRBUF_INIT;
742 int dirlen;
743 strbuf_addf(&packname, "%s/pack/", obj_dir);
744 dirlen = packname.len;
745 for (i = 0; i < pack_indexes->nr; i++) {
746 struct packed_git *p;
747 strbuf_setlen(&packname, dirlen);
748 strbuf_addstr(&packname, pack_indexes->items[i].string);
749 p = add_packed_git(packname.buf, packname.len, 1);
750 if (!p)
751 die("error adding pack %s", packname.buf);
752 if (open_pack_index(p))
753 die("error opening index for %s", packname.buf);
754 for_each_object_in_pack(p, add_packed_commits, &oids);
755 close_pack(p);
756 }
757 strbuf_release(&packname);
758 }
759
760 if (commit_hex) {
761 for (i = 0; i < commit_hex->nr; i++) {
762 const char *end;
763 struct object_id oid;
764 struct commit *result;
765
766 if (commit_hex->items[i].string &&
767 parse_oid_hex(commit_hex->items[i].string, &oid, &end))
768 continue;
769
770 result = lookup_commit_reference_gently(the_repository, &oid, 1);
771
772 if (result) {
773 ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
774 oidcpy(&oids.list[oids.nr], &(result->object.oid));
775 oids.nr++;
776 }
777 }
778 }
779
780 if (!pack_indexes && !commit_hex)
781 for_each_packed_object(add_packed_commits, &oids, 0);
782
783 close_reachable(&oids);
784
785 QSORT(oids.list, oids.nr, commit_compare);
786
787 count_distinct = 1;
788 for (i = 1; i < oids.nr; i++) {
789 if (oidcmp(&oids.list[i-1], &oids.list[i]))
790 count_distinct++;
791 }
792
793 if (count_distinct >= GRAPH_PARENT_MISSING)
794 die(_("the commit graph format cannot write %d commits"), count_distinct);
795
796 commits.nr = 0;
797 commits.alloc = count_distinct;
798 ALLOC_ARRAY(commits.list, commits.alloc);
799
800 num_extra_edges = 0;
801 for (i = 0; i < oids.nr; i++) {
802 int num_parents = 0;
803 if (i > 0 && !oidcmp(&oids.list[i-1], &oids.list[i]))
804 continue;
805
806 commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
807 parse_commit(commits.list[commits.nr]);
808
809 for (parent = commits.list[commits.nr]->parents;
810 parent; parent = parent->next)
811 num_parents++;
812
813 if (num_parents > 2)
814 num_extra_edges += num_parents - 1;
815
816 commits.nr++;
817 }
818 num_chunks = num_extra_edges ? 4 : 3;
819
820 if (commits.nr >= GRAPH_PARENT_MISSING)
821 die(_("too many commits to write graph"));
822
823 compute_generation_numbers(&commits);
824
825 graph_name = get_commit_graph_filename(obj_dir);
826 if (safe_create_leading_directories(graph_name))
827 die_errno(_("unable to create leading directories of %s"),
828 graph_name);
829
830 hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR);
831 f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
832
833 hashwrite_be32(f, GRAPH_SIGNATURE);
834
835 hashwrite_u8(f, GRAPH_VERSION);
836 hashwrite_u8(f, GRAPH_OID_VERSION);
837 hashwrite_u8(f, num_chunks);
838 hashwrite_u8(f, 0); /* unused padding byte */
839
840 chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT;
841 chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
842 chunk_ids[2] = GRAPH_CHUNKID_DATA;
843 if (num_extra_edges)
844 chunk_ids[3] = GRAPH_CHUNKID_LARGEEDGES;
845 else
846 chunk_ids[3] = 0;
847 chunk_ids[4] = 0;
848
849 chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH;
850 chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE;
851 chunk_offsets[2] = chunk_offsets[1] + GRAPH_OID_LEN * commits.nr;
852 chunk_offsets[3] = chunk_offsets[2] + (GRAPH_OID_LEN + 16) * commits.nr;
853 chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges;
854
855 for (i = 0; i <= num_chunks; i++) {
856 uint32_t chunk_write[3];
857
858 chunk_write[0] = htonl(chunk_ids[i]);
859 chunk_write[1] = htonl(chunk_offsets[i] >> 32);
860 chunk_write[2] = htonl(chunk_offsets[i] & 0xffffffff);
861 hashwrite(f, chunk_write, 12);
862 }
863
864 write_graph_chunk_fanout(f, commits.list, commits.nr);
865 write_graph_chunk_oids(f, GRAPH_OID_LEN, commits.list, commits.nr);
866 write_graph_chunk_data(f, GRAPH_OID_LEN, commits.list, commits.nr);
867 write_graph_chunk_large_edges(f, commits.list, commits.nr);
868
869 close_commit_graph();
870 finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
871 commit_lock_file(&lk);
872
873 free(oids.list);
874 oids.alloc = 0;
875 oids.nr = 0;
876 }
877
878 #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
879 static int verify_commit_graph_error;
880
881 static void graph_report(const char *fmt, ...)
882 {
883 va_list ap;
884
885 verify_commit_graph_error = 1;
886 va_start(ap, fmt);
887 vfprintf(stderr, fmt, ap);
888 fprintf(stderr, "\n");
889 va_end(ap);
890 }
891
892 #define GENERATION_ZERO_EXISTS 1
893 #define GENERATION_NUMBER_EXISTS 2
894
895 int verify_commit_graph(struct repository *r, struct commit_graph *g)
896 {
897 uint32_t i, cur_fanout_pos = 0;
898 struct object_id prev_oid, cur_oid, checksum;
899 int generation_zero = 0;
900 struct hashfile *f;
901 int devnull;
902
903 if (!g) {
904 graph_report("no commit-graph file loaded");
905 return 1;
906 }
907
908 verify_commit_graph_error = 0;
909
910 if (!g->chunk_oid_fanout)
911 graph_report("commit-graph is missing the OID Fanout chunk");
912 if (!g->chunk_oid_lookup)
913 graph_report("commit-graph is missing the OID Lookup chunk");
914 if (!g->chunk_commit_data)
915 graph_report("commit-graph is missing the Commit Data chunk");
916
917 if (verify_commit_graph_error)
918 return verify_commit_graph_error;
919
920 devnull = open("/dev/null", O_WRONLY);
921 f = hashfd(devnull, NULL);
922 hashwrite(f, g->data, g->data_len - g->hash_len);
923 finalize_hashfile(f, checksum.hash, CSUM_CLOSE);
924 if (hashcmp(checksum.hash, g->data + g->data_len - g->hash_len)) {
925 graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
926 verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
927 }
928
929 for (i = 0; i < g->num_commits; i++) {
930 struct commit *graph_commit;
931
932 hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
933
934 if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
935 graph_report("commit-graph has incorrect OID order: %s then %s",
936 oid_to_hex(&prev_oid),
937 oid_to_hex(&cur_oid));
938
939 oidcpy(&prev_oid, &cur_oid);
940
941 while (cur_oid.hash[0] > cur_fanout_pos) {
942 uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
943
944 if (i != fanout_value)
945 graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
946 cur_fanout_pos, fanout_value, i);
947 cur_fanout_pos++;
948 }
949
950 graph_commit = lookup_commit(r, &cur_oid);
951 if (!parse_commit_in_graph_one(g, graph_commit))
952 graph_report("failed to parse %s from commit-graph",
953 oid_to_hex(&cur_oid));
954 }
955
956 while (cur_fanout_pos < 256) {
957 uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
958
959 if (g->num_commits != fanout_value)
960 graph_report("commit-graph has incorrect fanout value: fanout[%d] = %u != %u",
961 cur_fanout_pos, fanout_value, i);
962
963 cur_fanout_pos++;
964 }
965
966 if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
967 return verify_commit_graph_error;
968
969 for (i = 0; i < g->num_commits; i++) {
970 struct commit *graph_commit, *odb_commit;
971 struct commit_list *graph_parents, *odb_parents;
972 uint32_t max_generation = 0;
973
974 hashcpy(cur_oid.hash, g->chunk_oid_lookup + g->hash_len * i);
975
976 graph_commit = lookup_commit(r, &cur_oid);
977 odb_commit = (struct commit *)create_object(r, cur_oid.hash, alloc_commit_node(r));
978 if (parse_commit_internal(odb_commit, 0, 0)) {
979 graph_report("failed to parse %s from object database",
980 oid_to_hex(&cur_oid));
981 continue;
982 }
983
984 if (oidcmp(&get_commit_tree_in_graph_one(g, graph_commit)->object.oid,
985 get_commit_tree_oid(odb_commit)))
986 graph_report("root tree OID for commit %s in commit-graph is %s != %s",
987 oid_to_hex(&cur_oid),
988 oid_to_hex(get_commit_tree_oid(graph_commit)),
989 oid_to_hex(get_commit_tree_oid(odb_commit)));
990
991 graph_parents = graph_commit->parents;
992 odb_parents = odb_commit->parents;
993
994 while (graph_parents) {
995 if (odb_parents == NULL) {
996 graph_report("commit-graph parent list for commit %s is too long",
997 oid_to_hex(&cur_oid));
998 break;
999 }
1000
1001 if (oidcmp(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
1002 graph_report("commit-graph parent for %s is %s != %s",
1003 oid_to_hex(&cur_oid),
1004 oid_to_hex(&graph_parents->item->object.oid),
1005 oid_to_hex(&odb_parents->item->object.oid));
1006
1007 if (graph_parents->item->generation > max_generation)
1008 max_generation = graph_parents->item->generation;
1009
1010 graph_parents = graph_parents->next;
1011 odb_parents = odb_parents->next;
1012 }
1013
1014 if (odb_parents != NULL)
1015 graph_report("commit-graph parent list for commit %s terminates early",
1016 oid_to_hex(&cur_oid));
1017
1018 if (!graph_commit->generation) {
1019 if (generation_zero == GENERATION_NUMBER_EXISTS)
1020 graph_report("commit-graph has generation number zero for commit %s, but non-zero elsewhere",
1021 oid_to_hex(&cur_oid));
1022 generation_zero = GENERATION_ZERO_EXISTS;
1023 } else if (generation_zero == GENERATION_ZERO_EXISTS)
1024 graph_report("commit-graph has non-zero generation number for commit %s, but zero elsewhere",
1025 oid_to_hex(&cur_oid));
1026
1027 if (generation_zero == GENERATION_ZERO_EXISTS)
1028 continue;
1029
1030 /*
1031 * If one of our parents has generation GENERATION_NUMBER_MAX, then
1032 * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid
1033 * extra logic in the following condition.
1034 */
1035 if (max_generation == GENERATION_NUMBER_MAX)
1036 max_generation--;
1037
1038 if (graph_commit->generation != max_generation + 1)
1039 graph_report("commit-graph generation for commit %s is %u != %u",
1040 oid_to_hex(&cur_oid),
1041 graph_commit->generation,
1042 max_generation + 1);
1043
1044 if (graph_commit->date != odb_commit->date)
1045 graph_report("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime,
1046 oid_to_hex(&cur_oid),
1047 graph_commit->date,
1048 odb_commit->date);
1049 }
1050
1051 return verify_commit_graph_error;
1052 }
1053
1054 void free_commit_graph(struct commit_graph *g)
1055 {
1056 if (!g)
1057 return;
1058 if (g->graph_fd >= 0) {
1059 munmap((void *)g->data, g->data_len);
1060 g->data = NULL;
1061 close(g->graph_fd);
1062 }
1063 free(g);
1064 }