object-store: move packed_git and packed_git_mru to object store
[git/git.git] / builtin / pack-redundant.c
1 /*
2 *
3 * Copyright 2005, Lukas Sandstrom <lukass@etek.chalmers.se>
4 *
5 * This file is licensed under the GPL v2.
6 *
7 */
8
9 #include "builtin.h"
10 #include "repository.h"
11 #include "packfile.h"
12 #include "object-store.h"
13
14 #define BLKSIZE 512
15
16 static const char pack_redundant_usage[] =
17 "git pack-redundant [--verbose] [--alt-odb] (--all | <filename.pack>...)";
18
19 static int load_all_packs, verbose, alt_odb;
20
21 struct llist_item {
22 struct llist_item *next;
23 const unsigned char *sha1;
24 };
25 static struct llist {
26 struct llist_item *front;
27 struct llist_item *back;
28 size_t size;
29 } *all_objects; /* all objects which must be present in local packfiles */
30
31 static struct pack_list {
32 struct pack_list *next;
33 struct packed_git *pack;
34 struct llist *unique_objects;
35 struct llist *all_objects;
36 } *local_packs = NULL, *altodb_packs = NULL;
37
38 struct pll {
39 struct pll *next;
40 struct pack_list *pl;
41 };
42
43 static struct llist_item *free_nodes;
44
45 static inline void llist_item_put(struct llist_item *item)
46 {
47 item->next = free_nodes;
48 free_nodes = item;
49 }
50
51 static inline struct llist_item *llist_item_get(void)
52 {
53 struct llist_item *new;
54 if ( free_nodes ) {
55 new = free_nodes;
56 free_nodes = free_nodes->next;
57 } else {
58 int i = 1;
59 ALLOC_ARRAY(new, BLKSIZE);
60 for (; i < BLKSIZE; i++)
61 llist_item_put(&new[i]);
62 }
63 return new;
64 }
65
66 static void llist_free(struct llist *list)
67 {
68 while ((list->back = list->front)) {
69 list->front = list->front->next;
70 llist_item_put(list->back);
71 }
72 free(list);
73 }
74
75 static inline void llist_init(struct llist **list)
76 {
77 *list = xmalloc(sizeof(struct llist));
78 (*list)->front = (*list)->back = NULL;
79 (*list)->size = 0;
80 }
81
82 static struct llist * llist_copy(struct llist *list)
83 {
84 struct llist *ret;
85 struct llist_item *new, *old, *prev;
86
87 llist_init(&ret);
88
89 if ((ret->size = list->size) == 0)
90 return ret;
91
92 new = ret->front = llist_item_get();
93 new->sha1 = list->front->sha1;
94
95 old = list->front->next;
96 while (old) {
97 prev = new;
98 new = llist_item_get();
99 prev->next = new;
100 new->sha1 = old->sha1;
101 old = old->next;
102 }
103 new->next = NULL;
104 ret->back = new;
105
106 return ret;
107 }
108
109 static inline struct llist_item *llist_insert(struct llist *list,
110 struct llist_item *after,
111 const unsigned char *sha1)
112 {
113 struct llist_item *new = llist_item_get();
114 new->sha1 = sha1;
115 new->next = NULL;
116
117 if (after != NULL) {
118 new->next = after->next;
119 after->next = new;
120 if (after == list->back)
121 list->back = new;
122 } else {/* insert in front */
123 if (list->size == 0)
124 list->back = new;
125 else
126 new->next = list->front;
127 list->front = new;
128 }
129 list->size++;
130 return new;
131 }
132
133 static inline struct llist_item *llist_insert_back(struct llist *list,
134 const unsigned char *sha1)
135 {
136 return llist_insert(list, list->back, sha1);
137 }
138
139 static inline struct llist_item *llist_insert_sorted_unique(struct llist *list,
140 const unsigned char *sha1, struct llist_item *hint)
141 {
142 struct llist_item *prev = NULL, *l;
143
144 l = (hint == NULL) ? list->front : hint;
145 while (l) {
146 int cmp = hashcmp(l->sha1, sha1);
147 if (cmp > 0) { /* we insert before this entry */
148 return llist_insert(list, prev, sha1);
149 }
150 if (!cmp) { /* already exists */
151 return l;
152 }
153 prev = l;
154 l = l->next;
155 }
156 /* insert at the end */
157 return llist_insert_back(list, sha1);
158 }
159
160 /* returns a pointer to an item in front of sha1 */
161 static inline struct llist_item * llist_sorted_remove(struct llist *list, const unsigned char *sha1, struct llist_item *hint)
162 {
163 struct llist_item *prev, *l;
164
165 redo_from_start:
166 l = (hint == NULL) ? list->front : hint;
167 prev = NULL;
168 while (l) {
169 int cmp = hashcmp(l->sha1, sha1);
170 if (cmp > 0) /* not in list, since sorted */
171 return prev;
172 if (!cmp) { /* found */
173 if (prev == NULL) {
174 if (hint != NULL && hint != list->front) {
175 /* we don't know the previous element */
176 hint = NULL;
177 goto redo_from_start;
178 }
179 list->front = l->next;
180 } else
181 prev->next = l->next;
182 if (l == list->back)
183 list->back = prev;
184 llist_item_put(l);
185 list->size--;
186 return prev;
187 }
188 prev = l;
189 l = l->next;
190 }
191 return prev;
192 }
193
194 /* computes A\B */
195 static void llist_sorted_difference_inplace(struct llist *A,
196 struct llist *B)
197 {
198 struct llist_item *hint, *b;
199
200 hint = NULL;
201 b = B->front;
202
203 while (b) {
204 hint = llist_sorted_remove(A, b->sha1, hint);
205 b = b->next;
206 }
207 }
208
209 static inline struct pack_list * pack_list_insert(struct pack_list **pl,
210 struct pack_list *entry)
211 {
212 struct pack_list *p = xmalloc(sizeof(struct pack_list));
213 memcpy(p, entry, sizeof(struct pack_list));
214 p->next = *pl;
215 *pl = p;
216 return p;
217 }
218
219 static inline size_t pack_list_size(struct pack_list *pl)
220 {
221 size_t ret = 0;
222 while (pl) {
223 ret++;
224 pl = pl->next;
225 }
226 return ret;
227 }
228
229 static struct pack_list * pack_list_difference(const struct pack_list *A,
230 const struct pack_list *B)
231 {
232 struct pack_list *ret;
233 const struct pack_list *pl;
234
235 if (A == NULL)
236 return NULL;
237
238 pl = B;
239 while (pl != NULL) {
240 if (A->pack == pl->pack)
241 return pack_list_difference(A->next, B);
242 pl = pl->next;
243 }
244 ret = xmalloc(sizeof(struct pack_list));
245 memcpy(ret, A, sizeof(struct pack_list));
246 ret->next = pack_list_difference(A->next, B);
247 return ret;
248 }
249
250 static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
251 {
252 unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
253 const unsigned char *p1_base, *p2_base;
254 struct llist_item *p1_hint = NULL, *p2_hint = NULL;
255
256 p1_base = p1->pack->index_data;
257 p2_base = p2->pack->index_data;
258 p1_base += 256 * 4 + ((p1->pack->index_version < 2) ? 4 : 8);
259 p2_base += 256 * 4 + ((p2->pack->index_version < 2) ? 4 : 8);
260 p1_step = (p1->pack->index_version < 2) ? 24 : 20;
261 p2_step = (p2->pack->index_version < 2) ? 24 : 20;
262
263 while (p1_off < p1->pack->num_objects * p1_step &&
264 p2_off < p2->pack->num_objects * p2_step)
265 {
266 int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
267 /* cmp ~ p1 - p2 */
268 if (cmp == 0) {
269 p1_hint = llist_sorted_remove(p1->unique_objects,
270 p1_base + p1_off, p1_hint);
271 p2_hint = llist_sorted_remove(p2->unique_objects,
272 p1_base + p1_off, p2_hint);
273 p1_off += p1_step;
274 p2_off += p2_step;
275 continue;
276 }
277 if (cmp < 0) { /* p1 has the object, p2 doesn't */
278 p1_off += p1_step;
279 } else { /* p2 has the object, p1 doesn't */
280 p2_off += p2_step;
281 }
282 }
283 }
284
285 static void pll_free(struct pll *l)
286 {
287 struct pll *old;
288 struct pack_list *opl;
289
290 while (l) {
291 old = l;
292 while (l->pl) {
293 opl = l->pl;
294 l->pl = opl->next;
295 free(opl);
296 }
297 l = l->next;
298 free(old);
299 }
300 }
301
302 /* all the permutations have to be free()d at the same time,
303 * since they refer to each other
304 */
305 static struct pll * get_permutations(struct pack_list *list, int n)
306 {
307 struct pll *subset, *ret = NULL, *new_pll = NULL;
308
309 if (list == NULL || pack_list_size(list) < n || n == 0)
310 return NULL;
311
312 if (n == 1) {
313 while (list) {
314 new_pll = xmalloc(sizeof(*new_pll));
315 new_pll->pl = NULL;
316 pack_list_insert(&new_pll->pl, list);
317 new_pll->next = ret;
318 ret = new_pll;
319 list = list->next;
320 }
321 return ret;
322 }
323
324 while (list->next) {
325 subset = get_permutations(list->next, n - 1);
326 while (subset) {
327 new_pll = xmalloc(sizeof(*new_pll));
328 new_pll->pl = subset->pl;
329 pack_list_insert(&new_pll->pl, list);
330 new_pll->next = ret;
331 ret = new_pll;
332 subset = subset->next;
333 }
334 list = list->next;
335 }
336 return ret;
337 }
338
339 static int is_superset(struct pack_list *pl, struct llist *list)
340 {
341 struct llist *diff;
342
343 diff = llist_copy(list);
344
345 while (pl) {
346 llist_sorted_difference_inplace(diff, pl->all_objects);
347 if (diff->size == 0) { /* we're done */
348 llist_free(diff);
349 return 1;
350 }
351 pl = pl->next;
352 }
353 llist_free(diff);
354 return 0;
355 }
356
357 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
358 {
359 size_t ret = 0;
360 unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
361 const unsigned char *p1_base, *p2_base;
362
363 p1_base = p1->index_data;
364 p2_base = p2->index_data;
365 p1_base += 256 * 4 + ((p1->index_version < 2) ? 4 : 8);
366 p2_base += 256 * 4 + ((p2->index_version < 2) ? 4 : 8);
367 p1_step = (p1->index_version < 2) ? 24 : 20;
368 p2_step = (p2->index_version < 2) ? 24 : 20;
369
370 while (p1_off < p1->num_objects * p1_step &&
371 p2_off < p2->num_objects * p2_step)
372 {
373 int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
374 /* cmp ~ p1 - p2 */
375 if (cmp == 0) {
376 ret++;
377 p1_off += p1_step;
378 p2_off += p2_step;
379 continue;
380 }
381 if (cmp < 0) { /* p1 has the object, p2 doesn't */
382 p1_off += p1_step;
383 } else { /* p2 has the object, p1 doesn't */
384 p2_off += p2_step;
385 }
386 }
387 return ret;
388 }
389
390 /* another O(n^2) function ... */
391 static size_t get_pack_redundancy(struct pack_list *pl)
392 {
393 struct pack_list *subset;
394 size_t ret = 0;
395
396 if (pl == NULL)
397 return 0;
398
399 while ((subset = pl->next)) {
400 while (subset) {
401 ret += sizeof_union(pl->pack, subset->pack);
402 subset = subset->next;
403 }
404 pl = pl->next;
405 }
406 return ret;
407 }
408
409 static inline off_t pack_set_bytecount(struct pack_list *pl)
410 {
411 off_t ret = 0;
412 while (pl) {
413 ret += pl->pack->pack_size;
414 ret += pl->pack->index_size;
415 pl = pl->next;
416 }
417 return ret;
418 }
419
420 static void minimize(struct pack_list **min)
421 {
422 struct pack_list *pl, *unique = NULL,
423 *non_unique = NULL, *min_perm = NULL;
424 struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
425 struct llist *missing;
426 off_t min_perm_size = 0, perm_size;
427 int n;
428
429 pl = local_packs;
430 while (pl) {
431 if (pl->unique_objects->size)
432 pack_list_insert(&unique, pl);
433 else
434 pack_list_insert(&non_unique, pl);
435 pl = pl->next;
436 }
437 /* find out which objects are missing from the set of unique packs */
438 missing = llist_copy(all_objects);
439 pl = unique;
440 while (pl) {
441 llist_sorted_difference_inplace(missing, pl->all_objects);
442 pl = pl->next;
443 }
444
445 /* return if there are no objects missing from the unique set */
446 if (missing->size == 0) {
447 *min = unique;
448 free(missing);
449 return;
450 }
451
452 /* find the permutations which contain all missing objects */
453 for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
454 perm_all = perm = get_permutations(non_unique, n);
455 while (perm) {
456 if (is_superset(perm->pl, missing)) {
457 new_perm = xmalloc(sizeof(struct pll));
458 memcpy(new_perm, perm, sizeof(struct pll));
459 new_perm->next = perm_ok;
460 perm_ok = new_perm;
461 }
462 perm = perm->next;
463 }
464 if (perm_ok)
465 break;
466 pll_free(perm_all);
467 }
468 if (perm_ok == NULL)
469 die("Internal error: No complete sets found!");
470
471 /* find the permutation with the smallest size */
472 perm = perm_ok;
473 while (perm) {
474 perm_size = pack_set_bytecount(perm->pl);
475 if (!min_perm_size || min_perm_size > perm_size) {
476 min_perm_size = perm_size;
477 min_perm = perm->pl;
478 }
479 perm = perm->next;
480 }
481 *min = min_perm;
482 /* add the unique packs to the list */
483 pl = unique;
484 while (pl) {
485 pack_list_insert(min, pl);
486 pl = pl->next;
487 }
488 }
489
490 static void load_all_objects(void)
491 {
492 struct pack_list *pl = local_packs;
493 struct llist_item *hint, *l;
494
495 llist_init(&all_objects);
496
497 while (pl) {
498 hint = NULL;
499 l = pl->all_objects->front;
500 while (l) {
501 hint = llist_insert_sorted_unique(all_objects,
502 l->sha1, hint);
503 l = l->next;
504 }
505 pl = pl->next;
506 }
507 /* remove objects present in remote packs */
508 pl = altodb_packs;
509 while (pl) {
510 llist_sorted_difference_inplace(all_objects, pl->all_objects);
511 pl = pl->next;
512 }
513 }
514
515 /* this scales like O(n^2) */
516 static void cmp_local_packs(void)
517 {
518 struct pack_list *subset, *pl = local_packs;
519
520 while ((subset = pl)) {
521 while ((subset = subset->next))
522 cmp_two_packs(pl, subset);
523 pl = pl->next;
524 }
525 }
526
527 static void scan_alt_odb_packs(void)
528 {
529 struct pack_list *local, *alt;
530
531 alt = altodb_packs;
532 while (alt) {
533 local = local_packs;
534 while (local) {
535 llist_sorted_difference_inplace(local->unique_objects,
536 alt->all_objects);
537 local = local->next;
538 }
539 llist_sorted_difference_inplace(all_objects, alt->all_objects);
540 alt = alt->next;
541 }
542 }
543
544 static struct pack_list * add_pack(struct packed_git *p)
545 {
546 struct pack_list l;
547 unsigned long off = 0, step;
548 const unsigned char *base;
549
550 if (!p->pack_local && !(alt_odb || verbose))
551 return NULL;
552
553 l.pack = p;
554 llist_init(&l.all_objects);
555
556 if (open_pack_index(p))
557 return NULL;
558
559 base = p->index_data;
560 base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
561 step = (p->index_version < 2) ? 24 : 20;
562 while (off < p->num_objects * step) {
563 llist_insert_back(l.all_objects, base + off);
564 off += step;
565 }
566 /* this list will be pruned in cmp_two_packs later */
567 l.unique_objects = llist_copy(l.all_objects);
568 if (p->pack_local)
569 return pack_list_insert(&local_packs, &l);
570 else
571 return pack_list_insert(&altodb_packs, &l);
572 }
573
574 static struct pack_list * add_pack_file(const char *filename)
575 {
576 struct packed_git *p = get_packed_git(the_repository);
577
578 if (strlen(filename) < 40)
579 die("Bad pack filename: %s", filename);
580
581 while (p) {
582 if (strstr(p->pack_name, filename))
583 return add_pack(p);
584 p = p->next;
585 }
586 die("Filename %s not found in packed_git", filename);
587 }
588
589 static void load_all(void)
590 {
591 struct packed_git *p = get_packed_git(the_repository);
592
593 while (p) {
594 add_pack(p);
595 p = p->next;
596 }
597 }
598
599 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
600 {
601 int i;
602 struct pack_list *min, *red, *pl;
603 struct llist *ignore;
604 unsigned char *sha1;
605 char buf[42]; /* 40 byte sha1 + \n + \0 */
606
607 if (argc == 2 && !strcmp(argv[1], "-h"))
608 usage(pack_redundant_usage);
609
610 for (i = 1; i < argc; i++) {
611 const char *arg = argv[i];
612 if (!strcmp(arg, "--")) {
613 i++;
614 break;
615 }
616 if (!strcmp(arg, "--all")) {
617 load_all_packs = 1;
618 continue;
619 }
620 if (!strcmp(arg, "--verbose")) {
621 verbose = 1;
622 continue;
623 }
624 if (!strcmp(arg, "--alt-odb")) {
625 alt_odb = 1;
626 continue;
627 }
628 if (*arg == '-')
629 usage(pack_redundant_usage);
630 else
631 break;
632 }
633
634 prepare_packed_git();
635
636 if (load_all_packs)
637 load_all();
638 else
639 while (*(argv + i) != NULL)
640 add_pack_file(*(argv + i++));
641
642 if (local_packs == NULL)
643 die("Zero packs found!");
644
645 load_all_objects();
646
647 cmp_local_packs();
648 if (alt_odb)
649 scan_alt_odb_packs();
650
651 /* ignore objects given on stdin */
652 llist_init(&ignore);
653 if (!isatty(0)) {
654 while (fgets(buf, sizeof(buf), stdin)) {
655 sha1 = xmalloc(20);
656 if (get_sha1_hex(buf, sha1))
657 die("Bad sha1 on stdin: %s", buf);
658 llist_insert_sorted_unique(ignore, sha1, NULL);
659 }
660 }
661 llist_sorted_difference_inplace(all_objects, ignore);
662 pl = local_packs;
663 while (pl) {
664 llist_sorted_difference_inplace(pl->unique_objects, ignore);
665 pl = pl->next;
666 }
667
668 minimize(&min);
669
670 if (verbose) {
671 fprintf(stderr, "There are %lu packs available in alt-odbs.\n",
672 (unsigned long)pack_list_size(altodb_packs));
673 fprintf(stderr, "The smallest (bytewise) set of packs is:\n");
674 pl = min;
675 while (pl) {
676 fprintf(stderr, "\t%s\n", pl->pack->pack_name);
677 pl = pl->next;
678 }
679 fprintf(stderr, "containing %lu duplicate objects "
680 "with a total size of %lukb.\n",
681 (unsigned long)get_pack_redundancy(min),
682 (unsigned long)pack_set_bytecount(min)/1024);
683 fprintf(stderr, "A total of %lu unique objects were considered.\n",
684 (unsigned long)all_objects->size);
685 fprintf(stderr, "Redundant packs (with indexes):\n");
686 }
687 pl = red = pack_list_difference(local_packs, min);
688 while (pl) {
689 printf("%s\n%s\n",
690 sha1_pack_index_name(pl->pack->sha1),
691 pl->pack->pack_name);
692 pl = pl->next;
693 }
694 if (verbose)
695 fprintf(stderr, "%luMB of redundant packs in total.\n",
696 (unsigned long)pack_set_bytecount(red)/(1024*1024));
697
698 return 0;
699 }