pack: move unpack_object_header()
[git/git.git] / packfile.c
CommitLineData
4f39cd82 1#include "cache.h"
6d6a80e0 2#include "mru.h"
0317f455 3#include "pack.h"
0abe14f6
JT
4#include "dir.h"
5#include "mergesort.h"
6#include "packfile.h"
7b3aa75d 7#include "delta.h"
4f39cd82
JT
8
9char *odb_pack_name(struct strbuf *buf,
10 const unsigned char *sha1,
11 const char *ext)
12{
13 strbuf_reset(buf);
14 strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
15 sha1_to_hex(sha1), ext);
16 return buf->buf;
17}
18
19char *sha1_pack_name(const unsigned char *sha1)
20{
21 static struct strbuf buf = STRBUF_INIT;
22 return odb_pack_name(&buf, sha1, "pack");
23}
24
25char *sha1_pack_index_name(const unsigned char *sha1)
26{
27 static struct strbuf buf = STRBUF_INIT;
28 return odb_pack_name(&buf, sha1, "idx");
29}
6d6a80e0 30
84f80ad5
JT
31static unsigned int pack_used_ctr;
32static unsigned int pack_mmap_calls;
33static unsigned int peak_pack_open_windows;
34static unsigned int pack_open_windows;
e65f1862 35static unsigned int pack_open_fds;
84f80ad5
JT
36static unsigned int pack_max_fds;
37static size_t peak_pack_mapped;
38static size_t pack_mapped;
6d6a80e0
JT
39struct packed_git *packed_git;
40
41static struct mru packed_git_mru_storage;
42struct mru *packed_git_mru = &packed_git_mru_storage;
8e21176c
JT
43
44#define SZ_FMT PRIuMAX
45static inline uintmax_t sz_fmt(size_t s) { return s; }
46
47void pack_report(void)
48{
49 fprintf(stderr,
50 "pack_report: getpagesize() = %10" SZ_FMT "\n"
51 "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
52 "pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
53 sz_fmt(getpagesize()),
54 sz_fmt(packed_git_window_size),
55 sz_fmt(packed_git_limit));
56 fprintf(stderr,
57 "pack_report: pack_used_ctr = %10u\n"
58 "pack_report: pack_mmap_calls = %10u\n"
59 "pack_report: pack_open_windows = %10u / %10u\n"
60 "pack_report: pack_mapped = "
61 "%10" SZ_FMT " / %10" SZ_FMT "\n",
62 pack_used_ctr,
63 pack_mmap_calls,
64 pack_open_windows, peak_pack_open_windows,
65 sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
66}
0317f455
JT
67
68/*
69 * Open and mmap the index file at path, perform a couple of
70 * consistency checks, then record its information to p. Return 0 on
71 * success.
72 */
73static int check_packed_git_idx(const char *path, struct packed_git *p)
74{
75 void *idx_map;
76 struct pack_idx_header *hdr;
77 size_t idx_size;
78 uint32_t version, nr, i, *index;
79 int fd = git_open(path);
80 struct stat st;
81
82 if (fd < 0)
83 return -1;
84 if (fstat(fd, &st)) {
85 close(fd);
86 return -1;
87 }
88 idx_size = xsize_t(st.st_size);
89 if (idx_size < 4 * 256 + 20 + 20) {
90 close(fd);
91 return error("index file %s is too small", path);
92 }
93 idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
94 close(fd);
95
96 hdr = idx_map;
97 if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
98 version = ntohl(hdr->idx_version);
99 if (version < 2 || version > 2) {
100 munmap(idx_map, idx_size);
101 return error("index file %s is version %"PRIu32
102 " and is not supported by this binary"
103 " (try upgrading GIT to a newer version)",
104 path, version);
105 }
106 } else
107 version = 1;
108
109 nr = 0;
110 index = idx_map;
111 if (version > 1)
112 index += 2; /* skip index header */
113 for (i = 0; i < 256; i++) {
114 uint32_t n = ntohl(index[i]);
115 if (n < nr) {
116 munmap(idx_map, idx_size);
117 return error("non-monotonic index %s", path);
118 }
119 nr = n;
120 }
121
122 if (version == 1) {
123 /*
124 * Total size:
125 * - 256 index entries 4 bytes each
126 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
127 * - 20-byte SHA1 of the packfile
128 * - 20-byte SHA1 file checksum
129 */
130 if (idx_size != 4*256 + nr * 24 + 20 + 20) {
131 munmap(idx_map, idx_size);
132 return error("wrong index v1 file size in %s", path);
133 }
134 } else if (version == 2) {
135 /*
136 * Minimum size:
137 * - 8 bytes of header
138 * - 256 index entries 4 bytes each
139 * - 20-byte sha1 entry * nr
140 * - 4-byte crc entry * nr
141 * - 4-byte offset entry * nr
142 * - 20-byte SHA1 of the packfile
143 * - 20-byte SHA1 file checksum
144 * And after the 4-byte offset table might be a
145 * variable sized table containing 8-byte entries
146 * for offsets larger than 2^31.
147 */
148 unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
149 unsigned long max_size = min_size;
150 if (nr)
151 max_size += (nr - 1)*8;
152 if (idx_size < min_size || idx_size > max_size) {
153 munmap(idx_map, idx_size);
154 return error("wrong index v2 file size in %s", path);
155 }
156 if (idx_size != min_size &&
157 /*
158 * make sure we can deal with large pack offsets.
159 * 31-bit signed offset won't be enough, neither
160 * 32-bit unsigned one will be.
161 */
162 (sizeof(off_t) <= 4)) {
163 munmap(idx_map, idx_size);
164 return error("pack too large for current definition of off_t in %s", path);
165 }
166 }
167
168 p->index_version = version;
169 p->index_data = idx_map;
170 p->index_size = idx_size;
171 p->num_objects = nr;
172 return 0;
173}
174
175int open_pack_index(struct packed_git *p)
176{
177 char *idx_name;
178 size_t len;
179 int ret;
180
181 if (p->index_data)
182 return 0;
183
184 if (!strip_suffix(p->pack_name, ".pack", &len))
185 die("BUG: pack_name does not end in .pack");
186 idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
187 ret = check_packed_git_idx(idx_name, p);
188 free(idx_name);
189 return ret;
190}
191
192static struct packed_git *alloc_packed_git(int extra)
193{
194 struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
195 memset(p, 0, sizeof(*p));
196 p->pack_fd = -1;
197 return p;
198}
199
200struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
201{
202 const char *path = sha1_pack_name(sha1);
203 size_t alloc = st_add(strlen(path), 1);
204 struct packed_git *p = alloc_packed_git(alloc);
205
206 memcpy(p->pack_name, path, alloc); /* includes NUL */
207 hashcpy(p->sha1, sha1);
208 if (check_packed_git_idx(idx_path, p)) {
209 free(p);
210 return NULL;
211 }
212
213 return p;
214}
f0e17e86
JT
215
216static void scan_windows(struct packed_git *p,
217 struct packed_git **lru_p,
218 struct pack_window **lru_w,
219 struct pack_window **lru_l)
220{
221 struct pack_window *w, *w_l;
222
223 for (w_l = NULL, w = p->windows; w; w = w->next) {
224 if (!w->inuse_cnt) {
225 if (!*lru_w || w->last_used < (*lru_w)->last_used) {
226 *lru_p = p;
227 *lru_w = w;
228 *lru_l = w_l;
229 }
230 }
231 w_l = w;
232 }
233}
234
84f80ad5 235static int unuse_one_window(struct packed_git *current)
f0e17e86
JT
236{
237 struct packed_git *p, *lru_p = NULL;
238 struct pack_window *lru_w = NULL, *lru_l = NULL;
239
240 if (current)
241 scan_windows(current, &lru_p, &lru_w, &lru_l);
242 for (p = packed_git; p; p = p->next)
243 scan_windows(p, &lru_p, &lru_w, &lru_l);
244 if (lru_p) {
245 munmap(lru_w->base, lru_w->len);
246 pack_mapped -= lru_w->len;
247 if (lru_l)
248 lru_l->next = lru_w->next;
249 else
250 lru_p->windows = lru_w->next;
251 free(lru_w);
252 pack_open_windows--;
253 return 1;
254 }
255 return 0;
256}
257
258void release_pack_memory(size_t need)
259{
260 size_t cur = pack_mapped;
261 while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
262 ; /* nothing */
263}
3836d88a
JT
264
265void close_pack_windows(struct packed_git *p)
266{
267 while (p->windows) {
268 struct pack_window *w = p->windows;
269
270 if (w->inuse_cnt)
271 die("pack '%s' still has open windows to it",
272 p->pack_name);
273 munmap(w->base, w->len);
274 pack_mapped -= w->len;
275 pack_open_windows--;
276 p->windows = w->next;
277 free(w);
278 }
279}
280
84f80ad5 281static int close_pack_fd(struct packed_git *p)
3836d88a
JT
282{
283 if (p->pack_fd < 0)
284 return 0;
285
286 close(p->pack_fd);
287 pack_open_fds--;
288 p->pack_fd = -1;
289
290 return 1;
291}
292
293void close_pack_index(struct packed_git *p)
294{
295 if (p->index_data) {
296 munmap((void *)p->index_data, p->index_size);
297 p->index_data = NULL;
298 }
299}
300
301static void close_pack(struct packed_git *p)
302{
303 close_pack_windows(p);
304 close_pack_fd(p);
305 close_pack_index(p);
306}
307
308void close_all_packs(void)
309{
310 struct packed_git *p;
311
312 for (p = packed_git; p; p = p->next)
313 if (p->do_not_close)
314 die("BUG: want to close pack marked 'do-not-close'");
315 else
316 close_pack(p);
317}
84f80ad5
JT
318
319/*
320 * The LRU pack is the one with the oldest MRU window, preferring packs
321 * with no used windows, or the oldest mtime if it has no windows allocated.
322 */
323static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
324{
325 struct pack_window *w, *this_mru_w;
326 int has_windows_inuse = 0;
327
328 /*
329 * Reject this pack if it has windows and the previously selected
330 * one does not. If this pack does not have windows, reject
331 * it if the pack file is newer than the previously selected one.
332 */
333 if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
334 return;
335
336 for (w = this_mru_w = p->windows; w; w = w->next) {
337 /*
338 * Reject this pack if any of its windows are in use,
339 * but the previously selected pack did not have any
340 * inuse windows. Otherwise, record that this pack
341 * has windows in use.
342 */
343 if (w->inuse_cnt) {
344 if (*accept_windows_inuse)
345 has_windows_inuse = 1;
346 else
347 return;
348 }
349
350 if (w->last_used > this_mru_w->last_used)
351 this_mru_w = w;
352
353 /*
354 * Reject this pack if it has windows that have been
355 * used more recently than the previously selected pack.
356 * If the previously selected pack had windows inuse and
357 * we have not encountered a window in this pack that is
358 * inuse, skip this check since we prefer a pack with no
359 * inuse windows to one that has inuse windows.
360 */
361 if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
362 this_mru_w->last_used > (*mru_w)->last_used)
363 return;
364 }
365
366 /*
367 * Select this pack.
368 */
369 *mru_w = this_mru_w;
370 *lru_p = p;
371 *accept_windows_inuse = has_windows_inuse;
372}
373
374static int close_one_pack(void)
375{
376 struct packed_git *p, *lru_p = NULL;
377 struct pack_window *mru_w = NULL;
378 int accept_windows_inuse = 1;
379
380 for (p = packed_git; p; p = p->next) {
381 if (p->pack_fd == -1)
382 continue;
383 find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
384 }
385
386 if (lru_p)
387 return close_pack_fd(lru_p);
388
389 return 0;
390}
391
392static unsigned int get_max_fd_limit(void)
393{
394#ifdef RLIMIT_NOFILE
395 {
396 struct rlimit lim;
397
398 if (!getrlimit(RLIMIT_NOFILE, &lim))
399 return lim.rlim_cur;
400 }
401#endif
402
403#ifdef _SC_OPEN_MAX
404 {
405 long open_max = sysconf(_SC_OPEN_MAX);
406 if (0 < open_max)
407 return open_max;
408 /*
409 * Otherwise, we got -1 for one of the two
410 * reasons:
411 *
412 * (1) sysconf() did not understand _SC_OPEN_MAX
413 * and signaled an error with -1; or
414 * (2) sysconf() said there is no limit.
415 *
416 * We _could_ clear errno before calling sysconf() to
417 * tell these two cases apart and return a huge number
418 * in the latter case to let the caller cap it to a
419 * value that is not so selfish, but letting the
420 * fallback OPEN_MAX codepath take care of these cases
421 * is a lot simpler.
422 */
423 }
424#endif
425
426#ifdef OPEN_MAX
427 return OPEN_MAX;
428#else
429 return 1; /* see the caller ;-) */
430#endif
431}
432
433/*
434 * Do not call this directly as this leaks p->pack_fd on error return;
435 * call open_packed_git() instead.
436 */
437static int open_packed_git_1(struct packed_git *p)
438{
439 struct stat st;
440 struct pack_header hdr;
441 unsigned char sha1[20];
442 unsigned char *idx_sha1;
443 long fd_flag;
444
445 if (!p->index_data && open_pack_index(p))
446 return error("packfile %s index unavailable", p->pack_name);
447
448 if (!pack_max_fds) {
449 unsigned int max_fds = get_max_fd_limit();
450
451 /* Save 3 for stdin/stdout/stderr, 22 for work */
452 if (25 < max_fds)
453 pack_max_fds = max_fds - 25;
454 else
455 pack_max_fds = 1;
456 }
457
458 while (pack_max_fds <= pack_open_fds && close_one_pack())
459 ; /* nothing */
460
461 p->pack_fd = git_open(p->pack_name);
462 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
463 return -1;
464 pack_open_fds++;
465
466 /* If we created the struct before we had the pack we lack size. */
467 if (!p->pack_size) {
468 if (!S_ISREG(st.st_mode))
469 return error("packfile %s not a regular file", p->pack_name);
470 p->pack_size = st.st_size;
471 } else if (p->pack_size != st.st_size)
472 return error("packfile %s size changed", p->pack_name);
473
474 /* We leave these file descriptors open with sliding mmap;
475 * there is no point keeping them open across exec(), though.
476 */
477 fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
478 if (fd_flag < 0)
479 return error("cannot determine file descriptor flags");
480 fd_flag |= FD_CLOEXEC;
481 if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
482 return error("cannot set FD_CLOEXEC");
483
484 /* Verify we recognize this pack file format. */
485 if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
486 return error("file %s is far too short to be a packfile", p->pack_name);
487 if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
488 return error("file %s is not a GIT packfile", p->pack_name);
489 if (!pack_version_ok(hdr.hdr_version))
490 return error("packfile %s is version %"PRIu32" and not"
491 " supported (try upgrading GIT to a newer version)",
492 p->pack_name, ntohl(hdr.hdr_version));
493
494 /* Verify the pack matches its index. */
495 if (p->num_objects != ntohl(hdr.hdr_entries))
496 return error("packfile %s claims to have %"PRIu32" objects"
497 " while index indicates %"PRIu32" objects",
498 p->pack_name, ntohl(hdr.hdr_entries),
499 p->num_objects);
500 if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
501 return error("end of packfile %s is unavailable", p->pack_name);
502 if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
503 return error("packfile %s signature is unavailable", p->pack_name);
504 idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
505 if (hashcmp(sha1, idx_sha1))
506 return error("packfile %s does not match index", p->pack_name);
507 return 0;
508}
509
510int open_packed_git(struct packed_git *p)
511{
512 if (!open_packed_git_1(p))
513 return 0;
514 close_pack_fd(p);
515 return -1;
516}
517
518static int in_window(struct pack_window *win, off_t offset)
519{
520 /* We must promise at least 20 bytes (one hash) after the
521 * offset is available from this window, otherwise the offset
522 * is not actually in this window and a different window (which
523 * has that one hash excess) must be used. This is to support
524 * the object header and delta base parsing routines below.
525 */
526 off_t win_off = win->offset;
527 return win_off <= offset
528 && (offset + 20) <= (win_off + win->len);
529}
530
531unsigned char *use_pack(struct packed_git *p,
532 struct pack_window **w_cursor,
533 off_t offset,
534 unsigned long *left)
535{
536 struct pack_window *win = *w_cursor;
537
538 /* Since packfiles end in a hash of their content and it's
539 * pointless to ask for an offset into the middle of that
540 * hash, and the in_window function above wouldn't match
541 * don't allow an offset too close to the end of the file.
542 */
543 if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
544 die("packfile %s cannot be accessed", p->pack_name);
545 if (offset > (p->pack_size - 20))
546 die("offset beyond end of packfile (truncated pack?)");
547 if (offset < 0)
548 die(_("offset before end of packfile (broken .idx?)"));
549
550 if (!win || !in_window(win, offset)) {
551 if (win)
552 win->inuse_cnt--;
553 for (win = p->windows; win; win = win->next) {
554 if (in_window(win, offset))
555 break;
556 }
557 if (!win) {
558 size_t window_align = packed_git_window_size / 2;
559 off_t len;
560
561 if (p->pack_fd == -1 && open_packed_git(p))
562 die("packfile %s cannot be accessed", p->pack_name);
563
564 win = xcalloc(1, sizeof(*win));
565 win->offset = (offset / window_align) * window_align;
566 len = p->pack_size - win->offset;
567 if (len > packed_git_window_size)
568 len = packed_git_window_size;
569 win->len = (size_t)len;
570 pack_mapped += win->len;
571 while (packed_git_limit < pack_mapped
572 && unuse_one_window(p))
573 ; /* nothing */
574 win->base = xmmap(NULL, win->len,
575 PROT_READ, MAP_PRIVATE,
576 p->pack_fd, win->offset);
577 if (win->base == MAP_FAILED)
578 die_errno("packfile %s cannot be mapped",
579 p->pack_name);
580 if (!win->offset && win->len == p->pack_size
581 && !p->do_not_close)
582 close_pack_fd(p);
583 pack_mmap_calls++;
584 pack_open_windows++;
585 if (pack_mapped > peak_pack_mapped)
586 peak_pack_mapped = pack_mapped;
587 if (pack_open_windows > peak_pack_open_windows)
588 peak_pack_open_windows = pack_open_windows;
589 win->next = p->windows;
590 p->windows = win;
591 }
592 }
593 if (win != *w_cursor) {
594 win->last_used = pack_used_ctr++;
595 win->inuse_cnt++;
596 *w_cursor = win;
597 }
598 offset -= win->offset;
599 if (left)
600 *left = win->len - xsize_t(offset);
601 return win->base + offset;
602}
97de1803
JT
603
604void unuse_pack(struct pack_window **w_cursor)
605{
606 struct pack_window *w = *w_cursor;
607 if (w) {
608 w->inuse_cnt--;
609 *w_cursor = NULL;
610 }
611}
9a428653
JT
612
613static void try_to_free_pack_memory(size_t size)
614{
615 release_pack_memory(size);
616}
617
618struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
619{
620 static int have_set_try_to_free_routine;
621 struct stat st;
622 size_t alloc;
623 struct packed_git *p;
624
625 if (!have_set_try_to_free_routine) {
626 have_set_try_to_free_routine = 1;
627 set_try_to_free_routine(try_to_free_pack_memory);
628 }
629
630 /*
631 * Make sure a corresponding .pack file exists and that
632 * the index looks sane.
633 */
634 if (!strip_suffix_mem(path, &path_len, ".idx"))
635 return NULL;
636
637 /*
638 * ".pack" is long enough to hold any suffix we're adding (and
639 * the use xsnprintf double-checks that)
640 */
641 alloc = st_add3(path_len, strlen(".pack"), 1);
642 p = alloc_packed_git(alloc);
643 memcpy(p->pack_name, path, path_len);
644
645 xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
646 if (!access(p->pack_name, F_OK))
647 p->pack_keep = 1;
648
649 xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
650 if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
651 free(p);
652 return NULL;
653 }
654
655 /* ok, it looks sane as far as we can check without
656 * actually mapping the pack file.
657 */
658 p->pack_size = st.st_size;
659 p->pack_local = local;
660 p->mtime = st.st_mtime;
661 if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
662 hashclr(p->sha1);
663 return p;
664}
e65f1862
JT
665
666void install_packed_git(struct packed_git *pack)
667{
668 if (pack->pack_fd != -1)
669 pack_open_fds++;
670
671 pack->next = packed_git;
672 packed_git = pack;
673}
0abe14f6
JT
674
675void (*report_garbage)(unsigned seen_bits, const char *path);
676
677static void report_helper(const struct string_list *list,
678 int seen_bits, int first, int last)
679{
680 if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
681 return;
682
683 for (; first < last; first++)
684 report_garbage(seen_bits, list->items[first].string);
685}
686
687static void report_pack_garbage(struct string_list *list)
688{
689 int i, baselen = -1, first = 0, seen_bits = 0;
690
691 if (!report_garbage)
692 return;
693
694 string_list_sort(list);
695
696 for (i = 0; i < list->nr; i++) {
697 const char *path = list->items[i].string;
698 if (baselen != -1 &&
699 strncmp(path, list->items[first].string, baselen)) {
700 report_helper(list, seen_bits, first, i);
701 baselen = -1;
702 seen_bits = 0;
703 }
704 if (baselen == -1) {
705 const char *dot = strrchr(path, '.');
706 if (!dot) {
707 report_garbage(PACKDIR_FILE_GARBAGE, path);
708 continue;
709 }
710 baselen = dot - path + 1;
711 first = i;
712 }
713 if (!strcmp(path + baselen, "pack"))
714 seen_bits |= 1;
715 else if (!strcmp(path + baselen, "idx"))
716 seen_bits |= 2;
717 }
718 report_helper(list, seen_bits, first, list->nr);
719}
720
721static void prepare_packed_git_one(char *objdir, int local)
722{
723 struct strbuf path = STRBUF_INIT;
724 size_t dirnamelen;
725 DIR *dir;
726 struct dirent *de;
727 struct string_list garbage = STRING_LIST_INIT_DUP;
728
729 strbuf_addstr(&path, objdir);
730 strbuf_addstr(&path, "/pack");
731 dir = opendir(path.buf);
732 if (!dir) {
733 if (errno != ENOENT)
734 error_errno("unable to open object pack directory: %s",
735 path.buf);
736 strbuf_release(&path);
737 return;
738 }
739 strbuf_addch(&path, '/');
740 dirnamelen = path.len;
741 while ((de = readdir(dir)) != NULL) {
742 struct packed_git *p;
743 size_t base_len;
744
745 if (is_dot_or_dotdot(de->d_name))
746 continue;
747
748 strbuf_setlen(&path, dirnamelen);
749 strbuf_addstr(&path, de->d_name);
750
751 base_len = path.len;
752 if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
753 /* Don't reopen a pack we already have. */
754 for (p = packed_git; p; p = p->next) {
755 size_t len;
756 if (strip_suffix(p->pack_name, ".pack", &len) &&
757 len == base_len &&
758 !memcmp(p->pack_name, path.buf, len))
759 break;
760 }
761 if (p == NULL &&
762 /*
763 * See if it really is a valid .idx file with
764 * corresponding .pack file that we can map.
765 */
766 (p = add_packed_git(path.buf, path.len, local)) != NULL)
767 install_packed_git(p);
768 }
769
770 if (!report_garbage)
771 continue;
772
773 if (ends_with(de->d_name, ".idx") ||
774 ends_with(de->d_name, ".pack") ||
775 ends_with(de->d_name, ".bitmap") ||
776 ends_with(de->d_name, ".keep"))
777 string_list_append(&garbage, path.buf);
778 else
779 report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
780 }
781 closedir(dir);
782 report_pack_garbage(&garbage);
783 string_list_clear(&garbage, 0);
784 strbuf_release(&path);
785}
786
787static int approximate_object_count_valid;
788
789/*
790 * Give a fast, rough count of the number of objects in the repository. This
791 * ignores loose objects completely. If you have a lot of them, then either
792 * you should repack because your performance will be awful, or they are
793 * all unreachable objects about to be pruned, in which case they're not really
794 * interesting as a measure of repo size in the first place.
795 */
796unsigned long approximate_object_count(void)
797{
798 static unsigned long count;
799 if (!approximate_object_count_valid) {
800 struct packed_git *p;
801
802 prepare_packed_git();
803 count = 0;
804 for (p = packed_git; p; p = p->next) {
805 if (open_pack_index(p))
806 continue;
807 count += p->num_objects;
808 }
809 }
810 return count;
811}
812
813static void *get_next_packed_git(const void *p)
814{
815 return ((const struct packed_git *)p)->next;
816}
817
818static void set_next_packed_git(void *p, void *next)
819{
820 ((struct packed_git *)p)->next = next;
821}
822
823static int sort_pack(const void *a_, const void *b_)
824{
825 const struct packed_git *a = a_;
826 const struct packed_git *b = b_;
827 int st;
828
829 /*
830 * Local packs tend to contain objects specific to our
831 * variant of the project than remote ones. In addition,
832 * remote ones could be on a network mounted filesystem.
833 * Favor local ones for these reasons.
834 */
835 st = a->pack_local - b->pack_local;
836 if (st)
837 return -st;
838
839 /*
840 * Younger packs tend to contain more recent objects,
841 * and more recent objects tend to get accessed more
842 * often.
843 */
844 if (a->mtime < b->mtime)
845 return 1;
846 else if (a->mtime == b->mtime)
847 return 0;
848 return -1;
849}
850
851static void rearrange_packed_git(void)
852{
853 packed_git = llist_mergesort(packed_git, get_next_packed_git,
854 set_next_packed_git, sort_pack);
855}
856
857static void prepare_packed_git_mru(void)
858{
859 struct packed_git *p;
860
861 mru_clear(packed_git_mru);
862 for (p = packed_git; p; p = p->next)
863 mru_append(packed_git_mru, p);
864}
865
866static int prepare_packed_git_run_once = 0;
867void prepare_packed_git(void)
868{
869 struct alternate_object_database *alt;
870
871 if (prepare_packed_git_run_once)
872 return;
873 prepare_packed_git_one(get_object_directory(), 1);
874 prepare_alt_odb();
875 for (alt = alt_odb_list; alt; alt = alt->next)
876 prepare_packed_git_one(alt->path, 0);
877 rearrange_packed_git();
878 prepare_packed_git_mru();
879 prepare_packed_git_run_once = 1;
880}
881
882void reprepare_packed_git(void)
883{
884 approximate_object_count_valid = 0;
885 prepare_packed_git_run_once = 0;
886 prepare_packed_git();
887}
32b42e15
JT
888
889unsigned long unpack_object_header_buffer(const unsigned char *buf,
890 unsigned long len, enum object_type *type, unsigned long *sizep)
891{
892 unsigned shift;
893 unsigned long size, c;
894 unsigned long used = 0;
895
896 c = buf[used++];
897 *type = (c >> 4) & 7;
898 size = c & 15;
899 shift = 4;
900 while (c & 0x80) {
901 if (len <= used || bitsizeof(long) <= shift) {
902 error("bad object header");
903 size = used = 0;
904 break;
905 }
906 c = buf[used++];
907 size += (c & 0x7f) << shift;
908 shift += 7;
909 }
910 *sizep = size;
911 return used;
912}
7b3aa75d
JT
913
914unsigned long get_size_from_delta(struct packed_git *p,
915 struct pack_window **w_curs,
916 off_t curpos)
917{
918 const unsigned char *data;
919 unsigned char delta_head[20], *in;
920 git_zstream stream;
921 int st;
922
923 memset(&stream, 0, sizeof(stream));
924 stream.next_out = delta_head;
925 stream.avail_out = sizeof(delta_head);
926
927 git_inflate_init(&stream);
928 do {
929 in = use_pack(p, w_curs, curpos, &stream.avail_in);
930 stream.next_in = in;
931 st = git_inflate(&stream, Z_FINISH);
932 curpos += stream.next_in - in;
933 } while ((st == Z_OK || st == Z_BUF_ERROR) &&
934 stream.total_out < sizeof(delta_head));
935 git_inflate_end(&stream);
936 if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
937 error("delta data unpack-initial failed");
938 return 0;
939 }
940
941 /* Examine the initial part of the delta to figure out
942 * the result size.
943 */
944 data = delta_head;
945
946 /* ignore base size */
947 get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
948
949 /* Read the result size */
950 return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
951}
3588dd6e
JT
952
953int unpack_object_header(struct packed_git *p,
954 struct pack_window **w_curs,
955 off_t *curpos,
956 unsigned long *sizep)
957{
958 unsigned char *base;
959 unsigned long left;
960 unsigned long used;
961 enum object_type type;
962
963 /* use_pack() assures us we have [base, base + 20) available
964 * as a range that we can look at. (Its actually the hash
965 * size that is assured.) With our object header encoding
966 * the maximum deflated object size is 2^137, which is just
967 * insane, so we know won't exceed what we have been given.
968 */
969 base = use_pack(p, w_curs, *curpos, &left);
970 used = unpack_object_header_buffer(base, left, &type, sizep);
971 if (!used) {
972 type = OBJ_BAD;
973 } else
974 *curpos += used;
975
976 return type;
977}