cache_or_unpack_entry: drop keep_cache parameter
[git/git.git] / sha1_file.c
CommitLineData
0fcfd160
LT
1/*
2 * GIT - The information manager from hell
3 *
4 * Copyright (C) Linus Torvalds, 2005
5 *
6 * This handles basic git sha1 object files - packing, unpacking,
7 * creation etc.
8 */
0fcfd160 9#include "cache.h"
6eac50d8 10#include "string-list.h"
697cc8ef 11#include "lockfile.h"
1f688557 12#include "delta.h"
a733cb60 13#include "pack.h"
8e440259
PE
14#include "blob.h"
15#include "commit.h"
4dd1fbc7 16#include "run-command.h"
8e440259
PE
17#include "tag.h"
18#include "tree.h"
c879daa2 19#include "tree-walk.h"
f35a6d3b 20#include "refs.h"
70f5d5d3 21#include "pack-revindex.h"
628522ec 22#include "sha1-lookup.h"
568508e7 23#include "bulk-checkin.h"
090ea126 24#include "streaming.h"
543c5caa 25#include "dir.h"
a73cdd21 26#include "mru.h"
0fcfd160 27
144bde78
LT
28#ifndef O_NOATIME
29#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
30#define O_NOATIME 01000000
31#else
32#define O_NOATIME 0
33#endif
34#endif
35
28bd70d8
JN
36#define SZ_FMT PRIuMAX
37static inline uintmax_t sz_fmt(size_t s) { return s; }
e05db0fd 38
96f1e58f 39const unsigned char null_sha1[20];
3e56e724 40const struct object_id null_oid;
88cd621d 41
c597ba80
NTND
42/*
43 * This is meant to hold a *small* number of objects that you would
44 * want read_sha1_file() to be able to return, but yet you do not want
45 * to write them into the object store (e.g. a browse-only
46 * application).
47 */
48static struct cached_object {
49 unsigned char sha1[20];
50 enum object_type type;
51 void *buf;
52 unsigned long size;
53} *cached_objects;
54static int cached_object_nr, cached_object_alloc;
55
56static struct cached_object empty_tree = {
dab0d410 57 EMPTY_TREE_SHA1_BIN_LITERAL,
c597ba80
NTND
58 OBJ_TREE,
59 "",
60 0
61};
62
63static struct cached_object *find_cached_object(const unsigned char *sha1)
64{
65 int i;
66 struct cached_object *co = cached_objects;
67
68 for (i = 0; i < cached_object_nr; i++, co++) {
69 if (!hashcmp(co->sha1, sha1))
70 return co;
71 }
72 if (!hashcmp(sha1, empty_tree.sha1))
73 return &empty_tree;
74 return NULL;
75}
76
90a6464b
JH
77int mkdir_in_gitdir(const char *path)
78{
79 if (mkdir(path, 0777)) {
80 int saved_errno = errno;
81 struct stat st;
82 struct strbuf sb = STRBUF_INIT;
83
84 if (errno != EEXIST)
85 return -1;
86 /*
87 * Are we looking at a path in a symlinked worktree
88 * whose original repository does not yet have it?
89 * e.g. .git/rr-cache pointing at its original
90 * repository in which the user hasn't performed any
91 * conflict resolution yet?
92 */
93 if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
94 strbuf_readlink(&sb, path, st.st_size) ||
95 !is_absolute_path(sb.buf) ||
96 mkdir(sb.buf, 0777)) {
97 strbuf_release(&sb);
98 errno = saved_errno;
99 return -1;
100 }
101 strbuf_release(&sb);
102 }
103 return adjust_shared_perm(path);
104}
105
0be0521b 106enum scld_error safe_create_leading_directories(char *path)
b2cb9425 107{
26c8ae2a 108 char *next_component = path + offset_1st_component(path);
0be0521b 109 enum scld_error ret = SCLD_OK;
67d42212 110
0be0521b 111 while (ret == SCLD_OK && next_component) {
f0502332 112 struct stat st;
0f527403 113 char *slash = next_component, slash_character;
f0502332 114
0f527403
MH
115 while (*slash && !is_dir_sep(*slash))
116 slash++;
117
118 if (!*slash)
b2cb9425 119 break;
bf10cf70 120
26c8ae2a 121 next_component = slash + 1;
0f527403 122 while (is_dir_sep(*next_component))
bf10cf70 123 next_component++;
26c8ae2a 124 if (!*next_component)
5f0bdf50 125 break;
831651fd 126
0f527403 127 slash_character = *slash;
831651fd 128 *slash = '\0';
67d42212
JR
129 if (!stat(path, &st)) {
130 /* path exists */
9e6f885d 131 if (!S_ISDIR(st.st_mode))
0be0521b 132 ret = SCLD_EXISTS;
53a39721 133 } else if (mkdir(path, 0777)) {
928734d9 134 if (errno == EEXIST &&
9e6f885d 135 !stat(path, &st) && S_ISDIR(st.st_mode))
928734d9 136 ; /* somebody created it since we checked */
18d37e86
MH
137 else if (errno == ENOENT)
138 /*
139 * Either mkdir() failed because
140 * somebody just pruned the containing
141 * directory, or stat() failed because
142 * the file that was in our way was
143 * just removed. Either way, inform
144 * the caller that it might be worth
145 * trying again:
146 */
147 ret = SCLD_VANISHED;
9e6f885d 148 else
0be0521b 149 ret = SCLD_FAILED;
53a39721 150 } else if (adjust_shared_perm(path)) {
0be0521b 151 ret = SCLD_PERMS;
457f06d6 152 }
0f527403 153 *slash = slash_character;
b2cb9425 154 }
9e6f885d 155 return ret;
b2cb9425 156}
723c31fe 157
0be0521b 158enum scld_error safe_create_leading_directories_const(const char *path)
8e21d63b
JK
159{
160 /* path points to cache entries, so xstrdup before messing with it */
161 char *buf = xstrdup(path);
0be0521b 162 enum scld_error result = safe_create_leading_directories(buf);
8e21d63b
JK
163 free(buf);
164 return result;
165}
166
ace1534d
JH
167static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
168{
169 int i;
170 for (i = 0; i < 20; i++) {
171 static char hex[] = "0123456789abcdef";
172 unsigned int val = sha1[i];
173 char *pos = pathbuf + i*2 + (i > 0);
174 *pos++ = hex[val >> 4];
175 *pos = hex[val & 0xf];
176 }
177}
178
30d6c6ea 179const char *sha1_file_name(const unsigned char *sha1)
0fcfd160 180{
560fb6a1
JK
181 static char buf[PATH_MAX];
182 const char *objdir;
183 int len;
0fcfd160 184
560fb6a1
JK
185 objdir = get_object_directory();
186 len = strlen(objdir);
187
188 /* '/' + sha1(2) + '/' + sha1(38) + '\0' */
189 if (len + 43 > PATH_MAX)
190 die("insanely long object directory %s", objdir);
191 memcpy(buf, objdir, len);
192 buf[len] = '/';
193 buf[len+3] = '/';
194 buf[len+42] = '\0';
195 fill_sha1_path(buf + len + 1, sha1);
196 return buf;
0fcfd160
LT
197}
198
d40d535b
MH
199/*
200 * Return the name of the pack or index file with the specified sha1
201 * in its filename. *base and *name are scratch space that must be
202 * provided by the caller. which should be "pack" or "idx".
203 */
633f43e1 204static char *sha1_get_pack_name(const unsigned char *sha1,
ac5190cc
JK
205 struct strbuf *buf,
206 const char *which)
bf592c50 207{
ac5190cc
JK
208 strbuf_reset(buf);
209 strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
210 sha1_to_hex(sha1), which);
211 return buf->buf;
bf592c50
DB
212}
213
633f43e1 214char *sha1_pack_name(const unsigned char *sha1)
bf592c50 215{
ac5190cc
JK
216 static struct strbuf buf = STRBUF_INIT;
217 return sha1_get_pack_name(sha1, &buf, "pack");
633f43e1 218}
bf592c50 219
633f43e1
HO
220char *sha1_pack_index_name(const unsigned char *sha1)
221{
ac5190cc
JK
222 static struct strbuf buf = STRBUF_INIT;
223 return sha1_get_pack_name(sha1, &buf, "idx");
bf592c50
DB
224}
225
d5a63b99
JH
226struct alternate_object_database *alt_odb_list;
227static struct alternate_object_database **alt_odb_tail;
ace1534d 228
ddd5d056
JH
229/*
230 * Prepare alternate object database registry.
d5a63b99
JH
231 *
232 * The variable alt_odb_list points at the list of struct
233 * alternate_object_database. The elements on this list come from
234 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
235 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
1494e038
JH
236 * whose contents is similar to that environment variable but can be
237 * LF separated. Its base points at a statically allocated buffer that
d5a63b99
JH
238 * contains "/the/directory/corresponding/to/.git/objects/...", while
239 * its name points just after the slash at the end of ".git/objects/"
240 * in the example above, and has enough space to hold 40-byte hex
241 * SHA1, an extra slash for the first level indirection, and the
242 * terminating NUL.
ddd5d056 243 */
539e7506
EK
244static int link_alt_odb_entry(const char *entry, const char *relative_base,
245 int depth, const char *normalized_objdir)
ace1534d 246{
c2f493a4
MW
247 struct alternate_object_database *ent;
248 struct alternate_object_database *alt;
50a6c8ef 249 size_t pfxlen, entlen;
5bdf0a84 250 struct strbuf pathbuf = STRBUF_INIT;
d5a63b99 251
85dadc38 252 if (!is_absolute_path(entry) && relative_base) {
5bdf0a84
HW
253 strbuf_addstr(&pathbuf, real_path(relative_base));
254 strbuf_addch(&pathbuf, '/');
c2f493a4 255 }
6eac50d8 256 strbuf_addstr(&pathbuf, entry);
c2f493a4 257
5bdf0a84
HW
258 normalize_path_copy(pathbuf.buf, pathbuf.buf);
259
260 pfxlen = strlen(pathbuf.buf);
261
262 /*
263 * The trailing slash after the directory name is given by
264 * this function at the end. Remove duplicates.
265 */
266 while (pfxlen && pathbuf.buf[pfxlen-1] == '/')
267 pfxlen -= 1;
268
50a6c8ef
JK
269 entlen = st_add(pfxlen, 43); /* '/' + 2 hex + '/' + 38 hex + NUL */
270 ent = xmalloc(st_add(sizeof(*ent), entlen));
5bdf0a84
HW
271 memcpy(ent->base, pathbuf.buf, pfxlen);
272 strbuf_release(&pathbuf);
c2f493a4
MW
273
274 ent->name = ent->base + pfxlen + 1;
275 ent->base[pfxlen + 3] = '/';
276 ent->base[pfxlen] = ent->base[entlen-1] = 0;
277
278 /* Detect cases where alternate disappeared */
90b4a71c 279 if (!is_directory(ent->base)) {
c2f493a4
MW
280 error("object directory %s does not exist; "
281 "check .git/objects/info/alternates.",
282 ent->base);
283 free(ent);
284 return -1;
285 }
286
287 /* Prevent the common mistake of listing the same
288 * thing twice, or object directory itself.
289 */
290 for (alt = alt_odb_list; alt; alt = alt->next) {
80b47854
RS
291 if (pfxlen == alt->name - alt->base - 1 &&
292 !memcmp(ent->base, alt->base, pfxlen)) {
c2f493a4
MW
293 free(ent);
294 return -1;
295 }
296 }
ba0897e6 297 if (!fspathcmp(ent->base, normalized_objdir)) {
c2f493a4
MW
298 free(ent);
299 return -1;
300 }
301
302 /* add the alternate entry */
303 *alt_odb_tail = ent;
304 alt_odb_tail = &(ent->next);
305 ent->next = NULL;
306
307 /* recursively add alternates */
308 read_info_alternates(ent->base, depth + 1);
309
310 ent->base[pfxlen] = '/';
311
312 return 0;
313}
314
c5950164 315static void link_alt_odb_entries(const char *alt, int len, int sep,
c2f493a4
MW
316 const char *relative_base, int depth)
317{
6eac50d8
MH
318 struct string_list entries = STRING_LIST_INIT_NODUP;
319 char *alt_copy;
320 int i;
539e7506 321 struct strbuf objdirbuf = STRBUF_INIT;
c2f493a4
MW
322
323 if (depth > 5) {
324 error("%s: ignoring alternate object stores, nesting too deep.",
325 relative_base);
326 return;
327 }
328
f655651e 329 strbuf_add_absolute_path(&objdirbuf, get_object_directory());
539e7506
EK
330 normalize_path_copy(objdirbuf.buf, objdirbuf.buf);
331
c5950164 332 alt_copy = xmemdupz(alt, len);
6eac50d8
MH
333 string_list_split_in_place(&entries, alt_copy, sep, -1);
334 for (i = 0; i < entries.nr; i++) {
335 const char *entry = entries.items[i].string;
336 if (entry[0] == '\0' || entry[0] == '#')
9577e7e3 337 continue;
6eac50d8
MH
338 if (!is_absolute_path(entry) && depth) {
339 error("%s: ignoring relative alternate object store %s",
340 relative_base, entry);
341 } else {
539e7506 342 link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf);
9577e7e3 343 }
9577e7e3 344 }
6eac50d8
MH
345 string_list_clear(&entries, 0);
346 free(alt_copy);
539e7506 347 strbuf_release(&objdirbuf);
d5a63b99
JH
348}
349
5e73633d 350void read_info_alternates(const char * relative_base, int depth)
d5a63b99 351{
9577e7e3 352 char *map;
dc49cd76 353 size_t mapsz;
d5a63b99 354 struct stat st;
5015f01c 355 char *path;
c2f493a4 356 int fd;
d5a63b99 357
5015f01c 358 path = xstrfmt("%s/info/alternates", relative_base);
f4e51683 359 fd = git_open_noatime(path);
5015f01c 360 free(path);
d5a63b99
JH
361 if (fd < 0)
362 return;
363 if (fstat(fd, &st) || (st.st_size == 0)) {
364 close(fd);
9a217f2a 365 return;
ace1534d 366 }
dc49cd76
SP
367 mapsz = xsize_t(st.st_size);
368 map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
d5a63b99 369 close(fd);
d5a63b99 370
c5950164 371 link_alt_odb_entries(map, mapsz, '\n', relative_base, depth);
c2f493a4 372
dc49cd76 373 munmap(map, mapsz);
ace1534d
JH
374}
375
bef70b22
DB
376void add_to_alternates_file(const char *reference)
377{
378 struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
77b9b1d1
JK
379 char *alts = git_pathdup("objects/info/alternates");
380 FILE *in, *out;
381
382 hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR);
383 out = fdopen_lock_file(lock, "w");
384 if (!out)
385 die_errno("unable to fdopen alternates lockfile");
386
387 in = fopen(alts, "r");
388 if (in) {
389 struct strbuf line = STRBUF_INIT;
390 int found = 0;
391
3f163962 392 while (strbuf_getline(&line, in) != EOF) {
77b9b1d1
JK
393 if (!strcmp(reference, line.buf)) {
394 found = 1;
395 break;
396 }
397 fprintf_or_die(out, "%s\n", line.buf);
398 }
399
400 strbuf_release(&line);
401 fclose(in);
402
403 if (found) {
404 rollback_lock_file(lock);
405 lock = NULL;
406 }
407 }
408 else if (errno != ENOENT)
409 die_errno("unable to read alternates file");
410
411 if (lock) {
412 fprintf_or_die(out, "%s\n", reference);
413 if (commit_lock_file(lock))
414 die_errno("unable to move new alternates file into place");
415 if (alt_odb_tail)
416 link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
417 }
418 free(alts);
bef70b22
DB
419}
420
fe1b2268 421int foreach_alt_odb(alt_odb_fn fn, void *cb)
d79796bc
JH
422{
423 struct alternate_object_database *ent;
fe1b2268 424 int r = 0;
d79796bc
JH
425
426 prepare_alt_odb();
fe1b2268
JK
427 for (ent = alt_odb_list; ent; ent = ent->next) {
428 r = fn(ent, cb);
429 if (r)
430 break;
431 }
432 return r;
d79796bc
JH
433}
434
c2f493a4
MW
435void prepare_alt_odb(void)
436{
554fe20d 437 const char *alt;
c2f493a4 438
7dc24aa5
SP
439 if (alt_odb_tail)
440 return;
441
c2f493a4
MW
442 alt = getenv(ALTERNATE_DB_ENVIRONMENT);
443 if (!alt) alt = "";
444
c2f493a4 445 alt_odb_tail = &alt_odb_list;
c5950164 446 link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
c2f493a4
MW
447
448 read_info_alternates(get_object_directory(), 0);
449}
450
3096b2ec 451/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
33d4221c 452static int freshen_file(const char *fn)
ace1534d 453{
33d4221c
JK
454 struct utimbuf t;
455 t.actime = t.modtime = time(NULL);
456 return !utime(fn, &t);
0f4dc14a 457}
ace1534d 458
3096b2ec
JK
459/*
460 * All of the check_and_freshen functions return 1 if the file exists and was
461 * freshened (if freshening was requested), 0 otherwise. If they return
462 * 0, you should not assume that it is safe to skip a write of the object (it
463 * either does not exist on disk, or has a stale mtime and may be subject to
464 * pruning).
465 */
33d4221c
JK
466static int check_and_freshen_file(const char *fn, int freshen)
467{
468 if (access(fn, F_OK))
469 return 0;
3096b2ec 470 if (freshen && !freshen_file(fn))
33d4221c
JK
471 return 0;
472 return 1;
473}
474
475static int check_and_freshen_local(const unsigned char *sha1, int freshen)
476{
477 return check_and_freshen_file(sha1_file_name(sha1), freshen);
478}
479
480static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
0f4dc14a
BC
481{
482 struct alternate_object_database *alt;
9a217f2a 483 prepare_alt_odb();
d5a63b99 484 for (alt = alt_odb_list; alt; alt = alt->next) {
0f4dc14a 485 fill_sha1_path(alt->name, sha1);
33d4221c 486 if (check_and_freshen_file(alt->base, freshen))
c529d75a 487 return 1;
ace1534d 488 }
c529d75a 489 return 0;
ace1534d
JH
490}
491
33d4221c
JK
492static int check_and_freshen(const unsigned char *sha1, int freshen)
493{
494 return check_and_freshen_local(sha1, freshen) ||
495 check_and_freshen_nonlocal(sha1, freshen);
496}
497
498int has_loose_object_nonlocal(const unsigned char *sha1)
499{
500 return check_and_freshen_nonlocal(sha1, 0);
501}
502
0f4dc14a
BC
503static int has_loose_object(const unsigned char *sha1)
504{
33d4221c 505 return check_and_freshen(sha1, 0);
0f4dc14a
BC
506}
507
60bb8b14 508static unsigned int pack_used_ctr;
a53128b6
SP
509static unsigned int pack_mmap_calls;
510static unsigned int peak_pack_open_windows;
511static unsigned int pack_open_windows;
c7934306
SP
512static unsigned int pack_open_fds;
513static unsigned int pack_max_fds;
a53128b6 514static size_t peak_pack_mapped;
60bb8b14 515static size_t pack_mapped;
9a217f2a 516struct packed_git *packed_git;
1f688557 517
a73cdd21
JK
518static struct mru packed_git_mru_storage;
519struct mru *packed_git_mru = &packed_git_mru_storage;
520
b79d18c9 521void pack_report(void)
a53128b6
SP
522{
523 fprintf(stderr,
e05db0fd
PR
524 "pack_report: getpagesize() = %10" SZ_FMT "\n"
525 "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
526 "pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
9e42d6a1
SP
527 sz_fmt(getpagesize()),
528 sz_fmt(packed_git_window_size),
529 sz_fmt(packed_git_limit));
a53128b6
SP
530 fprintf(stderr,
531 "pack_report: pack_used_ctr = %10u\n"
532 "pack_report: pack_mmap_calls = %10u\n"
533 "pack_report: pack_open_windows = %10u / %10u\n"
e05db0fd
PR
534 "pack_report: pack_mapped = "
535 "%10" SZ_FMT " / %10" SZ_FMT "\n",
a53128b6
SP
536 pack_used_ctr,
537 pack_mmap_calls,
538 pack_open_windows, peak_pack_open_windows,
9e42d6a1 539 sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
a53128b6
SP
540}
541
d40d535b
MH
542/*
543 * Open and mmap the index file at path, perform a couple of
544 * consistency checks, then record its information to p. Return 0 on
545 * success.
546 */
547static int check_packed_git_idx(const char *path, struct packed_git *p)
1f688557
JH
548{
549 void *idx_map;
42873078 550 struct pack_idx_header *hdr;
dc49cd76 551 size_t idx_size;
74e34e1f 552 uint32_t version, nr, i, *index;
f4e51683 553 int fd = git_open_noatime(path);
1f688557 554 struct stat st;
42873078 555
1f688557
JH
556 if (fd < 0)
557 return -1;
558 if (fstat(fd, &st)) {
559 close(fd);
560 return -1;
561 }
dc49cd76 562 idx_size = xsize_t(st.st_size);
2d88451b
SP
563 if (idx_size < 4 * 256 + 20 + 20) {
564 close(fd);
565 return error("index file %s is too small", path);
566 }
c4712e45 567 idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
1f688557 568 close(fd);
1f688557 569
42873078
NP
570 hdr = idx_map;
571 if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
74e34e1f
NP
572 version = ntohl(hdr->idx_version);
573 if (version < 2 || version > 2) {
574 munmap(idx_map, idx_size);
6e1c2344 575 return error("index file %s is version %"PRIu32
74e34e1f
NP
576 " and is not supported by this binary"
577 " (try upgrading GIT to a newer version)",
578 path, version);
579 }
580 } else
581 version = 1;
df1b059d 582
1f688557 583 nr = 0;
42873078 584 index = idx_map;
74e34e1f
NP
585 if (version > 1)
586 index += 2; /* skip index header */
1f688557 587 for (i = 0; i < 256; i++) {
326bf396 588 uint32_t n = ntohl(index[i]);
2d88451b
SP
589 if (n < nr) {
590 munmap(idx_map, idx_size);
df1b059d 591 return error("non-monotonic index %s", path);
2d88451b 592 }
1f688557
JH
593 nr = n;
594 }
595
74e34e1f
NP
596 if (version == 1) {
597 /*
598 * Total size:
599 * - 256 index entries 4 bytes each
600 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
601 * - 20-byte SHA1 of the packfile
602 * - 20-byte SHA1 file checksum
603 */
604 if (idx_size != 4*256 + nr * 24 + 20 + 20) {
605 munmap(idx_map, idx_size);
eef427a0 606 return error("wrong index v1 file size in %s", path);
74e34e1f
NP
607 }
608 } else if (version == 2) {
609 /*
610 * Minimum size:
611 * - 8 bytes of header
612 * - 256 index entries 4 bytes each
613 * - 20-byte sha1 entry * nr
614 * - 4-byte crc entry * nr
615 * - 4-byte offset entry * nr
616 * - 20-byte SHA1 of the packfile
617 * - 20-byte SHA1 file checksum
618 * And after the 4-byte offset table might be a
619 * variable sized table containing 8-byte entries
620 * for offsets larger than 2^31.
621 */
622 unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
1164f1e4
LT
623 unsigned long max_size = min_size;
624 if (nr)
625 max_size += (nr - 1)*8;
626 if (idx_size < min_size || idx_size > max_size) {
74e34e1f 627 munmap(idx_map, idx_size);
eef427a0 628 return error("wrong index v2 file size in %s", path);
74e34e1f 629 }
7109c889
JH
630 if (idx_size != min_size &&
631 /*
632 * make sure we can deal with large pack offsets.
633 * 31-bit signed offset won't be enough, neither
634 * 32-bit unsigned one will be.
635 */
636 (sizeof(off_t) <= 4)) {
637 munmap(idx_map, idx_size);
638 return error("pack too large for current definition of off_t in %s", path);
74e34e1f 639 }
2d88451b 640 }
1f688557 641
74e34e1f 642 p->index_version = version;
42873078
NP
643 p->index_data = idx_map;
644 p->index_size = idx_size;
57059091 645 p->num_objects = nr;
1f688557
JH
646 return 0;
647}
648
bc8e478a 649int open_pack_index(struct packed_git *p)
d079837e
SP
650{
651 char *idx_name;
9ae97018 652 size_t len;
d079837e
SP
653 int ret;
654
655 if (p->index_data)
656 return 0;
657
9ae97018
JK
658 if (!strip_suffix(p->pack_name, ".pack", &len))
659 die("BUG: pack_name does not end in .pack");
660 idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
d079837e
SP
661 ret = check_packed_git_idx(idx_name, p);
662 free(idx_name);
663 return ret;
664}
665
11daf39b
SP
666static void scan_windows(struct packed_git *p,
667 struct packed_git **lru_p,
668 struct pack_window **lru_w,
669 struct pack_window **lru_l)
1f688557 670{
11daf39b
SP
671 struct pack_window *w, *w_l;
672
673 for (w_l = NULL, w = p->windows; w; w = w->next) {
674 if (!w->inuse_cnt) {
675 if (!*lru_w || w->last_used < (*lru_w)->last_used) {
676 *lru_p = p;
677 *lru_w = w;
678 *lru_l = w_l;
54044bf8 679 }
54044bf8 680 }
11daf39b 681 w_l = w;
f9253394 682 }
11daf39b
SP
683}
684
7c3ecb32 685static int unuse_one_window(struct packed_git *current)
11daf39b
SP
686{
687 struct packed_git *p, *lru_p = NULL;
688 struct pack_window *lru_w = NULL, *lru_l = NULL;
689
690 if (current)
691 scan_windows(current, &lru_p, &lru_w, &lru_l);
692 for (p = packed_git; p; p = p->next)
693 scan_windows(p, &lru_p, &lru_w, &lru_l);
54044bf8
SP
694 if (lru_p) {
695 munmap(lru_w->base, lru_w->len);
696 pack_mapped -= lru_w->len;
697 if (lru_l)
698 lru_l->next = lru_w->next;
7c3ecb32 699 else
54044bf8 700 lru_p->windows = lru_w->next;
54044bf8 701 free(lru_w);
a53128b6 702 pack_open_windows--;
54044bf8
SP
703 return 1;
704 }
705 return 0;
f9253394
JH
706}
707
7c3ecb32 708void release_pack_memory(size_t need)
97bfeb34
SP
709{
710 size_t cur = pack_mapped;
7c3ecb32 711 while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
97bfeb34
SP
712 ; /* nothing */
713}
714
02710228
SP
715static void mmap_limit_check(size_t length)
716{
717 static size_t limit = 0;
718 if (!limit) {
719 limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
720 if (!limit)
721 limit = SIZE_MAX;
722 }
723 if (length > limit)
724 die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
725 (uintmax_t)length, (uintmax_t)limit);
726}
727
1570856b
JK
728void *xmmap_gently(void *start, size_t length,
729 int prot, int flags, int fd, off_t offset)
58ecbd5e 730{
02710228
SP
731 void *ret;
732
733 mmap_limit_check(length);
734 ret = mmap(start, length, prot, flags, fd, offset);
58ecbd5e
JN
735 if (ret == MAP_FAILED) {
736 if (!length)
737 return NULL;
7c3ecb32 738 release_pack_memory(length);
58ecbd5e 739 ret = mmap(start, length, prot, flags, fd, offset);
58ecbd5e
JN
740 }
741 return ret;
742}
743
1570856b
JK
744void *xmmap(void *start, size_t length,
745 int prot, int flags, int fd, off_t offset)
746{
747 void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
748 if (ret == MAP_FAILED)
9ca0aaf6 749 die_errno("mmap failed");
1570856b
JK
750 return ret;
751}
752
c9ced051
SP
753void close_pack_windows(struct packed_git *p)
754{
755 while (p->windows) {
756 struct pack_window *w = p->windows;
757
758 if (w->inuse_cnt)
759 die("pack '%s' still has open windows to it",
760 p->pack_name);
761 munmap(w->base, w->len);
762 pack_mapped -= w->len;
763 pack_open_windows--;
764 p->windows = w->next;
765 free(w);
766 }
767}
768
71fe5d7f
JS
769static int close_pack_fd(struct packed_git *p)
770{
771 if (p->pack_fd < 0)
772 return 0;
773
774 close(p->pack_fd);
775 pack_open_fds--;
776 p->pack_fd = -1;
777
778 return 1;
779}
780
38849a81
JS
781static void close_pack(struct packed_git *p)
782{
783 close_pack_windows(p);
784 close_pack_fd(p);
785 close_pack_index(p);
786}
787
788void close_all_packs(void)
789{
790 struct packed_git *p;
791
792 for (p = packed_git; p; p = p->next)
793 if (p->do_not_close)
ef1177d1 794 die("BUG: want to close pack marked 'do-not-close'");
38849a81
JS
795 else
796 close_pack(p);
797}
798
799
88d0db55
BC
800/*
801 * The LRU pack is the one with the oldest MRU window, preferring packs
802 * with no used windows, or the oldest mtime if it has no windows allocated.
803 */
804static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
805{
806 struct pack_window *w, *this_mru_w;
807 int has_windows_inuse = 0;
808
809 /*
810 * Reject this pack if it has windows and the previously selected
811 * one does not. If this pack does not have windows, reject
812 * it if the pack file is newer than the previously selected one.
813 */
814 if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
815 return;
816
817 for (w = this_mru_w = p->windows; w; w = w->next) {
818 /*
819 * Reject this pack if any of its windows are in use,
820 * but the previously selected pack did not have any
821 * inuse windows. Otherwise, record that this pack
822 * has windows in use.
823 */
824 if (w->inuse_cnt) {
825 if (*accept_windows_inuse)
826 has_windows_inuse = 1;
827 else
828 return;
829 }
830
831 if (w->last_used > this_mru_w->last_used)
832 this_mru_w = w;
833
834 /*
835 * Reject this pack if it has windows that have been
836 * used more recently than the previously selected pack.
837 * If the previously selected pack had windows inuse and
838 * we have not encountered a window in this pack that is
839 * inuse, skip this check since we prefer a pack with no
840 * inuse windows to one that has inuse windows.
841 */
842 if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
843 this_mru_w->last_used > (*mru_w)->last_used)
844 return;
845 }
846
847 /*
848 * Select this pack.
849 */
850 *mru_w = this_mru_w;
851 *lru_p = p;
852 *accept_windows_inuse = has_windows_inuse;
853}
854
855static int close_one_pack(void)
856{
857 struct packed_git *p, *lru_p = NULL;
858 struct pack_window *mru_w = NULL;
859 int accept_windows_inuse = 1;
860
861 for (p = packed_git; p; p = p->next) {
862 if (p->pack_fd == -1)
863 continue;
864 find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
865 }
866
71fe5d7f
JS
867 if (lru_p)
868 return close_pack_fd(lru_p);
88d0db55
BC
869
870 return 0;
871}
872
03e79c88 873void unuse_pack(struct pack_window **w_cursor)
f9253394 874{
03e79c88
SP
875 struct pack_window *w = *w_cursor;
876 if (w) {
877 w->inuse_cnt--;
878 *w_cursor = NULL;
879 }
1f688557
JH
880}
881
fa5fc15d
SP
882void close_pack_index(struct packed_git *p)
883{
884 if (p->index_data) {
885 munmap((void *)p->index_data, p->index_size);
886 p->index_data = NULL;
887 }
888}
889
a0788266
JS
890static unsigned int get_max_fd_limit(void)
891{
892#ifdef RLIMIT_NOFILE
491a8dec
JH
893 {
894 struct rlimit lim;
a0788266 895
491a8dec
JH
896 if (!getrlimit(RLIMIT_NOFILE, &lim))
897 return lim.rlim_cur;
898 }
899#endif
900
901#ifdef _SC_OPEN_MAX
902 {
903 long open_max = sysconf(_SC_OPEN_MAX);
904 if (0 < open_max)
905 return open_max;
906 /*
907 * Otherwise, we got -1 for one of the two
908 * reasons:
909 *
910 * (1) sysconf() did not understand _SC_OPEN_MAX
911 * and signaled an error with -1; or
912 * (2) sysconf() said there is no limit.
913 *
914 * We _could_ clear errno before calling sysconf() to
915 * tell these two cases apart and return a huge number
916 * in the latter case to let the caller cap it to a
917 * value that is not so selfish, but letting the
918 * fallback OPEN_MAX codepath take care of these cases
919 * is a lot simpler.
920 */
921 }
922#endif
a0788266 923
491a8dec 924#ifdef OPEN_MAX
a0788266
JS
925 return OPEN_MAX;
926#else
927 return 1; /* see the caller ;-) */
928#endif
929}
930
3cf8b462
SP
931/*
932 * Do not call this directly as this leaks p->pack_fd on error return;
933 * call open_packed_git() instead.
934 */
935static int open_packed_git_1(struct packed_git *p)
1f688557 936{
9bc879c1
SP
937 struct stat st;
938 struct pack_header hdr;
939 unsigned char sha1[20];
940 unsigned char *idx_sha1;
2c039da8 941 long fd_flag;
9bc879c1 942
d079837e
SP
943 if (!p->index_data && open_pack_index(p))
944 return error("packfile %s index unavailable", p->pack_name);
945
c7934306 946 if (!pack_max_fds) {
a0788266 947 unsigned int max_fds = get_max_fd_limit();
c7934306
SP
948
949 /* Save 3 for stdin/stdout/stderr, 22 for work */
950 if (25 < max_fds)
951 pack_max_fds = max_fds - 25;
952 else
953 pack_max_fds = 1;
954 }
955
88d0db55 956 while (pack_max_fds <= pack_open_fds && close_one_pack())
c7934306
SP
957 ; /* nothing */
958
f4e51683 959 p->pack_fd = git_open_noatime(p->pack_name);
9bc879c1 960 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
072db278 961 return -1;
c7934306 962 pack_open_fds++;
9bc879c1
SP
963
964 /* If we created the struct before we had the pack we lack size. */
bf592c50 965 if (!p->pack_size) {
bf592c50 966 if (!S_ISREG(st.st_mode))
072db278 967 return error("packfile %s not a regular file", p->pack_name);
bf592c50 968 p->pack_size = st.st_size;
9bc879c1 969 } else if (p->pack_size != st.st_size)
072db278 970 return error("packfile %s size changed", p->pack_name);
9bc879c1 971
2c039da8
JH
972 /* We leave these file descriptors open with sliding mmap;
973 * there is no point keeping them open across exec(), though.
974 */
975 fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
976 if (fd_flag < 0)
072db278 977 return error("cannot determine file descriptor flags");
2c039da8
JH
978 fd_flag |= FD_CLOEXEC;
979 if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
072db278 980 return error("cannot set FD_CLOEXEC");
2c039da8 981
9bc879c1 982 /* Verify we recognize this pack file format. */
e6e2bd62 983 if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
072db278 984 return error("file %s is far too short to be a packfile", p->pack_name);
9bc879c1 985 if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
072db278 986 return error("file %s is not a GIT packfile", p->pack_name);
9bc879c1 987 if (!pack_version_ok(hdr.hdr_version))
6e1c2344
RJ
988 return error("packfile %s is version %"PRIu32" and not"
989 " supported (try upgrading GIT to a newer version)",
9bc879c1
SP
990 p->pack_name, ntohl(hdr.hdr_version));
991
992 /* Verify the pack matches its index. */
57059091 993 if (p->num_objects != ntohl(hdr.hdr_entries))
6e1c2344
RJ
994 return error("packfile %s claims to have %"PRIu32" objects"
995 " while index indicates %"PRIu32" objects",
57059091
NP
996 p->pack_name, ntohl(hdr.hdr_entries),
997 p->num_objects);
9bc879c1 998 if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
072db278 999 return error("end of packfile %s is unavailable", p->pack_name);
e6e2bd62 1000 if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
072db278 1001 return error("packfile %s signature is unavailable", p->pack_name);
42873078 1002 idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
9bc879c1 1003 if (hashcmp(sha1, idx_sha1))
072db278
SP
1004 return error("packfile %s does not match index", p->pack_name);
1005 return 0;
9bc879c1
SP
1006}
1007
3cf8b462
SP
1008static int open_packed_git(struct packed_git *p)
1009{
1010 if (!open_packed_git_1(p))
1011 return 0;
71fe5d7f 1012 close_pack_fd(p);
3cf8b462
SP
1013 return -1;
1014}
1015
c4001d92 1016static int in_window(struct pack_window *win, off_t offset)
60bb8b14
SP
1017{
1018 /* We must promise at least 20 bytes (one hash) after the
1019 * offset is available from this window, otherwise the offset
1020 * is not actually in this window and a different window (which
1021 * has that one hash excess) must be used. This is to support
1022 * the object header and delta base parsing routines below.
1023 */
1024 off_t win_off = win->offset;
1025 return win_off <= offset
1026 && (offset + 20) <= (win_off + win->len);
1027}
1028
4b25d091 1029unsigned char *use_pack(struct packed_git *p,
03e79c88 1030 struct pack_window **w_cursor,
c4001d92 1031 off_t offset,
ef49a7a0 1032 unsigned long *left)
9bc879c1 1033{
60bb8b14 1034 struct pack_window *win = *w_cursor;
03e79c88 1035
a9d98a14 1036 /* Since packfiles end in a hash of their content and it's
60bb8b14
SP
1037 * pointless to ask for an offset into the middle of that
1038 * hash, and the in_window function above wouldn't match
1039 * don't allow an offset too close to the end of the file.
1040 */
d131b7af
SP
1041 if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
1042 die("packfile %s cannot be accessed", p->pack_name);
60bb8b14
SP
1043 if (offset > (p->pack_size - 20))
1044 die("offset beyond end of packfile (truncated pack?)");
13e0b0d3 1045 if (offset < 0)
7465feba 1046 die(_("offset before end of packfile (broken .idx?)"));
60bb8b14
SP
1047
1048 if (!win || !in_window(win, offset)) {
1049 if (win)
1050 win->inuse_cnt--;
1051 for (win = p->windows; win; win = win->next) {
1052 if (in_window(win, offset))
1053 break;
1054 }
1055 if (!win) {
78a28df9 1056 size_t window_align = packed_git_window_size / 2;
dc49cd76 1057 off_t len;
d131b7af
SP
1058
1059 if (p->pack_fd == -1 && open_packed_git(p))
1060 die("packfile %s cannot be accessed", p->pack_name);
1061
60bb8b14 1062 win = xcalloc(1, sizeof(*win));
78a28df9 1063 win->offset = (offset / window_align) * window_align;
dc49cd76
SP
1064 len = p->pack_size - win->offset;
1065 if (len > packed_git_window_size)
1066 len = packed_git_window_size;
1067 win->len = (size_t)len;
60bb8b14 1068 pack_mapped += win->len;
11daf39b 1069 while (packed_git_limit < pack_mapped
7c3ecb32 1070 && unuse_one_window(p))
60bb8b14 1071 ; /* nothing */
c4712e45 1072 win->base = xmmap(NULL, win->len,
60bb8b14
SP
1073 PROT_READ, MAP_PRIVATE,
1074 p->pack_fd, win->offset);
1075 if (win->base == MAP_FAILED)
7616c6ca
NTND
1076 die_errno("packfile %s cannot be mapped",
1077 p->pack_name);
d131b7af 1078 if (!win->offset && win->len == p->pack_size
71fe5d7f
JS
1079 && !p->do_not_close)
1080 close_pack_fd(p);
a53128b6
SP
1081 pack_mmap_calls++;
1082 pack_open_windows++;
1083 if (pack_mapped > peak_pack_mapped)
1084 peak_pack_mapped = pack_mapped;
1085 if (pack_open_windows > peak_pack_open_windows)
1086 peak_pack_open_windows = pack_open_windows;
60bb8b14
SP
1087 win->next = p->windows;
1088 p->windows = win;
1089 }
1f688557 1090 }
03e79c88
SP
1091 if (win != *w_cursor) {
1092 win->last_used = pack_used_ctr++;
1093 win->inuse_cnt++;
1094 *w_cursor = win;
1095 }
60bb8b14 1096 offset -= win->offset;
03e79c88 1097 if (left)
dc49cd76 1098 *left = win->len - xsize_t(offset);
03e79c88 1099 return win->base + offset;
1f688557
JH
1100}
1101
27d69a46
NP
1102static struct packed_git *alloc_packed_git(int extra)
1103{
50a6c8ef 1104 struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
27d69a46
NP
1105 memset(p, 0, sizeof(*p));
1106 p->pack_fd = -1;
1107 return p;
1108}
1109
e0500293
JN
1110static void try_to_free_pack_memory(size_t size)
1111{
7c3ecb32 1112 release_pack_memory(size);
e0500293
JN
1113}
1114
48bcc1c3 1115struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
1f688557 1116{
e0500293 1117 static int have_set_try_to_free_routine;
1f688557 1118 struct stat st;
48bcc1c3
JK
1119 size_t alloc;
1120 struct packed_git *p;
1f688557 1121
e0500293
JN
1122 if (!have_set_try_to_free_routine) {
1123 have_set_try_to_free_routine = 1;
1124 set_try_to_free_routine(try_to_free_pack_memory);
1125 }
1126
42873078
NP
1127 /*
1128 * Make sure a corresponding .pack file exists and that
1129 * the index looks sane.
1130 */
48bcc1c3 1131 if (!strip_suffix_mem(path, &path_len, ".idx"))
1f688557 1132 return NULL;
48bcc1c3
JK
1133
1134 /*
1135 * ".pack" is long enough to hold any suffix we're adding (and
1136 * the use xsnprintf double-checks that)
1137 */
50a6c8ef 1138 alloc = st_add3(path_len, strlen(".pack"), 1);
48bcc1c3 1139 p = alloc_packed_git(alloc);
42873078 1140 memcpy(p->pack_name, path, path_len);
8d25931d 1141
48bcc1c3 1142 xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
8d25931d
BC
1143 if (!access(p->pack_name, F_OK))
1144 p->pack_keep = 1;
1145
48bcc1c3 1146 xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
d079837e 1147 if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
42873078 1148 free(p);
1f688557
JH
1149 return NULL;
1150 }
42873078 1151
1f688557
JH
1152 /* ok, it looks sane as far as we can check without
1153 * actually mapping the pack file.
1154 */
1f688557 1155 p->pack_size = st.st_size;
9d835df2 1156 p->pack_local = local;
b867092f 1157 p->mtime = st.st_mtime;
42873078
NP
1158 if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
1159 hashclr(p->sha1);
1f688557
JH
1160 return p;
1161}
1162
7b64469a 1163struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
c508df5e 1164{
42873078 1165 const char *path = sha1_pack_name(sha1);
50a6c8ef 1166 size_t alloc = st_add(strlen(path), 1);
c7ab0ba3 1167 struct packed_git *p = alloc_packed_git(alloc);
bf592c50 1168
c7ab0ba3 1169 memcpy(p->pack_name, path, alloc); /* includes NUL */
27d69a46 1170 hashcpy(p->sha1, sha1);
42873078
NP
1171 if (check_packed_git_idx(idx_path, p)) {
1172 free(p);
bf592c50 1173 return NULL;
42873078 1174 }
bf592c50 1175
bf592c50
DB
1176 return p;
1177}
1178
1179void install_packed_git(struct packed_git *pack)
1180{
c7934306
SP
1181 if (pack->pack_fd != -1)
1182 pack_open_fds++;
1183
bf592c50
DB
1184 pack->next = packed_git;
1185 packed_git = pack;
1186}
1187
0a489b06 1188void (*report_garbage)(unsigned seen_bits, const char *path);
543c5caa
NTND
1189
1190static void report_helper(const struct string_list *list,
1191 int seen_bits, int first, int last)
1192{
0a489b06 1193 if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
543c5caa 1194 return;
0a489b06 1195
543c5caa 1196 for (; first < last; first++)
0a489b06 1197 report_garbage(seen_bits, list->items[first].string);
543c5caa
NTND
1198}
1199
1200static void report_pack_garbage(struct string_list *list)
1201{
1202 int i, baselen = -1, first = 0, seen_bits = 0;
1203
1204 if (!report_garbage)
1205 return;
1206
3383e199 1207 string_list_sort(list);
543c5caa
NTND
1208
1209 for (i = 0; i < list->nr; i++) {
1210 const char *path = list->items[i].string;
1211 if (baselen != -1 &&
1212 strncmp(path, list->items[first].string, baselen)) {
1213 report_helper(list, seen_bits, first, i);
1214 baselen = -1;
1215 seen_bits = 0;
1216 }
1217 if (baselen == -1) {
1218 const char *dot = strrchr(path, '.');
1219 if (!dot) {
0a489b06 1220 report_garbage(PACKDIR_FILE_GARBAGE, path);
543c5caa
NTND
1221 continue;
1222 }
1223 baselen = dot - path + 1;
1224 first = i;
1225 }
1226 if (!strcmp(path + baselen, "pack"))
1227 seen_bits |= 1;
1228 else if (!strcmp(path + baselen, "idx"))
1229 seen_bits |= 2;
1230 }
1231 report_helper(list, seen_bits, first, list->nr);
1232}
1233
9d835df2 1234static void prepare_packed_git_one(char *objdir, int local)
1f688557 1235{
880fb8de
RS
1236 struct strbuf path = STRBUF_INIT;
1237 size_t dirnamelen;
1f688557
JH
1238 DIR *dir;
1239 struct dirent *de;
543c5caa 1240 struct string_list garbage = STRING_LIST_INIT_DUP;
1f688557 1241
880fb8de
RS
1242 strbuf_addstr(&path, objdir);
1243 strbuf_addstr(&path, "/pack");
1244 dir = opendir(path.buf);
b5b16990 1245 if (!dir) {
26125f6b 1246 if (errno != ENOENT)
7616c6ca
NTND
1247 error_errno("unable to open object pack directory: %s",
1248 path.buf);
880fb8de 1249 strbuf_release(&path);
1f688557 1250 return;
b5b16990 1251 }
880fb8de
RS
1252 strbuf_addch(&path, '/');
1253 dirnamelen = path.len;
1f688557 1254 while ((de = readdir(dir)) != NULL) {
1f688557 1255 struct packed_git *p;
47bf4b0f 1256 size_t base_len;
543c5caa
NTND
1257
1258 if (is_dot_or_dotdot(de->d_name))
9cb18f56
JM
1259 continue;
1260
880fb8de
RS
1261 strbuf_setlen(&path, dirnamelen);
1262 strbuf_addstr(&path, de->d_name);
d90906a9 1263
47bf4b0f
JK
1264 base_len = path.len;
1265 if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
d90906a9
NTND
1266 /* Don't reopen a pack we already have. */
1267 for (p = packed_git; p; p = p->next) {
47bf4b0f
JK
1268 size_t len;
1269 if (strip_suffix(p->pack_name, ".pack", &len) &&
1270 len == base_len &&
1271 !memcmp(p->pack_name, path.buf, len))
d90906a9
NTND
1272 break;
1273 }
1274 if (p == NULL &&
1275 /*
1276 * See if it really is a valid .idx file with
1277 * corresponding .pack file that we can map.
1278 */
880fb8de 1279 (p = add_packed_git(path.buf, path.len, local)) != NULL)
d90906a9 1280 install_packed_git(p);
86f7780c 1281 }
543c5caa
NTND
1282
1283 if (!report_garbage)
1284 continue;
1285
2975c770
JK
1286 if (ends_with(de->d_name, ".idx") ||
1287 ends_with(de->d_name, ".pack") ||
1288 ends_with(de->d_name, ".bitmap") ||
1289 ends_with(de->d_name, ".keep"))
880fb8de 1290 string_list_append(&garbage, path.buf);
543c5caa 1291 else
0a489b06 1292 report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
1f688557 1293 }
5b35bcd5 1294 closedir(dir);
543c5caa
NTND
1295 report_pack_garbage(&garbage);
1296 string_list_clear(&garbage, 0);
880fb8de 1297 strbuf_release(&path);
1f688557
JH
1298}
1299
b867092f
JH
1300static int sort_pack(const void *a_, const void *b_)
1301{
1302 struct packed_git *a = *((struct packed_git **)a_);
1303 struct packed_git *b = *((struct packed_git **)b_);
1304 int st;
1305
1306 /*
1307 * Local packs tend to contain objects specific to our
1308 * variant of the project than remote ones. In addition,
1309 * remote ones could be on a network mounted filesystem.
1310 * Favor local ones for these reasons.
1311 */
1312 st = a->pack_local - b->pack_local;
1313 if (st)
1314 return -st;
1315
1316 /*
1317 * Younger packs tend to contain more recent objects,
1318 * and more recent objects tend to get accessed more
1319 * often.
1320 */
1321 if (a->mtime < b->mtime)
1322 return 1;
1323 else if (a->mtime == b->mtime)
1324 return 0;
1325 return -1;
1326}
1327
1328static void rearrange_packed_git(void)
1329{
1330 struct packed_git **ary, *p;
1331 int i, n;
1332
1333 for (n = 0, p = packed_git; p; p = p->next)
1334 n++;
1335 if (n < 2)
1336 return;
1337
1338 /* prepare an array of packed_git for easier sorting */
1339 ary = xcalloc(n, sizeof(struct packed_git *));
1340 for (n = 0, p = packed_git; p; p = p->next)
1341 ary[n++] = p;
1342
1343 qsort(ary, n, sizeof(struct packed_git *), sort_pack);
1344
1345 /* link them back again */
1346 for (i = 0; i < n - 1; i++)
1347 ary[i]->next = ary[i + 1];
1348 ary[n - 1]->next = NULL;
1349 packed_git = ary[0];
1350
1351 free(ary);
1352}
1353
a73cdd21
JK
1354static void prepare_packed_git_mru(void)
1355{
1356 struct packed_git *p;
1357
1358 mru_clear(packed_git_mru);
1359 for (p = packed_git; p; p = p->next)
1360 mru_append(packed_git_mru, p);
1361}
1362
637cdd9d 1363static int prepare_packed_git_run_once = 0;
9a217f2a 1364void prepare_packed_git(void)
1f688557 1365{
d5a63b99 1366 struct alternate_object_database *alt;
1f688557 1367
637cdd9d 1368 if (prepare_packed_git_run_once)
1f688557 1369 return;
9d835df2 1370 prepare_packed_git_one(get_object_directory(), 1);
9a217f2a 1371 prepare_alt_odb();
d5a63b99 1372 for (alt = alt_odb_list; alt; alt = alt->next) {
1494e038 1373 alt->name[-1] = 0;
9d835df2 1374 prepare_packed_git_one(alt->base, 0);
1494e038 1375 alt->name[-1] = '/';
1f688557 1376 }
b867092f 1377 rearrange_packed_git();
a73cdd21 1378 prepare_packed_git_mru();
637cdd9d
JK
1379 prepare_packed_git_run_once = 1;
1380}
1381
fc04c412 1382void reprepare_packed_git(void)
637cdd9d
JK
1383{
1384 prepare_packed_git_run_once = 0;
1385 prepare_packed_git();
1f688557
JH
1386}
1387
8eca0b47
NP
1388static void mark_bad_packed_object(struct packed_git *p,
1389 const unsigned char *sha1)
1390{
1391 unsigned i;
1392 for (i = 0; i < p->num_bad_objects; i++)
50a6c8ef 1393 if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
8eca0b47 1394 return;
50a6c8ef
JK
1395 p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
1396 st_mult(GIT_SHA1_RAWSZ,
1397 st_add(p->num_bad_objects, 1)));
1398 hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
8eca0b47
NP
1399 p->num_bad_objects++;
1400}
1401
b6c4cecc 1402static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
ac939109
NP
1403{
1404 struct packed_git *p;
1405 unsigned i;
1406
1407 for (p = packed_git; p; p = p->next)
1408 for (i = 0; i < p->num_bad_objects; i++)
1409 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
b6c4cecc
JH
1410 return p;
1411 return NULL;
ac939109
NP
1412}
1413
090ea126
NTND
1414/*
1415 * With an in-core object data in "map", rehash it to make sure the
1416 * object name actually matches "sha1" to detect object corruption.
1417 * With "map" == NULL, try reading the object named with "sha1" using
1418 * the streaming interface and rehash it to do the same.
1419 */
1420int check_sha1_signature(const unsigned char *sha1, void *map,
1421 unsigned long size, const char *type)
0fcfd160
LT
1422{
1423 unsigned char real_sha1[20];
090ea126
NTND
1424 enum object_type obj_type;
1425 struct git_istream *st;
1426 git_SHA_CTX c;
1427 char hdr[32];
1428 int hdrlen;
1429
1430 if (map) {
1431 hash_sha1_file(map, size, type, real_sha1);
1432 return hashcmp(sha1, real_sha1) ? -1 : 0;
1433 }
1434
1435 st = open_istream(sha1, &obj_type, &size, NULL);
1436 if (!st)
1437 return -1;
1438
1439 /* Generate the header */
ef1286d3 1440 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
090ea126
NTND
1441
1442 /* Sha1.. */
1443 git_SHA1_Init(&c);
1444 git_SHA1_Update(&c, hdr, hdrlen);
1445 for (;;) {
1446 char buf[1024 * 16];
1447 ssize_t readlen = read_istream(st, buf, sizeof(buf));
1448
f54fac53
JK
1449 if (readlen < 0) {
1450 close_istream(st);
1451 return -1;
1452 }
090ea126
NTND
1453 if (!readlen)
1454 break;
1455 git_SHA1_Update(&c, buf, readlen);
1456 }
1457 git_SHA1_Final(real_sha1, &c);
1458 close_istream(st);
a89fccd2 1459 return hashcmp(sha1, real_sha1) ? -1 : 0;
0fcfd160
LT
1460}
1461
ec73f580 1462int git_open_noatime(const char *name)
44d1c19e
LT
1463{
1464 static int sha1_file_open_flag = O_NOATIME;
44d1c19e 1465
f2e872aa 1466 for (;;) {
dff6f280
CB
1467 int fd;
1468
1469 errno = 0;
1470 fd = open(name, O_RDONLY | sha1_file_open_flag);
44d1c19e 1471 if (fd >= 0)
f2e872aa
SP
1472 return fd;
1473
f2e872aa
SP
1474 /* Might the failure be due to O_NOATIME? */
1475 if (errno != ENOENT && sha1_file_open_flag) {
44d1c19e 1476 sha1_file_open_flag = 0;
f2e872aa
SP
1477 continue;
1478 }
1479
1480 return -1;
44d1c19e 1481 }
44d1c19e
LT
1482}
1483
052fe5ea
JK
1484static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
1485{
052fe5ea
JK
1486 struct alternate_object_database *alt;
1487
30d6c6ea 1488 if (!lstat(sha1_file_name(sha1), st))
052fe5ea
JK
1489 return 0;
1490
1491 prepare_alt_odb();
1492 errno = ENOENT;
1493 for (alt = alt_odb_list; alt; alt = alt->next) {
30d6c6ea 1494 fill_sha1_path(alt->name, sha1);
052fe5ea
JK
1495 if (!lstat(alt->base, st))
1496 return 0;
1497 }
1498
1499 return -1;
1500}
1501
44d1c19e
LT
1502static int open_sha1_file(const unsigned char *sha1)
1503{
1504 int fd;
44d1c19e 1505 struct alternate_object_database *alt;
d6c8a05b 1506 int most_interesting_errno;
44d1c19e 1507
30d6c6ea 1508 fd = git_open_noatime(sha1_file_name(sha1));
44d1c19e
LT
1509 if (fd >= 0)
1510 return fd;
d6c8a05b 1511 most_interesting_errno = errno;
44d1c19e
LT
1512
1513 prepare_alt_odb();
44d1c19e 1514 for (alt = alt_odb_list; alt; alt = alt->next) {
30d6c6ea 1515 fill_sha1_path(alt->name, sha1);
f4e51683 1516 fd = git_open_noatime(alt->base);
44d1c19e
LT
1517 if (fd >= 0)
1518 return fd;
d6c8a05b
JK
1519 if (most_interesting_errno == ENOENT)
1520 most_interesting_errno = errno;
44d1c19e 1521 }
d6c8a05b 1522 errno = most_interesting_errno;
44d1c19e
LT
1523 return -1;
1524}
1525
f0270efd 1526void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
0fcfd160 1527{
0fcfd160 1528 void *map;
144bde78 1529 int fd;
ace1534d 1530
44d1c19e
LT
1531 fd = open_sha1_file(sha1);
1532 map = NULL;
1533 if (fd >= 0) {
1534 struct stat st;
0fcfd160 1535
44d1c19e
LT
1536 if (!fstat(fd, &st)) {
1537 *size = xsize_t(st.st_size);
33e42de0
MM
1538 if (!*size) {
1539 /* mmap() is forbidden on empty files */
1540 error("object file %s is empty", sha1_file_name(sha1));
1541 return NULL;
1542 }
44d1c19e 1543 map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
144bde78 1544 }
44d1c19e 1545 close(fd);
0fcfd160 1546 }
0fcfd160
LT
1547 return map;
1548}
1549
09ded04b
NP
1550unsigned long unpack_object_header_buffer(const unsigned char *buf,
1551 unsigned long len, enum object_type *type, unsigned long *sizep)
c4483576 1552{
ad1ed5ee 1553 unsigned shift;
48fb7deb 1554 unsigned long size, c;
ad1ed5ee
JH
1555 unsigned long used = 0;
1556
1557 c = buf[used++];
1558 *type = (c >> 4) & 7;
1559 size = c & 15;
1560 shift = 4;
1561 while (c & 0x80) {
f630cfda 1562 if (len <= used || bitsizeof(long) <= shift) {
09ded04b 1563 error("bad object header");
ea4f9685
JH
1564 size = used = 0;
1565 break;
09ded04b 1566 }
ad1ed5ee
JH
1567 c = buf[used++];
1568 size += (c & 0x7f) << shift;
1569 shift += 7;
1570 }
1571 *sizep = size;
1572 return used;
1573}
1574
eb4f4076 1575int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
ad1ed5ee 1576{
c4483576
LT
1577 /* Get the data stream */
1578 memset(stream, 0, sizeof(*stream));
1579 stream->next_in = map;
1580 stream->avail_in = mapsize;
1581 stream->next_out = buffer;
93821bd9
LT
1582 stream->avail_out = bufsiz;
1583
39c68542 1584 git_inflate_init(stream);
cc5c54e7 1585 return git_inflate(stream, 0);
c4483576
LT
1586}
1587
46f03448
KN
1588static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
1589 unsigned long mapsize, void *buffer,
1590 unsigned long bufsiz, struct strbuf *header)
1591{
1592 int status;
1593
1594 status = unpack_sha1_header(stream, map, mapsize, buffer, bufsiz);
1595
1596 /*
1597 * Check if entire header is unpacked in the first iteration.
1598 */
1599 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1600 return 0;
1601
1602 /*
1603 * buffer[0..bufsiz] was not large enough. Copy the partial
1604 * result out to header, and then append the result of further
1605 * reading the stream.
1606 */
1607 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1608 stream->next_out = buffer;
1609 stream->avail_out = bufsiz;
1610
1611 do {
1612 status = git_inflate(stream, 0);
1613 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1614 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1615 return 0;
1616 stream->next_out = buffer;
1617 stream->avail_out = bufsiz;
1618 } while (status != Z_STREAM_END);
1619 return -1;
1620}
1621
ef49a7a0 1622static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
5180cacc
LT
1623{
1624 int bytes = strlen(buffer) + 1;
3aee68aa 1625 unsigned char *buf = xmallocz(size);
93821bd9 1626 unsigned long n;
7efbff75 1627 int status = Z_OK;
5180cacc 1628
93821bd9
LT
1629 n = stream->total_out - bytes;
1630 if (n > size)
1631 n = size;
1632 memcpy(buf, (char *) buffer + bytes, n);
1633 bytes = n;
456cdf6e
LT
1634 if (bytes <= size) {
1635 /*
1636 * The above condition must be (bytes <= size), not
1637 * (bytes < size). In other words, even though we
ccf5ace0 1638 * expect no more output and set avail_out to zero,
456cdf6e
LT
1639 * the input zlib stream may have bytes that express
1640 * "this concludes the stream", and we *do* want to
1641 * eat that input.
1642 *
1643 * Otherwise we would not be able to test that we
1644 * consumed all the input to reach the expected size;
1645 * we also want to check that zlib tells us that all
1646 * went well with status == Z_STREAM_END at the end.
1647 */
5180cacc
LT
1648 stream->next_out = buf + bytes;
1649 stream->avail_out = size - bytes;
7efbff75 1650 while (status == Z_OK)
39c68542 1651 status = git_inflate(stream, Z_FINISH);
5180cacc 1652 }
456cdf6e 1653 if (status == Z_STREAM_END && !stream->avail_in) {
39c68542 1654 git_inflate_end(stream);
7efbff75
JH
1655 return buf;
1656 }
1657
1658 if (status < 0)
1659 error("corrupt loose object '%s'", sha1_to_hex(sha1));
1660 else if (stream->avail_in)
1661 error("garbage at end of loose object '%s'",
1662 sha1_to_hex(sha1));
1663 free(buf);
1664 return NULL;
5180cacc
LT
1665}
1666
1667/*
1668 * We used to just use "sscanf()", but that's actually way
1669 * too permissive for what we want to check. So do an anal
1670 * object header parse by hand.
1671 */
46f03448
KN
1672static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
1673 unsigned int flags)
5180cacc 1674{
46f03448 1675 const char *type_buf = hdr;
5180cacc 1676 unsigned long size;
46f03448 1677 int type, type_len = 0;
5180cacc
LT
1678
1679 /*
46f03448 1680 * The type can be of any size but is followed by
21666f1a 1681 * a space.
5180cacc 1682 */
5180cacc
LT
1683 for (;;) {
1684 char c = *hdr++;
1685 if (c == ' ')
1686 break;
46f03448 1687 type_len++;
5180cacc 1688 }
46f03448
KN
1689
1690 type = type_from_string_gently(type_buf, type_len, 1);
1691 if (oi->typename)
1692 strbuf_add(oi->typename, type_buf, type_len);
1693 /*
1694 * Set type to 0 if its an unknown object and
2e3a16b2 1695 * we're obtaining the type using '--allow-unknown-type'
46f03448
KN
1696 * option.
1697 */
1698 if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
1699 type = 0;
1700 else if (type < 0)
1701 die("invalid object type");
1702 if (oi->typep)
1703 *oi->typep = type;
5180cacc
LT
1704
1705 /*
1706 * The length must follow immediately, and be in canonical
1707 * decimal format (ie "010" is not valid).
1708 */
1709 size = *hdr++ - '0';
1710 if (size > 9)
1711 return -1;
1712 if (size) {
1713 for (;;) {
1714 unsigned long c = *hdr - '0';
1715 if (c > 9)
1716 break;
1717 hdr++;
1718 size = size * 10 + c;
1719 }
1720 }
46f03448
KN
1721
1722 if (oi->sizep)
1723 *oi->sizep = size;
5180cacc
LT
1724
1725 /*
1726 * The length must be followed by a zero byte
1727 */
46f03448
KN
1728 return *hdr ? -1 : type;
1729}
1730
1731int parse_sha1_header(const char *hdr, unsigned long *sizep)
1732{
1733 struct object_info oi;
1734
1735 oi.sizep = sizep;
1736 oi.typename = NULL;
1737 oi.typep = NULL;
1738 return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
5180cacc
LT
1739}
1740
7efbff75 1741static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
0fcfd160 1742{
5180cacc 1743 int ret;
ef49a7a0 1744 git_zstream stream;
5180cacc 1745 char hdr[8192];
0fcfd160 1746
5180cacc 1747 ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
21666f1a 1748 if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
0fcfd160
LT
1749 return NULL;
1750
7efbff75 1751 return unpack_sha1_rest(&stream, hdr, *size, sha1);
0fcfd160
LT
1752}
1753
54dab52a
NP
1754unsigned long get_size_from_delta(struct packed_git *p,
1755 struct pack_window **w_curs,
1756 off_t curpos)
1757{
1758 const unsigned char *data;
1759 unsigned char delta_head[20], *in;
ef49a7a0 1760 git_zstream stream;
54dab52a
NP
1761 int st;
1762
1763 memset(&stream, 0, sizeof(stream));
1764 stream.next_out = delta_head;
1765 stream.avail_out = sizeof(delta_head);
1766
39c68542 1767 git_inflate_init(&stream);
54dab52a
NP
1768 do {
1769 in = use_pack(p, w_curs, curpos, &stream.avail_in);
1770 stream.next_in = in;
39c68542 1771 st = git_inflate(&stream, Z_FINISH);
54dab52a
NP
1772 curpos += stream.next_in - in;
1773 } while ((st == Z_OK || st == Z_BUF_ERROR) &&
1774 stream.total_out < sizeof(delta_head));
39c68542 1775 git_inflate_end(&stream);
3d77d877
NP
1776 if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
1777 error("delta data unpack-initial failed");
1778 return 0;
1779 }
54dab52a
NP
1780
1781 /* Examine the initial part of the delta to figure out
1782 * the result size.
1783 */
1784 data = delta_head;
1785
1786 /* ignore base size */
1787 get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1788
1789 /* Read the result size */
1790 return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1791}
1792
c4001d92 1793static off_t get_delta_base(struct packed_git *p,
03e79c88 1794 struct pack_window **w_curs,
c4001d92 1795 off_t *curpos,
21666f1a 1796 enum object_type type,
c4001d92 1797 off_t delta_obj_offset)
eb32d236 1798{
2b87c45b 1799 unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
c4001d92 1800 off_t base_offset;
eb32d236 1801
8d8a4ea5
SP
1802 /* use_pack() assured us we have [base_info, base_info + 20)
1803 * as a range that we can look at without walking off the
1804 * end of the mapped window. Its actually the hash size
1805 * that is assured. An OFS_DELTA longer than the hash size
1806 * is stupid, as then a REF_DELTA would be smaller to store.
1807 */
21666f1a 1808 if (type == OBJ_OFS_DELTA) {
eb32d236
NP
1809 unsigned used = 0;
1810 unsigned char c = base_info[used++];
1811 base_offset = c & 127;
1812 while (c & 128) {
1813 base_offset += 1;
8723f216 1814 if (!base_offset || MSB(base_offset, 7))
8eca0b47 1815 return 0; /* overflow */
eb32d236
NP
1816 c = base_info[used++];
1817 base_offset = (base_offset << 7) + (c & 127);
1818 }
1819 base_offset = delta_obj_offset - base_offset;
d8f32556 1820 if (base_offset <= 0 || base_offset >= delta_obj_offset)
8eca0b47 1821 return 0; /* out of bound */
2b87c45b 1822 *curpos += used;
21666f1a 1823 } else if (type == OBJ_REF_DELTA) {
eb32d236
NP
1824 /* The base entry _must_ be in the same pack */
1825 base_offset = find_pack_entry_one(base_info, p);
2b87c45b 1826 *curpos += 20;
eb32d236
NP
1827 } else
1828 die("I am totally screwed");
2b87c45b 1829 return base_offset;
eb32d236
NP
1830}
1831
5d642e75
JK
1832/*
1833 * Like get_delta_base above, but we return the sha1 instead of the pack
1834 * offset. This means it is cheaper for REF deltas (we do not have to do
1835 * the final object lookup), but more expensive for OFS deltas (we
1836 * have to load the revidx to convert the offset back into a sha1).
1837 */
1838static const unsigned char *get_delta_base_sha1(struct packed_git *p,
1839 struct pack_window **w_curs,
1840 off_t curpos,
1841 enum object_type type,
1842 off_t delta_obj_offset)
1843{
1844 if (type == OBJ_REF_DELTA) {
1845 unsigned char *base = use_pack(p, w_curs, curpos, NULL);
1846 return base;
1847 } else if (type == OBJ_OFS_DELTA) {
1848 struct revindex_entry *revidx;
1849 off_t base_offset = get_delta_base(p, w_curs, &curpos,
1850 type, delta_obj_offset);
1851
1852 if (!base_offset)
1853 return NULL;
1854
1855 revidx = find_pack_revindex(p, base_offset);
1856 if (!revidx)
1857 return NULL;
1858
1859 return nth_packed_object_sha1(p, revidx->nr);
1860 } else
1861 return NULL;
1862}
1863
f8c8abc5
JH
1864int unpack_object_header(struct packed_git *p,
1865 struct pack_window **w_curs,
1866 off_t *curpos,
1867 unsigned long *sizep)
a733cb60 1868{
03e79c88 1869 unsigned char *base;
ef49a7a0 1870 unsigned long left;
ad1ed5ee 1871 unsigned long used;
2b87c45b 1872 enum object_type type;
a733cb60 1873
8d8a4ea5 1874 /* use_pack() assures us we have [base, base + 20) available
0353a0c4 1875 * as a range that we can look at. (Its actually the hash
3dff5379 1876 * size that is assured.) With our object header encoding
8d8a4ea5
SP
1877 * the maximum deflated object size is 2^137, which is just
1878 * insane, so we know won't exceed what we have been given.
1879 */
2b87c45b 1880 base = use_pack(p, w_curs, *curpos, &left);
09ded04b
NP
1881 used = unpack_object_header_buffer(base, left, &type, sizep);
1882 if (!used) {
1883 type = OBJ_BAD;
1884 } else
1885 *curpos += used;
ad1ed5ee 1886
2b87c45b 1887 return type;
a733cb60
LT
1888}
1889
790d96c0
TR
1890static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
1891{
1892 int type;
1893 struct revindex_entry *revidx;
1894 const unsigned char *sha1;
1895 revidx = find_pack_revindex(p, obj_offset);
1896 if (!revidx)
1897 return OBJ_BAD;
1898 sha1 = nth_packed_object_sha1(p, revidx->nr);
1899 mark_bad_packed_object(p, sha1);
1900 type = sha1_object_info(sha1, NULL);
1901 if (type <= OBJ_NONE)
1902 return OBJ_BAD;
1903 return type;
1904}
1905
790d96c0
TR
1906#define POI_STACK_PREALLOC 64
1907
90191d37
JK
1908static enum object_type packed_to_object_type(struct packed_git *p,
1909 off_t obj_offset,
1910 enum object_type type,
1911 struct pack_window **w_curs,
1912 off_t curpos)
1f688557 1913{
790d96c0
TR
1914 off_t small_poi_stack[POI_STACK_PREALLOC];
1915 off_t *poi_stack = small_poi_stack;
1916 int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
5db47c2b 1917
790d96c0
TR
1918 while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1919 off_t base_offset;
90191d37 1920 unsigned long size;
790d96c0
TR
1921 /* Push the object we're going to leave behind */
1922 if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
1923 poi_stack_alloc = alloc_nr(poi_stack_nr);
b32fa95f 1924 ALLOC_ARRAY(poi_stack, poi_stack_alloc);
790d96c0
TR
1925 memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
1926 } else {
1927 ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
1928 }
1929 poi_stack[poi_stack_nr++] = obj_offset;
1930 /* If parsing the base offset fails, just unwind */
90191d37 1931 base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
790d96c0
TR
1932 if (!base_offset)
1933 goto unwind;
1934 curpos = obj_offset = base_offset;
90191d37 1935 type = unpack_object_header(p, w_curs, &curpos, &size);
790d96c0
TR
1936 if (type <= OBJ_NONE) {
1937 /* If getting the base itself fails, we first
1938 * retry the base, otherwise unwind */
1939 type = retry_bad_packed_offset(p, base_offset);
1940 if (type > OBJ_NONE)
1941 goto out;
1942 goto unwind;
1943 }
1944 }
1945
21666f1a 1946 switch (type) {
790d96c0 1947 case OBJ_BAD:
a733cb60 1948 case OBJ_COMMIT:
a733cb60 1949 case OBJ_TREE:
a733cb60 1950 case OBJ_BLOB:
a733cb60 1951 case OBJ_TAG:
a69d0943 1952 break;
1f688557 1953 default:
3d77d877
NP
1954 error("unknown object type %i at offset %"PRIuMAX" in %s",
1955 type, (uintmax_t)obj_offset, p->pack_name);
1956 type = OBJ_BAD;
1f688557 1957 }
790d96c0
TR
1958
1959out:
1960 if (poi_stack != small_poi_stack)
1961 free(poi_stack);
21666f1a 1962 return type;
790d96c0
TR
1963
1964unwind:
1965 while (poi_stack_nr) {
1966 obj_offset = poi_stack[--poi_stack_nr];
1967 type = retry_bad_packed_offset(p, obj_offset);
1968 if (type > OBJ_NONE)
1969 goto out;
1970 }
1971 type = OBJ_BAD;
1972 goto out;
1f688557
JH
1973}
1974
90191d37 1975static int packed_object_info(struct packed_git *p, off_t obj_offset,
23c339c0 1976 struct object_info *oi)
90191d37
JK
1977{
1978 struct pack_window *w_curs = NULL;
1979 unsigned long size;
1980 off_t curpos = obj_offset;
1981 enum object_type type;
1982
412916ee
JK
1983 /*
1984 * We always get the representation type, but only convert it to
1985 * a "real" type later if the caller is interested.
1986 */
90191d37
JK
1987 type = unpack_object_header(p, &w_curs, &curpos, &size);
1988
23c339c0 1989 if (oi->sizep) {
90191d37
JK
1990 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1991 off_t tmp_pos = curpos;
1992 off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
1993 type, obj_offset);
1994 if (!base_offset) {
1995 type = OBJ_BAD;
1996 goto out;
1997 }
23c339c0
JK
1998 *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
1999 if (*oi->sizep == 0) {
90191d37
JK
2000 type = OBJ_BAD;
2001 goto out;
2002 }
2003 } else {
23c339c0 2004 *oi->sizep = size;
90191d37
JK
2005 }
2006 }
2007
23c339c0 2008 if (oi->disk_sizep) {
90191d37 2009 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
23c339c0 2010 *oi->disk_sizep = revidx[1].offset - obj_offset;
90191d37
JK
2011 }
2012
23c339c0
JK
2013 if (oi->typep) {
2014 *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos);
2015 if (*oi->typep < 0) {
412916ee
JK
2016 type = OBJ_BAD;
2017 goto out;
2018 }
2019 }
90191d37 2020
5d642e75
JK
2021 if (oi->delta_base_sha1) {
2022 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2023 const unsigned char *base;
2024
2025 base = get_delta_base_sha1(p, &w_curs, curpos,
2026 type, obj_offset);
2027 if (!base) {
2028 type = OBJ_BAD;
2029 goto out;
2030 }
2031
2032 hashcpy(oi->delta_base_sha1, base);
2033 } else
2034 hashclr(oi->delta_base_sha1);
2035 }
2036
90191d37
JK
2037out:
2038 unuse_pack(&w_curs);
2039 return type;
2040}
2041
eb950c19 2042static void *unpack_compressed_entry(struct packed_git *p,
03e79c88 2043 struct pack_window **w_curs,
c4001d92 2044 off_t curpos,
eb950c19 2045 unsigned long size)
de530aaa
SP
2046{
2047 int st;
ef49a7a0 2048 git_zstream stream;
079afb18 2049 unsigned char *buffer, *in;
de530aaa 2050
735efde8
NTND
2051 buffer = xmallocz_gently(size);
2052 if (!buffer)
2053 return NULL;
de530aaa 2054 memset(&stream, 0, sizeof(stream));
de530aaa 2055 stream.next_out = buffer;
39eea7bd 2056 stream.avail_out = size + 1;
de530aaa 2057
39c68542 2058 git_inflate_init(&stream);
079afb18 2059 do {
2b87c45b 2060 in = use_pack(p, w_curs, curpos, &stream.avail_in);
079afb18 2061 stream.next_in = in;
39c68542 2062 st = git_inflate(&stream, Z_FINISH);
39eea7bd
JH
2063 if (!stream.avail_out)
2064 break; /* the payload is larger than it should be */
2b87c45b 2065 curpos += stream.next_in - in;
079afb18 2066 } while (st == Z_OK || st == Z_BUF_ERROR);
39c68542 2067 git_inflate_end(&stream);
de530aaa
SP
2068 if ((st != Z_STREAM_END) || stream.total_out != size) {
2069 free(buffer);
2070 return NULL;
2071 }
2072
2073 return buffer;
2074}
2075
e5e01619
LT
2076#define MAX_DELTA_CACHE (256)
2077
18bdec11 2078static size_t delta_base_cached;
5e08ecbf
NP
2079
2080static struct delta_base_cache_lru_list {
2081 struct delta_base_cache_lru_list *prev;
2082 struct delta_base_cache_lru_list *next;
2083} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru };
2084
e5e01619 2085static struct delta_base_cache_entry {
5e08ecbf
NP
2086 struct delta_base_cache_lru_list lru;
2087 void *data;
e5e01619
LT
2088 struct packed_git *p;
2089 off_t base_offset;
2090 unsigned long size;
e5e01619
LT
2091 enum object_type type;
2092} delta_base_cache[MAX_DELTA_CACHE];
2093
2094static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset)
2095{
2096 unsigned long hash;
2097
56a1a3ab 2098 hash = (unsigned long)(intptr_t)p + (unsigned long)base_offset;
e5e01619 2099 hash += (hash >> 8) + (hash >> 16);
3358004a 2100 return hash % MAX_DELTA_CACHE;
e5e01619
LT
2101}
2102
84dd81c1
TR
2103static struct delta_base_cache_entry *
2104get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
5266d369
JH
2105{
2106 unsigned long hash = pack_entry_hash(p, base_offset);
84dd81c1
TR
2107 return delta_base_cache + hash;
2108}
2109
2110static int eq_delta_base_cache_entry(struct delta_base_cache_entry *ent,
2111 struct packed_git *p, off_t base_offset)
2112{
5266d369
JH
2113 return (ent->data && ent->p == p && ent->base_offset == base_offset);
2114}
2115
84dd81c1
TR
2116static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
2117{
2118 struct delta_base_cache_entry *ent;
2119 ent = get_delta_base_cache_entry(p, base_offset);
2120 return eq_delta_base_cache_entry(ent, p, base_offset);
2121}
2122
2123static void clear_delta_base_cache_entry(struct delta_base_cache_entry *ent)
2124{
2125 ent->data = NULL;
2126 ent->lru.next->prev = ent->lru.prev;
2127 ent->lru.prev->next = ent->lru.next;
2128 delta_base_cached -= ent->size;
2129}
2130
62f255ad 2131static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
85fe35ab 2132 unsigned long *base_size, enum object_type *type)
62f255ad 2133{
84dd81c1 2134 struct delta_base_cache_entry *ent;
e5e01619 2135
84dd81c1
TR
2136 ent = get_delta_base_cache_entry(p, base_offset);
2137
2138 if (!eq_delta_base_cache_entry(ent, p, base_offset))
749bc58c 2139 return unpack_entry(p, base_offset, type, base_size);
e5e01619 2140
e5e01619
LT
2141 *type = ent->type;
2142 *base_size = ent->size;
85fe35ab 2143 return xmemdupz(ent->data, ent->size);
62f255ad
LT
2144}
2145
18bdec11
SP
2146static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
2147{
2148 if (ent->data) {
2149 free(ent->data);
2150 ent->data = NULL;
5e08ecbf
NP
2151 ent->lru.next->prev = ent->lru.prev;
2152 ent->lru.prev->next = ent->lru.next;
18bdec11
SP
2153 delta_base_cached -= ent->size;
2154 }
2155}
2156
3d20c636
SP
2157void clear_delta_base_cache(void)
2158{
2159 unsigned long p;
2160 for (p = 0; p < MAX_DELTA_CACHE; p++)
2161 release_delta_base_cache(&delta_base_cache[p]);
2162}
2163
62f255ad
LT
2164static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
2165 void *base, unsigned long base_size, enum object_type type)
2166{
5e08ecbf 2167 unsigned long hash = pack_entry_hash(p, base_offset);
e5e01619 2168 struct delta_base_cache_entry *ent = delta_base_cache + hash;
5e08ecbf 2169 struct delta_base_cache_lru_list *lru;
e5e01619 2170
18bdec11
SP
2171 release_delta_base_cache(ent);
2172 delta_base_cached += base_size;
5e08ecbf
NP
2173
2174 for (lru = delta_base_cache_lru.next;
2175 delta_base_cached > delta_base_cache_limit
2176 && lru != &delta_base_cache_lru;
2177 lru = lru->next) {
2178 struct delta_base_cache_entry *f = (void *)lru;
18bdec11
SP
2179 if (f->type == OBJ_BLOB)
2180 release_delta_base_cache(f);
2181 }
5e08ecbf
NP
2182 for (lru = delta_base_cache_lru.next;
2183 delta_base_cached > delta_base_cache_limit
2184 && lru != &delta_base_cache_lru;
2185 lru = lru->next) {
2186 struct delta_base_cache_entry *f = (void *)lru;
2187 release_delta_base_cache(f);
2188 }
18bdec11 2189
e5e01619
LT
2190 ent->p = p;
2191 ent->base_offset = base_offset;
2192 ent->type = type;
2193 ent->data = base;
2194 ent->size = base_size;
5e08ecbf
NP
2195 ent->lru.next = &delta_base_cache_lru;
2196 ent->lru.prev = delta_base_cache_lru.prev;
2197 delta_base_cache_lru.prev->next = &ent->lru;
2198 delta_base_cache_lru.prev = &ent->lru;
62f255ad
LT
2199}
2200
c2c5b270
CC
2201static void *read_object(const unsigned char *sha1, enum object_type *type,
2202 unsigned long *size);
2203
5f44324d
JH
2204static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
2205{
67dc598e
KB
2206 static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
2207 trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
2208 p->pack_name, (uintmax_t)obj_offset);
5f44324d
JH
2209}
2210
0e8189e2
NP
2211int do_check_packed_object_crc;
2212
abe601bb
TR
2213#define UNPACK_ENTRY_STACK_PREALLOC 64
2214struct unpack_entry_stack_ent {
2215 off_t obj_offset;
2216 off_t curpos;
2217 unsigned long size;
2218};
2219
c4001d92 2220void *unpack_entry(struct packed_git *p, off_t obj_offset,
abe601bb 2221 enum object_type *final_type, unsigned long *final_size)
1f688557 2222{
03e79c88 2223 struct pack_window *w_curs = NULL;
c4001d92 2224 off_t curpos = obj_offset;
abe601bb
TR
2225 void *data = NULL;
2226 unsigned long size;
2227 enum object_type type;
2228 struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
2229 struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
2230 int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
2231 int base_from_cache = 0;
1f688557 2232
67dc598e 2233 write_pack_access_log(p, obj_offset);
5f44324d 2234
abe601bb
TR
2235 /* PHASE 1: drill down to the innermost base object */
2236 for (;;) {
2237 off_t base_offset;
2238 int i;
2239 struct delta_base_cache_entry *ent;
2240
77965f8b
NTND
2241 ent = get_delta_base_cache_entry(p, curpos);
2242 if (eq_delta_base_cache_entry(ent, p, curpos)) {
2243 type = ent->type;
2244 data = ent->data;
2245 size = ent->size;
2246 clear_delta_base_cache_entry(ent);
2247 base_from_cache = 1;
2248 break;
2249 }
2250
abe601bb
TR
2251 if (do_check_packed_object_crc && p->index_version > 1) {
2252 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
211c61c6 2253 off_t len = revidx[1].offset - obj_offset;
abe601bb
TR
2254 if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
2255 const unsigned char *sha1 =
2256 nth_packed_object_sha1(p, revidx->nr);
2257 error("bad packed object CRC for %s",
2258 sha1_to_hex(sha1));
2259 mark_bad_packed_object(p, sha1);
2260 unuse_pack(&w_curs);
2261 return NULL;
2262 }
2263 }
2264
abe601bb
TR
2265 type = unpack_object_header(p, &w_curs, &curpos, &size);
2266 if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
2267 break;
2268
2269 base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
2270 if (!base_offset) {
2271 error("failed to validate delta base reference "
2272 "at offset %"PRIuMAX" from %s",
2273 (uintmax_t)curpos, p->pack_name);
2274 /* bail to phase 2, in hopes of recovery */
2275 data = NULL;
2276 break;
2277 }
2278
2279 /* push object, proceed to base */
2280 if (delta_stack_nr >= delta_stack_alloc
2281 && delta_stack == small_delta_stack) {
2282 delta_stack_alloc = alloc_nr(delta_stack_nr);
b32fa95f 2283 ALLOC_ARRAY(delta_stack, delta_stack_alloc);
abe601bb
TR
2284 memcpy(delta_stack, small_delta_stack,
2285 sizeof(*delta_stack)*delta_stack_nr);
2286 } else {
2287 ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
2288 }
2289 i = delta_stack_nr++;
2290 delta_stack[i].obj_offset = obj_offset;
2291 delta_stack[i].curpos = curpos;
2292 delta_stack[i].size = size;
2293
2294 curpos = obj_offset = base_offset;
0e8189e2
NP
2295 }
2296
abe601bb
TR
2297 /* PHASE 2: handle the base */
2298 switch (type) {
eb32d236
NP
2299 case OBJ_OFS_DELTA:
2300 case OBJ_REF_DELTA:
abe601bb 2301 if (data)
ef1177d1 2302 die("BUG: unpack_entry: left loop at a valid delta");
4d703a1a 2303 break;
a733cb60 2304 case OBJ_COMMIT:
a733cb60 2305 case OBJ_TREE:
a733cb60 2306 case OBJ_BLOB:
a733cb60 2307 case OBJ_TAG:
abe601bb
TR
2308 if (!base_from_cache)
2309 data = unpack_compressed_entry(p, &w_curs, curpos, size);
4d703a1a 2310 break;
1f688557 2311 default:
8eca0b47
NP
2312 data = NULL;
2313 error("unknown object type %i at offset %"PRIuMAX" in %s",
abe601bb
TR
2314 type, (uintmax_t)obj_offset, p->pack_name);
2315 }
2316
2317 /* PHASE 3: apply deltas in order */
2318
2319 /* invariants:
2320 * 'data' holds the base data, or NULL if there was corruption
2321 */
2322 while (delta_stack_nr) {
2323 void *delta_data;
2324 void *base = data;
2325 unsigned long delta_size, base_size = size;
2326 int i;
2327
2328 data = NULL;
2329
2330 if (base)
2331 add_delta_base_cache(p, obj_offset, base, base_size, type);
2332
2333 if (!base) {
2334 /*
2335 * We're probably in deep shit, but let's try to fetch
2336 * the required base anyway from another pack or loose.
2337 * This is costly but should happen only in the presence
2338 * of a corrupted pack, and is better than failing outright.
2339 */
2340 struct revindex_entry *revidx;
2341 const unsigned char *base_sha1;
2342 revidx = find_pack_revindex(p, obj_offset);
2343 if (revidx) {
2344 base_sha1 = nth_packed_object_sha1(p, revidx->nr);
2345 error("failed to read delta base object %s"
2346 " at offset %"PRIuMAX" from %s",
2347 sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
2348 p->pack_name);
2349 mark_bad_packed_object(p, base_sha1);
2350 base = read_object(base_sha1, &type, &base_size);
2351 }
2352 }
2353
2354 i = --delta_stack_nr;
2355 obj_offset = delta_stack[i].obj_offset;
2356 curpos = delta_stack[i].curpos;
2357 delta_size = delta_stack[i].size;
2358
2359 if (!base)
2360 continue;
2361
2362 delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
2363
2364 if (!delta_data) {
2365 error("failed to unpack compressed delta "
2366 "at offset %"PRIuMAX" from %s",
2367 (uintmax_t)curpos, p->pack_name);
abe601bb
TR
2368 data = NULL;
2369 continue;
2370 }
2371
2372 data = patch_delta(base, base_size,
2373 delta_data, delta_size,
2374 &size);
1ee886c1
JK
2375
2376 /*
2377 * We could not apply the delta; warn the user, but keep going.
2378 * Our failure will be noticed either in the next iteration of
2379 * the loop, or if this is the final delta, in the caller when
2380 * we return NULL. Those code paths will take care of making
2381 * a more explicit warning and retrying with another copy of
2382 * the object.
2383 */
abe601bb 2384 if (!data)
1ee886c1 2385 error("failed to apply delta");
abe601bb 2386
4b8f772c 2387 free(delta_data);
1f688557 2388 }
abe601bb
TR
2389
2390 *final_type = type;
2391 *final_size = size;
2392
03e79c88 2393 unuse_pack(&w_curs);
019d1e65
NTND
2394
2395 if (delta_stack != small_delta_stack)
2396 free(delta_stack);
2397
21666f1a 2398 return data;
1f688557
JH
2399}
2400
d079837e 2401const unsigned char *nth_packed_object_sha1(struct packed_git *p,
d72308e0 2402 uint32_t n)
9a217f2a 2403{
42873078 2404 const unsigned char *index = p->index_data;
d079837e
SP
2405 if (!index) {
2406 if (open_pack_index(p))
2407 return NULL;
2408 index = p->index_data;
2409 }
57059091 2410 if (n >= p->num_objects)
d72308e0 2411 return NULL;
74e34e1f
NP
2412 index += 4 * 256;
2413 if (p->index_version == 1) {
2414 return index + 24 * n + 4;
2415 } else {
2416 index += 8;
2417 return index + 20 * n;
2418 }
2419}
2420
47fe3f6e
JK
2421void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
2422{
2423 const unsigned char *ptr = vptr;
2424 const unsigned char *start = p->index_data;
2425 const unsigned char *end = start + p->index_size;
2426 if (ptr < start)
7465feba 2427 die(_("offset before start of pack index for %s (corrupt index?)"),
47fe3f6e
JK
2428 p->pack_name);
2429 /* No need to check for underflow; .idx files must be at least 8 bytes */
2430 if (ptr >= end - 8)
7465feba 2431 die(_("offset beyond end of pack index for %s (truncated index?)"),
47fe3f6e
JK
2432 p->pack_name);
2433}
2434
99093238 2435off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
74e34e1f
NP
2436{
2437 const unsigned char *index = p->index_data;
2438 index += 4 * 256;
2439 if (p->index_version == 1) {
2440 return ntohl(*((uint32_t *)(index + 24 * n)));
2441 } else {
2442 uint32_t off;
2443 index += 8 + p->num_objects * (20 + 4);
2444 off = ntohl(*((uint32_t *)(index + 4 * n)));
2445 if (!(off & 0x80000000))
2446 return off;
2447 index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
47fe3f6e 2448 check_pack_index_ptr(p, index);
74e34e1f
NP
2449 return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
2450 ntohl(*((uint32_t *)(index + 4)));
2451 }
9a217f2a
JH
2452}
2453
c4001d92 2454off_t find_pack_entry_one(const unsigned char *sha1,
43057304 2455 struct packed_git *p)
1f688557 2456{
42873078 2457 const uint32_t *level1_ofs = p->index_data;
42873078 2458 const unsigned char *index = p->index_data;
628522ec
JH
2459 unsigned hi, lo, stride;
2460 static int use_lookup = -1;
2461 static int debug_lookup = -1;
2462
2463 if (debug_lookup < 0)
2464 debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
42873078 2465
d079837e
SP
2466 if (!index) {
2467 if (open_pack_index(p))
2468 return 0;
2469 level1_ofs = p->index_data;
2470 index = p->index_data;
2471 }
74e34e1f
NP
2472 if (p->index_version > 1) {
2473 level1_ofs += 2;
2474 index += 8;
2475 }
42873078 2476 index += 4 * 256;
74e34e1f
NP
2477 hi = ntohl(level1_ofs[*sha1]);
2478 lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
628522ec
JH
2479 if (p->index_version > 1) {
2480 stride = 20;
2481 } else {
2482 stride = 24;
2483 index += 4;
2484 }
2485
2486 if (debug_lookup)
6e1c2344 2487 printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
628522ec
JH
2488 sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
2489
2490 if (use_lookup < 0)
2491 use_lookup = !!getenv("GIT_USE_LOOKUP");
2492 if (use_lookup) {
2493 int pos = sha1_entry_pos(index, stride, 0,
2494 lo, hi, p->num_objects, sha1);
2495 if (pos < 0)
2496 return 0;
2497 return nth_packed_object_offset(p, pos);
2498 }
1f688557
JH
2499
2500 do {
74e34e1f 2501 unsigned mi = (lo + hi) / 2;
628522ec
JH
2502 int cmp = hashcmp(index + mi * stride, sha1);
2503
2504 if (debug_lookup)
2505 printf("lo %u hi %u rg %u mi %u\n",
2506 lo, hi, hi - lo, mi);
43057304 2507 if (!cmp)
74e34e1f 2508 return nth_packed_object_offset(p, mi);
1f688557
JH
2509 if (cmp > 0)
2510 hi = mi;
2511 else
2512 lo = mi+1;
2513 } while (lo < hi);
2514 return 0;
2515}
2516
4c080182 2517int is_pack_valid(struct packed_git *p)
d131b7af
SP
2518{
2519 /* An already open pack is known to be valid. */
2520 if (p->pack_fd != -1)
2521 return 1;
2522
2523 /* If the pack has one window completely covering the
2524 * file size, the pack is known to be valid even if
2525 * the descriptor is not currently open.
2526 */
2527 if (p->windows) {
2528 struct pack_window *w = p->windows;
2529
2530 if (!w->offset && w->len == p->pack_size)
2531 return 1;
2532 }
2533
2534 /* Force the pack to open to prove its valid. */
2535 return !open_packed_git(p);
2536}
2537
95099731
NTND
2538static int fill_pack_entry(const unsigned char *sha1,
2539 struct pack_entry *e,
2540 struct packed_git *p)
2541{
2542 off_t offset;
2543
2544 if (p->num_bad_objects) {
2545 unsigned i;
2546 for (i = 0; i < p->num_bad_objects; i++)
2547 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
2548 return 0;
2549 }
2550
2551 offset = find_pack_entry_one(sha1, p);
2552 if (!offset)
2553 return 0;
2554
2555 /*
2556 * We are about to tell the caller where they can locate the
2557 * requested object. We better make sure the packfile is
2558 * still here and can be accessed before supplying that
2559 * answer, as it may have been deleted since the index was
2560 * loaded!
2561 */
319b678a 2562 if (!is_pack_valid(p))
95099731 2563 return 0;
95099731
NTND
2564 e->offset = offset;
2565 e->p = p;
2566 hashcpy(e->sha1, sha1);
2567 return 1;
2568}
2569
d40d535b
MH
2570/*
2571 * Iff a pack file contains the object named by sha1, return true and
2572 * store its location to e.
2573 */
4d6acb70 2574static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
1f688557 2575{
a73cdd21 2576 struct mru_entry *p;
43057304 2577
1f688557 2578 prepare_packed_git();
f7c22cc6
NP
2579 if (!packed_git)
2580 return 0;
1f688557 2581
a73cdd21
JK
2582 for (p = packed_git_mru->head; p; p = p->next) {
2583 if (fill_pack_entry(sha1, e, p->item)) {
2584 mru_mark(packed_git_mru, p);
1b1005d1
MH
2585 return 1;
2586 }
c01f51cc 2587 }
1f688557
JH
2588 return 0;
2589}
2590
a6080a0a 2591struct packed_git *find_sha1_pack(const unsigned char *sha1,
bf592c50
DB
2592 struct packed_git *packs)
2593{
2594 struct packed_git *p;
bf592c50
DB
2595
2596 for (p = packs; p; p = p->next) {
43057304 2597 if (find_pack_entry_one(sha1, p))
bf592c50
DB
2598 return p;
2599 }
2600 return NULL;
21666f1a 2601
bf592c50
DB
2602}
2603
052fe5ea 2604static int sha1_loose_object_info(const unsigned char *sha1,
46f03448
KN
2605 struct object_info *oi,
2606 int flags)
65c2e0c3 2607{
46f03448
KN
2608 int status = 0;
2609 unsigned long mapsize;
65c2e0c3 2610 void *map;
ef49a7a0 2611 git_zstream stream;
d65a16f6 2612 char hdr[32];
46f03448 2613 struct strbuf hdrbuf = STRBUF_INIT;
65c2e0c3 2614
5d642e75
JK
2615 if (oi->delta_base_sha1)
2616 hashclr(oi->delta_base_sha1);
2617
052fe5ea
JK
2618 /*
2619 * If we don't care about type or size, then we don't
4ef8d1dd
JH
2620 * need to look inside the object at all. Note that we
2621 * do not optimize out the stat call, even if the
2622 * caller doesn't care about the disk-size, since our
2623 * return value implicitly indicates whether the
2624 * object even exists.
052fe5ea 2625 */
46f03448 2626 if (!oi->typep && !oi->typename && !oi->sizep) {
4ef8d1dd
JH
2627 struct stat st;
2628 if (stat_sha1_file(sha1, &st) < 0)
2629 return -1;
2630 if (oi->disk_sizep)
23c339c0 2631 *oi->disk_sizep = st.st_size;
052fe5ea
JK
2632 return 0;
2633 }
2634
bb6b8e4f 2635 map = map_sha1_file(sha1, &mapsize);
f0df4ed5 2636 if (!map)
dbea72a8 2637 return -1;
23c339c0
JK
2638 if (oi->disk_sizep)
2639 *oi->disk_sizep = mapsize;
46f03448
KN
2640 if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
2641 if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
2642 status = error("unable to unpack %s header with --allow-unknown-type",
2643 sha1_to_hex(sha1));
2644 } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
36e4d74a
JH
2645 status = error("unable to unpack %s header",
2646 sha1_to_hex(sha1));
46f03448
KN
2647 if (status < 0)
2648 ; /* Do nothing */
2649 else if (hdrbuf.len) {
2650 if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
2651 status = error("unable to parse %s header with --allow-unknown-type",
2652 sha1_to_hex(sha1));
2653 } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
36e4d74a 2654 status = error("unable to parse %s header", sha1_to_hex(sha1));
39c68542 2655 git_inflate_end(&stream);
65c2e0c3 2656 munmap(map, mapsize);
46f03448 2657 if (status && oi->typep)
23c339c0 2658 *oi->typep = status;
46f03448 2659 strbuf_release(&hdrbuf);
052fe5ea 2660 return 0;
65c2e0c3
JH
2661}
2662
de7b5d62 2663int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
f0df4ed5 2664{
c4d9986f 2665 struct cached_object *co;
f0df4ed5 2666 struct pack_entry e;
5b086407 2667 int rtype;
46f03448 2668 enum object_type real_type;
1f7117ef 2669 const unsigned char *real = lookup_replace_object_extended(sha1, flags);
f0df4ed5 2670
1f7117ef 2671 co = find_cached_object(real);
c4d9986f 2672 if (co) {
5b086407
JK
2673 if (oi->typep)
2674 *(oi->typep) = co->type;
9a490590
JH
2675 if (oi->sizep)
2676 *(oi->sizep) = co->size;
161f00e7
JK
2677 if (oi->disk_sizep)
2678 *(oi->disk_sizep) = 0;
5d642e75
JK
2679 if (oi->delta_base_sha1)
2680 hashclr(oi->delta_base_sha1);
46f03448
KN
2681 if (oi->typename)
2682 strbuf_addstr(oi->typename, typename(co->type));
9a490590 2683 oi->whence = OI_CACHED;
5b086407 2684 return 0;
c4d9986f
NTND
2685 }
2686
1f7117ef 2687 if (!find_pack_entry(real, &e)) {
ddd63e64 2688 /* Most likely it's a loose object. */
46f03448 2689 if (!sha1_loose_object_info(real, oi, flags)) {
9a490590 2690 oi->whence = OI_LOOSE;
5b086407 2691 return 0;
9a490590 2692 }
ddd63e64
SG
2693
2694 /* Not a loose object; someone else may have just packed it. */
f0df4ed5 2695 reprepare_packed_git();
1f7117ef 2696 if (!find_pack_entry(real, &e))
052fe5ea 2697 return -1;
f0df4ed5 2698 }
3d77d877 2699
46f03448
KN
2700 /*
2701 * packed_object_info() does not follow the delta chain to
2702 * find out the real type, unless it is given oi->typep.
2703 */
2704 if (oi->typename && !oi->typep)
2705 oi->typep = &real_type;
2706
23c339c0 2707 rtype = packed_object_info(e.p, e.offset, oi);
412916ee 2708 if (rtype < 0) {
1f7117ef 2709 mark_bad_packed_object(e.p, real);
46f03448
KN
2710 if (oi->typep == &real_type)
2711 oi->typep = NULL;
1f7117ef 2712 return sha1_object_info_extended(real, oi, 0);
5266d369
JH
2713 } else if (in_delta_base_cache(e.p, e.offset)) {
2714 oi->whence = OI_DBCACHED;
9a490590
JH
2715 } else {
2716 oi->whence = OI_PACKED;
2717 oi->u.packed.offset = e.offset;
2718 oi->u.packed.pack = e.p;
2719 oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
2720 rtype == OBJ_OFS_DELTA);
3d77d877 2721 }
46f03448
KN
2722 if (oi->typename)
2723 strbuf_addstr(oi->typename, typename(*oi->typep));
2724 if (oi->typep == &real_type)
2725 oi->typep = NULL;
3d77d877 2726
5b086407 2727 return 0;
f0df4ed5
JS
2728}
2729
3fc0dca9 2730/* returns enum object_type or negative */
9a490590
JH
2731int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
2732{
5b086407 2733 enum object_type type;
d099b717 2734 struct object_info oi = {NULL};
9a490590 2735
5b086407 2736 oi.typep = &type;
9a490590 2737 oi.sizep = sizep;
de7b5d62 2738 if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
5b086407
JK
2739 return -1;
2740 return type;
9a490590
JH
2741}
2742
21666f1a
NP
2743static void *read_packed_sha1(const unsigned char *sha1,
2744 enum object_type *type, unsigned long *size)
1f688557
JH
2745{
2746 struct pack_entry e;
8eca0b47 2747 void *data;
1f688557 2748
cd673c1f 2749 if (!find_pack_entry(sha1, &e))
1f688557 2750 return NULL;
85fe35ab 2751 data = cache_or_unpack_entry(e.p, e.offset, size, type);
8eca0b47
NP
2752 if (!data) {
2753 /*
2754 * We're probably in deep shit, but let's try to fetch
2755 * the required object anyway from another pack or loose.
2756 * This should happen only in the presence of a corrupted
2757 * pack, and is better than failing outright.
2758 */
2759 error("failed to read object %s at offset %"PRIuMAX" from %s",
2760 sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
2761 mark_bad_packed_object(e.p, sha1);
ac939109 2762 data = read_object(sha1, type, size);
8eca0b47
NP
2763 }
2764 return data;
1f688557
JH
2765}
2766
21666f1a
NP
2767int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
2768 unsigned char *sha1)
d66b37bb
JH
2769{
2770 struct cached_object *co;
2771
21666f1a 2772 hash_sha1_file(buf, len, typename(type), sha1);
d66b37bb
JH
2773 if (has_sha1_file(sha1) || find_cached_object(sha1))
2774 return 0;
c7353967 2775 ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
d66b37bb
JH
2776 co = &cached_objects[cached_object_nr++];
2777 co->size = len;
21666f1a 2778 co->type = type;
efa13f7b
JH
2779 co->buf = xmalloc(len);
2780 memcpy(co->buf, buf, len);
d66b37bb
JH
2781 hashcpy(co->sha1, sha1);
2782 return 0;
2783}
2784
c2c5b270
CC
2785static void *read_object(const unsigned char *sha1, enum object_type *type,
2786 unsigned long *size)
0fcfd160
LT
2787{
2788 unsigned long mapsize;
2789 void *map, *buf;
d66b37bb
JH
2790 struct cached_object *co;
2791
2792 co = find_cached_object(sha1);
2793 if (co) {
21666f1a 2794 *type = co->type;
d66b37bb 2795 *size = co->size;
182af834 2796 return xmemdupz(co->buf, co->size);
d66b37bb 2797 }
0fcfd160 2798
8276c007
PE
2799 buf = read_packed_sha1(sha1, type, size);
2800 if (buf)
2801 return buf;
bb6b8e4f 2802 map = map_sha1_file(sha1, &mapsize);
0fcfd160 2803 if (map) {
7efbff75 2804 buf = unpack_sha1_file(map, mapsize, type, size, sha1);
0fcfd160
LT
2805 munmap(map, mapsize);
2806 return buf;
2807 }
637cdd9d 2808 reprepare_packed_git();
8276c007 2809 return read_packed_sha1(sha1, type, size);
0fcfd160
LT
2810}
2811
b6c4cecc
JH
2812/*
2813 * This function dies on corrupt objects; the callers who want to
2814 * deal with them should arrange to call read_object() and give error
2815 * messages themselves.
2816 */
5bf29b95
JH
2817void *read_sha1_file_extended(const unsigned char *sha1,
2818 enum object_type *type,
2819 unsigned long *size,
2820 unsigned flag)
ac939109 2821{
3ba7a065 2822 void *data;
b6c4cecc 2823 const struct packed_git *p;
bf93eea0 2824 const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
b6c4cecc 2825
3ba7a065
JH
2826 errno = 0;
2827 data = read_object(repl, type, size);
4bbf5a26 2828 if (data)
b6c4cecc 2829 return data;
68095570 2830
25f3af3f 2831 if (errno && errno != ENOENT)
3ba7a065
JH
2832 die_errno("failed to read object %s", sha1_to_hex(sha1));
2833
68095570 2834 /* die if we replaced an object with one that does not exist */
b6c4cecc 2835 if (repl != sha1)
68095570
CC
2836 die("replacement %s not found for %s",
2837 sha1_to_hex(repl), sha1_to_hex(sha1));
2838
b6c4cecc 2839 if (has_loose_object(repl)) {
30d6c6ea
MH
2840 const char *path = sha1_file_name(sha1);
2841
b6c4cecc
JH
2842 die("loose object %s (stored in %s) is corrupt",
2843 sha1_to_hex(repl), path);
e8b15e61 2844 }
68095570 2845
b6c4cecc
JH
2846 if ((p = has_packed_and_bad(repl)) != NULL)
2847 die("packed object %s (stored in %s) is corrupt",
2848 sha1_to_hex(repl), p->pack_name);
f5552aee 2849
b6c4cecc 2850 return NULL;
ac939109
NP
2851}
2852
40469ee9 2853void *read_object_with_reference(const unsigned char *sha1,
21666f1a 2854 const char *required_type_name,
40469ee9
JH
2855 unsigned long *size,
2856 unsigned char *actual_sha1_return)
f4913f91 2857{
21666f1a 2858 enum object_type type, required_type;
f4913f91
JH
2859 void *buffer;
2860 unsigned long isize;
40469ee9 2861 unsigned char actual_sha1[20];
f4913f91 2862
21666f1a 2863 required_type = type_from_string(required_type_name);
e702496e 2864 hashcpy(actual_sha1, sha1);
40469ee9
JH
2865 while (1) {
2866 int ref_length = -1;
2867 const char *ref_type = NULL;
f4913f91 2868
21666f1a 2869 buffer = read_sha1_file(actual_sha1, &type, &isize);
40469ee9
JH
2870 if (!buffer)
2871 return NULL;
21666f1a 2872 if (type == required_type) {
40469ee9
JH
2873 *size = isize;
2874 if (actual_sha1_return)
e702496e 2875 hashcpy(actual_sha1_return, actual_sha1);
40469ee9
JH
2876 return buffer;
2877 }
2878 /* Handle references */
21666f1a 2879 else if (type == OBJ_COMMIT)
40469ee9 2880 ref_type = "tree ";
21666f1a 2881 else if (type == OBJ_TAG)
40469ee9
JH
2882 ref_type = "object ";
2883 else {
2884 free(buffer);
2885 return NULL;
2886 }
2887 ref_length = strlen(ref_type);
f4913f91 2888
50974ec9
MK
2889 if (ref_length + 40 > isize ||
2890 memcmp(buffer, ref_type, ref_length) ||
1d7f171c 2891 get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
40469ee9
JH
2892 free(buffer);
2893 return NULL;
2894 }
1cf58e72 2895 free(buffer);
40469ee9
JH
2896 /* Now we have the ID of the referred-to object in
2897 * actual_sha1. Check again. */
f4913f91 2898 }
f4913f91
JH
2899}
2900
ce9fbf16 2901static void write_sha1_file_prepare(const void *buf, unsigned long len,
972a9155 2902 const char *type, unsigned char *sha1,
d65a16f6 2903 char *hdr, int *hdrlen)
d410c0f5 2904{
9126f009 2905 git_SHA_CTX c;
d410c0f5
JH
2906
2907 /* Generate the header */
ef1286d3 2908 *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1;
d410c0f5
JH
2909
2910 /* Sha1.. */
9126f009
NP
2911 git_SHA1_Init(&c);
2912 git_SHA1_Update(&c, hdr, *hdrlen);
2913 git_SHA1_Update(&c, buf, len);
2914 git_SHA1_Final(sha1, &c);
d410c0f5
JH
2915}
2916
230f1322 2917/*
5a688fe4 2918 * Move the just written object into its final resting place.
230f1322 2919 */
cb5add58 2920int finalize_object_file(const char *tmpfile, const char *filename)
230f1322 2921{
e32c0a9c 2922 int ret = 0;
5a688fe4 2923
348df166 2924 if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
be66a6c4
JS
2925 goto try_rename;
2926 else if (link(tmpfile, filename))
e32c0a9c 2927 ret = errno;
7ebb6fca
LT
2928
2929 /*
2930 * Coda hack - coda doesn't like cross-directory links,
2931 * so we fall back to a rename, which will mean that it
2932 * won't be able to check collisions, but that's not a
2933 * big deal.
2934 *
2935 * The same holds for FAT formatted media.
2936 *
3be1f18e 2937 * When this succeeds, we just return. We have nothing
7ebb6fca
LT
2938 * left to unlink.
2939 */
2940 if (ret && ret != EEXIST) {
be66a6c4 2941 try_rename:
7ebb6fca 2942 if (!rename(tmpfile, filename))
3be1f18e 2943 goto out;
9e48b389 2944 ret = errno;
230f1322 2945 }
691f1a28 2946 unlink_or_warn(tmpfile);
230f1322
LT
2947 if (ret) {
2948 if (ret != EEXIST) {
7616c6ca 2949 return error_errno("unable to write sha1 filename %s", filename);
230f1322
LT
2950 }
2951 /* FIXME!!! Collision check here ? */
2952 }
2953
3be1f18e 2954out:
5256b006 2955 if (adjust_shared_perm(filename))
5a688fe4 2956 return error("unable to set permission to '%s'", filename);
230f1322
LT
2957 return 0;
2958}
2959
4d548150
LT
2960static int write_buffer(int fd, const void *buf, size_t len)
2961{
d34cf19b 2962 if (write_in_full(fd, buf, len) < 0)
7616c6ca 2963 return error_errno("file write error");
4d548150
LT
2964 return 0;
2965}
2966
ce9fbf16 2967int hash_sha1_file(const void *buf, unsigned long len, const char *type,
abdc3fc8
RS
2968 unsigned char *sha1)
2969{
d65a16f6 2970 char hdr[32];
ef1286d3 2971 int hdrlen = sizeof(hdr);
abdc3fc8
RS
2972 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
2973 return 0;
2974}
2975
e9039dd3
LT
2976/* Finalize a file on disk, and close it. */
2977static void close_sha1_file(int fd)
2978{
aafe9fba
LT
2979 if (fsync_object_files)
2980 fsync_or_die(fd, "sha1 file");
e9039dd3 2981 if (close(fd) != 0)
d824cbba 2982 die_errno("error when closing sha1 file");
e9039dd3
LT
2983}
2984
5723fe7e
LT
2985/* Size of directory component, including the ending '/' */
2986static inline int directory_size(const char *filename)
2987{
2988 const char *s = strrchr(filename, '/');
2989 if (!s)
2990 return 0;
2991 return s - filename + 1;
2992}
2993
2994/*
2995 * This creates a temporary file in the same directory as the final
2996 * 'filename'
2997 *
2998 * We want to avoid cross-directory filename renames, because those
2999 * can have problems on various filesystems (FAT, NFS, Coda).
3000 */
d4b3d11a 3001static int create_tmpfile(struct strbuf *tmp, const char *filename)
5723fe7e
LT
3002{
3003 int fd, dirlen = directory_size(filename);
3004
d4b3d11a
JK
3005 strbuf_reset(tmp);
3006 strbuf_add(tmp, filename, dirlen);
3007 strbuf_addstr(tmp, "tmp_obj_XXXXXX");
3008 fd = git_mkstemp_mode(tmp->buf, 0444);
cbacbf4e 3009 if (fd < 0 && dirlen && errno == ENOENT) {
d4b3d11a
JK
3010 /*
3011 * Make sure the directory exists; note that the contents
3012 * of the buffer are undefined after mkstemp returns an
3013 * error, so we have to rewrite the whole buffer from
3014 * scratch.
3015 */
3016 strbuf_reset(tmp);
3017 strbuf_add(tmp, filename, dirlen - 1);
3018 if (mkdir(tmp->buf, 0777) && errno != EEXIST)
b2476a60 3019 return -1;
d4b3d11a 3020 if (adjust_shared_perm(tmp->buf))
5723fe7e
LT
3021 return -1;
3022
3023 /* Try again */
d4b3d11a
JK
3024 strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
3025 fd = git_mkstemp_mode(tmp->buf, 0444);
5723fe7e
LT
3026 }
3027 return fd;
3028}
3029
bbac7311 3030static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
c00e657d 3031 const void *buf, unsigned long len, time_t mtime)
0fcfd160 3032{
915308b1 3033 int fd, ret;
9892beba 3034 unsigned char compressed[4096];
ef49a7a0 3035 git_zstream stream;
748af44c
NP
3036 git_SHA_CTX c;
3037 unsigned char parano_sha1[20];
d4b3d11a 3038 static struct strbuf tmp_file = STRBUF_INIT;
30d6c6ea 3039 const char *filename = sha1_file_name(sha1);
a44c9a5e 3040
d4b3d11a 3041 fd = create_tmpfile(&tmp_file, filename);
aac17941 3042 if (fd < 0) {
35243577 3043 if (errno == EACCES)
82247e9b 3044 return error("insufficient permission for adding an object to repository database %s", get_object_directory());
916d081b 3045 else
7616c6ca 3046 return error_errno("unable to create temporary file");
aac17941
LT
3047 }
3048
0fcfd160 3049 /* Set it up */
55bb5c91 3050 git_deflate_init(&stream, zlib_compression_level);
0fcfd160 3051 stream.next_out = compressed;
9892beba 3052 stream.avail_out = sizeof(compressed);
748af44c 3053 git_SHA1_Init(&c);
a44c9a5e
LT
3054
3055 /* First header.. */
d65a16f6 3056 stream.next_in = (unsigned char *)hdr;
a44c9a5e 3057 stream.avail_in = hdrlen;
55bb5c91
JH
3058 while (git_deflate(&stream, 0) == Z_OK)
3059 ; /* nothing */
748af44c 3060 git_SHA1_Update(&c, hdr, hdrlen);
a44c9a5e
LT
3061
3062 /* Then the data itself.. */
c00e657d 3063 stream.next_in = (void *)buf;
a44c9a5e 3064 stream.avail_in = len;
9892beba 3065 do {
748af44c 3066 unsigned char *in0 = stream.next_in;
55bb5c91 3067 ret = git_deflate(&stream, Z_FINISH);
748af44c 3068 git_SHA1_Update(&c, in0, stream.next_in - in0);
9892beba
NP
3069 if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
3070 die("unable to write sha1 file");
3071 stream.next_out = compressed;
3072 stream.avail_out = sizeof(compressed);
3073 } while (ret == Z_OK);
3074
ac54c277
LT
3075 if (ret != Z_STREAM_END)
3076 die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
55bb5c91 3077 ret = git_deflate_end_gently(&stream);
ac54c277
LT
3078 if (ret != Z_OK)
3079 die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
748af44c
NP
3080 git_SHA1_Final(parano_sha1, &c);
3081 if (hashcmp(sha1, parano_sha1) != 0)
3082 die("confused by unstable object source data for %s", sha1_to_hex(sha1));
ac54c277 3083
e9039dd3 3084 close_sha1_file(fd);
0fcfd160 3085
bbac7311
NP
3086 if (mtime) {
3087 struct utimbuf utb;
3088 utb.actime = mtime;
3089 utb.modtime = mtime;
d4b3d11a 3090 if (utime(tmp_file.buf, &utb) < 0)
7616c6ca 3091 warning_errno("failed utime() on %s", tmp_file.buf);
bbac7311
NP
3092 }
3093
d4b3d11a 3094 return finalize_object_file(tmp_file.buf, filename);
0fcfd160 3095}
8237b185 3096
33d4221c
JK
3097static int freshen_loose_object(const unsigned char *sha1)
3098{
3099 return check_and_freshen(sha1, 1);
3100}
3101
3102static int freshen_packed_object(const unsigned char *sha1)
3103{
3104 struct pack_entry e;
ee1c6c34
JK
3105 if (!find_pack_entry(sha1, &e))
3106 return 0;
3107 if (e.p->freshened)
3108 return 1;
3109 if (!freshen_file(e.p->pack_name))
3110 return 0;
3111 e.p->freshened = 1;
3112 return 1;
33d4221c
JK
3113}
3114
1427a7ff 3115int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
bbac7311 3116{
bbac7311 3117 char hdr[32];
ef1286d3 3118 int hdrlen = sizeof(hdr);
bbac7311
NP
3119
3120 /* Normally if we have it in the pack then we do not bother writing
3121 * it out into .git/objects/??/?{38} file.
3122 */
3123 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
b5f52f37 3124 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
bbac7311
NP
3125 return 0;
3126 return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
3127}
3128
0c3db67c
ES
3129int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
3130 unsigned char *sha1, unsigned flags)
3131{
3132 char *header;
3133 int hdrlen, status = 0;
3134
3135 /* type string, SP, %lu of the length plus NUL must fit this */
ef1286d3
JK
3136 hdrlen = strlen(type) + 32;
3137 header = xmalloc(hdrlen);
0c3db67c
ES
3138 write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
3139
3140 if (!(flags & HASH_WRITE_OBJECT))
3141 goto cleanup;
051086b9 3142 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
0c3db67c
ES
3143 goto cleanup;
3144 status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
3145
3146cleanup:
3147 free(header);
3148 return status;
3149}
3150
bbac7311
NP
3151int force_object_loose(const unsigned char *sha1, time_t mtime)
3152{
bbac7311
NP
3153 void *buf;
3154 unsigned long len;
3155 enum object_type type;
3156 char hdr[32];
3157 int hdrlen;
1fb23e65 3158 int ret;
bbac7311 3159
c529d75a 3160 if (has_loose_object(sha1))
bbac7311
NP
3161 return 0;
3162 buf = read_packed_sha1(sha1, &type, &len);
3163 if (!buf)
3164 return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
ef1286d3 3165 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
1fb23e65
BS
3166 ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
3167 free(buf);
3168
3169 return ret;
bbac7311
NP
3170}
3171
bf592c50
DB
3172int has_pack_index(const unsigned char *sha1)
3173{
3174 struct stat st;
3175 if (stat(sha1_pack_index_name(sha1), &st))
3176 return 0;
3177 return 1;
bf592c50
DB
3178}
3179
cd673c1f
JH
3180int has_sha1_pack(const unsigned char *sha1)
3181{
3182 struct pack_entry e;
3183 return find_pack_entry(sha1, &e);
3184}
3185
0eeb077b 3186int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
8237b185 3187{
1f688557
JH
3188 struct pack_entry e;
3189
cd673c1f 3190 if (find_pack_entry(sha1, &e))
1f688557 3191 return 1;
45e8a748
JK
3192 if (has_loose_object(sha1))
3193 return 1;
0eeb077b
JK
3194 if (flags & HAS_SHA1_QUICK)
3195 return 0;
45e8a748
JK
3196 reprepare_packed_git();
3197 return find_pack_entry(sha1, &e);
b419aa25 3198}
3199
3200int has_object_file(const struct object_id *oid)
3201{
3202 return has_sha1_file(oid->hash);
8237b185 3203}
74400e71 3204
c879daa2
NTND
3205static void check_tree(const void *buf, size_t size)
3206{
3207 struct tree_desc desc;
3208 struct name_entry entry;
3209
3210 init_tree_desc(&desc, buf, size);
3211 while (tree_entry(&desc, &entry))
3212 /* do nothing
3213 * tree_entry() will die() on malformed entries */
3214 ;
3215}
3216
3217static void check_commit(const void *buf, size_t size)
3218{
3219 struct commit c;
3220 memset(&c, 0, sizeof(c));
3221 if (parse_commit_buffer(&c, buf, size))
3222 die("corrupt commit");
3223}
3224
3225static void check_tag(const void *buf, size_t size)
3226{
3227 struct tag t;
3228 memset(&t, 0, sizeof(t));
3229 if (parse_tag_buffer(&t, buf, size))
3230 die("corrupt tag");
3231}
3232
43df4f86 3233static int index_mem(unsigned char *sha1, void *buf, size_t size,
c4ce46fc
JH
3234 enum object_type type,
3235 const char *path, unsigned flags)
e7332f96 3236{
6c510bee 3237 int ret, re_allocated = 0;
c4ce46fc 3238 int write_object = flags & HASH_WRITE_OBJECT;
74400e71 3239
7672db20 3240 if (!type)
edaec3fb 3241 type = OBJ_BLOB;
6c510bee
LT
3242
3243 /*
3244 * Convert blobs to git internal format
3245 */
43df4f86 3246 if ((type == OBJ_BLOB) && path) {
f285a2d7 3247 struct strbuf nbuf = STRBUF_INIT;
21e5ad50 3248 if (convert_to_git(path, buf, size, &nbuf,
5e12e78e 3249 write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
b315c5c0 3250 buf = strbuf_detach(&nbuf, &size);
6c510bee
LT
3251 re_allocated = 1;
3252 }
3253 }
c4ce46fc 3254 if (flags & HASH_FORMAT_CHECK) {
c879daa2
NTND
3255 if (type == OBJ_TREE)
3256 check_tree(buf, size);
3257 if (type == OBJ_COMMIT)
3258 check_commit(buf, size);
3259 if (type == OBJ_TAG)
3260 check_tag(buf, size);
3261 }
6c510bee 3262
7672db20 3263 if (write_object)
edaec3fb 3264 ret = write_sha1_file(buf, size, typename(type), sha1);
abdc3fc8 3265 else
edaec3fb 3266 ret = hash_sha1_file(buf, size, typename(type), sha1);
43df4f86 3267 if (re_allocated)
6c510bee 3268 free(buf);
43df4f86
DP
3269 return ret;
3270}
3271
9035d75a
SP
3272static int index_stream_convert_blob(unsigned char *sha1, int fd,
3273 const char *path, unsigned flags)
3274{
3275 int ret;
3276 const int write_object = flags & HASH_WRITE_OBJECT;
3277 struct strbuf sbuf = STRBUF_INIT;
3278
3279 assert(path);
3280 assert(would_convert_to_git_filter_fd(path));
3281
3282 convert_to_git_filter_fd(path, fd, &sbuf,
3283 write_object ? safe_crlf : SAFE_CRLF_FALSE);
3284
3285 if (write_object)
3286 ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3287 sha1);
3288 else
3289 ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3290 sha1);
3291 strbuf_release(&sbuf);
3292 return ret;
3293}
3294
7b41e1e1
JH
3295static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
3296 const char *path, unsigned flags)
3297{
3298 struct strbuf sbuf = STRBUF_INIT;
3299 int ret;
3300
3301 if (strbuf_read(&sbuf, fd, 4096) >= 0)
3302 ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
3303 else
3304 ret = -1;
3305 strbuf_release(&sbuf);
3306 return ret;
3307}
3308
ea68b0ce
DP
3309#define SMALL_FILE_SIZE (32*1024)
3310
7b41e1e1
JH
3311static int index_core(unsigned char *sha1, int fd, size_t size,
3312 enum object_type type, const char *path,
3313 unsigned flags)
43df4f86
DP
3314{
3315 int ret;
43df4f86 3316
7b41e1e1 3317 if (!size) {
f6a1e1e2 3318 ret = index_mem(sha1, "", size, type, path, flags);
ea68b0ce
DP
3319 } else if (size <= SMALL_FILE_SIZE) {
3320 char *buf = xmalloc(size);
3321 if (size == read_in_full(fd, buf, size))
c4ce46fc 3322 ret = index_mem(sha1, buf, size, type, path, flags);
ea68b0ce 3323 else
7616c6ca 3324 ret = error_errno("short read");
ea68b0ce 3325 free(buf);
08bda208 3326 } else {
43df4f86 3327 void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
c4ce46fc 3328 ret = index_mem(sha1, buf, size, type, path, flags);
aac17941 3329 munmap(buf, size);
08bda208 3330 }
7b41e1e1
JH
3331 return ret;
3332}
3333
4dd1fbc7 3334/*
568508e7
JH
3335 * This creates one packfile per large blob unless bulk-checkin
3336 * machinery is "plugged".
4dd1fbc7
JH
3337 *
3338 * This also bypasses the usual "convert-to-git" dance, and that is on
3339 * purpose. We could write a streaming version of the converting
3340 * functions and insert that before feeding the data to fast-import
4f22b101
JK
3341 * (or equivalent in-core API described above). However, that is
3342 * somewhat complicated, as we do not know the size of the filter
3343 * result, which we need to know beforehand when writing a git object.
3344 * Since the primary motivation for trying to stream from the working
3345 * tree file and to avoid mmaping it in core is to deal with large
3346 * binary blobs, they generally do not want to get any conversion, and
3347 * callers should avoid this code path when filters are requested.
4dd1fbc7
JH
3348 */
3349static int index_stream(unsigned char *sha1, int fd, size_t size,
3350 enum object_type type, const char *path,
3351 unsigned flags)
3352{
568508e7 3353 return index_bulk_checkin(sha1, fd, size, type, path, flags);
4dd1fbc7
JH
3354}
3355
7b41e1e1
JH
3356int index_fd(unsigned char *sha1, int fd, struct stat *st,
3357 enum object_type type, const char *path, unsigned flags)
3358{
3359 int ret;
7b41e1e1 3360
9079ab7c
SP
3361 /*
3362 * Call xsize_t() only when needed to avoid potentially unnecessary
3363 * die() for large files.
3364 */
9035d75a
SP
3365 if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
3366 ret = index_stream_convert_blob(sha1, fd, path, flags);
3367 else if (!S_ISREG(st->st_mode))
7b41e1e1 3368 ret = index_pipe(sha1, fd, type, path, flags);
9079ab7c 3369 else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
7ce7c760 3370 (path && would_convert_to_git(path)))
9079ab7c
SP
3371 ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
3372 flags);
4dd1fbc7 3373 else
9079ab7c
SP
3374 ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
3375 flags);
43df4f86 3376 close(fd);
aac17941 3377 return ret;
74400e71 3378}
ec1fcc16 3379
c4ce46fc 3380int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
ec1fcc16
JH
3381{
3382 int fd;
b760d3aa 3383 struct strbuf sb = STRBUF_INIT;
ec1fcc16
JH
3384
3385 switch (st->st_mode & S_IFMT) {
3386 case S_IFREG:
3387 fd = open(path, O_RDONLY);
3388 if (fd < 0)
7616c6ca 3389 return error_errno("open(\"%s\")", path);
c4ce46fc 3390 if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
ec1fcc16
JH
3391 return error("%s: failed to insert into database",
3392 path);
3393 break;
3394 case S_IFLNK:
7616c6ca
NTND
3395 if (strbuf_readlink(&sb, path, st->st_size))
3396 return error_errno("readlink(\"%s\")", path);
c4ce46fc 3397 if (!(flags & HASH_WRITE_OBJECT))
b760d3aa
LT
3398 hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
3399 else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
ec1fcc16
JH
3400 return error("%s: failed to insert into database",
3401 path);
b760d3aa 3402 strbuf_release(&sb);
ec1fcc16 3403 break;
f35a6d3b
LT
3404 case S_IFDIR:
3405 return resolve_gitlink_ref(path, "HEAD", sha1);
ec1fcc16
JH
3406 default:
3407 return error("%s: unsupported file type", path);
3408 }
3409 return 0;
3410}
a69e5429
JH
3411
3412int read_pack_header(int fd, struct pack_header *header)
3413{
c697ad14
HO
3414 if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
3415 /* "eof before pack header was fully read" */
3416 return PH_ERROR_EOF;
3417
a69e5429
JH
3418 if (header->hdr_signature != htonl(PACK_SIGNATURE))
3419 /* "protocol error (pack signature mismatch detected)" */
3420 return PH_ERROR_PACK_SIGNATURE;
3421 if (!pack_version_ok(header->hdr_version))
3422 /* "protocol error (pack version unsupported)" */
3423 return PH_ERROR_PROTOCOL;
3424 return 0;
3425}
40d52ff7
JK
3426
3427void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
3428{
3429 enum object_type type = sha1_object_info(sha1, NULL);
3430 if (type < 0)
3431 die("%s is not a valid object", sha1_to_hex(sha1));
3432 if (type != expect)
3433 die("%s is not a valid '%s' object", sha1_to_hex(sha1),
3434 typename(expect));
3435}
27e1e22d
JK
3436
3437static int for_each_file_in_obj_subdir(int subdir_nr,
3438 struct strbuf *path,
3439 each_loose_object_fn obj_cb,
3440 each_loose_cruft_fn cruft_cb,
3441 each_loose_subdir_fn subdir_cb,
3442 void *data)
3443{
3444 size_t baselen = path->len;
3445 DIR *dir = opendir(path->buf);
3446 struct dirent *de;
3447 int r = 0;
3448
3449 if (!dir) {
3450 if (errno == ENOENT)
3451 return 0;
7616c6ca 3452 return error_errno("unable to open %s", path->buf);
27e1e22d
JK
3453 }
3454
3455 while ((de = readdir(dir))) {
3456 if (is_dot_or_dotdot(de->d_name))
3457 continue;
3458
3459 strbuf_setlen(path, baselen);
3460 strbuf_addf(path, "/%s", de->d_name);
3461
3462 if (strlen(de->d_name) == 38) {
3463 char hex[41];
3464 unsigned char sha1[20];
3465
3466 snprintf(hex, sizeof(hex), "%02x%s",
3467 subdir_nr, de->d_name);
3468 if (!get_sha1_hex(hex, sha1)) {
3469 if (obj_cb) {
3470 r = obj_cb(sha1, path->buf, data);
3471 if (r)
3472 break;
3473 }
3474 continue;
3475 }
3476 }
3477
3478 if (cruft_cb) {
3479 r = cruft_cb(de->d_name, path->buf, data);
3480 if (r)
3481 break;
3482 }
3483 }
094c7e63 3484 closedir(dir);
27e1e22d 3485
094c7e63 3486 strbuf_setlen(path, baselen);
27e1e22d
JK
3487 if (!r && subdir_cb)
3488 r = subdir_cb(subdir_nr, path->buf, data);
3489
27e1e22d
JK
3490 return r;
3491}
3492
e6f875e0 3493int for_each_loose_file_in_objdir_buf(struct strbuf *path,
27e1e22d
JK
3494 each_loose_object_fn obj_cb,
3495 each_loose_cruft_fn cruft_cb,
3496 each_loose_subdir_fn subdir_cb,
3497 void *data)
3498{
e6f875e0 3499 size_t baselen = path->len;
27e1e22d
JK
3500 int r = 0;
3501 int i;
3502
27e1e22d 3503 for (i = 0; i < 256; i++) {
e6f875e0
JK
3504 strbuf_addf(path, "/%02x", i);
3505 r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
27e1e22d 3506 subdir_cb, data);
e6f875e0 3507 strbuf_setlen(path, baselen);
27e1e22d
JK
3508 if (r)
3509 break;
3510 }
3511
e6f875e0
JK
3512 return r;
3513}
3514
3515int for_each_loose_file_in_objdir(const char *path,
3516 each_loose_object_fn obj_cb,
3517 each_loose_cruft_fn cruft_cb,
3518 each_loose_subdir_fn subdir_cb,
3519 void *data)
3520{
3521 struct strbuf buf = STRBUF_INIT;
3522 int r;
3523
3524 strbuf_addstr(&buf, path);
3525 r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
3526 subdir_cb, data);
27e1e22d 3527 strbuf_release(&buf);