Git 2.14.2
[git/git.git] / sha1_file.c
CommitLineData
0fcfd160
LT
1/*
2 * GIT - The information manager from hell
3 *
4 * Copyright (C) Linus Torvalds, 2005
5 *
6 * This handles basic git sha1 object files - packing, unpacking,
7 * creation etc.
8 */
0fcfd160 9#include "cache.h"
b2141fc1 10#include "config.h"
6eac50d8 11#include "string-list.h"
697cc8ef 12#include "lockfile.h"
1f688557 13#include "delta.h"
a733cb60 14#include "pack.h"
8e440259
PE
15#include "blob.h"
16#include "commit.h"
4dd1fbc7 17#include "run-command.h"
8e440259
PE
18#include "tag.h"
19#include "tree.h"
c879daa2 20#include "tree-walk.h"
f35a6d3b 21#include "refs.h"
70f5d5d3 22#include "pack-revindex.h"
628522ec 23#include "sha1-lookup.h"
568508e7 24#include "bulk-checkin.h"
090ea126 25#include "streaming.h"
543c5caa 26#include "dir.h"
a73cdd21 27#include "mru.h"
12d95ef6 28#include "list.h"
c4c6effa 29#include "mergesort.h"
cf3c6352 30#include "quote.h"
0fcfd160 31
28bd70d8
JN
32#define SZ_FMT PRIuMAX
33static inline uintmax_t sz_fmt(size_t s) { return s; }
e05db0fd 34
96f1e58f 35const unsigned char null_sha1[20];
3e56e724 36const struct object_id null_oid;
8576fde6
JK
37const struct object_id empty_tree_oid = {
38 EMPTY_TREE_SHA1_BIN_LITERAL
39};
40const struct object_id empty_blob_oid = {
41 EMPTY_BLOB_SHA1_BIN_LITERAL
42};
88cd621d 43
c597ba80
NTND
44/*
45 * This is meant to hold a *small* number of objects that you would
46 * want read_sha1_file() to be able to return, but yet you do not want
47 * to write them into the object store (e.g. a browse-only
48 * application).
49 */
50static struct cached_object {
51 unsigned char sha1[20];
52 enum object_type type;
53 void *buf;
54 unsigned long size;
55} *cached_objects;
56static int cached_object_nr, cached_object_alloc;
57
58static struct cached_object empty_tree = {
dab0d410 59 EMPTY_TREE_SHA1_BIN_LITERAL,
c597ba80
NTND
60 OBJ_TREE,
61 "",
62 0
63};
64
65static struct cached_object *find_cached_object(const unsigned char *sha1)
66{
67 int i;
68 struct cached_object *co = cached_objects;
69
70 for (i = 0; i < cached_object_nr; i++, co++) {
71 if (!hashcmp(co->sha1, sha1))
72 return co;
73 }
74 if (!hashcmp(sha1, empty_tree.sha1))
75 return &empty_tree;
76 return NULL;
77}
78
90a6464b
JH
79int mkdir_in_gitdir(const char *path)
80{
81 if (mkdir(path, 0777)) {
82 int saved_errno = errno;
83 struct stat st;
84 struct strbuf sb = STRBUF_INIT;
85
86 if (errno != EEXIST)
87 return -1;
88 /*
89 * Are we looking at a path in a symlinked worktree
90 * whose original repository does not yet have it?
91 * e.g. .git/rr-cache pointing at its original
92 * repository in which the user hasn't performed any
93 * conflict resolution yet?
94 */
95 if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
96 strbuf_readlink(&sb, path, st.st_size) ||
97 !is_absolute_path(sb.buf) ||
98 mkdir(sb.buf, 0777)) {
99 strbuf_release(&sb);
100 errno = saved_errno;
101 return -1;
102 }
103 strbuf_release(&sb);
104 }
105 return adjust_shared_perm(path);
106}
107
0be0521b 108enum scld_error safe_create_leading_directories(char *path)
b2cb9425 109{
26c8ae2a 110 char *next_component = path + offset_1st_component(path);
0be0521b 111 enum scld_error ret = SCLD_OK;
67d42212 112
0be0521b 113 while (ret == SCLD_OK && next_component) {
f0502332 114 struct stat st;
0f527403 115 char *slash = next_component, slash_character;
f0502332 116
0f527403
MH
117 while (*slash && !is_dir_sep(*slash))
118 slash++;
119
120 if (!*slash)
b2cb9425 121 break;
bf10cf70 122
26c8ae2a 123 next_component = slash + 1;
0f527403 124 while (is_dir_sep(*next_component))
bf10cf70 125 next_component++;
26c8ae2a 126 if (!*next_component)
5f0bdf50 127 break;
831651fd 128
0f527403 129 slash_character = *slash;
831651fd 130 *slash = '\0';
67d42212
JR
131 if (!stat(path, &st)) {
132 /* path exists */
204a047f
MH
133 if (!S_ISDIR(st.st_mode)) {
134 errno = ENOTDIR;
0be0521b 135 ret = SCLD_EXISTS;
204a047f 136 }
53a39721 137 } else if (mkdir(path, 0777)) {
928734d9 138 if (errno == EEXIST &&
9e6f885d 139 !stat(path, &st) && S_ISDIR(st.st_mode))
928734d9 140 ; /* somebody created it since we checked */
18d37e86
MH
141 else if (errno == ENOENT)
142 /*
143 * Either mkdir() failed because
144 * somebody just pruned the containing
145 * directory, or stat() failed because
146 * the file that was in our way was
147 * just removed. Either way, inform
148 * the caller that it might be worth
149 * trying again:
150 */
151 ret = SCLD_VANISHED;
9e6f885d 152 else
0be0521b 153 ret = SCLD_FAILED;
53a39721 154 } else if (adjust_shared_perm(path)) {
0be0521b 155 ret = SCLD_PERMS;
457f06d6 156 }
0f527403 157 *slash = slash_character;
b2cb9425 158 }
9e6f885d 159 return ret;
b2cb9425 160}
723c31fe 161
0be0521b 162enum scld_error safe_create_leading_directories_const(const char *path)
8e21d63b 163{
02944307 164 int save_errno;
8e21d63b
JK
165 /* path points to cache entries, so xstrdup before messing with it */
166 char *buf = xstrdup(path);
0be0521b 167 enum scld_error result = safe_create_leading_directories(buf);
02944307
MH
168
169 save_errno = errno;
8e21d63b 170 free(buf);
02944307 171 errno = save_errno;
8e21d63b
JK
172 return result;
173}
174
177978f5
MH
175int raceproof_create_file(const char *path, create_file_fn fn, void *cb)
176{
177 /*
178 * The number of times we will try to remove empty directories
179 * in the way of path. This is only 1 because if another
180 * process is racily creating directories that conflict with
181 * us, we don't want to fight against them.
182 */
183 int remove_directories_remaining = 1;
184
185 /*
186 * The number of times that we will try to create the
187 * directories containing path. We are willing to attempt this
188 * more than once, because another process could be trying to
189 * clean up empty directories at the same time as we are
190 * trying to create them.
191 */
192 int create_directories_remaining = 3;
193
194 /* A scratch copy of path, filled lazily if we need it: */
195 struct strbuf path_copy = STRBUF_INIT;
196
197 int ret, save_errno;
198
199 /* Sanity check: */
200 assert(*path);
201
202retry_fn:
203 ret = fn(path, cb);
204 save_errno = errno;
205 if (!ret)
206 goto out;
207
208 if (errno == EISDIR && remove_directories_remaining-- > 0) {
209 /*
210 * A directory is in the way. Maybe it is empty; try
211 * to remove it:
212 */
213 if (!path_copy.len)
214 strbuf_addstr(&path_copy, path);
215
216 if (!remove_dir_recursively(&path_copy, REMOVE_DIR_EMPTY_ONLY))
217 goto retry_fn;
218 } else if (errno == ENOENT && create_directories_remaining-- > 0) {
219 /*
220 * Maybe the containing directory didn't exist, or
221 * maybe it was just deleted by a process that is
222 * racing with us to clean up empty directories. Try
223 * to create it:
224 */
225 enum scld_error scld_result;
226
227 if (!path_copy.len)
228 strbuf_addstr(&path_copy, path);
229
230 do {
231 scld_result = safe_create_leading_directories(path_copy.buf);
232 if (scld_result == SCLD_OK)
233 goto retry_fn;
234 } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0);
235 }
236
237out:
238 strbuf_release(&path_copy);
239 errno = save_errno;
240 return ret;
241}
242
f7b7774f 243static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1)
ace1534d
JH
244{
245 int i;
246 for (i = 0; i < 20; i++) {
247 static char hex[] = "0123456789abcdef";
248 unsigned int val = sha1[i];
f7b7774f
JK
249 strbuf_addch(buf, hex[val >> 4]);
250 strbuf_addch(buf, hex[val & 0xf]);
afbba2f0 251 if (!i)
f7b7774f 252 strbuf_addch(buf, '/');
ace1534d
JH
253 }
254}
255
30d6c6ea 256const char *sha1_file_name(const unsigned char *sha1)
0fcfd160 257{
f7b7774f 258 static struct strbuf buf = STRBUF_INIT;
0fcfd160 259
f7b7774f
JK
260 strbuf_reset(&buf);
261 strbuf_addf(&buf, "%s/", get_object_directory());
560fb6a1 262
f7b7774f
JK
263 fill_sha1_path(&buf, sha1);
264 return buf.buf;
0fcfd160
LT
265}
266
38dbe5f0
JK
267struct strbuf *alt_scratch_buf(struct alternate_object_database *alt)
268{
269 strbuf_setlen(&alt->scratch, alt->base_len);
270 return &alt->scratch;
271}
272
29ec6af2
JK
273static const char *alt_sha1_path(struct alternate_object_database *alt,
274 const unsigned char *sha1)
275{
38dbe5f0 276 struct strbuf *buf = alt_scratch_buf(alt);
f7b7774f 277 fill_sha1_path(buf, sha1);
38dbe5f0 278 return buf->buf;
0fcfd160
LT
279}
280
1cec8c63
JK
281 char *odb_pack_name(struct strbuf *buf,
282 const unsigned char *sha1,
283 const char *ext)
bf592c50 284{
ac5190cc
JK
285 strbuf_reset(buf);
286 strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
1cec8c63 287 sha1_to_hex(sha1), ext);
ac5190cc 288 return buf->buf;
bf592c50
DB
289}
290
633f43e1 291char *sha1_pack_name(const unsigned char *sha1)
bf592c50 292{
ac5190cc 293 static struct strbuf buf = STRBUF_INIT;
1cec8c63 294 return odb_pack_name(&buf, sha1, "pack");
633f43e1 295}
bf592c50 296
633f43e1
HO
297char *sha1_pack_index_name(const unsigned char *sha1)
298{
ac5190cc 299 static struct strbuf buf = STRBUF_INIT;
1cec8c63 300 return odb_pack_name(&buf, sha1, "idx");
bf592c50
DB
301}
302
d5a63b99
JH
303struct alternate_object_database *alt_odb_list;
304static struct alternate_object_database **alt_odb_tail;
ace1534d 305
4ea82473
JK
306/*
307 * Return non-zero iff the path is usable as an alternate object database.
308 */
309static int alt_odb_usable(struct strbuf *path, const char *normalized_objdir)
310{
311 struct alternate_object_database *alt;
312
313 /* Detect cases where alternate disappeared */
314 if (!is_directory(path->buf)) {
315 error("object directory %s does not exist; "
316 "check .git/objects/info/alternates.",
317 path->buf);
318 return 0;
319 }
320
321 /*
322 * Prevent the common mistake of listing the same
323 * thing twice, or object directory itself.
324 */
325 for (alt = alt_odb_list; alt; alt = alt->next) {
ea0fc3b4 326 if (!fspathcmp(path->buf, alt->path))
4ea82473
JK
327 return 0;
328 }
329 if (!fspathcmp(path->buf, normalized_objdir))
330 return 0;
331
332 return 1;
333}
334
ddd5d056
JH
335/*
336 * Prepare alternate object database registry.
d5a63b99
JH
337 *
338 * The variable alt_odb_list points at the list of struct
339 * alternate_object_database. The elements on this list come from
340 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
341 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
1494e038
JH
342 * whose contents is similar to that environment variable but can be
343 * LF separated. Its base points at a statically allocated buffer that
d5a63b99
JH
344 * contains "/the/directory/corresponding/to/.git/objects/...", while
345 * its name points just after the slash at the end of ".git/objects/"
346 * in the example above, and has enough space to hold 40-byte hex
347 * SHA1, an extra slash for the first level indirection, and the
348 * terminating NUL.
ddd5d056 349 */
2456990d 350static void read_info_alternates(const char * relative_base, int depth);
539e7506
EK
351static int link_alt_odb_entry(const char *entry, const char *relative_base,
352 int depth, const char *normalized_objdir)
ace1534d 353{
c2f493a4 354 struct alternate_object_database *ent;
5bdf0a84 355 struct strbuf pathbuf = STRBUF_INIT;
d5a63b99 356
85dadc38 357 if (!is_absolute_path(entry) && relative_base) {
4ac9006f 358 strbuf_realpath(&pathbuf, relative_base, 1);
5bdf0a84 359 strbuf_addch(&pathbuf, '/');
c2f493a4 360 }
6eac50d8 361 strbuf_addstr(&pathbuf, entry);
c2f493a4 362
37a95862 363 if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
670c359d
JK
364 error("unable to normalize alternate object path: %s",
365 pathbuf.buf);
366 strbuf_release(&pathbuf);
367 return -1;
368 }
5bdf0a84
HW
369
370 /*
371 * The trailing slash after the directory name is given by
372 * this function at the end. Remove duplicates.
373 */
4ea82473
JK
374 while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
375 strbuf_setlen(&pathbuf, pathbuf.len - 1);
5bdf0a84 376
4ea82473
JK
377 if (!alt_odb_usable(&pathbuf, normalized_objdir)) {
378 strbuf_release(&pathbuf);
c2f493a4
MW
379 return -1;
380 }
381
7f0fa2c0 382 ent = alloc_alt_odb(pathbuf.buf);
c2f493a4
MW
383
384 /* add the alternate entry */
385 *alt_odb_tail = ent;
386 alt_odb_tail = &(ent->next);
387 ent->next = NULL;
388
389 /* recursively add alternates */
4ea82473 390 read_info_alternates(pathbuf.buf, depth + 1);
c2f493a4 391
4ea82473 392 strbuf_release(&pathbuf);
c2f493a4
MW
393 return 0;
394}
395
cf3c6352
JK
396static const char *parse_alt_odb_entry(const char *string,
397 int sep,
398 struct strbuf *out)
399{
400 const char *end;
401
402 strbuf_reset(out);
403
404 if (*string == '#') {
405 /* comment; consume up to next separator */
406 end = strchrnul(string, sep);
407 } else if (*string == '"' && !unquote_c_style(out, string, &end)) {
408 /*
409 * quoted path; unquote_c_style has copied the
410 * data for us and set "end". Broken quoting (e.g.,
411 * an entry that doesn't end with a quote) falls
412 * back to the unquoted case below.
413 */
414 } else {
415 /* normal, unquoted path */
416 end = strchrnul(string, sep);
417 strbuf_add(out, string, end - string);
418 }
419
420 if (*end)
421 end++;
422 return end;
423}
424
c5950164 425static void link_alt_odb_entries(const char *alt, int len, int sep,
c2f493a4
MW
426 const char *relative_base, int depth)
427{
539e7506 428 struct strbuf objdirbuf = STRBUF_INIT;
cf3c6352 429 struct strbuf entry = STRBUF_INIT;
c2f493a4
MW
430
431 if (depth > 5) {
432 error("%s: ignoring alternate object stores, nesting too deep.",
433 relative_base);
434 return;
435 }
436
f655651e 437 strbuf_add_absolute_path(&objdirbuf, get_object_directory());
670c359d
JK
438 if (strbuf_normalize_path(&objdirbuf) < 0)
439 die("unable to normalize object directory: %s",
440 objdirbuf.buf);
539e7506 441
cf3c6352
JK
442 while (*alt) {
443 alt = parse_alt_odb_entry(alt, sep, &entry);
444 if (!entry.len)
9577e7e3 445 continue;
cf3c6352 446 link_alt_odb_entry(entry.buf, relative_base, depth, objdirbuf.buf);
9577e7e3 447 }
cf3c6352 448 strbuf_release(&entry);
539e7506 449 strbuf_release(&objdirbuf);
d5a63b99
JH
450}
451
2456990d 452static void read_info_alternates(const char * relative_base, int depth)
d5a63b99 453{
9577e7e3 454 char *map;
dc49cd76 455 size_t mapsz;
d5a63b99 456 struct stat st;
5015f01c 457 char *path;
c2f493a4 458 int fd;
d5a63b99 459
5015f01c 460 path = xstrfmt("%s/info/alternates", relative_base);
a5436b57 461 fd = git_open(path);
5015f01c 462 free(path);
d5a63b99
JH
463 if (fd < 0)
464 return;
465 if (fstat(fd, &st) || (st.st_size == 0)) {
466 close(fd);
9a217f2a 467 return;
ace1534d 468 }
dc49cd76
SP
469 mapsz = xsize_t(st.st_size);
470 map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
d5a63b99 471 close(fd);
d5a63b99 472
c5950164 473 link_alt_odb_entries(map, mapsz, '\n', relative_base, depth);
c2f493a4 474
dc49cd76 475 munmap(map, mapsz);
ace1534d
JH
476}
477
7f0fa2c0
JK
478struct alternate_object_database *alloc_alt_odb(const char *dir)
479{
480 struct alternate_object_database *ent;
7f0fa2c0 481
597f9134 482 FLEX_ALLOC_STR(ent, path, dir);
38dbe5f0
JK
483 strbuf_init(&ent->scratch, 0);
484 strbuf_addf(&ent->scratch, "%s/", dir);
485 ent->base_len = ent->scratch.len;
7f0fa2c0
JK
486
487 return ent;
488}
489
bef70b22
DB
490void add_to_alternates_file(const char *reference)
491{
492 struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
77b9b1d1
JK
493 char *alts = git_pathdup("objects/info/alternates");
494 FILE *in, *out;
495
496 hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR);
497 out = fdopen_lock_file(lock, "w");
498 if (!out)
499 die_errno("unable to fdopen alternates lockfile");
500
501 in = fopen(alts, "r");
502 if (in) {
503 struct strbuf line = STRBUF_INIT;
504 int found = 0;
505
3f163962 506 while (strbuf_getline(&line, in) != EOF) {
77b9b1d1
JK
507 if (!strcmp(reference, line.buf)) {
508 found = 1;
509 break;
510 }
511 fprintf_or_die(out, "%s\n", line.buf);
512 }
513
514 strbuf_release(&line);
515 fclose(in);
516
517 if (found) {
518 rollback_lock_file(lock);
519 lock = NULL;
520 }
521 }
522 else if (errno != ENOENT)
523 die_errno("unable to read alternates file");
524
525 if (lock) {
526 fprintf_or_die(out, "%s\n", reference);
527 if (commit_lock_file(lock))
528 die_errno("unable to move new alternates file into place");
529 if (alt_odb_tail)
530 link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
531 }
532 free(alts);
bef70b22
DB
533}
534
a5b34d21
JK
535void add_to_alternates_memory(const char *reference)
536{
537 /*
538 * Make sure alternates are initialized, or else our entry may be
539 * overwritten when they are.
540 */
541 prepare_alt_odb();
542
543 link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
544}
545
9eeea7d2
SB
546/*
547 * Compute the exact path an alternate is at and returns it. In case of
548 * error NULL is returned and the human readable error is added to `err`
549 * `path` may be relative and should point to $GITDIR.
550 * `err` must not be null.
551 */
552char *compute_alternate_path(const char *path, struct strbuf *err)
553{
554 char *ref_git = NULL;
555 const char *repo, *ref_git_s;
556 int seen_error = 0;
557
558 ref_git_s = real_path_if_valid(path);
559 if (!ref_git_s) {
560 seen_error = 1;
561 strbuf_addf(err, _("path '%s' does not exist"), path);
562 goto out;
563 } else
564 /*
565 * Beware: read_gitfile(), real_path() and mkpath()
566 * return static buffer
567 */
568 ref_git = xstrdup(ref_git_s);
569
570 repo = read_gitfile(ref_git);
571 if (!repo)
572 repo = read_gitfile(mkpath("%s/.git", ref_git));
573 if (repo) {
574 free(ref_git);
575 ref_git = xstrdup(repo);
576 }
577
578 if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
579 char *ref_git_git = mkpathdup("%s/.git", ref_git);
580 free(ref_git);
581 ref_git = ref_git_git;
582 } else if (!is_directory(mkpath("%s/objects", ref_git))) {
583 struct strbuf sb = STRBUF_INIT;
584 seen_error = 1;
585 if (get_common_dir(&sb, ref_git)) {
586 strbuf_addf(err,
587 _("reference repository '%s' as a linked "
588 "checkout is not supported yet."),
589 path);
590 goto out;
591 }
592
593 strbuf_addf(err, _("reference repository '%s' is not a "
594 "local repository."), path);
595 goto out;
596 }
597
598 if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
599 strbuf_addf(err, _("reference repository '%s' is shallow"),
600 path);
601 seen_error = 1;
602 goto out;
603 }
604
605 if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
606 strbuf_addf(err,
607 _("reference repository '%s' is grafted"),
608 path);
609 seen_error = 1;
610 goto out;
611 }
612
613out:
614 if (seen_error) {
6a83d902 615 FREE_AND_NULL(ref_git);
9eeea7d2
SB
616 }
617
618 return ref_git;
619}
620
fe1b2268 621int foreach_alt_odb(alt_odb_fn fn, void *cb)
d79796bc
JH
622{
623 struct alternate_object_database *ent;
fe1b2268 624 int r = 0;
d79796bc
JH
625
626 prepare_alt_odb();
fe1b2268
JK
627 for (ent = alt_odb_list; ent; ent = ent->next) {
628 r = fn(ent, cb);
629 if (r)
630 break;
631 }
632 return r;
d79796bc
JH
633}
634
c2f493a4
MW
635void prepare_alt_odb(void)
636{
554fe20d 637 const char *alt;
c2f493a4 638
7dc24aa5
SP
639 if (alt_odb_tail)
640 return;
641
c2f493a4
MW
642 alt = getenv(ALTERNATE_DB_ENVIRONMENT);
643 if (!alt) alt = "";
644
c2f493a4 645 alt_odb_tail = &alt_odb_list;
c5950164 646 link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
c2f493a4
MW
647
648 read_info_alternates(get_object_directory(), 0);
649}
650
3096b2ec 651/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
33d4221c 652static int freshen_file(const char *fn)
ace1534d 653{
33d4221c
JK
654 struct utimbuf t;
655 t.actime = t.modtime = time(NULL);
656 return !utime(fn, &t);
0f4dc14a 657}
ace1534d 658
3096b2ec
JK
659/*
660 * All of the check_and_freshen functions return 1 if the file exists and was
661 * freshened (if freshening was requested), 0 otherwise. If they return
662 * 0, you should not assume that it is safe to skip a write of the object (it
663 * either does not exist on disk, or has a stale mtime and may be subject to
664 * pruning).
665 */
6a5e6f5e 666int check_and_freshen_file(const char *fn, int freshen)
33d4221c
JK
667{
668 if (access(fn, F_OK))
669 return 0;
3096b2ec 670 if (freshen && !freshen_file(fn))
33d4221c
JK
671 return 0;
672 return 1;
673}
674
675static int check_and_freshen_local(const unsigned char *sha1, int freshen)
676{
677 return check_and_freshen_file(sha1_file_name(sha1), freshen);
678}
679
680static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
0f4dc14a
BC
681{
682 struct alternate_object_database *alt;
9a217f2a 683 prepare_alt_odb();
d5a63b99 684 for (alt = alt_odb_list; alt; alt = alt->next) {
29ec6af2
JK
685 const char *path = alt_sha1_path(alt, sha1);
686 if (check_and_freshen_file(path, freshen))
c529d75a 687 return 1;
ace1534d 688 }
c529d75a 689 return 0;
ace1534d
JH
690}
691
33d4221c
JK
692static int check_and_freshen(const unsigned char *sha1, int freshen)
693{
694 return check_and_freshen_local(sha1, freshen) ||
695 check_and_freshen_nonlocal(sha1, freshen);
696}
697
698int has_loose_object_nonlocal(const unsigned char *sha1)
699{
700 return check_and_freshen_nonlocal(sha1, 0);
701}
702
0f4dc14a
BC
703static int has_loose_object(const unsigned char *sha1)
704{
33d4221c 705 return check_and_freshen(sha1, 0);
0f4dc14a
BC
706}
707
60bb8b14 708static unsigned int pack_used_ctr;
a53128b6
SP
709static unsigned int pack_mmap_calls;
710static unsigned int peak_pack_open_windows;
711static unsigned int pack_open_windows;
c7934306
SP
712static unsigned int pack_open_fds;
713static unsigned int pack_max_fds;
a53128b6 714static size_t peak_pack_mapped;
60bb8b14 715static size_t pack_mapped;
9a217f2a 716struct packed_git *packed_git;
1f688557 717
a73cdd21
JK
718static struct mru packed_git_mru_storage;
719struct mru *packed_git_mru = &packed_git_mru_storage;
720
b79d18c9 721void pack_report(void)
a53128b6
SP
722{
723 fprintf(stderr,
e05db0fd
PR
724 "pack_report: getpagesize() = %10" SZ_FMT "\n"
725 "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
726 "pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
9e42d6a1
SP
727 sz_fmt(getpagesize()),
728 sz_fmt(packed_git_window_size),
729 sz_fmt(packed_git_limit));
a53128b6
SP
730 fprintf(stderr,
731 "pack_report: pack_used_ctr = %10u\n"
732 "pack_report: pack_mmap_calls = %10u\n"
733 "pack_report: pack_open_windows = %10u / %10u\n"
e05db0fd
PR
734 "pack_report: pack_mapped = "
735 "%10" SZ_FMT " / %10" SZ_FMT "\n",
a53128b6
SP
736 pack_used_ctr,
737 pack_mmap_calls,
738 pack_open_windows, peak_pack_open_windows,
9e42d6a1 739 sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
a53128b6
SP
740}
741
d40d535b
MH
742/*
743 * Open and mmap the index file at path, perform a couple of
744 * consistency checks, then record its information to p. Return 0 on
745 * success.
746 */
747static int check_packed_git_idx(const char *path, struct packed_git *p)
1f688557
JH
748{
749 void *idx_map;
42873078 750 struct pack_idx_header *hdr;
dc49cd76 751 size_t idx_size;
74e34e1f 752 uint32_t version, nr, i, *index;
a5436b57 753 int fd = git_open(path);
1f688557 754 struct stat st;
42873078 755
1f688557
JH
756 if (fd < 0)
757 return -1;
758 if (fstat(fd, &st)) {
759 close(fd);
760 return -1;
761 }
dc49cd76 762 idx_size = xsize_t(st.st_size);
2d88451b
SP
763 if (idx_size < 4 * 256 + 20 + 20) {
764 close(fd);
765 return error("index file %s is too small", path);
766 }
c4712e45 767 idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
1f688557 768 close(fd);
1f688557 769
42873078
NP
770 hdr = idx_map;
771 if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
74e34e1f
NP
772 version = ntohl(hdr->idx_version);
773 if (version < 2 || version > 2) {
774 munmap(idx_map, idx_size);
6e1c2344 775 return error("index file %s is version %"PRIu32
74e34e1f
NP
776 " and is not supported by this binary"
777 " (try upgrading GIT to a newer version)",
778 path, version);
779 }
780 } else
781 version = 1;
df1b059d 782
1f688557 783 nr = 0;
42873078 784 index = idx_map;
74e34e1f
NP
785 if (version > 1)
786 index += 2; /* skip index header */
1f688557 787 for (i = 0; i < 256; i++) {
326bf396 788 uint32_t n = ntohl(index[i]);
2d88451b
SP
789 if (n < nr) {
790 munmap(idx_map, idx_size);
df1b059d 791 return error("non-monotonic index %s", path);
2d88451b 792 }
1f688557
JH
793 nr = n;
794 }
795
74e34e1f
NP
796 if (version == 1) {
797 /*
798 * Total size:
799 * - 256 index entries 4 bytes each
800 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
801 * - 20-byte SHA1 of the packfile
802 * - 20-byte SHA1 file checksum
803 */
804 if (idx_size != 4*256 + nr * 24 + 20 + 20) {
805 munmap(idx_map, idx_size);
eef427a0 806 return error("wrong index v1 file size in %s", path);
74e34e1f
NP
807 }
808 } else if (version == 2) {
809 /*
810 * Minimum size:
811 * - 8 bytes of header
812 * - 256 index entries 4 bytes each
813 * - 20-byte sha1 entry * nr
814 * - 4-byte crc entry * nr
815 * - 4-byte offset entry * nr
816 * - 20-byte SHA1 of the packfile
817 * - 20-byte SHA1 file checksum
818 * And after the 4-byte offset table might be a
819 * variable sized table containing 8-byte entries
820 * for offsets larger than 2^31.
821 */
822 unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
1164f1e4
LT
823 unsigned long max_size = min_size;
824 if (nr)
825 max_size += (nr - 1)*8;
826 if (idx_size < min_size || idx_size > max_size) {
74e34e1f 827 munmap(idx_map, idx_size);
eef427a0 828 return error("wrong index v2 file size in %s", path);
74e34e1f 829 }
7109c889
JH
830 if (idx_size != min_size &&
831 /*
832 * make sure we can deal with large pack offsets.
833 * 31-bit signed offset won't be enough, neither
834 * 32-bit unsigned one will be.
835 */
836 (sizeof(off_t) <= 4)) {
837 munmap(idx_map, idx_size);
838 return error("pack too large for current definition of off_t in %s", path);
74e34e1f 839 }
2d88451b 840 }
1f688557 841
74e34e1f 842 p->index_version = version;
42873078
NP
843 p->index_data = idx_map;
844 p->index_size = idx_size;
57059091 845 p->num_objects = nr;
1f688557
JH
846 return 0;
847}
848
bc8e478a 849int open_pack_index(struct packed_git *p)
d079837e
SP
850{
851 char *idx_name;
9ae97018 852 size_t len;
d079837e
SP
853 int ret;
854
855 if (p->index_data)
856 return 0;
857
9ae97018
JK
858 if (!strip_suffix(p->pack_name, ".pack", &len))
859 die("BUG: pack_name does not end in .pack");
860 idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
d079837e
SP
861 ret = check_packed_git_idx(idx_name, p);
862 free(idx_name);
863 return ret;
864}
865
11daf39b
SP
866static void scan_windows(struct packed_git *p,
867 struct packed_git **lru_p,
868 struct pack_window **lru_w,
869 struct pack_window **lru_l)
1f688557 870{
11daf39b
SP
871 struct pack_window *w, *w_l;
872
873 for (w_l = NULL, w = p->windows; w; w = w->next) {
874 if (!w->inuse_cnt) {
875 if (!*lru_w || w->last_used < (*lru_w)->last_used) {
876 *lru_p = p;
877 *lru_w = w;
878 *lru_l = w_l;
54044bf8 879 }
54044bf8 880 }
11daf39b 881 w_l = w;
f9253394 882 }
11daf39b
SP
883}
884
7c3ecb32 885static int unuse_one_window(struct packed_git *current)
11daf39b
SP
886{
887 struct packed_git *p, *lru_p = NULL;
888 struct pack_window *lru_w = NULL, *lru_l = NULL;
889
890 if (current)
891 scan_windows(current, &lru_p, &lru_w, &lru_l);
892 for (p = packed_git; p; p = p->next)
893 scan_windows(p, &lru_p, &lru_w, &lru_l);
54044bf8
SP
894 if (lru_p) {
895 munmap(lru_w->base, lru_w->len);
896 pack_mapped -= lru_w->len;
897 if (lru_l)
898 lru_l->next = lru_w->next;
7c3ecb32 899 else
54044bf8 900 lru_p->windows = lru_w->next;
54044bf8 901 free(lru_w);
a53128b6 902 pack_open_windows--;
54044bf8
SP
903 return 1;
904 }
905 return 0;
f9253394
JH
906}
907
7c3ecb32 908void release_pack_memory(size_t need)
97bfeb34
SP
909{
910 size_t cur = pack_mapped;
7c3ecb32 911 while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
97bfeb34
SP
912 ; /* nothing */
913}
914
02710228
SP
915static void mmap_limit_check(size_t length)
916{
917 static size_t limit = 0;
918 if (!limit) {
919 limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
920 if (!limit)
921 limit = SIZE_MAX;
922 }
923 if (length > limit)
924 die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
925 (uintmax_t)length, (uintmax_t)limit);
926}
927
1570856b
JK
928void *xmmap_gently(void *start, size_t length,
929 int prot, int flags, int fd, off_t offset)
58ecbd5e 930{
02710228
SP
931 void *ret;
932
933 mmap_limit_check(length);
934 ret = mmap(start, length, prot, flags, fd, offset);
58ecbd5e
JN
935 if (ret == MAP_FAILED) {
936 if (!length)
937 return NULL;
7c3ecb32 938 release_pack_memory(length);
58ecbd5e 939 ret = mmap(start, length, prot, flags, fd, offset);
58ecbd5e
JN
940 }
941 return ret;
942}
943
1570856b
JK
944void *xmmap(void *start, size_t length,
945 int prot, int flags, int fd, off_t offset)
946{
947 void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
948 if (ret == MAP_FAILED)
9ca0aaf6 949 die_errno("mmap failed");
1570856b
JK
950 return ret;
951}
952
c9ced051
SP
953void close_pack_windows(struct packed_git *p)
954{
955 while (p->windows) {
956 struct pack_window *w = p->windows;
957
958 if (w->inuse_cnt)
959 die("pack '%s' still has open windows to it",
960 p->pack_name);
961 munmap(w->base, w->len);
962 pack_mapped -= w->len;
963 pack_open_windows--;
964 p->windows = w->next;
965 free(w);
966 }
967}
968
71fe5d7f
JS
969static int close_pack_fd(struct packed_git *p)
970{
971 if (p->pack_fd < 0)
972 return 0;
973
974 close(p->pack_fd);
975 pack_open_fds--;
976 p->pack_fd = -1;
977
978 return 1;
979}
980
38849a81
JS
981static void close_pack(struct packed_git *p)
982{
983 close_pack_windows(p);
984 close_pack_fd(p);
985 close_pack_index(p);
986}
987
988void close_all_packs(void)
989{
990 struct packed_git *p;
991
992 for (p = packed_git; p; p = p->next)
993 if (p->do_not_close)
ef1177d1 994 die("BUG: want to close pack marked 'do-not-close'");
38849a81
JS
995 else
996 close_pack(p);
997}
998
999
88d0db55
BC
1000/*
1001 * The LRU pack is the one with the oldest MRU window, preferring packs
1002 * with no used windows, or the oldest mtime if it has no windows allocated.
1003 */
1004static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
1005{
1006 struct pack_window *w, *this_mru_w;
1007 int has_windows_inuse = 0;
1008
1009 /*
1010 * Reject this pack if it has windows and the previously selected
1011 * one does not. If this pack does not have windows, reject
1012 * it if the pack file is newer than the previously selected one.
1013 */
1014 if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
1015 return;
1016
1017 for (w = this_mru_w = p->windows; w; w = w->next) {
1018 /*
1019 * Reject this pack if any of its windows are in use,
1020 * but the previously selected pack did not have any
1021 * inuse windows. Otherwise, record that this pack
1022 * has windows in use.
1023 */
1024 if (w->inuse_cnt) {
1025 if (*accept_windows_inuse)
1026 has_windows_inuse = 1;
1027 else
1028 return;
1029 }
1030
1031 if (w->last_used > this_mru_w->last_used)
1032 this_mru_w = w;
1033
1034 /*
1035 * Reject this pack if it has windows that have been
1036 * used more recently than the previously selected pack.
1037 * If the previously selected pack had windows inuse and
1038 * we have not encountered a window in this pack that is
1039 * inuse, skip this check since we prefer a pack with no
1040 * inuse windows to one that has inuse windows.
1041 */
1042 if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
1043 this_mru_w->last_used > (*mru_w)->last_used)
1044 return;
1045 }
1046
1047 /*
1048 * Select this pack.
1049 */
1050 *mru_w = this_mru_w;
1051 *lru_p = p;
1052 *accept_windows_inuse = has_windows_inuse;
1053}
1054
1055static int close_one_pack(void)
1056{
1057 struct packed_git *p, *lru_p = NULL;
1058 struct pack_window *mru_w = NULL;
1059 int accept_windows_inuse = 1;
1060
1061 for (p = packed_git; p; p = p->next) {
1062 if (p->pack_fd == -1)
1063 continue;
1064 find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
1065 }
1066
71fe5d7f
JS
1067 if (lru_p)
1068 return close_pack_fd(lru_p);
88d0db55
BC
1069
1070 return 0;
1071}
1072
03e79c88 1073void unuse_pack(struct pack_window **w_cursor)
f9253394 1074{
03e79c88
SP
1075 struct pack_window *w = *w_cursor;
1076 if (w) {
1077 w->inuse_cnt--;
1078 *w_cursor = NULL;
1079 }
1f688557
JH
1080}
1081
fa5fc15d
SP
1082void close_pack_index(struct packed_git *p)
1083{
1084 if (p->index_data) {
1085 munmap((void *)p->index_data, p->index_size);
1086 p->index_data = NULL;
1087 }
1088}
1089
a0788266
JS
1090static unsigned int get_max_fd_limit(void)
1091{
1092#ifdef RLIMIT_NOFILE
491a8dec
JH
1093 {
1094 struct rlimit lim;
a0788266 1095
491a8dec
JH
1096 if (!getrlimit(RLIMIT_NOFILE, &lim))
1097 return lim.rlim_cur;
1098 }
1099#endif
1100
1101#ifdef _SC_OPEN_MAX
1102 {
1103 long open_max = sysconf(_SC_OPEN_MAX);
1104 if (0 < open_max)
1105 return open_max;
1106 /*
1107 * Otherwise, we got -1 for one of the two
1108 * reasons:
1109 *
1110 * (1) sysconf() did not understand _SC_OPEN_MAX
1111 * and signaled an error with -1; or
1112 * (2) sysconf() said there is no limit.
1113 *
1114 * We _could_ clear errno before calling sysconf() to
1115 * tell these two cases apart and return a huge number
1116 * in the latter case to let the caller cap it to a
1117 * value that is not so selfish, but letting the
1118 * fallback OPEN_MAX codepath take care of these cases
1119 * is a lot simpler.
1120 */
1121 }
1122#endif
a0788266 1123
491a8dec 1124#ifdef OPEN_MAX
a0788266
JS
1125 return OPEN_MAX;
1126#else
1127 return 1; /* see the caller ;-) */
1128#endif
1129}
1130
3cf8b462
SP
1131/*
1132 * Do not call this directly as this leaks p->pack_fd on error return;
1133 * call open_packed_git() instead.
1134 */
1135static int open_packed_git_1(struct packed_git *p)
1f688557 1136{
9bc879c1
SP
1137 struct stat st;
1138 struct pack_header hdr;
1139 unsigned char sha1[20];
1140 unsigned char *idx_sha1;
2c039da8 1141 long fd_flag;
9bc879c1 1142
d079837e
SP
1143 if (!p->index_data && open_pack_index(p))
1144 return error("packfile %s index unavailable", p->pack_name);
1145
c7934306 1146 if (!pack_max_fds) {
a0788266 1147 unsigned int max_fds = get_max_fd_limit();
c7934306
SP
1148
1149 /* Save 3 for stdin/stdout/stderr, 22 for work */
1150 if (25 < max_fds)
1151 pack_max_fds = max_fds - 25;
1152 else
1153 pack_max_fds = 1;
1154 }
1155
88d0db55 1156 while (pack_max_fds <= pack_open_fds && close_one_pack())
c7934306
SP
1157 ; /* nothing */
1158
a5436b57 1159 p->pack_fd = git_open(p->pack_name);
9bc879c1 1160 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
072db278 1161 return -1;
c7934306 1162 pack_open_fds++;
9bc879c1
SP
1163
1164 /* If we created the struct before we had the pack we lack size. */
bf592c50 1165 if (!p->pack_size) {
bf592c50 1166 if (!S_ISREG(st.st_mode))
072db278 1167 return error("packfile %s not a regular file", p->pack_name);
bf592c50 1168 p->pack_size = st.st_size;
9bc879c1 1169 } else if (p->pack_size != st.st_size)
072db278 1170 return error("packfile %s size changed", p->pack_name);
9bc879c1 1171
2c039da8
JH
1172 /* We leave these file descriptors open with sliding mmap;
1173 * there is no point keeping them open across exec(), though.
1174 */
1175 fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
1176 if (fd_flag < 0)
072db278 1177 return error("cannot determine file descriptor flags");
2c039da8
JH
1178 fd_flag |= FD_CLOEXEC;
1179 if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
072db278 1180 return error("cannot set FD_CLOEXEC");
2c039da8 1181
9bc879c1 1182 /* Verify we recognize this pack file format. */
e6e2bd62 1183 if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
072db278 1184 return error("file %s is far too short to be a packfile", p->pack_name);
9bc879c1 1185 if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
072db278 1186 return error("file %s is not a GIT packfile", p->pack_name);
9bc879c1 1187 if (!pack_version_ok(hdr.hdr_version))
6e1c2344
RJ
1188 return error("packfile %s is version %"PRIu32" and not"
1189 " supported (try upgrading GIT to a newer version)",
9bc879c1
SP
1190 p->pack_name, ntohl(hdr.hdr_version));
1191
1192 /* Verify the pack matches its index. */
57059091 1193 if (p->num_objects != ntohl(hdr.hdr_entries))
6e1c2344
RJ
1194 return error("packfile %s claims to have %"PRIu32" objects"
1195 " while index indicates %"PRIu32" objects",
57059091
NP
1196 p->pack_name, ntohl(hdr.hdr_entries),
1197 p->num_objects);
9bc879c1 1198 if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
072db278 1199 return error("end of packfile %s is unavailable", p->pack_name);
e6e2bd62 1200 if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
072db278 1201 return error("packfile %s signature is unavailable", p->pack_name);
42873078 1202 idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
9bc879c1 1203 if (hashcmp(sha1, idx_sha1))
072db278
SP
1204 return error("packfile %s does not match index", p->pack_name);
1205 return 0;
9bc879c1
SP
1206}
1207
3cf8b462
SP
1208static int open_packed_git(struct packed_git *p)
1209{
1210 if (!open_packed_git_1(p))
1211 return 0;
71fe5d7f 1212 close_pack_fd(p);
3cf8b462
SP
1213 return -1;
1214}
1215
c4001d92 1216static int in_window(struct pack_window *win, off_t offset)
60bb8b14
SP
1217{
1218 /* We must promise at least 20 bytes (one hash) after the
1219 * offset is available from this window, otherwise the offset
1220 * is not actually in this window and a different window (which
1221 * has that one hash excess) must be used. This is to support
1222 * the object header and delta base parsing routines below.
1223 */
1224 off_t win_off = win->offset;
1225 return win_off <= offset
1226 && (offset + 20) <= (win_off + win->len);
1227}
1228
4b25d091 1229unsigned char *use_pack(struct packed_git *p,
03e79c88 1230 struct pack_window **w_cursor,
c4001d92 1231 off_t offset,
ef49a7a0 1232 unsigned long *left)
9bc879c1 1233{
60bb8b14 1234 struct pack_window *win = *w_cursor;
03e79c88 1235
a9d98a14 1236 /* Since packfiles end in a hash of their content and it's
60bb8b14
SP
1237 * pointless to ask for an offset into the middle of that
1238 * hash, and the in_window function above wouldn't match
1239 * don't allow an offset too close to the end of the file.
1240 */
d131b7af
SP
1241 if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
1242 die("packfile %s cannot be accessed", p->pack_name);
60bb8b14
SP
1243 if (offset > (p->pack_size - 20))
1244 die("offset beyond end of packfile (truncated pack?)");
13e0b0d3 1245 if (offset < 0)
7465feba 1246 die(_("offset before end of packfile (broken .idx?)"));
60bb8b14
SP
1247
1248 if (!win || !in_window(win, offset)) {
1249 if (win)
1250 win->inuse_cnt--;
1251 for (win = p->windows; win; win = win->next) {
1252 if (in_window(win, offset))
1253 break;
1254 }
1255 if (!win) {
78a28df9 1256 size_t window_align = packed_git_window_size / 2;
dc49cd76 1257 off_t len;
d131b7af
SP
1258
1259 if (p->pack_fd == -1 && open_packed_git(p))
1260 die("packfile %s cannot be accessed", p->pack_name);
1261
60bb8b14 1262 win = xcalloc(1, sizeof(*win));
78a28df9 1263 win->offset = (offset / window_align) * window_align;
dc49cd76
SP
1264 len = p->pack_size - win->offset;
1265 if (len > packed_git_window_size)
1266 len = packed_git_window_size;
1267 win->len = (size_t)len;
60bb8b14 1268 pack_mapped += win->len;
11daf39b 1269 while (packed_git_limit < pack_mapped
7c3ecb32 1270 && unuse_one_window(p))
60bb8b14 1271 ; /* nothing */
c4712e45 1272 win->base = xmmap(NULL, win->len,
60bb8b14
SP
1273 PROT_READ, MAP_PRIVATE,
1274 p->pack_fd, win->offset);
1275 if (win->base == MAP_FAILED)
7616c6ca
NTND
1276 die_errno("packfile %s cannot be mapped",
1277 p->pack_name);
d131b7af 1278 if (!win->offset && win->len == p->pack_size
71fe5d7f
JS
1279 && !p->do_not_close)
1280 close_pack_fd(p);
a53128b6
SP
1281 pack_mmap_calls++;
1282 pack_open_windows++;
1283 if (pack_mapped > peak_pack_mapped)
1284 peak_pack_mapped = pack_mapped;
1285 if (pack_open_windows > peak_pack_open_windows)
1286 peak_pack_open_windows = pack_open_windows;
60bb8b14
SP
1287 win->next = p->windows;
1288 p->windows = win;
1289 }
1f688557 1290 }
03e79c88
SP
1291 if (win != *w_cursor) {
1292 win->last_used = pack_used_ctr++;
1293 win->inuse_cnt++;
1294 *w_cursor = win;
1295 }
60bb8b14 1296 offset -= win->offset;
03e79c88 1297 if (left)
dc49cd76 1298 *left = win->len - xsize_t(offset);
03e79c88 1299 return win->base + offset;
1f688557
JH
1300}
1301
27d69a46
NP
1302static struct packed_git *alloc_packed_git(int extra)
1303{
50a6c8ef 1304 struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
27d69a46
NP
1305 memset(p, 0, sizeof(*p));
1306 p->pack_fd = -1;
1307 return p;
1308}
1309
e0500293
JN
1310static void try_to_free_pack_memory(size_t size)
1311{
7c3ecb32 1312 release_pack_memory(size);
e0500293
JN
1313}
1314
48bcc1c3 1315struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
1f688557 1316{
e0500293 1317 static int have_set_try_to_free_routine;
1f688557 1318 struct stat st;
48bcc1c3
JK
1319 size_t alloc;
1320 struct packed_git *p;
1f688557 1321
e0500293
JN
1322 if (!have_set_try_to_free_routine) {
1323 have_set_try_to_free_routine = 1;
1324 set_try_to_free_routine(try_to_free_pack_memory);
1325 }
1326
42873078
NP
1327 /*
1328 * Make sure a corresponding .pack file exists and that
1329 * the index looks sane.
1330 */
48bcc1c3 1331 if (!strip_suffix_mem(path, &path_len, ".idx"))
1f688557 1332 return NULL;
48bcc1c3
JK
1333
1334 /*
1335 * ".pack" is long enough to hold any suffix we're adding (and
1336 * the use xsnprintf double-checks that)
1337 */
50a6c8ef 1338 alloc = st_add3(path_len, strlen(".pack"), 1);
48bcc1c3 1339 p = alloc_packed_git(alloc);
42873078 1340 memcpy(p->pack_name, path, path_len);
8d25931d 1341
48bcc1c3 1342 xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
8d25931d
BC
1343 if (!access(p->pack_name, F_OK))
1344 p->pack_keep = 1;
1345
48bcc1c3 1346 xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
d079837e 1347 if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
42873078 1348 free(p);
1f688557
JH
1349 return NULL;
1350 }
42873078 1351
1f688557
JH
1352 /* ok, it looks sane as far as we can check without
1353 * actually mapping the pack file.
1354 */
1f688557 1355 p->pack_size = st.st_size;
9d835df2 1356 p->pack_local = local;
b867092f 1357 p->mtime = st.st_mtime;
42873078
NP
1358 if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
1359 hashclr(p->sha1);
1f688557
JH
1360 return p;
1361}
1362
7b64469a 1363struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
c508df5e 1364{
42873078 1365 const char *path = sha1_pack_name(sha1);
50a6c8ef 1366 size_t alloc = st_add(strlen(path), 1);
c7ab0ba3 1367 struct packed_git *p = alloc_packed_git(alloc);
bf592c50 1368
c7ab0ba3 1369 memcpy(p->pack_name, path, alloc); /* includes NUL */
27d69a46 1370 hashcpy(p->sha1, sha1);
42873078
NP
1371 if (check_packed_git_idx(idx_path, p)) {
1372 free(p);
bf592c50 1373 return NULL;
42873078 1374 }
bf592c50 1375
bf592c50
DB
1376 return p;
1377}
1378
1379void install_packed_git(struct packed_git *pack)
1380{
c7934306
SP
1381 if (pack->pack_fd != -1)
1382 pack_open_fds++;
1383
bf592c50
DB
1384 pack->next = packed_git;
1385 packed_git = pack;
1386}
1387
0a489b06 1388void (*report_garbage)(unsigned seen_bits, const char *path);
543c5caa
NTND
1389
1390static void report_helper(const struct string_list *list,
1391 int seen_bits, int first, int last)
1392{
0a489b06 1393 if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
543c5caa 1394 return;
0a489b06 1395
543c5caa 1396 for (; first < last; first++)
0a489b06 1397 report_garbage(seen_bits, list->items[first].string);
543c5caa
NTND
1398}
1399
1400static void report_pack_garbage(struct string_list *list)
1401{
1402 int i, baselen = -1, first = 0, seen_bits = 0;
1403
1404 if (!report_garbage)
1405 return;
1406
3383e199 1407 string_list_sort(list);
543c5caa
NTND
1408
1409 for (i = 0; i < list->nr; i++) {
1410 const char *path = list->items[i].string;
1411 if (baselen != -1 &&
1412 strncmp(path, list->items[first].string, baselen)) {
1413 report_helper(list, seen_bits, first, i);
1414 baselen = -1;
1415 seen_bits = 0;
1416 }
1417 if (baselen == -1) {
1418 const char *dot = strrchr(path, '.');
1419 if (!dot) {
0a489b06 1420 report_garbage(PACKDIR_FILE_GARBAGE, path);
543c5caa
NTND
1421 continue;
1422 }
1423 baselen = dot - path + 1;
1424 first = i;
1425 }
1426 if (!strcmp(path + baselen, "pack"))
1427 seen_bits |= 1;
1428 else if (!strcmp(path + baselen, "idx"))
1429 seen_bits |= 2;
1430 }
1431 report_helper(list, seen_bits, first, list->nr);
1432}
1433
9d835df2 1434static void prepare_packed_git_one(char *objdir, int local)
1f688557 1435{
880fb8de
RS
1436 struct strbuf path = STRBUF_INIT;
1437 size_t dirnamelen;
1f688557
JH
1438 DIR *dir;
1439 struct dirent *de;
543c5caa 1440 struct string_list garbage = STRING_LIST_INIT_DUP;
1f688557 1441
880fb8de
RS
1442 strbuf_addstr(&path, objdir);
1443 strbuf_addstr(&path, "/pack");
1444 dir = opendir(path.buf);
b5b16990 1445 if (!dir) {
26125f6b 1446 if (errno != ENOENT)
7616c6ca
NTND
1447 error_errno("unable to open object pack directory: %s",
1448 path.buf);
880fb8de 1449 strbuf_release(&path);
1f688557 1450 return;
b5b16990 1451 }
880fb8de
RS
1452 strbuf_addch(&path, '/');
1453 dirnamelen = path.len;
1f688557 1454 while ((de = readdir(dir)) != NULL) {
1f688557 1455 struct packed_git *p;
47bf4b0f 1456 size_t base_len;
543c5caa
NTND
1457
1458 if (is_dot_or_dotdot(de->d_name))
9cb18f56
JM
1459 continue;
1460
880fb8de
RS
1461 strbuf_setlen(&path, dirnamelen);
1462 strbuf_addstr(&path, de->d_name);
d90906a9 1463
47bf4b0f
JK
1464 base_len = path.len;
1465 if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
d90906a9
NTND
1466 /* Don't reopen a pack we already have. */
1467 for (p = packed_git; p; p = p->next) {
47bf4b0f
JK
1468 size_t len;
1469 if (strip_suffix(p->pack_name, ".pack", &len) &&
1470 len == base_len &&
1471 !memcmp(p->pack_name, path.buf, len))
d90906a9
NTND
1472 break;
1473 }
1474 if (p == NULL &&
1475 /*
1476 * See if it really is a valid .idx file with
1477 * corresponding .pack file that we can map.
1478 */
880fb8de 1479 (p = add_packed_git(path.buf, path.len, local)) != NULL)
d90906a9 1480 install_packed_git(p);
86f7780c 1481 }
543c5caa
NTND
1482
1483 if (!report_garbage)
1484 continue;
1485
2975c770
JK
1486 if (ends_with(de->d_name, ".idx") ||
1487 ends_with(de->d_name, ".pack") ||
1488 ends_with(de->d_name, ".bitmap") ||
1489 ends_with(de->d_name, ".keep"))
880fb8de 1490 string_list_append(&garbage, path.buf);
543c5caa 1491 else
0a489b06 1492 report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
1f688557 1493 }
5b35bcd5 1494 closedir(dir);
543c5caa
NTND
1495 report_pack_garbage(&garbage);
1496 string_list_clear(&garbage, 0);
880fb8de 1497 strbuf_release(&path);
1f688557
JH
1498}
1499
8e3f52d7
JK
1500static int approximate_object_count_valid;
1501
1502/*
1503 * Give a fast, rough count of the number of objects in the repository. This
1504 * ignores loose objects completely. If you have a lot of them, then either
1505 * you should repack because your performance will be awful, or they are
1506 * all unreachable objects about to be pruned, in which case they're not really
1507 * interesting as a measure of repo size in the first place.
1508 */
1509unsigned long approximate_object_count(void)
1510{
1511 static unsigned long count;
1512 if (!approximate_object_count_valid) {
1513 struct packed_git *p;
1514
1515 prepare_packed_git();
1516 count = 0;
1517 for (p = packed_git; p; p = p->next) {
1518 if (open_pack_index(p))
1519 continue;
1520 count += p->num_objects;
1521 }
1522 }
1523 return count;
1524}
1525
c4c6effa
RS
1526static void *get_next_packed_git(const void *p)
1527{
1528 return ((const struct packed_git *)p)->next;
1529}
1530
1531static void set_next_packed_git(void *p, void *next)
1532{
1533 ((struct packed_git *)p)->next = next;
1534}
1535
b867092f
JH
1536static int sort_pack(const void *a_, const void *b_)
1537{
c4c6effa
RS
1538 const struct packed_git *a = a_;
1539 const struct packed_git *b = b_;
b867092f
JH
1540 int st;
1541
1542 /*
1543 * Local packs tend to contain objects specific to our
1544 * variant of the project than remote ones. In addition,
1545 * remote ones could be on a network mounted filesystem.
1546 * Favor local ones for these reasons.
1547 */
1548 st = a->pack_local - b->pack_local;
1549 if (st)
1550 return -st;
1551
1552 /*
1553 * Younger packs tend to contain more recent objects,
1554 * and more recent objects tend to get accessed more
1555 * often.
1556 */
1557 if (a->mtime < b->mtime)
1558 return 1;
1559 else if (a->mtime == b->mtime)
1560 return 0;
1561 return -1;
1562}
1563
1564static void rearrange_packed_git(void)
1565{
c4c6effa
RS
1566 packed_git = llist_mergesort(packed_git, get_next_packed_git,
1567 set_next_packed_git, sort_pack);
b867092f
JH
1568}
1569
a73cdd21
JK
1570static void prepare_packed_git_mru(void)
1571{
1572 struct packed_git *p;
1573
1574 mru_clear(packed_git_mru);
1575 for (p = packed_git; p; p = p->next)
1576 mru_append(packed_git_mru, p);
1577}
1578
637cdd9d 1579static int prepare_packed_git_run_once = 0;
9a217f2a 1580void prepare_packed_git(void)
1f688557 1581{
d5a63b99 1582 struct alternate_object_database *alt;
1f688557 1583
637cdd9d 1584 if (prepare_packed_git_run_once)
1f688557 1585 return;
9d835df2 1586 prepare_packed_git_one(get_object_directory(), 1);
9a217f2a 1587 prepare_alt_odb();
597f9134
JK
1588 for (alt = alt_odb_list; alt; alt = alt->next)
1589 prepare_packed_git_one(alt->path, 0);
b867092f 1590 rearrange_packed_git();
a73cdd21 1591 prepare_packed_git_mru();
637cdd9d
JK
1592 prepare_packed_git_run_once = 1;
1593}
1594
fc04c412 1595void reprepare_packed_git(void)
637cdd9d 1596{
8e3f52d7 1597 approximate_object_count_valid = 0;
637cdd9d
JK
1598 prepare_packed_git_run_once = 0;
1599 prepare_packed_git();
1f688557
JH
1600}
1601
8eca0b47
NP
1602static void mark_bad_packed_object(struct packed_git *p,
1603 const unsigned char *sha1)
1604{
1605 unsigned i;
1606 for (i = 0; i < p->num_bad_objects; i++)
50a6c8ef 1607 if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
8eca0b47 1608 return;
50a6c8ef 1609 p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
cd02599c 1610 st_mult(GIT_MAX_RAWSZ,
50a6c8ef
JK
1611 st_add(p->num_bad_objects, 1)));
1612 hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
8eca0b47
NP
1613 p->num_bad_objects++;
1614}
1615
b6c4cecc 1616static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
ac939109
NP
1617{
1618 struct packed_git *p;
1619 unsigned i;
1620
1621 for (p = packed_git; p; p = p->next)
1622 for (i = 0; i < p->num_bad_objects; i++)
1623 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
b6c4cecc
JH
1624 return p;
1625 return NULL;
ac939109
NP
1626}
1627
090ea126
NTND
1628/*
1629 * With an in-core object data in "map", rehash it to make sure the
1630 * object name actually matches "sha1" to detect object corruption.
1631 * With "map" == NULL, try reading the object named with "sha1" using
1632 * the streaming interface and rehash it to do the same.
1633 */
1634int check_sha1_signature(const unsigned char *sha1, void *map,
1635 unsigned long size, const char *type)
0fcfd160
LT
1636{
1637 unsigned char real_sha1[20];
090ea126
NTND
1638 enum object_type obj_type;
1639 struct git_istream *st;
1640 git_SHA_CTX c;
1641 char hdr[32];
1642 int hdrlen;
1643
1644 if (map) {
1645 hash_sha1_file(map, size, type, real_sha1);
1646 return hashcmp(sha1, real_sha1) ? -1 : 0;
1647 }
1648
1649 st = open_istream(sha1, &obj_type, &size, NULL);
1650 if (!st)
1651 return -1;
1652
1653 /* Generate the header */
ef1286d3 1654 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
090ea126
NTND
1655
1656 /* Sha1.. */
1657 git_SHA1_Init(&c);
1658 git_SHA1_Update(&c, hdr, hdrlen);
1659 for (;;) {
1660 char buf[1024 * 16];
1661 ssize_t readlen = read_istream(st, buf, sizeof(buf));
1662
f54fac53
JK
1663 if (readlen < 0) {
1664 close_istream(st);
1665 return -1;
1666 }
090ea126
NTND
1667 if (!readlen)
1668 break;
1669 git_SHA1_Update(&c, buf, readlen);
1670 }
1671 git_SHA1_Final(real_sha1, &c);
1672 close_istream(st);
a89fccd2 1673 return hashcmp(sha1, real_sha1) ? -1 : 0;
0fcfd160
LT
1674}
1675
1b8ac5ea 1676int git_open_cloexec(const char *name, int flags)
44d1c19e 1677{
1e3001a8
JH
1678 int fd;
1679 static int o_cloexec = O_CLOEXEC;
f2e872aa 1680
1e3001a8
JH
1681 fd = open(name, flags | o_cloexec);
1682 if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) {
cd66ada0 1683 /* Try again w/o O_CLOEXEC: the kernel might not support it */
1e3001a8
JH
1684 o_cloexec &= ~O_CLOEXEC;
1685 fd = open(name, flags | o_cloexec);
1b8ac5ea 1686 }
f2e872aa 1687
9fb9495d 1688#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC)
1e3001a8
JH
1689 {
1690 static int fd_cloexec = FD_CLOEXEC;
1691
1692 if (!o_cloexec && 0 <= fd && fd_cloexec) {
1693 /* Opened w/o O_CLOEXEC? try with fcntl(2) to add it */
9fb9495d
EW
1694 int flags = fcntl(fd, F_GETFD);
1695 if (fcntl(fd, F_SETFD, flags | fd_cloexec))
1e3001a8 1696 fd_cloexec = 0;
cd66ada0 1697 }
44d1c19e 1698 }
1e3001a8 1699#endif
1b8ac5ea 1700 return fd;
44d1c19e
LT
1701}
1702
771e7d57
JK
1703/*
1704 * Find "sha1" as a loose object in the local repository or in an alternate.
1705 * Returns 0 on success, negative on failure.
1706 *
1707 * The "path" out-parameter will give the path of the object we found (if any).
1708 * Note that it may point to static storage and is only valid until another
1709 * call to sha1_file_name(), etc.
1710 */
1711static int stat_sha1_file(const unsigned char *sha1, struct stat *st,
1712 const char **path)
052fe5ea 1713{
052fe5ea
JK
1714 struct alternate_object_database *alt;
1715
771e7d57
JK
1716 *path = sha1_file_name(sha1);
1717 if (!lstat(*path, st))
052fe5ea
JK
1718 return 0;
1719
1720 prepare_alt_odb();
1721 errno = ENOENT;
1722 for (alt = alt_odb_list; alt; alt = alt->next) {
771e7d57
JK
1723 *path = alt_sha1_path(alt, sha1);
1724 if (!lstat(*path, st))
052fe5ea
JK
1725 return 0;
1726 }
1727
1728 return -1;
1729}
1730
771e7d57
JK
1731/*
1732 * Like stat_sha1_file(), but actually open the object and return the
1733 * descriptor. See the caveats on the "path" parameter above.
1734 */
1735static int open_sha1_file(const unsigned char *sha1, const char **path)
44d1c19e
LT
1736{
1737 int fd;
44d1c19e 1738 struct alternate_object_database *alt;
d6c8a05b 1739 int most_interesting_errno;
44d1c19e 1740
771e7d57
JK
1741 *path = sha1_file_name(sha1);
1742 fd = git_open(*path);
44d1c19e
LT
1743 if (fd >= 0)
1744 return fd;
d6c8a05b 1745 most_interesting_errno = errno;
44d1c19e
LT
1746
1747 prepare_alt_odb();
44d1c19e 1748 for (alt = alt_odb_list; alt; alt = alt->next) {
771e7d57
JK
1749 *path = alt_sha1_path(alt, sha1);
1750 fd = git_open(*path);
44d1c19e
LT
1751 if (fd >= 0)
1752 return fd;
d6c8a05b
JK
1753 if (most_interesting_errno == ENOENT)
1754 most_interesting_errno = errno;
44d1c19e 1755 }
d6c8a05b 1756 errno = most_interesting_errno;
44d1c19e
LT
1757 return -1;
1758}
1759
f6371f92
JK
1760/*
1761 * Map the loose object at "path" if it is not NULL, or the path found by
1762 * searching for a loose object named "sha1".
1763 */
1764static void *map_sha1_file_1(const char *path,
1765 const unsigned char *sha1,
1766 unsigned long *size)
0fcfd160 1767{
0fcfd160 1768 void *map;
144bde78 1769 int fd;
ace1534d 1770
f6371f92
JK
1771 if (path)
1772 fd = git_open(path);
1773 else
1774 fd = open_sha1_file(sha1, &path);
44d1c19e
LT
1775 map = NULL;
1776 if (fd >= 0) {
1777 struct stat st;
0fcfd160 1778
44d1c19e
LT
1779 if (!fstat(fd, &st)) {
1780 *size = xsize_t(st.st_size);
33e42de0
MM
1781 if (!*size) {
1782 /* mmap() is forbidden on empty files */
771e7d57 1783 error("object file %s is empty", path);
33e42de0
MM
1784 return NULL;
1785 }
44d1c19e 1786 map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
144bde78 1787 }
44d1c19e 1788 close(fd);
0fcfd160 1789 }
0fcfd160
LT
1790 return map;
1791}
1792
f6371f92
JK
1793void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
1794{
1795 return map_sha1_file_1(NULL, sha1, size);
1796}
1797
09ded04b
NP
1798unsigned long unpack_object_header_buffer(const unsigned char *buf,
1799 unsigned long len, enum object_type *type, unsigned long *sizep)
c4483576 1800{
ad1ed5ee 1801 unsigned shift;
48fb7deb 1802 unsigned long size, c;
ad1ed5ee
JH
1803 unsigned long used = 0;
1804
1805 c = buf[used++];
1806 *type = (c >> 4) & 7;
1807 size = c & 15;
1808 shift = 4;
1809 while (c & 0x80) {
f630cfda 1810 if (len <= used || bitsizeof(long) <= shift) {
09ded04b 1811 error("bad object header");
ea4f9685
JH
1812 size = used = 0;
1813 break;
09ded04b 1814 }
ad1ed5ee
JH
1815 c = buf[used++];
1816 size += (c & 0x7f) << shift;
1817 shift += 7;
1818 }
1819 *sizep = size;
1820 return used;
1821}
1822
d21f8426
JH
1823static int unpack_sha1_short_header(git_zstream *stream,
1824 unsigned char *map, unsigned long mapsize,
1825 void *buffer, unsigned long bufsiz)
ad1ed5ee 1826{
c4483576
LT
1827 /* Get the data stream */
1828 memset(stream, 0, sizeof(*stream));
1829 stream->next_in = map;
1830 stream->avail_in = mapsize;
1831 stream->next_out = buffer;
93821bd9
LT
1832 stream->avail_out = bufsiz;
1833
39c68542 1834 git_inflate_init(stream);
cc5c54e7 1835 return git_inflate(stream, 0);
c4483576
LT
1836}
1837
d21f8426
JH
1838int unpack_sha1_header(git_zstream *stream,
1839 unsigned char *map, unsigned long mapsize,
1840 void *buffer, unsigned long bufsiz)
1841{
1842 int status = unpack_sha1_short_header(stream, map, mapsize,
1843 buffer, bufsiz);
1844
1845 if (status < Z_OK)
1846 return status;
1847
1848 /* Make sure we have the terminating NUL */
1849 if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1850 return -1;
1851 return 0;
1852}
1853
46f03448
KN
1854static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
1855 unsigned long mapsize, void *buffer,
1856 unsigned long bufsiz, struct strbuf *header)
1857{
1858 int status;
1859
d21f8426
JH
1860 status = unpack_sha1_short_header(stream, map, mapsize, buffer, bufsiz);
1861 if (status < Z_OK)
1862 return -1;
46f03448
KN
1863
1864 /*
1865 * Check if entire header is unpacked in the first iteration.
1866 */
1867 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1868 return 0;
1869
1870 /*
1871 * buffer[0..bufsiz] was not large enough. Copy the partial
1872 * result out to header, and then append the result of further
1873 * reading the stream.
1874 */
1875 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1876 stream->next_out = buffer;
1877 stream->avail_out = bufsiz;
1878
1879 do {
1880 status = git_inflate(stream, 0);
1881 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1882 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1883 return 0;
1884 stream->next_out = buffer;
1885 stream->avail_out = bufsiz;
1886 } while (status != Z_STREAM_END);
1887 return -1;
1888}
1889
ef49a7a0 1890static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
5180cacc
LT
1891{
1892 int bytes = strlen(buffer) + 1;
3aee68aa 1893 unsigned char *buf = xmallocz(size);
93821bd9 1894 unsigned long n;
7efbff75 1895 int status = Z_OK;
5180cacc 1896
93821bd9
LT
1897 n = stream->total_out - bytes;
1898 if (n > size)
1899 n = size;
1900 memcpy(buf, (char *) buffer + bytes, n);
1901 bytes = n;
456cdf6e
LT
1902 if (bytes <= size) {
1903 /*
1904 * The above condition must be (bytes <= size), not
1905 * (bytes < size). In other words, even though we
ccf5ace0 1906 * expect no more output and set avail_out to zero,
456cdf6e
LT
1907 * the input zlib stream may have bytes that express
1908 * "this concludes the stream", and we *do* want to
1909 * eat that input.
1910 *
1911 * Otherwise we would not be able to test that we
1912 * consumed all the input to reach the expected size;
1913 * we also want to check that zlib tells us that all
1914 * went well with status == Z_STREAM_END at the end.
1915 */
5180cacc
LT
1916 stream->next_out = buf + bytes;
1917 stream->avail_out = size - bytes;
7efbff75 1918 while (status == Z_OK)
39c68542 1919 status = git_inflate(stream, Z_FINISH);
5180cacc 1920 }
456cdf6e 1921 if (status == Z_STREAM_END && !stream->avail_in) {
39c68542 1922 git_inflate_end(stream);
7efbff75
JH
1923 return buf;
1924 }
1925
1926 if (status < 0)
1927 error("corrupt loose object '%s'", sha1_to_hex(sha1));
1928 else if (stream->avail_in)
1929 error("garbage at end of loose object '%s'",
1930 sha1_to_hex(sha1));
1931 free(buf);
1932 return NULL;
5180cacc
LT
1933}
1934
1935/*
1936 * We used to just use "sscanf()", but that's actually way
1937 * too permissive for what we want to check. So do an anal
1938 * object header parse by hand.
1939 */
46f03448
KN
1940static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
1941 unsigned int flags)
5180cacc 1942{
46f03448 1943 const char *type_buf = hdr;
5180cacc 1944 unsigned long size;
46f03448 1945 int type, type_len = 0;
5180cacc
LT
1946
1947 /*
46f03448 1948 * The type can be of any size but is followed by
21666f1a 1949 * a space.
5180cacc 1950 */
5180cacc
LT
1951 for (;;) {
1952 char c = *hdr++;
d21f8426
JH
1953 if (!c)
1954 return -1;
5180cacc
LT
1955 if (c == ' ')
1956 break;
46f03448 1957 type_len++;
5180cacc 1958 }
46f03448
KN
1959
1960 type = type_from_string_gently(type_buf, type_len, 1);
1961 if (oi->typename)
1962 strbuf_add(oi->typename, type_buf, type_len);
1963 /*
1964 * Set type to 0 if its an unknown object and
2e3a16b2 1965 * we're obtaining the type using '--allow-unknown-type'
46f03448
KN
1966 * option.
1967 */
19fc5e84 1968 if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
46f03448
KN
1969 type = 0;
1970 else if (type < 0)
1971 die("invalid object type");
1972 if (oi->typep)
1973 *oi->typep = type;
5180cacc
LT
1974
1975 /*
1976 * The length must follow immediately, and be in canonical
1977 * decimal format (ie "010" is not valid).
1978 */
1979 size = *hdr++ - '0';
1980 if (size > 9)
1981 return -1;
1982 if (size) {
1983 for (;;) {
1984 unsigned long c = *hdr - '0';
1985 if (c > 9)
1986 break;
1987 hdr++;
1988 size = size * 10 + c;
1989 }
1990 }
46f03448
KN
1991
1992 if (oi->sizep)
1993 *oi->sizep = size;
5180cacc
LT
1994
1995 /*
1996 * The length must be followed by a zero byte
1997 */
46f03448
KN
1998 return *hdr ? -1 : type;
1999}
2000
2001int parse_sha1_header(const char *hdr, unsigned long *sizep)
2002{
27b5c1a0 2003 struct object_info oi = OBJECT_INFO_INIT;
46f03448
KN
2004
2005 oi.sizep = sizep;
1f0c0d36 2006 return parse_sha1_header_extended(hdr, &oi, 0);
0fcfd160
LT
2007}
2008
54dab52a
NP
2009unsigned long get_size_from_delta(struct packed_git *p,
2010 struct pack_window **w_curs,
2011 off_t curpos)
2012{
2013 const unsigned char *data;
2014 unsigned char delta_head[20], *in;
ef49a7a0 2015 git_zstream stream;
54dab52a
NP
2016 int st;
2017
2018 memset(&stream, 0, sizeof(stream));
2019 stream.next_out = delta_head;
2020 stream.avail_out = sizeof(delta_head);
2021
39c68542 2022 git_inflate_init(&stream);
54dab52a
NP
2023 do {
2024 in = use_pack(p, w_curs, curpos, &stream.avail_in);
2025 stream.next_in = in;
39c68542 2026 st = git_inflate(&stream, Z_FINISH);
54dab52a
NP
2027 curpos += stream.next_in - in;
2028 } while ((st == Z_OK || st == Z_BUF_ERROR) &&
2029 stream.total_out < sizeof(delta_head));
39c68542 2030 git_inflate_end(&stream);
3d77d877
NP
2031 if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
2032 error("delta data unpack-initial failed");
2033 return 0;
2034 }
54dab52a
NP
2035
2036 /* Examine the initial part of the delta to figure out
2037 * the result size.
2038 */
2039 data = delta_head;
2040
2041 /* ignore base size */
2042 get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
2043
2044 /* Read the result size */
2045 return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
2046}
2047
c4001d92 2048static off_t get_delta_base(struct packed_git *p,
03e79c88 2049 struct pack_window **w_curs,
c4001d92 2050 off_t *curpos,
21666f1a 2051 enum object_type type,
c4001d92 2052 off_t delta_obj_offset)
eb32d236 2053{
2b87c45b 2054 unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
c4001d92 2055 off_t base_offset;
eb32d236 2056
8d8a4ea5
SP
2057 /* use_pack() assured us we have [base_info, base_info + 20)
2058 * as a range that we can look at without walking off the
2059 * end of the mapped window. Its actually the hash size
2060 * that is assured. An OFS_DELTA longer than the hash size
2061 * is stupid, as then a REF_DELTA would be smaller to store.
2062 */
21666f1a 2063 if (type == OBJ_OFS_DELTA) {
eb32d236
NP
2064 unsigned used = 0;
2065 unsigned char c = base_info[used++];
2066 base_offset = c & 127;
2067 while (c & 128) {
2068 base_offset += 1;
8723f216 2069 if (!base_offset || MSB(base_offset, 7))
8eca0b47 2070 return 0; /* overflow */
eb32d236
NP
2071 c = base_info[used++];
2072 base_offset = (base_offset << 7) + (c & 127);
2073 }
2074 base_offset = delta_obj_offset - base_offset;
d8f32556 2075 if (base_offset <= 0 || base_offset >= delta_obj_offset)
8eca0b47 2076 return 0; /* out of bound */
2b87c45b 2077 *curpos += used;
21666f1a 2078 } else if (type == OBJ_REF_DELTA) {
eb32d236
NP
2079 /* The base entry _must_ be in the same pack */
2080 base_offset = find_pack_entry_one(base_info, p);
2b87c45b 2081 *curpos += 20;
eb32d236
NP
2082 } else
2083 die("I am totally screwed");
2b87c45b 2084 return base_offset;
eb32d236
NP
2085}
2086
5d642e75
JK
2087/*
2088 * Like get_delta_base above, but we return the sha1 instead of the pack
2089 * offset. This means it is cheaper for REF deltas (we do not have to do
2090 * the final object lookup), but more expensive for OFS deltas (we
2091 * have to load the revidx to convert the offset back into a sha1).
2092 */
2093static const unsigned char *get_delta_base_sha1(struct packed_git *p,
2094 struct pack_window **w_curs,
2095 off_t curpos,
2096 enum object_type type,
2097 off_t delta_obj_offset)
2098{
2099 if (type == OBJ_REF_DELTA) {
2100 unsigned char *base = use_pack(p, w_curs, curpos, NULL);
2101 return base;
2102 } else if (type == OBJ_OFS_DELTA) {
2103 struct revindex_entry *revidx;
2104 off_t base_offset = get_delta_base(p, w_curs, &curpos,
2105 type, delta_obj_offset);
2106
2107 if (!base_offset)
2108 return NULL;
2109
2110 revidx = find_pack_revindex(p, base_offset);
2111 if (!revidx)
2112 return NULL;
2113
2114 return nth_packed_object_sha1(p, revidx->nr);
2115 } else
2116 return NULL;
2117}
2118
f8c8abc5
JH
2119int unpack_object_header(struct packed_git *p,
2120 struct pack_window **w_curs,
2121 off_t *curpos,
2122 unsigned long *sizep)
a733cb60 2123{
03e79c88 2124 unsigned char *base;
ef49a7a0 2125 unsigned long left;
ad1ed5ee 2126 unsigned long used;
2b87c45b 2127 enum object_type type;
a733cb60 2128
8d8a4ea5 2129 /* use_pack() assures us we have [base, base + 20) available
0353a0c4 2130 * as a range that we can look at. (Its actually the hash
3dff5379 2131 * size that is assured.) With our object header encoding
8d8a4ea5
SP
2132 * the maximum deflated object size is 2^137, which is just
2133 * insane, so we know won't exceed what we have been given.
2134 */
2b87c45b 2135 base = use_pack(p, w_curs, *curpos, &left);
09ded04b
NP
2136 used = unpack_object_header_buffer(base, left, &type, sizep);
2137 if (!used) {
2138 type = OBJ_BAD;
2139 } else
2140 *curpos += used;
ad1ed5ee 2141
2b87c45b 2142 return type;
a733cb60
LT
2143}
2144
790d96c0
TR
2145static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
2146{
2147 int type;
2148 struct revindex_entry *revidx;
2149 const unsigned char *sha1;
2150 revidx = find_pack_revindex(p, obj_offset);
2151 if (!revidx)
2152 return OBJ_BAD;
2153 sha1 = nth_packed_object_sha1(p, revidx->nr);
2154 mark_bad_packed_object(p, sha1);
2155 type = sha1_object_info(sha1, NULL);
2156 if (type <= OBJ_NONE)
2157 return OBJ_BAD;
2158 return type;
2159}
2160
790d96c0
TR
2161#define POI_STACK_PREALLOC 64
2162
90191d37
JK
2163static enum object_type packed_to_object_type(struct packed_git *p,
2164 off_t obj_offset,
2165 enum object_type type,
2166 struct pack_window **w_curs,
2167 off_t curpos)
1f688557 2168{
790d96c0
TR
2169 off_t small_poi_stack[POI_STACK_PREALLOC];
2170 off_t *poi_stack = small_poi_stack;
2171 int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
5db47c2b 2172
790d96c0
TR
2173 while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2174 off_t base_offset;
90191d37 2175 unsigned long size;
790d96c0
TR
2176 /* Push the object we're going to leave behind */
2177 if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
2178 poi_stack_alloc = alloc_nr(poi_stack_nr);
b32fa95f 2179 ALLOC_ARRAY(poi_stack, poi_stack_alloc);
790d96c0
TR
2180 memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
2181 } else {
2182 ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
2183 }
2184 poi_stack[poi_stack_nr++] = obj_offset;
2185 /* If parsing the base offset fails, just unwind */
90191d37 2186 base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
790d96c0
TR
2187 if (!base_offset)
2188 goto unwind;
2189 curpos = obj_offset = base_offset;
90191d37 2190 type = unpack_object_header(p, w_curs, &curpos, &size);
790d96c0
TR
2191 if (type <= OBJ_NONE) {
2192 /* If getting the base itself fails, we first
2193 * retry the base, otherwise unwind */
2194 type = retry_bad_packed_offset(p, base_offset);
2195 if (type > OBJ_NONE)
2196 goto out;
2197 goto unwind;
2198 }
2199 }
2200
21666f1a 2201 switch (type) {
790d96c0 2202 case OBJ_BAD:
a733cb60 2203 case OBJ_COMMIT:
a733cb60 2204 case OBJ_TREE:
a733cb60 2205 case OBJ_BLOB:
a733cb60 2206 case OBJ_TAG:
a69d0943 2207 break;
1f688557 2208 default:
3d77d877
NP
2209 error("unknown object type %i at offset %"PRIuMAX" in %s",
2210 type, (uintmax_t)obj_offset, p->pack_name);
2211 type = OBJ_BAD;
1f688557 2212 }
790d96c0
TR
2213
2214out:
2215 if (poi_stack != small_poi_stack)
2216 free(poi_stack);
21666f1a 2217 return type;
790d96c0
TR
2218
2219unwind:
2220 while (poi_stack_nr) {
2221 obj_offset = poi_stack[--poi_stack_nr];
2222 type = retry_bad_packed_offset(p, obj_offset);
2223 if (type > OBJ_NONE)
2224 goto out;
2225 }
2226 type = OBJ_BAD;
2227 goto out;
1f688557
JH
2228}
2229
8261e1f1 2230static struct hashmap delta_base_cache;
18bdec11 2231static size_t delta_base_cached;
5e08ecbf 2232
12d95ef6 2233static LIST_HEAD(delta_base_cache_lru);
5e08ecbf 2234
8261e1f1 2235struct delta_base_cache_key {
e5e01619
LT
2236 struct packed_git *p;
2237 off_t base_offset;
8261e1f1
JK
2238};
2239
2240struct delta_base_cache_entry {
2241 struct hashmap hash;
2242 struct delta_base_cache_key key;
2243 struct list_head lru;
2244 void *data;
e5e01619 2245 unsigned long size;
e5e01619 2246 enum object_type type;
8261e1f1 2247};
e5e01619 2248
8261e1f1 2249static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
e5e01619 2250{
8261e1f1 2251 unsigned int hash;
e5e01619 2252
8261e1f1 2253 hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
e5e01619 2254 hash += (hash >> 8) + (hash >> 16);
8261e1f1 2255 return hash;
e5e01619
LT
2256}
2257
84dd81c1
TR
2258static struct delta_base_cache_entry *
2259get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
5266d369 2260{
8261e1f1
JK
2261 struct hashmap_entry entry;
2262 struct delta_base_cache_key key;
2263
2264 if (!delta_base_cache.cmpfn)
2265 return NULL;
2266
2267 hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
2268 key.p = p;
2269 key.base_offset = base_offset;
2270 return hashmap_get(&delta_base_cache, &entry, &key);
2271}
2272
2273static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
2274 const struct delta_base_cache_key *b)
2275{
2276 return a->p == b->p && a->base_offset == b->base_offset;
84dd81c1
TR
2277}
2278
7663cdc8
SB
2279static int delta_base_cache_hash_cmp(const void *unused_cmp_data,
2280 const void *va, const void *vb,
8261e1f1 2281 const void *vkey)
84dd81c1 2282{
8261e1f1
JK
2283 const struct delta_base_cache_entry *a = va, *b = vb;
2284 const struct delta_base_cache_key *key = vkey;
2285 if (key)
2286 return !delta_base_cache_key_eq(&a->key, key);
2287 else
2288 return !delta_base_cache_key_eq(&a->key, &b->key);
5266d369
JH
2289}
2290
84dd81c1
TR
2291static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
2292{
8261e1f1 2293 return !!get_delta_base_cache_entry(p, base_offset);
84dd81c1
TR
2294}
2295
4a5397ca
JK
2296/*
2297 * Remove the entry from the cache, but do _not_ free the associated
2298 * entry data. The caller takes ownership of the "data" buffer, and
2299 * should copy out any fields it wants before detaching.
2300 */
2301static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
84dd81c1 2302{
8261e1f1 2303 hashmap_remove(&delta_base_cache, ent, &ent->key);
12d95ef6 2304 list_del(&ent->lru);
84dd81c1 2305 delta_base_cached -= ent->size;
8261e1f1 2306 free(ent);
84dd81c1
TR
2307}
2308
62f255ad 2309static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
85fe35ab 2310 unsigned long *base_size, enum object_type *type)
62f255ad 2311{
84dd81c1 2312 struct delta_base_cache_entry *ent;
e5e01619 2313
84dd81c1 2314 ent = get_delta_base_cache_entry(p, base_offset);
8261e1f1 2315 if (!ent)
749bc58c 2316 return unpack_entry(p, base_offset, type, base_size);
e5e01619 2317
c84a1f3e
JT
2318 if (type)
2319 *type = ent->type;
2320 if (base_size)
2321 *base_size = ent->size;
85fe35ab 2322 return xmemdupz(ent->data, ent->size);
62f255ad
LT
2323}
2324
18bdec11
SP
2325static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
2326{
8261e1f1
JK
2327 free(ent->data);
2328 detach_delta_base_cache_entry(ent);
18bdec11
SP
2329}
2330
3d20c636
SP
2331void clear_delta_base_cache(void)
2332{
abd5a002
JK
2333 struct list_head *lru, *tmp;
2334 list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
2335 struct delta_base_cache_entry *entry =
2336 list_entry(lru, struct delta_base_cache_entry, lru);
8261e1f1
JK
2337 release_delta_base_cache(entry);
2338 }
3d20c636
SP
2339}
2340
62f255ad
LT
2341static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
2342 void *base, unsigned long base_size, enum object_type type)
2343{
8261e1f1 2344 struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
4b92bae7 2345 struct list_head *lru, *tmp;
e5e01619 2346
18bdec11 2347 delta_base_cached += base_size;
5e08ecbf 2348
4b92bae7 2349 list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
12d95ef6
JK
2350 struct delta_base_cache_entry *f =
2351 list_entry(lru, struct delta_base_cache_entry, lru);
2352 if (delta_base_cached <= delta_base_cache_limit)
2353 break;
5e08ecbf
NP
2354 release_delta_base_cache(f);
2355 }
18bdec11 2356
8261e1f1
JK
2357 ent->key.p = p;
2358 ent->key.base_offset = base_offset;
e5e01619
LT
2359 ent->type = type;
2360 ent->data = base;
2361 ent->size = base_size;
12d95ef6 2362 list_add_tail(&ent->lru, &delta_base_cache_lru);
8261e1f1
JK
2363
2364 if (!delta_base_cache.cmpfn)
7663cdc8 2365 hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0);
8261e1f1
JK
2366 hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
2367 hashmap_add(&delta_base_cache, ent);
62f255ad
LT
2368}
2369
845b102b
JT
2370int packed_object_info(struct packed_git *p, off_t obj_offset,
2371 struct object_info *oi)
2372{
2373 struct pack_window *w_curs = NULL;
2374 unsigned long size;
2375 off_t curpos = obj_offset;
2376 enum object_type type;
2377
2378 /*
2379 * We always get the representation type, but only convert it to
2380 * a "real" type later if the caller is interested.
2381 */
c84a1f3e
JT
2382 if (oi->contentp) {
2383 *oi->contentp = cache_or_unpack_entry(p, obj_offset, oi->sizep,
2384 &type);
2385 if (!*oi->contentp)
2386 type = OBJ_BAD;
2387 } else {
2388 type = unpack_object_header(p, &w_curs, &curpos, &size);
2389 }
845b102b 2390
c84a1f3e 2391 if (!oi->contentp && oi->sizep) {
845b102b
JT
2392 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2393 off_t tmp_pos = curpos;
2394 off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
2395 type, obj_offset);
2396 if (!base_offset) {
2397 type = OBJ_BAD;
2398 goto out;
2399 }
2400 *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
2401 if (*oi->sizep == 0) {
2402 type = OBJ_BAD;
2403 goto out;
2404 }
2405 } else {
2406 *oi->sizep = size;
2407 }
2408 }
2409
2410 if (oi->disk_sizep) {
2411 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2412 *oi->disk_sizep = revidx[1].offset - obj_offset;
2413 }
2414
2415 if (oi->typep || oi->typename) {
2416 enum object_type ptot;
2417 ptot = packed_to_object_type(p, obj_offset, type, &w_curs,
2418 curpos);
2419 if (oi->typep)
2420 *oi->typep = ptot;
2421 if (oi->typename) {
2422 const char *tn = typename(ptot);
2423 if (tn)
2424 strbuf_addstr(oi->typename, tn);
2425 }
2426 if (ptot < 0) {
2427 type = OBJ_BAD;
2428 goto out;
2429 }
2430 }
2431
2432 if (oi->delta_base_sha1) {
2433 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2434 const unsigned char *base;
2435
2436 base = get_delta_base_sha1(p, &w_curs, curpos,
2437 type, obj_offset);
2438 if (!base) {
2439 type = OBJ_BAD;
2440 goto out;
2441 }
2442
2443 hashcpy(oi->delta_base_sha1, base);
2444 } else
2445 hashclr(oi->delta_base_sha1);
2446 }
2447
2448out:
2449 unuse_pack(&w_curs);
2450 return type;
2451}
2452
2453static void *unpack_compressed_entry(struct packed_git *p,
2454 struct pack_window **w_curs,
2455 off_t curpos,
2456 unsigned long size)
2457{
2458 int st;
2459 git_zstream stream;
2460 unsigned char *buffer, *in;
2461
2462 buffer = xmallocz_gently(size);
2463 if (!buffer)
2464 return NULL;
2465 memset(&stream, 0, sizeof(stream));
2466 stream.next_out = buffer;
2467 stream.avail_out = size + 1;
2468
2469 git_inflate_init(&stream);
2470 do {
2471 in = use_pack(p, w_curs, curpos, &stream.avail_in);
2472 stream.next_in = in;
2473 st = git_inflate(&stream, Z_FINISH);
2474 if (!stream.avail_out)
2475 break; /* the payload is larger than it should be */
2476 curpos += stream.next_in - in;
2477 } while (st == Z_OK || st == Z_BUF_ERROR);
2478 git_inflate_end(&stream);
2479 if ((st != Z_STREAM_END) || stream.total_out != size) {
2480 free(buffer);
2481 return NULL;
2482 }
2483
2484 return buffer;
2485}
2486
c2c5b270
CC
2487static void *read_object(const unsigned char *sha1, enum object_type *type,
2488 unsigned long *size);
2489
5f44324d
JH
2490static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
2491{
67dc598e
KB
2492 static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
2493 trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
2494 p->pack_name, (uintmax_t)obj_offset);
5f44324d
JH
2495}
2496
0e8189e2
NP
2497int do_check_packed_object_crc;
2498
abe601bb
TR
2499#define UNPACK_ENTRY_STACK_PREALLOC 64
2500struct unpack_entry_stack_ent {
2501 off_t obj_offset;
2502 off_t curpos;
2503 unsigned long size;
2504};
2505
c4001d92 2506void *unpack_entry(struct packed_git *p, off_t obj_offset,
abe601bb 2507 enum object_type *final_type, unsigned long *final_size)
1f688557 2508{
03e79c88 2509 struct pack_window *w_curs = NULL;
c4001d92 2510 off_t curpos = obj_offset;
abe601bb
TR
2511 void *data = NULL;
2512 unsigned long size;
2513 enum object_type type;
2514 struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
2515 struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
2516 int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
2517 int base_from_cache = 0;
1f688557 2518
67dc598e 2519 write_pack_access_log(p, obj_offset);
5f44324d 2520
abe601bb
TR
2521 /* PHASE 1: drill down to the innermost base object */
2522 for (;;) {
2523 off_t base_offset;
2524 int i;
2525 struct delta_base_cache_entry *ent;
2526
77965f8b 2527 ent = get_delta_base_cache_entry(p, curpos);
8261e1f1 2528 if (ent) {
77965f8b
NTND
2529 type = ent->type;
2530 data = ent->data;
2531 size = ent->size;
4a5397ca 2532 detach_delta_base_cache_entry(ent);
77965f8b
NTND
2533 base_from_cache = 1;
2534 break;
2535 }
2536
abe601bb
TR
2537 if (do_check_packed_object_crc && p->index_version > 1) {
2538 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
211c61c6 2539 off_t len = revidx[1].offset - obj_offset;
abe601bb
TR
2540 if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
2541 const unsigned char *sha1 =
2542 nth_packed_object_sha1(p, revidx->nr);
2543 error("bad packed object CRC for %s",
2544 sha1_to_hex(sha1));
2545 mark_bad_packed_object(p, sha1);
896dca3a
RS
2546 data = NULL;
2547 goto out;
abe601bb
TR
2548 }
2549 }
2550
abe601bb
TR
2551 type = unpack_object_header(p, &w_curs, &curpos, &size);
2552 if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
2553 break;
2554
2555 base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
2556 if (!base_offset) {
2557 error("failed to validate delta base reference "
2558 "at offset %"PRIuMAX" from %s",
2559 (uintmax_t)curpos, p->pack_name);
2560 /* bail to phase 2, in hopes of recovery */
2561 data = NULL;
2562 break;
2563 }
2564
2565 /* push object, proceed to base */
2566 if (delta_stack_nr >= delta_stack_alloc
2567 && delta_stack == small_delta_stack) {
2568 delta_stack_alloc = alloc_nr(delta_stack_nr);
b32fa95f 2569 ALLOC_ARRAY(delta_stack, delta_stack_alloc);
abe601bb
TR
2570 memcpy(delta_stack, small_delta_stack,
2571 sizeof(*delta_stack)*delta_stack_nr);
2572 } else {
2573 ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
2574 }
2575 i = delta_stack_nr++;
2576 delta_stack[i].obj_offset = obj_offset;
2577 delta_stack[i].curpos = curpos;
2578 delta_stack[i].size = size;
2579
2580 curpos = obj_offset = base_offset;
0e8189e2
NP
2581 }
2582
abe601bb
TR
2583 /* PHASE 2: handle the base */
2584 switch (type) {
eb32d236
NP
2585 case OBJ_OFS_DELTA:
2586 case OBJ_REF_DELTA:
abe601bb 2587 if (data)
ef1177d1 2588 die("BUG: unpack_entry: left loop at a valid delta");
4d703a1a 2589 break;
a733cb60 2590 case OBJ_COMMIT:
a733cb60 2591 case OBJ_TREE:
a733cb60 2592 case OBJ_BLOB:
a733cb60 2593 case OBJ_TAG:
abe601bb
TR
2594 if (!base_from_cache)
2595 data = unpack_compressed_entry(p, &w_curs, curpos, size);
4d703a1a 2596 break;
1f688557 2597 default:
8eca0b47
NP
2598 data = NULL;
2599 error("unknown object type %i at offset %"PRIuMAX" in %s",
abe601bb
TR
2600 type, (uintmax_t)obj_offset, p->pack_name);
2601 }
2602
2603 /* PHASE 3: apply deltas in order */
2604
2605 /* invariants:
2606 * 'data' holds the base data, or NULL if there was corruption
2607 */
2608 while (delta_stack_nr) {
2609 void *delta_data;
2610 void *base = data;
886ddf47 2611 void *external_base = NULL;
abe601bb
TR
2612 unsigned long delta_size, base_size = size;
2613 int i;
2614
2615 data = NULL;
2616
2617 if (base)
2618 add_delta_base_cache(p, obj_offset, base, base_size, type);
2619
2620 if (!base) {
2621 /*
2622 * We're probably in deep shit, but let's try to fetch
2623 * the required base anyway from another pack or loose.
2624 * This is costly but should happen only in the presence
2625 * of a corrupted pack, and is better than failing outright.
2626 */
2627 struct revindex_entry *revidx;
2628 const unsigned char *base_sha1;
2629 revidx = find_pack_revindex(p, obj_offset);
2630 if (revidx) {
2631 base_sha1 = nth_packed_object_sha1(p, revidx->nr);
2632 error("failed to read delta base object %s"
2633 " at offset %"PRIuMAX" from %s",
2634 sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
2635 p->pack_name);
2636 mark_bad_packed_object(p, base_sha1);
2637 base = read_object(base_sha1, &type, &base_size);
886ddf47 2638 external_base = base;
abe601bb
TR
2639 }
2640 }
2641
2642 i = --delta_stack_nr;
2643 obj_offset = delta_stack[i].obj_offset;
2644 curpos = delta_stack[i].curpos;
2645 delta_size = delta_stack[i].size;
2646
2647 if (!base)
2648 continue;
2649
2650 delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
2651
2652 if (!delta_data) {
2653 error("failed to unpack compressed delta "
2654 "at offset %"PRIuMAX" from %s",
2655 (uintmax_t)curpos, p->pack_name);
abe601bb 2656 data = NULL;
886ddf47 2657 free(external_base);
abe601bb
TR
2658 continue;
2659 }
2660
2661 data = patch_delta(base, base_size,
2662 delta_data, delta_size,
2663 &size);
1ee886c1
JK
2664
2665 /*
2666 * We could not apply the delta; warn the user, but keep going.
2667 * Our failure will be noticed either in the next iteration of
2668 * the loop, or if this is the final delta, in the caller when
2669 * we return NULL. Those code paths will take care of making
2670 * a more explicit warning and retrying with another copy of
2671 * the object.
2672 */
abe601bb 2673 if (!data)
1ee886c1 2674 error("failed to apply delta");
abe601bb 2675
4b8f772c 2676 free(delta_data);
886ddf47 2677 free(external_base);
1f688557 2678 }
abe601bb 2679
c84a1f3e
JT
2680 if (final_type)
2681 *final_type = type;
2682 if (final_size)
2683 *final_size = size;
abe601bb 2684
896dca3a 2685out:
03e79c88 2686 unuse_pack(&w_curs);
019d1e65
NTND
2687
2688 if (delta_stack != small_delta_stack)
2689 free(delta_stack);
2690
21666f1a 2691 return data;
1f688557
JH
2692}
2693
d079837e 2694const unsigned char *nth_packed_object_sha1(struct packed_git *p,
d72308e0 2695 uint32_t n)
9a217f2a 2696{
42873078 2697 const unsigned char *index = p->index_data;
d079837e
SP
2698 if (!index) {
2699 if (open_pack_index(p))
2700 return NULL;
2701 index = p->index_data;
2702 }
57059091 2703 if (n >= p->num_objects)
d72308e0 2704 return NULL;
74e34e1f
NP
2705 index += 4 * 256;
2706 if (p->index_version == 1) {
2707 return index + 24 * n + 4;
2708 } else {
2709 index += 8;
2710 return index + 20 * n;
2711 }
2712}
2713
068f85e3 2714const struct object_id *nth_packed_object_oid(struct object_id *oid,
2715 struct packed_git *p,
2716 uint32_t n)
2717{
2718 const unsigned char *hash = nth_packed_object_sha1(p, n);
2719 if (!hash)
2720 return NULL;
2721 hashcpy(oid->hash, hash);
2722 return oid;
2723}
2724
47fe3f6e
JK
2725void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
2726{
2727 const unsigned char *ptr = vptr;
2728 const unsigned char *start = p->index_data;
2729 const unsigned char *end = start + p->index_size;
2730 if (ptr < start)
7465feba 2731 die(_("offset before start of pack index for %s (corrupt index?)"),
47fe3f6e
JK
2732 p->pack_name);
2733 /* No need to check for underflow; .idx files must be at least 8 bytes */
2734 if (ptr >= end - 8)
7465feba 2735 die(_("offset beyond end of pack index for %s (truncated index?)"),
47fe3f6e
JK
2736 p->pack_name);
2737}
2738
99093238 2739off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
74e34e1f
NP
2740{
2741 const unsigned char *index = p->index_data;
2742 index += 4 * 256;
2743 if (p->index_version == 1) {
2744 return ntohl(*((uint32_t *)(index + 24 * n)));
2745 } else {
2746 uint32_t off;
2747 index += 8 + p->num_objects * (20 + 4);
2748 off = ntohl(*((uint32_t *)(index + 4 * n)));
2749 if (!(off & 0x80000000))
2750 return off;
2751 index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
47fe3f6e 2752 check_pack_index_ptr(p, index);
74e34e1f
NP
2753 return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
2754 ntohl(*((uint32_t *)(index + 4)));
2755 }
9a217f2a
JH
2756}
2757
c4001d92 2758off_t find_pack_entry_one(const unsigned char *sha1,
43057304 2759 struct packed_git *p)
1f688557 2760{
42873078 2761 const uint32_t *level1_ofs = p->index_data;
42873078 2762 const unsigned char *index = p->index_data;
628522ec
JH
2763 unsigned hi, lo, stride;
2764 static int use_lookup = -1;
2765 static int debug_lookup = -1;
2766
2767 if (debug_lookup < 0)
2768 debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
42873078 2769
d079837e
SP
2770 if (!index) {
2771 if (open_pack_index(p))
2772 return 0;
2773 level1_ofs = p->index_data;
2774 index = p->index_data;
2775 }
74e34e1f
NP
2776 if (p->index_version > 1) {
2777 level1_ofs += 2;
2778 index += 8;
2779 }
42873078 2780 index += 4 * 256;
74e34e1f
NP
2781 hi = ntohl(level1_ofs[*sha1]);
2782 lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
628522ec
JH
2783 if (p->index_version > 1) {
2784 stride = 20;
2785 } else {
2786 stride = 24;
2787 index += 4;
2788 }
2789
2790 if (debug_lookup)
6e1c2344 2791 printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
628522ec
JH
2792 sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
2793
2794 if (use_lookup < 0)
2795 use_lookup = !!getenv("GIT_USE_LOOKUP");
2796 if (use_lookup) {
2797 int pos = sha1_entry_pos(index, stride, 0,
2798 lo, hi, p->num_objects, sha1);
2799 if (pos < 0)
2800 return 0;
2801 return nth_packed_object_offset(p, pos);
2802 }
1f688557 2803
6355a768 2804 while (lo < hi) {
74e34e1f 2805 unsigned mi = (lo + hi) / 2;
628522ec
JH
2806 int cmp = hashcmp(index + mi * stride, sha1);
2807
2808 if (debug_lookup)
2809 printf("lo %u hi %u rg %u mi %u\n",
2810 lo, hi, hi - lo, mi);
43057304 2811 if (!cmp)
74e34e1f 2812 return nth_packed_object_offset(p, mi);
1f688557
JH
2813 if (cmp > 0)
2814 hi = mi;
2815 else
2816 lo = mi+1;
6355a768 2817 }
1f688557
JH
2818 return 0;
2819}
2820
4c080182 2821int is_pack_valid(struct packed_git *p)
d131b7af
SP
2822{
2823 /* An already open pack is known to be valid. */
2824 if (p->pack_fd != -1)
2825 return 1;
2826
2827 /* If the pack has one window completely covering the
2828 * file size, the pack is known to be valid even if
2829 * the descriptor is not currently open.
2830 */
2831 if (p->windows) {
2832 struct pack_window *w = p->windows;
2833
2834 if (!w->offset && w->len == p->pack_size)
2835 return 1;
2836 }
2837
2838 /* Force the pack to open to prove its valid. */
2839 return !open_packed_git(p);
2840}
2841
95099731
NTND
2842static int fill_pack_entry(const unsigned char *sha1,
2843 struct pack_entry *e,
2844 struct packed_git *p)
2845{
2846 off_t offset;
2847
2848 if (p->num_bad_objects) {
2849 unsigned i;
2850 for (i = 0; i < p->num_bad_objects; i++)
2851 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
2852 return 0;
2853 }
2854
2855 offset = find_pack_entry_one(sha1, p);
2856 if (!offset)
2857 return 0;
2858
2859 /*
2860 * We are about to tell the caller where they can locate the
2861 * requested object. We better make sure the packfile is
2862 * still here and can be accessed before supplying that
2863 * answer, as it may have been deleted since the index was
2864 * loaded!
2865 */
319b678a 2866 if (!is_pack_valid(p))
95099731 2867 return 0;
95099731
NTND
2868 e->offset = offset;
2869 e->p = p;
2870 hashcpy(e->sha1, sha1);
2871 return 1;
2872}
2873
d40d535b
MH
2874/*
2875 * Iff a pack file contains the object named by sha1, return true and
2876 * store its location to e.
2877 */
4d6acb70 2878static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
1f688557 2879{
a73cdd21 2880 struct mru_entry *p;
43057304 2881
1f688557 2882 prepare_packed_git();
f7c22cc6
NP
2883 if (!packed_git)
2884 return 0;
1f688557 2885
a73cdd21
JK
2886 for (p = packed_git_mru->head; p; p = p->next) {
2887 if (fill_pack_entry(sha1, e, p->item)) {
2888 mru_mark(packed_git_mru, p);
1b1005d1
MH
2889 return 1;
2890 }
c01f51cc 2891 }
1f688557
JH
2892 return 0;
2893}
2894
a6080a0a 2895struct packed_git *find_sha1_pack(const unsigned char *sha1,
bf592c50
DB
2896 struct packed_git *packs)
2897{
2898 struct packed_git *p;
bf592c50
DB
2899
2900 for (p = packs; p; p = p->next) {
43057304 2901 if (find_pack_entry_one(sha1, p))
bf592c50
DB
2902 return p;
2903 }
2904 return NULL;
21666f1a 2905
bf592c50
DB
2906}
2907
052fe5ea 2908static int sha1_loose_object_info(const unsigned char *sha1,
46f03448
KN
2909 struct object_info *oi,
2910 int flags)
65c2e0c3 2911{
46f03448
KN
2912 int status = 0;
2913 unsigned long mapsize;
65c2e0c3 2914 void *map;
ef49a7a0 2915 git_zstream stream;
d65a16f6 2916 char hdr[32];
46f03448 2917 struct strbuf hdrbuf = STRBUF_INIT;
c84a1f3e 2918 unsigned long size_scratch;
65c2e0c3 2919
5d642e75
JK
2920 if (oi->delta_base_sha1)
2921 hashclr(oi->delta_base_sha1);
2922
052fe5ea
JK
2923 /*
2924 * If we don't care about type or size, then we don't
4ef8d1dd
JH
2925 * need to look inside the object at all. Note that we
2926 * do not optimize out the stat call, even if the
2927 * caller doesn't care about the disk-size, since our
2928 * return value implicitly indicates whether the
2929 * object even exists.
052fe5ea 2930 */
c84a1f3e 2931 if (!oi->typep && !oi->typename && !oi->sizep && !oi->contentp) {
771e7d57 2932 const char *path;
4ef8d1dd 2933 struct stat st;
771e7d57 2934 if (stat_sha1_file(sha1, &st, &path) < 0)
4ef8d1dd
JH
2935 return -1;
2936 if (oi->disk_sizep)
23c339c0 2937 *oi->disk_sizep = st.st_size;
052fe5ea
JK
2938 return 0;
2939 }
2940
bb6b8e4f 2941 map = map_sha1_file(sha1, &mapsize);
f0df4ed5 2942 if (!map)
dbea72a8 2943 return -1;
c84a1f3e
JT
2944
2945 if (!oi->sizep)
2946 oi->sizep = &size_scratch;
2947
23c339c0
JK
2948 if (oi->disk_sizep)
2949 *oi->disk_sizep = mapsize;
19fc5e84 2950 if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) {
46f03448
KN
2951 if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
2952 status = error("unable to unpack %s header with --allow-unknown-type",
2953 sha1_to_hex(sha1));
2954 } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
36e4d74a
JH
2955 status = error("unable to unpack %s header",
2956 sha1_to_hex(sha1));
46f03448
KN
2957 if (status < 0)
2958 ; /* Do nothing */
2959 else if (hdrbuf.len) {
2960 if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
2961 status = error("unable to parse %s header with --allow-unknown-type",
2962 sha1_to_hex(sha1));
2963 } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
36e4d74a 2964 status = error("unable to parse %s header", sha1_to_hex(sha1));
c84a1f3e
JT
2965
2966 if (status >= 0 && oi->contentp)
2967 *oi->contentp = unpack_sha1_rest(&stream, hdr,
2968 *oi->sizep, sha1);
2969 else
2970 git_inflate_end(&stream);
2971
65c2e0c3 2972 munmap(map, mapsize);
46f03448 2973 if (status && oi->typep)
23c339c0 2974 *oi->typep = status;
c84a1f3e
JT
2975 if (oi->sizep == &size_scratch)
2976 oi->sizep = NULL;
46f03448 2977 strbuf_release(&hdrbuf);
93cff9a9 2978 return (status < 0) ? status : 0;
65c2e0c3
JH
2979}
2980
de7b5d62 2981int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
f0df4ed5 2982{
cd585e2a 2983 static struct object_info blank_oi = OBJECT_INFO_INIT;
f0df4ed5 2984 struct pack_entry e;
5b086407 2985 int rtype;
1f0c0d36
JT
2986 const unsigned char *real = (flags & OBJECT_INFO_LOOKUP_REPLACE) ?
2987 lookup_replace_object(sha1) :
2988 sha1;
f0df4ed5 2989
cd585e2a
JT
2990 if (!oi)
2991 oi = &blank_oi;
2992
dfdd4afc
JT
2993 if (!(flags & OBJECT_INFO_SKIP_CACHED)) {
2994 struct cached_object *co = find_cached_object(real);
2995 if (co) {
2996 if (oi->typep)
2997 *(oi->typep) = co->type;
2998 if (oi->sizep)
2999 *(oi->sizep) = co->size;
3000 if (oi->disk_sizep)
3001 *(oi->disk_sizep) = 0;
3002 if (oi->delta_base_sha1)
3003 hashclr(oi->delta_base_sha1);
3004 if (oi->typename)
3005 strbuf_addstr(oi->typename, typename(co->type));
3006 if (oi->contentp)
3007 *oi->contentp = xmemdupz(co->buf, co->size);
3008 oi->whence = OI_CACHED;
3009 return 0;
3010 }
c4d9986f
NTND
3011 }
3012
1f7117ef 3013 if (!find_pack_entry(real, &e)) {
ddd63e64 3014 /* Most likely it's a loose object. */
46f03448 3015 if (!sha1_loose_object_info(real, oi, flags)) {
9a490590 3016 oi->whence = OI_LOOSE;
5b086407 3017 return 0;
9a490590 3018 }
ddd63e64
SG
3019
3020 /* Not a loose object; someone else may have just packed it. */
dfdd4afc 3021 if (flags & OBJECT_INFO_QUICK) {
052fe5ea 3022 return -1;
dfdd4afc
JT
3023 } else {
3024 reprepare_packed_git();
3025 if (!find_pack_entry(real, &e))
3026 return -1;
3027 }
f0df4ed5 3028 }
3d77d877 3029
cd585e2a
JT
3030 if (oi == &blank_oi)
3031 /*
3032 * We know that the caller doesn't actually need the
3033 * information below, so return early.
3034 */
3035 return 0;
46f03448 3036
23c339c0 3037 rtype = packed_object_info(e.p, e.offset, oi);
412916ee 3038 if (rtype < 0) {
1f7117ef
CC
3039 mark_bad_packed_object(e.p, real);
3040 return sha1_object_info_extended(real, oi, 0);
5266d369
JH
3041 } else if (in_delta_base_cache(e.p, e.offset)) {
3042 oi->whence = OI_DBCACHED;
9a490590
JH
3043 } else {
3044 oi->whence = OI_PACKED;
3045 oi->u.packed.offset = e.offset;
3046 oi->u.packed.pack = e.p;
3047 oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
3048 rtype == OBJ_OFS_DELTA);
3d77d877
NP
3049 }
3050
5b086407 3051 return 0;
f0df4ed5
JS
3052}
3053
3fc0dca9 3054/* returns enum object_type or negative */
9a490590
JH
3055int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
3056{
5b086407 3057 enum object_type type;
27b5c1a0 3058 struct object_info oi = OBJECT_INFO_INIT;
9a490590 3059
5b086407 3060 oi.typep = &type;
9a490590 3061 oi.sizep = sizep;
1f0c0d36
JT
3062 if (sha1_object_info_extended(sha1, &oi,
3063 OBJECT_INFO_LOOKUP_REPLACE) < 0)
5b086407
JK
3064 return -1;
3065 return type;
9a490590
JH
3066}
3067
21666f1a
NP
3068static void *read_packed_sha1(const unsigned char *sha1,
3069 enum object_type *type, unsigned long *size)
1f688557
JH
3070{
3071 struct pack_entry e;
8eca0b47 3072 void *data;
1f688557 3073
cd673c1f 3074 if (!find_pack_entry(sha1, &e))
1f688557 3075 return NULL;
85fe35ab 3076 data = cache_or_unpack_entry(e.p, e.offset, size, type);
8eca0b47
NP
3077 if (!data) {
3078 /*
3079 * We're probably in deep shit, but let's try to fetch
3080 * the required object anyway from another pack or loose.
3081 * This should happen only in the presence of a corrupted
3082 * pack, and is better than failing outright.
3083 */
3084 error("failed to read object %s at offset %"PRIuMAX" from %s",
3085 sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
3086 mark_bad_packed_object(e.p, sha1);
ac939109 3087 data = read_object(sha1, type, size);
8eca0b47
NP
3088 }
3089 return data;
1f688557
JH
3090}
3091
21666f1a
NP
3092int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
3093 unsigned char *sha1)
d66b37bb
JH
3094{
3095 struct cached_object *co;
3096
21666f1a 3097 hash_sha1_file(buf, len, typename(type), sha1);
d66b37bb
JH
3098 if (has_sha1_file(sha1) || find_cached_object(sha1))
3099 return 0;
c7353967 3100 ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
d66b37bb
JH
3101 co = &cached_objects[cached_object_nr++];
3102 co->size = len;
21666f1a 3103 co->type = type;
efa13f7b
JH
3104 co->buf = xmalloc(len);
3105 memcpy(co->buf, buf, len);
d66b37bb
JH
3106 hashcpy(co->sha1, sha1);
3107 return 0;
3108}
3109
c2c5b270
CC
3110static void *read_object(const unsigned char *sha1, enum object_type *type,
3111 unsigned long *size)
0fcfd160 3112{
c84a1f3e
JT
3113 struct object_info oi = OBJECT_INFO_INIT;
3114 void *content;
3115 oi.typep = type;
3116 oi.sizep = size;
3117 oi.contentp = &content;
0fcfd160 3118
c84a1f3e
JT
3119 if (sha1_object_info_extended(sha1, &oi, 0) < 0)
3120 return NULL;
3121 return content;
0fcfd160
LT
3122}
3123
b6c4cecc
JH
3124/*
3125 * This function dies on corrupt objects; the callers who want to
3126 * deal with them should arrange to call read_object() and give error
3127 * messages themselves.
3128 */
5bf29b95
JH
3129void *read_sha1_file_extended(const unsigned char *sha1,
3130 enum object_type *type,
3131 unsigned long *size,
1f0c0d36 3132 int lookup_replace)
ac939109 3133{
3ba7a065 3134 void *data;
b6c4cecc 3135 const struct packed_git *p;
771e7d57
JK
3136 const char *path;
3137 struct stat st;
1f0c0d36
JT
3138 const unsigned char *repl = lookup_replace ? lookup_replace_object(sha1)
3139 : sha1;
b6c4cecc 3140
3ba7a065
JH
3141 errno = 0;
3142 data = read_object(repl, type, size);
4bbf5a26 3143 if (data)
b6c4cecc 3144 return data;
68095570 3145
25f3af3f 3146 if (errno && errno != ENOENT)
3ba7a065
JH
3147 die_errno("failed to read object %s", sha1_to_hex(sha1));
3148
68095570 3149 /* die if we replaced an object with one that does not exist */
b6c4cecc 3150 if (repl != sha1)
68095570
CC
3151 die("replacement %s not found for %s",
3152 sha1_to_hex(repl), sha1_to_hex(sha1));
3153
771e7d57 3154 if (!stat_sha1_file(repl, &st, &path))
b6c4cecc
JH
3155 die("loose object %s (stored in %s) is corrupt",
3156 sha1_to_hex(repl), path);
68095570 3157
b6c4cecc
JH
3158 if ((p = has_packed_and_bad(repl)) != NULL)
3159 die("packed object %s (stored in %s) is corrupt",
3160 sha1_to_hex(repl), p->pack_name);
f5552aee 3161
b6c4cecc 3162 return NULL;
ac939109
NP
3163}
3164
40469ee9 3165void *read_object_with_reference(const unsigned char *sha1,
21666f1a 3166 const char *required_type_name,
40469ee9
JH
3167 unsigned long *size,
3168 unsigned char *actual_sha1_return)
f4913f91 3169{
21666f1a 3170 enum object_type type, required_type;
f4913f91
JH
3171 void *buffer;
3172 unsigned long isize;
40469ee9 3173 unsigned char actual_sha1[20];
f4913f91 3174
21666f1a 3175 required_type = type_from_string(required_type_name);
e702496e 3176 hashcpy(actual_sha1, sha1);
40469ee9
JH
3177 while (1) {
3178 int ref_length = -1;
3179 const char *ref_type = NULL;
f4913f91 3180
21666f1a 3181 buffer = read_sha1_file(actual_sha1, &type, &isize);
40469ee9
JH
3182 if (!buffer)
3183 return NULL;
21666f1a 3184 if (type == required_type) {
40469ee9
JH
3185 *size = isize;
3186 if (actual_sha1_return)
e702496e 3187 hashcpy(actual_sha1_return, actual_sha1);
40469ee9
JH
3188 return buffer;
3189 }
3190 /* Handle references */
21666f1a 3191 else if (type == OBJ_COMMIT)
40469ee9 3192 ref_type = "tree ";
21666f1a 3193 else if (type == OBJ_TAG)
40469ee9
JH
3194 ref_type = "object ";
3195 else {
3196 free(buffer);
3197 return NULL;
3198 }
3199 ref_length = strlen(ref_type);
f4913f91 3200
50974ec9
MK
3201 if (ref_length + 40 > isize ||
3202 memcmp(buffer, ref_type, ref_length) ||
1d7f171c 3203 get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
40469ee9
JH
3204 free(buffer);
3205 return NULL;
3206 }
1cf58e72 3207 free(buffer);
40469ee9
JH
3208 /* Now we have the ID of the referred-to object in
3209 * actual_sha1. Check again. */
f4913f91 3210 }
f4913f91
JH
3211}
3212
ce9fbf16 3213static void write_sha1_file_prepare(const void *buf, unsigned long len,
972a9155 3214 const char *type, unsigned char *sha1,
d65a16f6 3215 char *hdr, int *hdrlen)
d410c0f5 3216{
9126f009 3217 git_SHA_CTX c;
d410c0f5
JH
3218
3219 /* Generate the header */
ef1286d3 3220 *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1;
d410c0f5
JH
3221
3222 /* Sha1.. */
9126f009
NP
3223 git_SHA1_Init(&c);
3224 git_SHA1_Update(&c, hdr, *hdrlen);
3225 git_SHA1_Update(&c, buf, len);
3226 git_SHA1_Final(sha1, &c);
d410c0f5
JH
3227}
3228
230f1322 3229/*
5a688fe4 3230 * Move the just written object into its final resting place.
230f1322 3231 */
cb5add58 3232int finalize_object_file(const char *tmpfile, const char *filename)
230f1322 3233{
e32c0a9c 3234 int ret = 0;
5a688fe4 3235
348df166 3236 if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
be66a6c4
JS
3237 goto try_rename;
3238 else if (link(tmpfile, filename))
e32c0a9c 3239 ret = errno;
7ebb6fca
LT
3240
3241 /*
3242 * Coda hack - coda doesn't like cross-directory links,
3243 * so we fall back to a rename, which will mean that it
3244 * won't be able to check collisions, but that's not a
3245 * big deal.
3246 *
3247 * The same holds for FAT formatted media.
3248 *
3be1f18e 3249 * When this succeeds, we just return. We have nothing
7ebb6fca
LT
3250 * left to unlink.
3251 */
3252 if (ret && ret != EEXIST) {
be66a6c4 3253 try_rename:
7ebb6fca 3254 if (!rename(tmpfile, filename))
3be1f18e 3255 goto out;
9e48b389 3256 ret = errno;
230f1322 3257 }
691f1a28 3258 unlink_or_warn(tmpfile);
230f1322
LT
3259 if (ret) {
3260 if (ret != EEXIST) {
7616c6ca 3261 return error_errno("unable to write sha1 filename %s", filename);
230f1322
LT
3262 }
3263 /* FIXME!!! Collision check here ? */
3264 }
3265
3be1f18e 3266out:
5256b006 3267 if (adjust_shared_perm(filename))
5a688fe4 3268 return error("unable to set permission to '%s'", filename);
230f1322
LT
3269 return 0;
3270}
3271
4d548150
LT
3272static int write_buffer(int fd, const void *buf, size_t len)
3273{
d34cf19b 3274 if (write_in_full(fd, buf, len) < 0)
7616c6ca 3275 return error_errno("file write error");
4d548150
LT
3276 return 0;
3277}
3278
ce9fbf16 3279int hash_sha1_file(const void *buf, unsigned long len, const char *type,
abdc3fc8
RS
3280 unsigned char *sha1)
3281{
d65a16f6 3282 char hdr[32];
ef1286d3 3283 int hdrlen = sizeof(hdr);
abdc3fc8
RS
3284 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
3285 return 0;
3286}
3287
e9039dd3
LT
3288/* Finalize a file on disk, and close it. */
3289static void close_sha1_file(int fd)
3290{
aafe9fba
LT
3291 if (fsync_object_files)
3292 fsync_or_die(fd, "sha1 file");
e9039dd3 3293 if (close(fd) != 0)
d824cbba 3294 die_errno("error when closing sha1 file");
e9039dd3
LT
3295}
3296
5723fe7e
LT
3297/* Size of directory component, including the ending '/' */
3298static inline int directory_size(const char *filename)
3299{
3300 const char *s = strrchr(filename, '/');
3301 if (!s)
3302 return 0;
3303 return s - filename + 1;
3304}
3305
3306/*
3307 * This creates a temporary file in the same directory as the final
3308 * 'filename'
3309 *
3310 * We want to avoid cross-directory filename renames, because those
3311 * can have problems on various filesystems (FAT, NFS, Coda).
3312 */
d4b3d11a 3313static int create_tmpfile(struct strbuf *tmp, const char *filename)
5723fe7e
LT
3314{
3315 int fd, dirlen = directory_size(filename);
3316
d4b3d11a
JK
3317 strbuf_reset(tmp);
3318 strbuf_add(tmp, filename, dirlen);
3319 strbuf_addstr(tmp, "tmp_obj_XXXXXX");
3320 fd = git_mkstemp_mode(tmp->buf, 0444);
cbacbf4e 3321 if (fd < 0 && dirlen && errno == ENOENT) {
d4b3d11a
JK
3322 /*
3323 * Make sure the directory exists; note that the contents
3324 * of the buffer are undefined after mkstemp returns an
3325 * error, so we have to rewrite the whole buffer from
3326 * scratch.
3327 */
3328 strbuf_reset(tmp);
3329 strbuf_add(tmp, filename, dirlen - 1);
3330 if (mkdir(tmp->buf, 0777) && errno != EEXIST)
b2476a60 3331 return -1;
d4b3d11a 3332 if (adjust_shared_perm(tmp->buf))
5723fe7e
LT
3333 return -1;
3334
3335 /* Try again */
d4b3d11a
JK
3336 strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
3337 fd = git_mkstemp_mode(tmp->buf, 0444);
5723fe7e
LT
3338 }
3339 return fd;
3340}
3341
bbac7311 3342static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
c00e657d 3343 const void *buf, unsigned long len, time_t mtime)
0fcfd160 3344{
915308b1 3345 int fd, ret;
9892beba 3346 unsigned char compressed[4096];
ef49a7a0 3347 git_zstream stream;
748af44c
NP
3348 git_SHA_CTX c;
3349 unsigned char parano_sha1[20];
d4b3d11a 3350 static struct strbuf tmp_file = STRBUF_INIT;
30d6c6ea 3351 const char *filename = sha1_file_name(sha1);
a44c9a5e 3352
d4b3d11a 3353 fd = create_tmpfile(&tmp_file, filename);
aac17941 3354 if (fd < 0) {
35243577 3355 if (errno == EACCES)
82247e9b 3356 return error("insufficient permission for adding an object to repository database %s", get_object_directory());
916d081b 3357 else
7616c6ca 3358 return error_errno("unable to create temporary file");
aac17941
LT
3359 }
3360
0fcfd160 3361 /* Set it up */
55bb5c91 3362 git_deflate_init(&stream, zlib_compression_level);
0fcfd160 3363 stream.next_out = compressed;
9892beba 3364 stream.avail_out = sizeof(compressed);
748af44c 3365 git_SHA1_Init(&c);
a44c9a5e
LT
3366
3367 /* First header.. */
d65a16f6 3368 stream.next_in = (unsigned char *)hdr;
a44c9a5e 3369 stream.avail_in = hdrlen;
55bb5c91
JH
3370 while (git_deflate(&stream, 0) == Z_OK)
3371 ; /* nothing */
748af44c 3372 git_SHA1_Update(&c, hdr, hdrlen);
a44c9a5e
LT
3373
3374 /* Then the data itself.. */
c00e657d 3375 stream.next_in = (void *)buf;
a44c9a5e 3376 stream.avail_in = len;
9892beba 3377 do {
748af44c 3378 unsigned char *in0 = stream.next_in;
55bb5c91 3379 ret = git_deflate(&stream, Z_FINISH);
748af44c 3380 git_SHA1_Update(&c, in0, stream.next_in - in0);
9892beba
NP
3381 if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
3382 die("unable to write sha1 file");
3383 stream.next_out = compressed;
3384 stream.avail_out = sizeof(compressed);
3385 } while (ret == Z_OK);
3386
ac54c277
LT
3387 if (ret != Z_STREAM_END)
3388 die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
55bb5c91 3389 ret = git_deflate_end_gently(&stream);
ac54c277
LT
3390 if (ret != Z_OK)
3391 die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
748af44c
NP
3392 git_SHA1_Final(parano_sha1, &c);
3393 if (hashcmp(sha1, parano_sha1) != 0)
3394 die("confused by unstable object source data for %s", sha1_to_hex(sha1));
ac54c277 3395
e9039dd3 3396 close_sha1_file(fd);
0fcfd160 3397
bbac7311
NP
3398 if (mtime) {
3399 struct utimbuf utb;
3400 utb.actime = mtime;
3401 utb.modtime = mtime;
d4b3d11a 3402 if (utime(tmp_file.buf, &utb) < 0)
7616c6ca 3403 warning_errno("failed utime() on %s", tmp_file.buf);
bbac7311
NP
3404 }
3405
d4b3d11a 3406 return finalize_object_file(tmp_file.buf, filename);
0fcfd160 3407}
8237b185 3408
33d4221c
JK
3409static int freshen_loose_object(const unsigned char *sha1)
3410{
3411 return check_and_freshen(sha1, 1);
3412}
3413
3414static int freshen_packed_object(const unsigned char *sha1)
3415{
3416 struct pack_entry e;
ee1c6c34
JK
3417 if (!find_pack_entry(sha1, &e))
3418 return 0;
3419 if (e.p->freshened)
3420 return 1;
3421 if (!freshen_file(e.p->pack_name))
3422 return 0;
3423 e.p->freshened = 1;
3424 return 1;
33d4221c
JK
3425}
3426
1427a7ff 3427int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
bbac7311 3428{
bbac7311 3429 char hdr[32];
ef1286d3 3430 int hdrlen = sizeof(hdr);
bbac7311
NP
3431
3432 /* Normally if we have it in the pack then we do not bother writing
3433 * it out into .git/objects/??/?{38} file.
3434 */
3435 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
b5f52f37 3436 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
bbac7311
NP
3437 return 0;
3438 return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
3439}
3440
0c3db67c
ES
3441int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
3442 unsigned char *sha1, unsigned flags)
3443{
3444 char *header;
3445 int hdrlen, status = 0;
3446
3447 /* type string, SP, %lu of the length plus NUL must fit this */
ef1286d3
JK
3448 hdrlen = strlen(type) + 32;
3449 header = xmalloc(hdrlen);
0c3db67c
ES
3450 write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
3451
3452 if (!(flags & HASH_WRITE_OBJECT))
3453 goto cleanup;
051086b9 3454 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
0c3db67c
ES
3455 goto cleanup;
3456 status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
3457
3458cleanup:
3459 free(header);
3460 return status;
3461}
3462
bbac7311
NP
3463int force_object_loose(const unsigned char *sha1, time_t mtime)
3464{
bbac7311
NP
3465 void *buf;
3466 unsigned long len;
3467 enum object_type type;
3468 char hdr[32];
3469 int hdrlen;
1fb23e65 3470 int ret;
bbac7311 3471
c529d75a 3472 if (has_loose_object(sha1))
bbac7311
NP
3473 return 0;
3474 buf = read_packed_sha1(sha1, &type, &len);
3475 if (!buf)
3476 return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
ef1286d3 3477 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
1fb23e65
BS
3478 ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
3479 free(buf);
3480
3481 return ret;
bbac7311
NP
3482}
3483
bf592c50
DB
3484int has_pack_index(const unsigned char *sha1)
3485{
3486 struct stat st;
3487 if (stat(sha1_pack_index_name(sha1), &st))
3488 return 0;
3489 return 1;
bf592c50
DB
3490}
3491
cd673c1f
JH
3492int has_sha1_pack(const unsigned char *sha1)
3493{
3494 struct pack_entry e;
3495 return find_pack_entry(sha1, &e);
3496}
3497
0eeb077b 3498int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
8237b185 3499{
3e8b7d3c
JN
3500 if (!startup_info->have_repository)
3501 return 0;
e83e71c5
JT
3502 return sha1_object_info_extended(sha1, NULL,
3503 flags | OBJECT_INFO_SKIP_CACHED) >= 0;
b419aa25 3504}
3505
3506int has_object_file(const struct object_id *oid)
3507{
3508 return has_sha1_file(oid->hash);
8237b185 3509}
74400e71 3510
5827a035
JK
3511int has_object_file_with_flags(const struct object_id *oid, int flags)
3512{
3513 return has_sha1_file_with_flags(oid->hash, flags);
3514}
3515
c879daa2
NTND
3516static void check_tree(const void *buf, size_t size)
3517{
3518 struct tree_desc desc;
3519 struct name_entry entry;
3520
3521 init_tree_desc(&desc, buf, size);
3522 while (tree_entry(&desc, &entry))
3523 /* do nothing
3524 * tree_entry() will die() on malformed entries */
3525 ;
3526}
3527
3528static void check_commit(const void *buf, size_t size)
3529{
3530 struct commit c;
3531 memset(&c, 0, sizeof(c));
3532 if (parse_commit_buffer(&c, buf, size))
3533 die("corrupt commit");
3534}
3535
3536static void check_tag(const void *buf, size_t size)
3537{
3538 struct tag t;
3539 memset(&t, 0, sizeof(t));
3540 if (parse_tag_buffer(&t, buf, size))
3541 die("corrupt tag");
3542}
3543
43df4f86 3544static int index_mem(unsigned char *sha1, void *buf, size_t size,
c4ce46fc
JH
3545 enum object_type type,
3546 const char *path, unsigned flags)
e7332f96 3547{
6c510bee 3548 int ret, re_allocated = 0;
c4ce46fc 3549 int write_object = flags & HASH_WRITE_OBJECT;
74400e71 3550
7672db20 3551 if (!type)
edaec3fb 3552 type = OBJ_BLOB;
6c510bee
LT
3553
3554 /*
3555 * Convert blobs to git internal format
3556 */
43df4f86 3557 if ((type == OBJ_BLOB) && path) {
f285a2d7 3558 struct strbuf nbuf = STRBUF_INIT;
82b474e0 3559 if (convert_to_git(&the_index, path, buf, size, &nbuf,
5e12e78e 3560 write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
b315c5c0 3561 buf = strbuf_detach(&nbuf, &size);
6c510bee
LT
3562 re_allocated = 1;