object: add repository argument to lookup_object
[git/git.git] / builtin / unpack-objects.c
CommitLineData
64413630 1#include "builtin.h"
bad50dc8 2#include "cache.h"
b2141fc1 3#include "config.h"
cbd53a21 4#include "object-store.h"
74536958 5#include "object.h"
8ee378a0 6#include "delta.h"
a733cb60 7#include "pack.h"
8e440259
PE
8#include "blob.h"
9#include "commit.h"
10#include "tag.h"
11#include "tree.h"
b41860bf 12#include "tree-walk.h"
96a02f8f 13#include "progress.h"
2add1e6d 14#include "decorate.h"
b41860bf 15#include "fsck.h"
bad50dc8 16
b41860bf 17static int dry_run, quiet, recover, has_errors, strict;
33e8fc87 18static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
bad50dc8 19
67e5a5ec
LT
20/* We always read in 4kB chunks. */
21static unsigned char buffer[4096];
d7dd0223
NP
22static unsigned int offset, len;
23static off_t consumed_bytes;
5ad21867 24static off_t max_input_size;
3206b6bd 25static git_hash_ctx ctx;
22410549 26static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
bad50dc8 27
f2898cfa
JH
28/*
29 * When running under --strict mode, objects whose reachability are
30 * suspect are kept in core without getting written in the object
31 * store.
32 */
2add1e6d
MK
33struct obj_buffer {
34 char *buffer;
35 unsigned long size;
36};
37
38static struct decoration obj_decorate;
39
40static struct obj_buffer *lookup_object_buffer(struct object *base)
41{
42 return lookup_decoration(&obj_decorate, base);
43}
44
b41860bf
MK
45static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
46{
47 struct obj_buffer *obj;
48 obj = xcalloc(1, sizeof(struct obj_buffer));
49 obj->buffer = buffer;
50 obj->size = size;
51 if (add_decoration(&obj_decorate, object, obj))
f2fd0760 52 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
b41860bf
MK
53}
54
67e5a5ec
LT
55/*
56 * Make sure at least "min" bytes are available in the buffer, and
57 * return the pointer to the buffer.
58 */
79a65697 59static void *fill(int min)
67e5a5ec
LT
60{
61 if (min <= len)
62 return buffer + offset;
67e5a5ec
LT
63 if (min > sizeof(buffer))
64 die("cannot fill %d bytes", min);
65 if (offset) {
3206b6bd 66 the_hash_algo->update_fn(&ctx, buffer, offset);
79a65697 67 memmove(buffer, buffer + offset, len);
67e5a5ec
LT
68 offset = 0;
69 }
70 do {
8a912bcb 71 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
67e5a5ec
LT
72 if (ret <= 0) {
73 if (!ret)
74 die("early EOF");
d824cbba 75 die_errno("read error on input");
67e5a5ec
LT
76 }
77 len += ret;
78 } while (len < min);
79 return buffer;
80}
74536958 81
67e5a5ec
LT
82static void use(int bytes)
83{
84 if (bytes > len)
85 die("used more bytes than were available");
86 len -= bytes;
87 offset += bytes;
d7dd0223
NP
88
89 /* make sure off_t is sufficiently large not to wrap */
c03c8315 90 if (signed_add_overflows(consumed_bytes, bytes))
d7dd0223 91 die("pack too large for current definition of off_t");
209c554a 92 consumed_bytes += bytes;
5ad21867
CC
93 if (max_input_size && consumed_bytes > max_input_size)
94 die(_("pack exceeds maximum allowed size"));
67e5a5ec 95}
bad50dc8 96
67e5a5ec 97static void *get_data(unsigned long size)
bad50dc8 98{
ef49a7a0 99 git_zstream stream;
a1e920a0 100 void *buf = xmallocz(size);
67e5a5ec 101
67e5a5ec
LT
102 memset(&stream, 0, sizeof(stream));
103
104 stream.next_out = buf;
105 stream.avail_out = size;
106 stream.next_in = fill(1);
107 stream.avail_in = len;
39c68542 108 git_inflate_init(&stream);
67e5a5ec
LT
109
110 for (;;) {
39c68542 111 int ret = git_inflate(&stream, 0);
67e5a5ec
LT
112 use(len - stream.avail_in);
113 if (stream.total_out == size && ret == Z_STREAM_END)
114 break;
f986f2c8 115 if (ret != Z_OK) {
82247e9b 116 error("inflate returned %d", ret);
6a83d902 117 FREE_AND_NULL(buf);
3b67d291 118 if (!recover)
f986f2c8
JH
119 exit(1);
120 has_errors = 1;
121 break;
122 }
67e5a5ec
LT
123 stream.next_in = fill(1);
124 stream.avail_in = len;
125 }
39c68542 126 git_inflate_end(&stream);
67e5a5ec 127 return buf;
bad50dc8
LT
128}
129
67e5a5ec 130struct delta_info {
834bc47b 131 struct object_id base_oid;
d7dd0223
NP
132 unsigned nr;
133 off_t base_offset;
67e5a5ec
LT
134 unsigned long size;
135 void *delta;
136 struct delta_info *next;
137};
138
139static struct delta_info *delta_list;
140
834bc47b 141static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
d7dd0223 142 off_t base_offset,
209c554a 143 void *delta, unsigned long size)
bad50dc8 144{
67e5a5ec 145 struct delta_info *info = xmalloc(sizeof(*info));
bad50dc8 146
834bc47b 147 oidcpy(&info->base_oid, base_oid);
209c554a 148 info->base_offset = base_offset;
67e5a5ec
LT
149 info->size = size;
150 info->delta = delta;
209c554a 151 info->nr = nr;
67e5a5ec
LT
152 info->next = delta_list;
153 delta_list = info;
bad50dc8
LT
154}
155
209c554a 156struct obj_info {
d7dd0223 157 off_t offset;
834bc47b 158 struct object_id oid;
b41860bf 159 struct object *obj;
209c554a
NP
160};
161
95308d64 162/* Remember to update object flag allocation in object.h */
b41860bf
MK
163#define FLAG_OPEN (1u<<20)
164#define FLAG_WRITTEN (1u<<21)
165
209c554a 166static struct obj_info *obj_list;
2af202be 167static unsigned nr_objects;
b41860bf 168
f2898cfa
JH
169/*
170 * Called only from check_object() after it verified this object
171 * is Ok.
172 */
90a398bb 173static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
b41860bf 174{
834bc47b 175 struct object_id oid;
90a398bb 176
a09c985e 177 if (write_object_file(obj_buf->buffer, obj_buf->size,
169c9c01 178 type_name(obj->type), &oid) < 0)
f2fd0760 179 die("failed to write object %s", oid_to_hex(&obj->oid));
b41860bf
MK
180 obj->flags |= FLAG_WRITTEN;
181}
182
f2898cfa
JH
183/*
184 * At the very end of the processing, write_rest() scans the objects
185 * that have reachability requirements and calls this function.
186 * Verify its reachability and validity recursively and write it out.
187 */
22410549 188static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
b41860bf 189{
90a398bb
JS
190 struct obj_buffer *obj_buf;
191
b41860bf 192 if (!obj)
9a217391 193 return 1;
b41860bf
MK
194
195 if (obj->flags & FLAG_WRITTEN)
9a217391 196 return 0;
b41860bf
MK
197
198 if (type != OBJ_ANY && obj->type != type)
199 die("object type mismatch");
200
201 if (!(obj->flags & FLAG_OPEN)) {
202 unsigned long size;
0df8e965 203 int type = oid_object_info(the_repository, &obj->oid, &size);
b41860bf
MK
204 if (type != obj->type || type <= 0)
205 die("object of unexpected type");
206 obj->flags |= FLAG_WRITTEN;
9a217391 207 return 0;
b41860bf
MK
208 }
209
90a398bb
JS
210 obj_buf = lookup_object_buffer(obj);
211 if (!obj_buf)
f2fd0760 212 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
22410549 213 if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
db5a58c1 214 die("fsck error in packed object");
22410549
JS
215 fsck_options.walk = check_object;
216 if (fsck_walk(obj, NULL, &fsck_options))
f2fd0760 217 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
90a398bb 218 write_cached_object(obj, obj_buf);
9a217391 219 return 0;
b41860bf
MK
220}
221
222static void write_rest(void)
223{
224 unsigned i;
9a217391
JH
225 for (i = 0; i < nr_objects; i++) {
226 if (obj_list[i].obj)
22410549 227 check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
9a217391 228 }
b41860bf 229}
67e5a5ec 230
21666f1a
NP
231static void added_object(unsigned nr, enum object_type type,
232 void *data, unsigned long size);
209c554a 233
f2898cfa
JH
234/*
235 * Write out nr-th object from the list, now we know the contents
236 * of it. Under --strict, this buffers structured objects in-core,
237 * to be checked at the end.
238 */
21666f1a
NP
239static void write_object(unsigned nr, enum object_type type,
240 void *buf, unsigned long size)
cca7081a 241{
b41860bf 242 if (!strict) {
169c9c01 243 if (write_object_file(buf, size, type_name(type),
a09c985e 244 &obj_list[nr].oid) < 0)
b41860bf 245 die("failed to write object");
f2898cfa 246 added_object(nr, type, buf, size);
b41860bf 247 free(buf);
f2898cfa 248 obj_list[nr].obj = NULL;
b41860bf
MK
249 } else if (type == OBJ_BLOB) {
250 struct blob *blob;
169c9c01 251 if (write_object_file(buf, size, type_name(type),
a09c985e 252 &obj_list[nr].oid) < 0)
b41860bf 253 die("failed to write object");
f2898cfa 254 added_object(nr, type, buf, size);
b41860bf
MK
255 free(buf);
256
3aca1fc6 257 blob = lookup_blob(&obj_list[nr].oid);
b41860bf
MK
258 if (blob)
259 blob->object.flags |= FLAG_WRITTEN;
260 else
261 die("invalid blob object");
f2898cfa 262 obj_list[nr].obj = NULL;
b41860bf
MK
263 } else {
264 struct object *obj;
265 int eaten;
169c9c01 266 hash_object_file(buf, size, type_name(type), &obj_list[nr].oid);
f2898cfa 267 added_object(nr, type, buf, size);
c251c83d 268 obj = parse_object_buffer(&obj_list[nr].oid, type, size, buf,
269 &eaten);
b41860bf 270 if (!obj)
debca9d2 271 die("invalid %s", type_name(type));
b41860bf
MK
272 add_object_buffer(obj, buf, size);
273 obj->flags |= FLAG_OPEN;
274 obj_list[nr].obj = obj;
275 }
cca7081a
LT
276}
277
21666f1a 278static void resolve_delta(unsigned nr, enum object_type type,
f986f2c8
JH
279 void *base, unsigned long base_size,
280 void *delta, unsigned long delta_size)
bad50dc8 281{
67e5a5ec
LT
282 void *result;
283 unsigned long result_size;
bad50dc8 284
67e5a5ec
LT
285 result = patch_delta(base, base_size,
286 delta, delta_size,
287 &result_size);
288 if (!result)
289 die("failed to apply delta");
290 free(delta);
21666f1a 291 write_object(nr, type, result, result_size);
bad50dc8
LT
292}
293
f2898cfa
JH
294/*
295 * We now know the contents of an object (which is nr-th in the pack);
296 * resolve all the deltified objects that are based on it.
297 */
21666f1a
NP
298static void added_object(unsigned nr, enum object_type type,
299 void *data, unsigned long size)
74536958 300{
67e5a5ec
LT
301 struct delta_info **p = &delta_list;
302 struct delta_info *info;
303
304 while ((info = *p) != NULL) {
834bc47b 305 if (!oidcmp(&info->base_oid, &obj_list[nr].oid) ||
209c554a 306 info->base_offset == obj_list[nr].offset) {
67e5a5ec
LT
307 *p = info->next;
308 p = &delta_list;
209c554a
NP
309 resolve_delta(info->nr, type, data, size,
310 info->delta, info->size);
67e5a5ec
LT
311 free(info);
312 continue;
313 }
314 p = &info->next;
315 }
316}
317
21666f1a 318static void unpack_non_delta_entry(enum object_type type, unsigned long size,
209c554a 319 unsigned nr)
67e5a5ec
LT
320{
321 void *buf = get_data(size);
8ee378a0 322
f986f2c8 323 if (!dry_run && buf)
21666f1a 324 write_object(nr, type, buf, size);
b41860bf
MK
325 else
326 free(buf);
8ee378a0
JH
327}
328
834bc47b 329static int resolve_against_held(unsigned nr, const struct object_id *base,
f2898cfa
JH
330 void *delta_data, unsigned long delta_size)
331{
332 struct object *obj;
333 struct obj_buffer *obj_buffer;
5abddd1e 334 obj = lookup_object(the_repository, base->hash);
f2898cfa
JH
335 if (!obj)
336 return 0;
337 obj_buffer = lookup_object_buffer(obj);
338 if (!obj_buffer)
339 return 0;
340 resolve_delta(nr, obj->type, obj_buffer->buffer,
341 obj_buffer->size, delta_data, delta_size);
342 return 1;
343}
344
21666f1a 345static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
209c554a 346 unsigned nr)
8ee378a0 347{
67e5a5ec
LT
348 void *delta_data, *base;
349 unsigned long base_size;
834bc47b 350 struct object_id base_oid;
8ee378a0 351
21666f1a 352 if (type == OBJ_REF_DELTA) {
3206b6bd 353 hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
354 use(the_hash_algo->rawsz);
209c554a
NP
355 delta_data = get_data(delta_size);
356 if (dry_run || !delta_data) {
357 free(delta_data);
358 return;
359 }
834bc47b 360 if (has_object_file(&base_oid))
f2898cfa 361 ; /* Ok we have this one */
834bc47b 362 else if (resolve_against_held(nr, &base_oid,
f2898cfa
JH
363 delta_data, delta_size))
364 return; /* we are done */
365 else {
366 /* cannot resolve yet --- queue it */
834bc47b 367 oidclr(&obj_list[nr].oid);
368 add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
209c554a
NP
369 return;
370 }
371 } else {
372 unsigned base_found = 0;
373 unsigned char *pack, c;
d7dd0223 374 off_t base_offset;
209c554a 375 unsigned lo, mid, hi;
c4fb06c0 376
209c554a
NP
377 pack = fill(1);
378 c = *pack;
379 use(1);
380 base_offset = c & 127;
381 while (c & 128) {
382 base_offset += 1;
8723f216 383 if (!base_offset || MSB(base_offset, 7))
209c554a
NP
384 die("offset value overflow for delta base object");
385 pack = fill(1);
386 c = *pack;
387 use(1);
388 base_offset = (base_offset << 7) + (c & 127);
389 }
390 base_offset = obj_list[nr].offset - base_offset;
d8f32556
NP
391 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
392 die("offset value out of bound for delta base object");
8ee378a0 393
209c554a
NP
394 delta_data = get_data(delta_size);
395 if (dry_run || !delta_data) {
396 free(delta_data);
397 return;
398 }
399 lo = 0;
400 hi = nr;
401 while (lo < hi) {
19716b21 402 mid = lo + (hi - lo) / 2;
209c554a
NP
403 if (base_offset < obj_list[mid].offset) {
404 hi = mid;
405 } else if (base_offset > obj_list[mid].offset) {
406 lo = mid + 1;
407 } else {
834bc47b 408 oidcpy(&base_oid, &obj_list[mid].oid);
409 base_found = !is_null_oid(&base_oid);
209c554a
NP
410 break;
411 }
412 }
413 if (!base_found) {
f2898cfa
JH
414 /*
415 * The delta base object is itself a delta that
416 * has not been resolved yet.
417 */
834bc47b 418 oidclr(&obj_list[nr].oid);
419 add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
209c554a 420 return;
2add1e6d
MK
421 }
422 }
423
834bc47b 424 if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
f2898cfa 425 return;
209c554a 426
b4f5aca4 427 base = read_object_file(&base_oid, &type, &base_size);
f986f2c8
JH
428 if (!base) {
429 error("failed to read delta-pack base object %s",
834bc47b 430 oid_to_hex(&base_oid));
3b67d291 431 if (!recover)
f986f2c8
JH
432 exit(1);
433 has_errors = 1;
434 return;
435 }
209c554a 436 resolve_delta(nr, type, base, base_size, delta_data, delta_size);
ee639140 437 free(base);
8ee378a0
JH
438}
439
96a02f8f 440static void unpack_one(unsigned nr)
8ee378a0 441{
01247d87 442 unsigned shift;
48fb7deb
LT
443 unsigned char *pack;
444 unsigned long size, c;
67e5a5ec 445 enum object_type type;
74536958 446
209c554a
NP
447 obj_list[nr].offset = consumed_bytes;
448
67e5a5ec
LT
449 pack = fill(1);
450 c = *pack;
451 use(1);
a733cb60
LT
452 type = (c >> 4) & 7;
453 size = (c & 15);
01247d87 454 shift = 4;
a733cb60 455 while (c & 0x80) {
67e5a5ec 456 pack = fill(1);
209c554a 457 c = *pack;
67e5a5ec 458 use(1);
01247d87
LT
459 size += (c & 0x7f) << shift;
460 shift += 7;
a733cb60 461 }
96a02f8f 462
a733cb60
LT
463 switch (type) {
464 case OBJ_COMMIT:
465 case OBJ_TREE:
466 case OBJ_BLOB:
467 case OBJ_TAG:
209c554a 468 unpack_non_delta_entry(type, size, nr);
a733cb60 469 return;
eb32d236 470 case OBJ_REF_DELTA:
209c554a
NP
471 case OBJ_OFS_DELTA:
472 unpack_delta_entry(type, size, nr);
a733cb60 473 return;
67e5a5ec 474 default:
f986f2c8
JH
475 error("bad object type %d", type);
476 has_errors = 1;
3b67d291 477 if (recover)
f986f2c8
JH
478 return;
479 exit(1);
74536958
LT
480 }
481}
482
74536958
LT
483static void unpack_all(void)
484{
67e5a5ec 485 int i;
dc6a0757 486 struct progress *progress = NULL;
67e5a5ec 487 struct pack_header *hdr = fill(sizeof(struct pack_header));
b41860bf
MK
488
489 nr_objects = ntohl(hdr->hdr_entries);
67e5a5ec
LT
490
491 if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
492 die("bad pack file");
d60fc1c8 493 if (!pack_version_ok(hdr->hdr_version))
6e1c2344
RJ
494 die("unknown pack file version %"PRIu32,
495 ntohl(hdr->hdr_version));
96a02f8f 496 use(sizeof(struct pack_header));
67e5a5ec 497
13aaf148 498 if (!quiet)
754dbc43 499 progress = start_progress(_("Unpacking objects"), nr_objects);
19d4b416 500 obj_list = xcalloc(nr_objects, sizeof(*obj_list));
96a02f8f
NP
501 for (i = 0; i < nr_objects; i++) {
502 unpack_one(i);
4d4fcc54 503 display_progress(progress, i + 1);
96a02f8f 504 }
4d4fcc54 505 stop_progress(&progress);
96a02f8f 506
67e5a5ec
LT
507 if (delta_list)
508 die("unresolved deltas left after unpacking");
74536958
LT
509}
510
64413630 511int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
bad50dc8
LT
512{
513 int i;
834bc47b 514 struct object_id oid;
bad50dc8 515
afc711b8 516 check_replace_refs = 0;
dae556bd 517
ef90d6d4 518 git_config(git_default_config, NULL);
53228a5f 519
476e8011
JH
520 quiet = !isatty(2);
521
bad50dc8
LT
522 for (i = 1 ; i < argc; i++) {
523 const char *arg = argv[i];
524
525 if (*arg == '-') {
74536958
LT
526 if (!strcmp(arg, "-n")) {
527 dry_run = 1;
528 continue;
529 }
d36f7b80
LT
530 if (!strcmp(arg, "-q")) {
531 quiet = 1;
532 continue;
533 }
f986f2c8 534 if (!strcmp(arg, "-r")) {
3b67d291 535 recover = 1;
f986f2c8
JH
536 continue;
537 }
b41860bf
MK
538 if (!strcmp(arg, "--strict")) {
539 strict = 1;
540 continue;
541 }
5d477a33
JS
542 if (skip_prefix(arg, "--strict=", &arg)) {
543 strict = 1;
544 fsck_set_msg_types(&fsck_options, arg);
545 continue;
546 }
59556548 547 if (starts_with(arg, "--pack_header=")) {
bed006fb
NP
548 struct pack_header *hdr;
549 char *c;
550
551 hdr = (struct pack_header *)buffer;
552 hdr->hdr_signature = htonl(PACK_SIGNATURE);
553 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
554 if (*c != ',')
555 die("bad %s", arg);
556 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
557 if (*c)
558 die("bad %s", arg);
559 len = sizeof(*hdr);
560 continue;
561 }
5ad21867
CC
562 if (skip_prefix(arg, "--max-input-size=", &arg)) {
563 max_input_size = strtoumax(arg, NULL, 10);
564 continue;
565 }
bad50dc8
LT
566 usage(unpack_usage);
567 }
67e5a5ec
LT
568
569 /* We don't take any non-flag arguments now.. Maybe some day */
bad50dc8 570 usage(unpack_usage);
67e5a5ec 571 }
3206b6bd 572 the_hash_algo->init_fn(&ctx);
74536958 573 unpack_all();
3206b6bd 574 the_hash_algo->update_fn(&ctx, buffer, offset);
575 the_hash_algo->final_fn(oid.hash, &ctx);
6e328d6c 576 if (strict) {
b41860bf 577 write_rest();
6e328d6c
JK
578 if (fsck_finish(&fsck_options))
579 die(_("fsck error in pack objects"));
580 }
3206b6bd 581 if (hashcmp(fill(the_hash_algo->rawsz), oid.hash))
67e5a5ec 582 die("final sha1 did not match");
3206b6bd 583 use(the_hash_algo->rawsz);
67e5a5ec
LT
584
585 /* Write the last part of the buffer to stdout */
586 while (len) {
1c15afb9
JH
587 int ret = xwrite(1, buffer + offset, len);
588 if (ret <= 0)
67e5a5ec 589 break;
67e5a5ec
LT
590 len -= ret;
591 offset += ret;
592 }
593
594 /* All done */
f986f2c8 595 return has_errors;
bad50dc8 596}