Merge branch 'sg/test-i18ngrep'
[git/git.git] / builtin / unpack-objects.c
1 #include "builtin.h"
2 #include "cache.h"
3 #include "config.h"
4 #include "object.h"
5 #include "delta.h"
6 #include "pack.h"
7 #include "blob.h"
8 #include "commit.h"
9 #include "tag.h"
10 #include "tree.h"
11 #include "tree-walk.h"
12 #include "progress.h"
13 #include "decorate.h"
14 #include "fsck.h"
15
16 static int dry_run, quiet, recover, has_errors, strict;
17 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
18
19 /* We always read in 4kB chunks. */
20 static unsigned char buffer[4096];
21 static unsigned int offset, len;
22 static off_t consumed_bytes;
23 static off_t max_input_size;
24 static git_hash_ctx ctx;
25 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
26
27 /*
28 * When running under --strict mode, objects whose reachability are
29 * suspect are kept in core without getting written in the object
30 * store.
31 */
32 struct obj_buffer {
33 char *buffer;
34 unsigned long size;
35 };
36
37 static struct decoration obj_decorate;
38
39 static struct obj_buffer *lookup_object_buffer(struct object *base)
40 {
41 return lookup_decoration(&obj_decorate, base);
42 }
43
44 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
45 {
46 struct obj_buffer *obj;
47 obj = xcalloc(1, sizeof(struct obj_buffer));
48 obj->buffer = buffer;
49 obj->size = size;
50 if (add_decoration(&obj_decorate, object, obj))
51 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
52 }
53
54 /*
55 * Make sure at least "min" bytes are available in the buffer, and
56 * return the pointer to the buffer.
57 */
58 static void *fill(int min)
59 {
60 if (min <= len)
61 return buffer + offset;
62 if (min > sizeof(buffer))
63 die("cannot fill %d bytes", min);
64 if (offset) {
65 the_hash_algo->update_fn(&ctx, buffer, offset);
66 memmove(buffer, buffer + offset, len);
67 offset = 0;
68 }
69 do {
70 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
71 if (ret <= 0) {
72 if (!ret)
73 die("early EOF");
74 die_errno("read error on input");
75 }
76 len += ret;
77 } while (len < min);
78 return buffer;
79 }
80
81 static void use(int bytes)
82 {
83 if (bytes > len)
84 die("used more bytes than were available");
85 len -= bytes;
86 offset += bytes;
87
88 /* make sure off_t is sufficiently large not to wrap */
89 if (signed_add_overflows(consumed_bytes, bytes))
90 die("pack too large for current definition of off_t");
91 consumed_bytes += bytes;
92 if (max_input_size && consumed_bytes > max_input_size)
93 die(_("pack exceeds maximum allowed size"));
94 }
95
96 static void *get_data(unsigned long size)
97 {
98 git_zstream stream;
99 void *buf = xmallocz(size);
100
101 memset(&stream, 0, sizeof(stream));
102
103 stream.next_out = buf;
104 stream.avail_out = size;
105 stream.next_in = fill(1);
106 stream.avail_in = len;
107 git_inflate_init(&stream);
108
109 for (;;) {
110 int ret = git_inflate(&stream, 0);
111 use(len - stream.avail_in);
112 if (stream.total_out == size && ret == Z_STREAM_END)
113 break;
114 if (ret != Z_OK) {
115 error("inflate returned %d", ret);
116 FREE_AND_NULL(buf);
117 if (!recover)
118 exit(1);
119 has_errors = 1;
120 break;
121 }
122 stream.next_in = fill(1);
123 stream.avail_in = len;
124 }
125 git_inflate_end(&stream);
126 return buf;
127 }
128
129 struct delta_info {
130 struct object_id base_oid;
131 unsigned nr;
132 off_t base_offset;
133 unsigned long size;
134 void *delta;
135 struct delta_info *next;
136 };
137
138 static struct delta_info *delta_list;
139
140 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
141 off_t base_offset,
142 void *delta, unsigned long size)
143 {
144 struct delta_info *info = xmalloc(sizeof(*info));
145
146 oidcpy(&info->base_oid, base_oid);
147 info->base_offset = base_offset;
148 info->size = size;
149 info->delta = delta;
150 info->nr = nr;
151 info->next = delta_list;
152 delta_list = info;
153 }
154
155 struct obj_info {
156 off_t offset;
157 struct object_id oid;
158 struct object *obj;
159 };
160
161 #define FLAG_OPEN (1u<<20)
162 #define FLAG_WRITTEN (1u<<21)
163
164 static struct obj_info *obj_list;
165 static unsigned nr_objects;
166
167 /*
168 * Called only from check_object() after it verified this object
169 * is Ok.
170 */
171 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
172 {
173 struct object_id oid;
174
175 if (write_object_file(obj_buf->buffer, obj_buf->size,
176 typename(obj->type), &oid) < 0)
177 die("failed to write object %s", oid_to_hex(&obj->oid));
178 obj->flags |= FLAG_WRITTEN;
179 }
180
181 /*
182 * At the very end of the processing, write_rest() scans the objects
183 * that have reachability requirements and calls this function.
184 * Verify its reachability and validity recursively and write it out.
185 */
186 static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
187 {
188 struct obj_buffer *obj_buf;
189
190 if (!obj)
191 return 1;
192
193 if (obj->flags & FLAG_WRITTEN)
194 return 0;
195
196 if (type != OBJ_ANY && obj->type != type)
197 die("object type mismatch");
198
199 if (!(obj->flags & FLAG_OPEN)) {
200 unsigned long size;
201 int type = sha1_object_info(obj->oid.hash, &size);
202 if (type != obj->type || type <= 0)
203 die("object of unexpected type");
204 obj->flags |= FLAG_WRITTEN;
205 return 0;
206 }
207
208 obj_buf = lookup_object_buffer(obj);
209 if (!obj_buf)
210 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
211 if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
212 die("Error in object");
213 fsck_options.walk = check_object;
214 if (fsck_walk(obj, NULL, &fsck_options))
215 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
216 write_cached_object(obj, obj_buf);
217 return 0;
218 }
219
220 static void write_rest(void)
221 {
222 unsigned i;
223 for (i = 0; i < nr_objects; i++) {
224 if (obj_list[i].obj)
225 check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
226 }
227 }
228
229 static void added_object(unsigned nr, enum object_type type,
230 void *data, unsigned long size);
231
232 /*
233 * Write out nr-th object from the list, now we know the contents
234 * of it. Under --strict, this buffers structured objects in-core,
235 * to be checked at the end.
236 */
237 static void write_object(unsigned nr, enum object_type type,
238 void *buf, unsigned long size)
239 {
240 if (!strict) {
241 if (write_object_file(buf, size, typename(type),
242 &obj_list[nr].oid) < 0)
243 die("failed to write object");
244 added_object(nr, type, buf, size);
245 free(buf);
246 obj_list[nr].obj = NULL;
247 } else if (type == OBJ_BLOB) {
248 struct blob *blob;
249 if (write_object_file(buf, size, typename(type),
250 &obj_list[nr].oid) < 0)
251 die("failed to write object");
252 added_object(nr, type, buf, size);
253 free(buf);
254
255 blob = lookup_blob(&obj_list[nr].oid);
256 if (blob)
257 blob->object.flags |= FLAG_WRITTEN;
258 else
259 die("invalid blob object");
260 obj_list[nr].obj = NULL;
261 } else {
262 struct object *obj;
263 int eaten;
264 hash_object_file(buf, size, typename(type), &obj_list[nr].oid);
265 added_object(nr, type, buf, size);
266 obj = parse_object_buffer(&obj_list[nr].oid, type, size, buf,
267 &eaten);
268 if (!obj)
269 die("invalid %s", typename(type));
270 add_object_buffer(obj, buf, size);
271 obj->flags |= FLAG_OPEN;
272 obj_list[nr].obj = obj;
273 }
274 }
275
276 static void resolve_delta(unsigned nr, enum object_type type,
277 void *base, unsigned long base_size,
278 void *delta, unsigned long delta_size)
279 {
280 void *result;
281 unsigned long result_size;
282
283 result = patch_delta(base, base_size,
284 delta, delta_size,
285 &result_size);
286 if (!result)
287 die("failed to apply delta");
288 free(delta);
289 write_object(nr, type, result, result_size);
290 }
291
292 /*
293 * We now know the contents of an object (which is nr-th in the pack);
294 * resolve all the deltified objects that are based on it.
295 */
296 static void added_object(unsigned nr, enum object_type type,
297 void *data, unsigned long size)
298 {
299 struct delta_info **p = &delta_list;
300 struct delta_info *info;
301
302 while ((info = *p) != NULL) {
303 if (!oidcmp(&info->base_oid, &obj_list[nr].oid) ||
304 info->base_offset == obj_list[nr].offset) {
305 *p = info->next;
306 p = &delta_list;
307 resolve_delta(info->nr, type, data, size,
308 info->delta, info->size);
309 free(info);
310 continue;
311 }
312 p = &info->next;
313 }
314 }
315
316 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
317 unsigned nr)
318 {
319 void *buf = get_data(size);
320
321 if (!dry_run && buf)
322 write_object(nr, type, buf, size);
323 else
324 free(buf);
325 }
326
327 static int resolve_against_held(unsigned nr, const struct object_id *base,
328 void *delta_data, unsigned long delta_size)
329 {
330 struct object *obj;
331 struct obj_buffer *obj_buffer;
332 obj = lookup_object(base->hash);
333 if (!obj)
334 return 0;
335 obj_buffer = lookup_object_buffer(obj);
336 if (!obj_buffer)
337 return 0;
338 resolve_delta(nr, obj->type, obj_buffer->buffer,
339 obj_buffer->size, delta_data, delta_size);
340 return 1;
341 }
342
343 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
344 unsigned nr)
345 {
346 void *delta_data, *base;
347 unsigned long base_size;
348 struct object_id base_oid;
349
350 if (type == OBJ_REF_DELTA) {
351 hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
352 use(the_hash_algo->rawsz);
353 delta_data = get_data(delta_size);
354 if (dry_run || !delta_data) {
355 free(delta_data);
356 return;
357 }
358 if (has_object_file(&base_oid))
359 ; /* Ok we have this one */
360 else if (resolve_against_held(nr, &base_oid,
361 delta_data, delta_size))
362 return; /* we are done */
363 else {
364 /* cannot resolve yet --- queue it */
365 oidclr(&obj_list[nr].oid);
366 add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
367 return;
368 }
369 } else {
370 unsigned base_found = 0;
371 unsigned char *pack, c;
372 off_t base_offset;
373 unsigned lo, mid, hi;
374
375 pack = fill(1);
376 c = *pack;
377 use(1);
378 base_offset = c & 127;
379 while (c & 128) {
380 base_offset += 1;
381 if (!base_offset || MSB(base_offset, 7))
382 die("offset value overflow for delta base object");
383 pack = fill(1);
384 c = *pack;
385 use(1);
386 base_offset = (base_offset << 7) + (c & 127);
387 }
388 base_offset = obj_list[nr].offset - base_offset;
389 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
390 die("offset value out of bound for delta base object");
391
392 delta_data = get_data(delta_size);
393 if (dry_run || !delta_data) {
394 free(delta_data);
395 return;
396 }
397 lo = 0;
398 hi = nr;
399 while (lo < hi) {
400 mid = lo + (hi - lo) / 2;
401 if (base_offset < obj_list[mid].offset) {
402 hi = mid;
403 } else if (base_offset > obj_list[mid].offset) {
404 lo = mid + 1;
405 } else {
406 oidcpy(&base_oid, &obj_list[mid].oid);
407 base_found = !is_null_oid(&base_oid);
408 break;
409 }
410 }
411 if (!base_found) {
412 /*
413 * The delta base object is itself a delta that
414 * has not been resolved yet.
415 */
416 oidclr(&obj_list[nr].oid);
417 add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
418 return;
419 }
420 }
421
422 if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
423 return;
424
425 base = read_sha1_file(base_oid.hash, &type, &base_size);
426 if (!base) {
427 error("failed to read delta-pack base object %s",
428 oid_to_hex(&base_oid));
429 if (!recover)
430 exit(1);
431 has_errors = 1;
432 return;
433 }
434 resolve_delta(nr, type, base, base_size, delta_data, delta_size);
435 free(base);
436 }
437
438 static void unpack_one(unsigned nr)
439 {
440 unsigned shift;
441 unsigned char *pack;
442 unsigned long size, c;
443 enum object_type type;
444
445 obj_list[nr].offset = consumed_bytes;
446
447 pack = fill(1);
448 c = *pack;
449 use(1);
450 type = (c >> 4) & 7;
451 size = (c & 15);
452 shift = 4;
453 while (c & 0x80) {
454 pack = fill(1);
455 c = *pack;
456 use(1);
457 size += (c & 0x7f) << shift;
458 shift += 7;
459 }
460
461 switch (type) {
462 case OBJ_COMMIT:
463 case OBJ_TREE:
464 case OBJ_BLOB:
465 case OBJ_TAG:
466 unpack_non_delta_entry(type, size, nr);
467 return;
468 case OBJ_REF_DELTA:
469 case OBJ_OFS_DELTA:
470 unpack_delta_entry(type, size, nr);
471 return;
472 default:
473 error("bad object type %d", type);
474 has_errors = 1;
475 if (recover)
476 return;
477 exit(1);
478 }
479 }
480
481 static void unpack_all(void)
482 {
483 int i;
484 struct progress *progress = NULL;
485 struct pack_header *hdr = fill(sizeof(struct pack_header));
486
487 nr_objects = ntohl(hdr->hdr_entries);
488
489 if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
490 die("bad pack file");
491 if (!pack_version_ok(hdr->hdr_version))
492 die("unknown pack file version %"PRIu32,
493 ntohl(hdr->hdr_version));
494 use(sizeof(struct pack_header));
495
496 if (!quiet)
497 progress = start_progress(_("Unpacking objects"), nr_objects);
498 obj_list = xcalloc(nr_objects, sizeof(*obj_list));
499 for (i = 0; i < nr_objects; i++) {
500 unpack_one(i);
501 display_progress(progress, i + 1);
502 }
503 stop_progress(&progress);
504
505 if (delta_list)
506 die("unresolved deltas left after unpacking");
507 }
508
509 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
510 {
511 int i;
512 struct object_id oid;
513
514 check_replace_refs = 0;
515
516 git_config(git_default_config, NULL);
517
518 quiet = !isatty(2);
519
520 for (i = 1 ; i < argc; i++) {
521 const char *arg = argv[i];
522
523 if (*arg == '-') {
524 if (!strcmp(arg, "-n")) {
525 dry_run = 1;
526 continue;
527 }
528 if (!strcmp(arg, "-q")) {
529 quiet = 1;
530 continue;
531 }
532 if (!strcmp(arg, "-r")) {
533 recover = 1;
534 continue;
535 }
536 if (!strcmp(arg, "--strict")) {
537 strict = 1;
538 continue;
539 }
540 if (skip_prefix(arg, "--strict=", &arg)) {
541 strict = 1;
542 fsck_set_msg_types(&fsck_options, arg);
543 continue;
544 }
545 if (starts_with(arg, "--pack_header=")) {
546 struct pack_header *hdr;
547 char *c;
548
549 hdr = (struct pack_header *)buffer;
550 hdr->hdr_signature = htonl(PACK_SIGNATURE);
551 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
552 if (*c != ',')
553 die("bad %s", arg);
554 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
555 if (*c)
556 die("bad %s", arg);
557 len = sizeof(*hdr);
558 continue;
559 }
560 if (skip_prefix(arg, "--max-input-size=", &arg)) {
561 max_input_size = strtoumax(arg, NULL, 10);
562 continue;
563 }
564 usage(unpack_usage);
565 }
566
567 /* We don't take any non-flag arguments now.. Maybe some day */
568 usage(unpack_usage);
569 }
570 the_hash_algo->init_fn(&ctx);
571 unpack_all();
572 the_hash_algo->update_fn(&ctx, buffer, offset);
573 the_hash_algo->final_fn(oid.hash, &ctx);
574 if (strict)
575 write_rest();
576 if (hashcmp(fill(the_hash_algo->rawsz), oid.hash))
577 die("final sha1 did not match");
578 use(the_hash_algo->rawsz);
579
580 /* Write the last part of the buffer to stdout */
581 while (len) {
582 int ret = xwrite(1, buffer + offset, len);
583 if (ret <= 0)
584 break;
585 len -= ret;
586 offset += ret;
587 }
588
589 /* All done */
590 return has_errors;
591 }