Allow frontends to bidirectionally communicate with fast-import
[git/git.git] / fast-import.c
CommitLineData
463acbe1
SP
1/*
2Format of STDIN stream:
3
4 stream ::= cmd*;
5
6 cmd ::= new_blob
c44cdc7e 7 | new_commit
463acbe1 8 | new_tag
5fced8dc 9 | reset_branch
8c1f22da 10 | checkpoint
ac053c02 11 | progress
463acbe1
SP
12 ;
13
c44cdc7e 14 new_blob ::= 'blob' lf
c905e090 15 mark?
c44cdc7e
SP
16 file_content;
17 file_content ::= data;
463acbe1 18
c44cdc7e 19 new_commit ::= 'commit' sp ref_str lf
00e2b884 20 mark?
63e0c8b3
SP
21 ('author' sp name '<' email '>' when lf)?
22 'committer' sp name '<' email '>' when lf
00e2b884 23 commit_msg
02f3389d 24 ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
62b6f483 25 ('merge' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)*
c44cdc7e 26 file_change*
1fdb649c 27 lf?;
c44cdc7e 28 commit_msg ::= data;
463acbe1 29
b6f3481b
SP
30 file_change ::= file_clr
31 | file_del
32 | file_rnm
33 | file_cpy
34 | file_obm
35 | file_inm;
825769a8 36 file_clr ::= 'deleteall' lf;
b715cfbb 37 file_del ::= 'D' sp path_str lf;
f39a946a 38 file_rnm ::= 'R' sp path_str sp path_str lf;
b6f3481b 39 file_cpy ::= 'C' sp path_str sp path_str lf;
b715cfbb
SP
40 file_obm ::= 'M' sp mode sp (hexsha1 | idnum) sp path_str lf;
41 file_inm ::= 'M' sp mode sp 'inline' sp path_str lf
42 data;
c44cdc7e
SP
43
44 new_tag ::= 'tag' sp tag_str lf
45 'from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf
c905e090 46 'tagger' sp name '<' email '>' when lf
c44cdc7e
SP
47 tag_msg;
48 tag_msg ::= data;
49
9938ffc5
SP
50 reset_branch ::= 'reset' sp ref_str lf
51 ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
1fdb649c 52 lf?;
5fced8dc 53
7bfe6e26 54 checkpoint ::= 'checkpoint' lf
1fdb649c 55 lf?;
7bfe6e26 56
ac053c02
SP
57 progress ::= 'progress' sp not_lf* lf
58 lf?;
59
c44cdc7e
SP
60 # note: the first idnum in a stream should be 1 and subsequent
61 # idnums should not have gaps between values as this will cause
62 # the stream parser to reserve space for the gapped values. An
c905e090 63 # idnum can be updated in the future to a new object by issuing
c44cdc7e 64 # a new mark directive with the old idnum.
c905e090 65 #
c44cdc7e 66 mark ::= 'mark' sp idnum lf;
3b4dce02 67 data ::= (delimited_data | exact_data)
2c570cde 68 lf?;
3b4dce02
SP
69
70 # note: delim may be any string but must not contain lf.
71 # data_line may contain any data but must not be exactly
72 # delim.
73 delimited_data ::= 'data' sp '<<' delim lf
74 (data_line lf)*
c905e090 75 delim lf;
c44cdc7e
SP
76
77 # note: declen indicates the length of binary_data in bytes.
3b4dce02 78 # declen does not include the lf preceeding the binary data.
c44cdc7e 79 #
3b4dce02
SP
80 exact_data ::= 'data' sp declen lf
81 binary_data;
c44cdc7e
SP
82
83 # note: quoted strings are C-style quoting supporting \c for
84 # common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn
c905e090 85 # is the signed byte value in octal. Note that the only
c44cdc7e
SP
86 # characters which must actually be escaped to protect the
87 # stream formatting is: \, " and LF. Otherwise these values
c905e090 88 # are UTF8.
c44cdc7e 89 #
6c3aac1c
SP
90 ref_str ::= ref;
91 sha1exp_str ::= sha1exp;
92 tag_str ::= tag;
c44cdc7e 93 path_str ::= path | '"' quoted(path) '"' ;
b715cfbb
SP
94 mode ::= '100644' | '644'
95 | '100755' | '755'
9981b6d9 96 | '120000'
b715cfbb 97 ;
c44cdc7e
SP
98
99 declen ::= # unsigned 32 bit value, ascii base10 notation;
2104838b 100 bigint ::= # unsigned integer value, ascii base10 notation;
463acbe1 101 binary_data ::= # file content, not interpreted;
c44cdc7e 102
63e0c8b3
SP
103 when ::= raw_when | rfc2822_when;
104 raw_when ::= ts sp tz;
105 rfc2822_when ::= # Valid RFC 2822 date and time;
106
463acbe1
SP
107 sp ::= # ASCII space character;
108 lf ::= # ASCII newline (LF) character;
c44cdc7e
SP
109
110 # note: a colon (':') must precede the numerical value assigned to
c905e090 111 # an idnum. This is to distinguish it from a ref or tag name as
c44cdc7e 112 # GIT does not permit ':' in ref or tag strings.
c905e090 113 #
2104838b 114 idnum ::= ':' bigint;
c44cdc7e
SP
115 path ::= # GIT style file path, e.g. "a/b/c";
116 ref ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
117 tag ::= # GIT tag name, e.g. "FIREFOX_1_5";
463acbe1
SP
118 sha1exp ::= # Any valid GIT SHA1 expression;
119 hexsha1 ::= # SHA1 in hexadecimal format;
c44cdc7e
SP
120
121 # note: name and email are UTF8 strings, however name must not
c905e090 122 # contain '<' or lf and email must not contain any of the
c44cdc7e 123 # following: '<', '>', lf.
c905e090 124 #
c44cdc7e 125 name ::= # valid GIT author/committer name;
463acbe1 126 email ::= # valid GIT author/committer email;
c44cdc7e
SP
127 ts ::= # time since the epoch in seconds, ascii base10 notation;
128 tz ::= # GIT style timezone;
401d53fa
SP
129
130 # note: comments may appear anywhere in the input, except
131 # within a data command. Any form of the data command
132 # always escapes the related input from comment processing.
133 #
134 # In case it is not clear, the '#' that starts the comment
135 # must be the first character on that the line (an lf have
136 # preceeded it).
137 #
138 comment ::= '#' not_lf* lf;
139 not_lf ::= # Any byte that is not ASCII newline (LF);
463acbe1
SP
140*/
141
db5e523f
SP
142#include "builtin.h"
143#include "cache.h"
144#include "object.h"
145#include "blob.h"
463acbe1 146#include "tree.h"
7073e69e 147#include "commit.h"
db5e523f
SP
148#include "delta.h"
149#include "pack.h"
463acbe1 150#include "refs.h"
db5e523f 151#include "csum-file.h"
c44cdc7e
SP
152#include "strbuf.h"
153#include "quote.h"
db5e523f 154
69e74e74
SP
155#define PACK_ID_BITS 16
156#define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
157
27d6d290
SP
158struct object_entry
159{
160 struct object_entry *next;
10831c55 161 uint32_t offset;
03842d8e 162 unsigned type : TYPE_BITS;
69e74e74 163 unsigned pack_id : PACK_ID_BITS;
27d6d290
SP
164 unsigned char sha1[20];
165};
166
463acbe1 167struct object_entry_pool
27d6d290 168{
463acbe1 169 struct object_entry_pool *next_pool;
27d6d290
SP
170 struct object_entry *next_free;
171 struct object_entry *end;
ac47a738 172 struct object_entry entries[FLEX_ARRAY]; /* more */
27d6d290
SP
173};
174
d8397168
SP
175struct mark_set
176{
d8397168
SP
177 union {
178 struct object_entry *marked[1024];
179 struct mark_set *sets[1024];
180 } data;
6f64f6d9 181 unsigned int shift;
d8397168
SP
182};
183
ac47a738
SP
184struct last_object
185{
186 void *data;
4cabf858 187 unsigned long len;
10831c55 188 uint32_t offset;
6bb5b329 189 unsigned int depth;
d489bc14 190 unsigned no_free:1;
ac47a738
SP
191};
192
463acbe1
SP
193struct mem_pool
194{
195 struct mem_pool *next_pool;
196 char *next_free;
197 char *end;
198 char space[FLEX_ARRAY]; /* more */
199};
200
201struct atom_str
202{
203 struct atom_str *next_atom;
10831c55 204 unsigned short str_len;
463acbe1
SP
205 char str_dat[FLEX_ARRAY]; /* more */
206};
207
208struct tree_content;
6bb5b329
SP
209struct tree_entry
210{
463acbe1
SP
211 struct tree_content *tree;
212 struct atom_str* name;
4cabf858
SP
213 struct tree_entry_ms
214 {
10831c55 215 uint16_t mode;
4cabf858
SP
216 unsigned char sha1[20];
217 } versions[2];
6bb5b329
SP
218};
219
463acbe1 220struct tree_content
6bb5b329 221{
463acbe1
SP
222 unsigned int entry_capacity; /* must match avail_tree_content */
223 unsigned int entry_count;
4cabf858 224 unsigned int delta_depth;
463acbe1
SP
225 struct tree_entry *entries[FLEX_ARRAY]; /* more */
226};
227
228struct avail_tree_content
229{
230 unsigned int entry_capacity; /* must match tree_content */
231 struct avail_tree_content *next_avail;
6bb5b329
SP
232};
233
234struct branch
235{
463acbe1
SP
236 struct branch *table_next_branch;
237 struct branch *active_next_branch;
6bb5b329 238 const char *name;
463acbe1 239 struct tree_entry branch_tree;
69e74e74 240 uintmax_t last_commit;
734c91f9
SP
241 unsigned active : 1;
242 unsigned pack_id : PACK_ID_BITS;
463acbe1 243 unsigned char sha1[20];
6bb5b329
SP
244};
245
72303d44
SP
246struct tag
247{
248 struct tag *next_tag;
249 const char *name;
2369ed79 250 unsigned int pack_id;
72303d44
SP
251 unsigned char sha1[20];
252};
253
e2eb469d
SP
254struct dbuf
255{
256 void *buffer;
257 size_t capacity;
258};
259
62b6f483
SP
260struct hash_list
261{
262 struct hash_list *next;
263 unsigned char sha1[20];
264};
463acbe1 265
63e0c8b3
SP
266typedef enum {
267 WHENSPEC_RAW = 1,
268 WHENSPEC_RFC2822,
269 WHENSPEC_NOW,
270} whenspec_type;
271
0ea9f045 272/* Configured limits on output */
d5c57b28 273static unsigned long max_depth = 10;
6777a59f 274static off_t max_packsize = (1LL << 32) - 1;
7073e69e 275static int force_update;
0ea9f045
SP
276
277/* Stats and misc. counters */
278static uintmax_t alloc_count;
0ea9f045
SP
279static uintmax_t marks_set_count;
280static uintmax_t object_count_by_type[1 << TYPE_BITS];
281static uintmax_t duplicate_count_by_type[1 << TYPE_BITS];
282static uintmax_t delta_count_by_type[1 << TYPE_BITS];
a7ddc487 283static unsigned long object_count;
6bb5b329 284static unsigned long branch_count;
d6c7eb2c 285static unsigned long branch_load_count;
7073e69e 286static int failure;
bdf1c06d 287static FILE *pack_edges;
ac47a738 288
463acbe1
SP
289/* Memory pools */
290static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
291static size_t total_allocd;
292static struct mem_pool *mem_pool;
293
c44cdc7e 294/* Atom management */
463acbe1
SP
295static unsigned int atom_table_sz = 4451;
296static unsigned int atom_cnt;
297static struct atom_str **atom_table;
298
299/* The .pack file being generated */
7bfe6e26 300static unsigned int pack_id;
d489bc14 301static struct packed_git *pack_data;
7bfe6e26 302static struct packed_git **all_packs;
41e5257f 303static unsigned long pack_size;
ac47a738
SP
304
305/* Table of objects we've written. */
4cabf858 306static unsigned int object_entry_alloc = 5000;
463acbe1
SP
307static struct object_entry_pool *blocks;
308static struct object_entry *object_table[1 << 16];
d8397168 309static struct mark_set *marks;
a6a1a831 310static const char* mark_file;
ac47a738
SP
311
312/* Our last blob */
463acbe1
SP
313static struct last_object last_blob;
314
315/* Tree management */
316static unsigned int tree_entry_alloc = 1000;
317static void *avail_tree_entry;
318static unsigned int avail_tree_table_sz = 100;
319static struct avail_tree_content **avail_tree_table;
e2eb469d
SP
320static struct dbuf old_tree;
321static struct dbuf new_tree;
8bcce301 322
6bb5b329 323/* Branch data */
d5c57b28
SP
324static unsigned long max_active_branches = 5;
325static unsigned long cur_active_branches;
326static unsigned long branch_table_sz = 1039;
463acbe1
SP
327static struct branch **branch_table;
328static struct branch *active_branches;
329
72303d44
SP
330/* Tag data */
331static struct tag *first_tag;
332static struct tag *last_tag;
333
c44cdc7e 334/* Input stream parsing */
63e0c8b3 335static whenspec_type whenspec = WHENSPEC_RAW;
c44cdc7e 336static struct strbuf command_buf;
1fdb649c 337static int unread_command_buf;
0ea9f045 338static uintmax_t next_mark;
243f801d 339static struct dbuf new_data;
c44cdc7e 340
6bb5b329 341
03842d8e 342static void alloc_objects(unsigned int cnt)
8bcce301 343{
463acbe1 344 struct object_entry_pool *b;
27d6d290 345
463acbe1 346 b = xmalloc(sizeof(struct object_entry_pool)
27d6d290 347 + cnt * sizeof(struct object_entry));
463acbe1 348 b->next_pool = blocks;
27d6d290
SP
349 b->next_free = b->entries;
350 b->end = b->entries + cnt;
351 blocks = b;
352 alloc_count += cnt;
353}
8bcce301 354
e5b1444b 355static struct object_entry *new_object(unsigned char *sha1)
8bcce301 356{
27d6d290 357 struct object_entry *e;
8bcce301 358
27d6d290 359 if (blocks->next_free == blocks->end)
463acbe1 360 alloc_objects(object_entry_alloc);
8bcce301 361
27d6d290 362 e = blocks->next_free++;
445b8599 363 hashcpy(e->sha1, sha1);
27d6d290 364 return e;
8bcce301
SP
365}
366
e5b1444b 367static struct object_entry *find_object(unsigned char *sha1)
463acbe1
SP
368{
369 unsigned int h = sha1[0] << 8 | sha1[1];
370 struct object_entry *e;
371 for (e = object_table[h]; e; e = e->next)
445b8599 372 if (!hashcmp(sha1, e->sha1))
463acbe1
SP
373 return e;
374 return NULL;
375}
376
e5b1444b 377static struct object_entry *insert_object(unsigned char *sha1)
8bcce301
SP
378{
379 unsigned int h = sha1[0] << 8 | sha1[1];
27d6d290 380 struct object_entry *e = object_table[h];
463acbe1 381 struct object_entry *p = NULL;
8bcce301
SP
382
383 while (e) {
445b8599 384 if (!hashcmp(sha1, e->sha1))
8bcce301
SP
385 return e;
386 p = e;
387 e = e->next;
388 }
389
390 e = new_object(sha1);
463acbe1 391 e->next = NULL;
8bcce301
SP
392 e->offset = 0;
393 if (p)
394 p->next = e;
395 else
27d6d290 396 object_table[h] = e;
8bcce301
SP
397 return e;
398}
db5e523f 399
463acbe1
SP
400static unsigned int hc_str(const char *s, size_t len)
401{
402 unsigned int r = 0;
403 while (len-- > 0)
404 r = r * 31 + *s++;
405 return r;
406}
407
e5b1444b 408static void *pool_alloc(size_t len)
463acbe1
SP
409{
410 struct mem_pool *p;
411 void *r;
412
413 for (p = mem_pool; p; p = p->next_pool)
414 if ((p->end - p->next_free >= len))
415 break;
416
417 if (!p) {
418 if (len >= (mem_pool_alloc/2)) {
419 total_allocd += len;
420 return xmalloc(len);
421 }
422 total_allocd += sizeof(struct mem_pool) + mem_pool_alloc;
423 p = xmalloc(sizeof(struct mem_pool) + mem_pool_alloc);
424 p->next_pool = mem_pool;
425 p->next_free = p->space;
426 p->end = p->next_free + mem_pool_alloc;
427 mem_pool = p;
428 }
429
430 r = p->next_free;
8d8928b0
SP
431 /* round out to a pointer alignment */
432 if (len & (sizeof(void*) - 1))
433 len += sizeof(void*) - (len & (sizeof(void*) - 1));
463acbe1
SP
434 p->next_free += len;
435 return r;
436}
437
e5b1444b 438static void *pool_calloc(size_t count, size_t size)
463acbe1
SP
439{
440 size_t len = count * size;
441 void *r = pool_alloc(len);
442 memset(r, 0, len);
443 return r;
444}
445
e5b1444b 446static char *pool_strdup(const char *s)
463acbe1
SP
447{
448 char *r = pool_alloc(strlen(s) + 1);
449 strcpy(r, s);
450 return r;
451}
452
243f801d
SP
453static void size_dbuf(struct dbuf *b, size_t maxlen)
454{
455 if (b->buffer) {
456 if (b->capacity >= maxlen)
457 return;
458 free(b->buffer);
459 }
460 b->capacity = ((maxlen / 1024) + 1) * 1024;
461 b->buffer = xmalloc(b->capacity);
462}
463
0ea9f045 464static void insert_mark(uintmax_t idnum, struct object_entry *oe)
d8397168
SP
465{
466 struct mark_set *s = marks;
467 while ((idnum >> s->shift) >= 1024) {
468 s = pool_calloc(1, sizeof(struct mark_set));
469 s->shift = marks->shift + 10;
470 s->data.sets[0] = marks;
471 marks = s;
472 }
473 while (s->shift) {
0ea9f045 474 uintmax_t i = idnum >> s->shift;
d8397168
SP
475 idnum -= i << s->shift;
476 if (!s->data.sets[i]) {
477 s->data.sets[i] = pool_calloc(1, sizeof(struct mark_set));
478 s->data.sets[i]->shift = s->shift - 10;
479 }
480 s = s->data.sets[i];
481 }
482 if (!s->data.marked[idnum])
483 marks_set_count++;
484 s->data.marked[idnum] = oe;
485}
486
e5b1444b 487static struct object_entry *find_mark(uintmax_t idnum)
d8397168 488{
0ea9f045 489 uintmax_t orig_idnum = idnum;
d8397168
SP
490 struct mark_set *s = marks;
491 struct object_entry *oe = NULL;
492 if ((idnum >> s->shift) < 1024) {
493 while (s && s->shift) {
0ea9f045 494 uintmax_t i = idnum >> s->shift;
d8397168
SP
495 idnum -= i << s->shift;
496 s = s->data.sets[i];
497 }
498 if (s)
499 oe = s->data.marked[idnum];
500 }
501 if (!oe)
3efb1f34 502 die("mark :%" PRIuMAX " not declared", orig_idnum);
d8397168
SP
503 return oe;
504}
505
e5b1444b 506static struct atom_str *to_atom(const char *s, unsigned short len)
463acbe1
SP
507{
508 unsigned int hc = hc_str(s, len) % atom_table_sz;
509 struct atom_str *c;
510
511 for (c = atom_table[hc]; c; c = c->next_atom)
512 if (c->str_len == len && !strncmp(s, c->str_dat, len))
513 return c;
514
515 c = pool_alloc(sizeof(struct atom_str) + len + 1);
516 c->str_len = len;
517 strncpy(c->str_dat, s, len);
518 c->str_dat[len] = 0;
519 c->next_atom = atom_table[hc];
520 atom_table[hc] = c;
521 atom_cnt++;
522 return c;
523}
524
e5b1444b 525static struct branch *lookup_branch(const char *name)
463acbe1
SP
526{
527 unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
528 struct branch *b;
529
530 for (b = branch_table[hc]; b; b = b->table_next_branch)
531 if (!strcmp(name, b->name))
532 return b;
533 return NULL;
534}
535
e5b1444b 536static struct branch *new_branch(const char *name)
463acbe1
SP
537{
538 unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
539 struct branch* b = lookup_branch(name);
540
541 if (b)
542 die("Invalid attempt to create duplicate branch: %s", name);
ea08a6fd
SP
543 switch (check_ref_format(name)) {
544 case 0: break; /* its valid */
545 case -2: break; /* valid, but too few '/', allow anyway */
546 default:
c44cdc7e 547 die("Branch name doesn't conform to GIT standards: %s", name);
ea08a6fd 548 }
463acbe1
SP
549
550 b = pool_calloc(1, sizeof(struct branch));
551 b->name = pool_strdup(name);
552 b->table_next_branch = branch_table[hc];
8a8c55ea
SP
553 b->branch_tree.versions[0].mode = S_IFDIR;
554 b->branch_tree.versions[1].mode = S_IFDIR;
734c91f9 555 b->active = 0;
69e74e74 556 b->pack_id = MAX_PACK_ID;
463acbe1
SP
557 branch_table[hc] = b;
558 branch_count++;
559 return b;
560}
561
562static unsigned int hc_entries(unsigned int cnt)
563{
564 cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8;
565 return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1;
566}
567
e5b1444b 568static struct tree_content *new_tree_content(unsigned int cnt)
463acbe1
SP
569{
570 struct avail_tree_content *f, *l = NULL;
571 struct tree_content *t;
572 unsigned int hc = hc_entries(cnt);
573
574 for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail)
575 if (f->entry_capacity >= cnt)
576 break;
577
578 if (f) {
579 if (l)
580 l->next_avail = f->next_avail;
581 else
582 avail_tree_table[hc] = f->next_avail;
583 } else {
584 cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
585 f = pool_alloc(sizeof(*t) + sizeof(t->entries[0]) * cnt);
586 f->entry_capacity = cnt;
587 }
588
589 t = (struct tree_content*)f;
590 t->entry_count = 0;
4cabf858 591 t->delta_depth = 0;
463acbe1
SP
592 return t;
593}
594
595static void release_tree_entry(struct tree_entry *e);
596static void release_tree_content(struct tree_content *t)
597{
598 struct avail_tree_content *f = (struct avail_tree_content*)t;
599 unsigned int hc = hc_entries(f->entry_capacity);
afde8dd9
SP
600 f->next_avail = avail_tree_table[hc];
601 avail_tree_table[hc] = f;
602}
603
604static void release_tree_content_recursive(struct tree_content *t)
605{
463acbe1
SP
606 unsigned int i;
607 for (i = 0; i < t->entry_count; i++)
608 release_tree_entry(t->entries[i]);
afde8dd9 609 release_tree_content(t);
463acbe1
SP
610}
611
e5b1444b 612static struct tree_content *grow_tree_content(
463acbe1
SP
613 struct tree_content *t,
614 int amt)
615{
616 struct tree_content *r = new_tree_content(t->entry_count + amt);
617 r->entry_count = t->entry_count;
4cabf858 618 r->delta_depth = t->delta_depth;
463acbe1
SP
619 memcpy(r->entries,t->entries,t->entry_count*sizeof(t->entries[0]));
620 release_tree_content(t);
621 return r;
622}
623
e5b1444b 624static struct tree_entry *new_tree_entry(void)
463acbe1
SP
625{
626 struct tree_entry *e;
627
628 if (!avail_tree_entry) {
629 unsigned int n = tree_entry_alloc;
8435a9cb 630 total_allocd += n * sizeof(struct tree_entry);
463acbe1 631 avail_tree_entry = e = xmalloc(n * sizeof(struct tree_entry));
2eb26d84 632 while (n-- > 1) {
463acbe1
SP
633 *((void**)e) = e + 1;
634 e++;
635 }
35ef237c 636 *((void**)e) = NULL;
463acbe1
SP
637 }
638
639 e = avail_tree_entry;
640 avail_tree_entry = *((void**)e);
641 return e;
642}
643
644static void release_tree_entry(struct tree_entry *e)
645{
646 if (e->tree)
afde8dd9 647 release_tree_content_recursive(e->tree);
463acbe1
SP
648 *((void**)e) = avail_tree_entry;
649 avail_tree_entry = e;
650}
651
b6f3481b
SP
652static struct tree_content *dup_tree_content(struct tree_content *s)
653{
654 struct tree_content *d;
655 struct tree_entry *a, *b;
656 unsigned int i;
657
658 if (!s)
659 return NULL;
660 d = new_tree_content(s->entry_count);
661 for (i = 0; i < s->entry_count; i++) {
662 a = s->entries[i];
663 b = new_tree_entry();
664 memcpy(b, a, sizeof(*a));
665 if (a->tree && is_null_sha1(b->versions[1].sha1))
666 b->tree = dup_tree_content(a->tree);
667 else
668 b->tree = NULL;
669 d->entries[i] = b;
670 }
671 d->entry_count = s->entry_count;
672 d->delta_depth = s->delta_depth;
673
674 return d;
675}
676
fd99224e 677static void start_packfile(void)
f70b6534 678{
8455e484 679 static char tmpfile[PATH_MAX];
7bfe6e26 680 struct packed_git *p;
f70b6534 681 struct pack_header hdr;
0fcbcae7 682 int pack_fd;
f70b6534 683
8455e484 684 snprintf(tmpfile, sizeof(tmpfile),
0e55181f 685 "%s/tmp_pack_XXXXXX", get_object_directory());
7647b17f 686 pack_fd = xmkstemp(tmpfile);
8455e484
SP
687 p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2);
688 strcpy(p->pack_name, tmpfile);
7bfe6e26 689 p->pack_fd = pack_fd;
f70b6534
SP
690
691 hdr.hdr_signature = htonl(PACK_SIGNATURE);
692 hdr.hdr_version = htonl(2);
693 hdr.hdr_entries = 0;
0fcbcae7 694 write_or_die(p->pack_fd, &hdr, sizeof(hdr));
7bfe6e26
SP
695
696 pack_data = p;
f70b6534
SP
697 pack_size = sizeof(hdr);
698 object_count = 0;
7bfe6e26
SP
699
700 all_packs = xrealloc(all_packs, sizeof(*all_packs) * (pack_id + 1));
701 all_packs[pack_id] = p;
f70b6534
SP
702}
703
f70b6534
SP
704static int oecmp (const void *a_, const void *b_)
705{
706 struct object_entry *a = *((struct object_entry**)a_);
707 struct object_entry *b = *((struct object_entry**)b_);
708 return hashcmp(a->sha1, b->sha1);
709}
710
e5b1444b 711static char *create_index(void)
f70b6534 712{
8455e484
SP
713 static char tmpfile[PATH_MAX];
714 SHA_CTX ctx;
f70b6534
SP
715 struct sha1file *f;
716 struct object_entry **idx, **c, **last, *e;
717 struct object_entry_pool *o;
ebea9dd4 718 uint32_t array[256];
8455e484 719 int i, idx_fd;
f70b6534
SP
720
721 /* Build the sorted table of object IDs. */
722 idx = xmalloc(object_count * sizeof(struct object_entry*));
723 c = idx;
724 for (o = blocks; o; o = o->next_pool)
d9ee53ce
SP
725 for (e = o->next_free; e-- != o->entries;)
726 if (pack_id == e->pack_id)
727 *c++ = e;
f70b6534 728 last = idx + object_count;
2fce1f3c
SP
729 if (c != last)
730 die("internal consistency error creating the index");
f70b6534
SP
731 qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
732
733 /* Generate the fan-out array. */
734 c = idx;
735 for (i = 0; i < 256; i++) {
736 struct object_entry **next = c;;
737 while (next < last) {
738 if ((*next)->sha1[0] != i)
739 break;
740 next++;
741 }
742 array[i] = htonl(next - idx);
743 c = next;
744 }
745
8455e484 746 snprintf(tmpfile, sizeof(tmpfile),
0e55181f 747 "%s/tmp_idx_XXXXXX", get_object_directory());
7647b17f 748 idx_fd = xmkstemp(tmpfile);
8455e484 749 f = sha1fd(idx_fd, tmpfile);
f70b6534 750 sha1write(f, array, 256 * sizeof(int));
8455e484 751 SHA1_Init(&ctx);
f70b6534 752 for (c = idx; c != last; c++) {
ebea9dd4 753 uint32_t offset = htonl((*c)->offset);
f70b6534
SP
754 sha1write(f, &offset, 4);
755 sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
8455e484 756 SHA1_Update(&ctx, (*c)->sha1, 20);
f70b6534 757 }
09543c96 758 sha1write(f, pack_data->sha1, sizeof(pack_data->sha1));
8455e484 759 sha1close(f, NULL, 1);
f70b6534 760 free(idx);
8455e484
SP
761 SHA1_Final(pack_data->sha1, &ctx);
762 return tmpfile;
763}
764
e5b1444b 765static char *keep_pack(char *curr_index_name)
8455e484
SP
766{
767 static char name[PATH_MAX];
3a55602e 768 static const char *keep_msg = "fast-import";
8455e484
SP
769 int keep_fd;
770
771 chmod(pack_data->pack_name, 0444);
772 chmod(curr_index_name, 0444);
773
774 snprintf(name, sizeof(name), "%s/pack/pack-%s.keep",
775 get_object_directory(), sha1_to_hex(pack_data->sha1));
776 keep_fd = open(name, O_RDWR|O_CREAT|O_EXCL, 0600);
777 if (keep_fd < 0)
778 die("cannot create keep file");
779 write(keep_fd, keep_msg, strlen(keep_msg));
780 close(keep_fd);
781
782 snprintf(name, sizeof(name), "%s/pack/pack-%s.pack",
783 get_object_directory(), sha1_to_hex(pack_data->sha1));
784 if (move_temp_to_file(pack_data->pack_name, name))
785 die("cannot store pack file");
8455e484
SP
786
787 snprintf(name, sizeof(name), "%s/pack/pack-%s.idx",
788 get_object_directory(), sha1_to_hex(pack_data->sha1));
789 if (move_temp_to_file(curr_index_name, name))
790 die("cannot store index file");
791 return name;
792}
793
fd99224e 794static void unkeep_all_packs(void)
8455e484
SP
795{
796 static char name[PATH_MAX];
797 int k;
798
799 for (k = 0; k < pack_id; k++) {
800 struct packed_git *p = all_packs[k];
801 snprintf(name, sizeof(name), "%s/pack/pack-%s.keep",
802 get_object_directory(), sha1_to_hex(p->sha1));
803 unlink(name);
804 }
f70b6534
SP
805}
806
fd99224e 807static void end_packfile(void)
f70b6534 808{
7bfe6e26
SP
809 struct packed_git *old_p = pack_data, *new_p;
810
3e005baf 811 if (object_count) {
8455e484 812 char *idx_name;
2369ed79
SP
813 int i;
814 struct branch *b;
815 struct tag *t;
8455e484 816
8b0eca7c
DH
817 fixup_pack_header_footer(pack_data->pack_fd, pack_data->sha1,
818 pack_data->pack_name, object_count);
819 close(pack_data->pack_fd);
8455e484 820 idx_name = keep_pack(create_index());
3e005baf
SP
821
822 /* Register the packfile with core git's machinary. */
823 new_p = add_packed_git(idx_name, strlen(idx_name), 1);
824 if (!new_p)
825 die("core git rejected index %s", idx_name);
826 new_p->windows = old_p->windows;
2369ed79 827 all_packs[pack_id] = new_p;
3e005baf 828 install_packed_git(new_p);
2369ed79
SP
829
830 /* Print the boundary */
bdf1c06d
SP
831 if (pack_edges) {
832 fprintf(pack_edges, "%s:", new_p->pack_name);
833 for (i = 0; i < branch_table_sz; i++) {
834 for (b = branch_table[i]; b; b = b->table_next_branch) {
835 if (b->pack_id == pack_id)
836 fprintf(pack_edges, " %s", sha1_to_hex(b->sha1));
837 }
2369ed79 838 }
bdf1c06d
SP
839 for (t = first_tag; t; t = t->next_tag) {
840 if (t->pack_id == pack_id)
841 fprintf(pack_edges, " %s", sha1_to_hex(t->sha1));
842 }
843 fputc('\n', pack_edges);
844 fflush(pack_edges);
2369ed79 845 }
2369ed79
SP
846
847 pack_id++;
3e005baf 848 }
12801587 849 else
3e005baf 850 unlink(old_p->pack_name);
7bfe6e26 851 free(old_p);
7bfe6e26
SP
852
853 /* We can't carry a delta across packfiles. */
854 free(last_blob.data);
855 last_blob.data = NULL;
856 last_blob.len = 0;
857 last_blob.offset = 0;
858 last_blob.depth = 0;
f70b6534
SP
859}
860
820b9310 861static void cycle_packfile(void)
d9ee53ce
SP
862{
863 end_packfile();
864 start_packfile();
865}
866
c44cdc7e 867static size_t encode_header(
ac47a738 868 enum object_type type,
c44cdc7e 869 size_t size,
ac47a738 870 unsigned char *hdr)
db5e523f
SP
871{
872 int n = 1;
873 unsigned char c;
874
1fcdd62a 875 if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
db5e523f
SP
876 die("bad type %d", type);
877
878 c = (type << 4) | (size & 15);
879 size >>= 4;
880 while (size) {
881 *hdr++ = c | 0x80;
882 c = size & 0x7f;
883 size >>= 7;
884 n++;
885 }
886 *hdr = c;
887 return n;
888}
889
ac47a738
SP
890static int store_object(
891 enum object_type type,
892 void *dat,
c44cdc7e 893 size_t datlen,
6bb5b329 894 struct last_object *last,
d8397168 895 unsigned char *sha1out,
0ea9f045 896 uintmax_t mark)
db5e523f 897{
db5e523f 898 void *out, *delta;
ac47a738
SP
899 struct object_entry *e;
900 unsigned char hdr[96];
901 unsigned char sha1[20];
db5e523f 902 unsigned long hdrlen, deltalen;
ac47a738
SP
903 SHA_CTX c;
904 z_stream s;
905
df843662 906 hdrlen = sprintf((char*)hdr,"%s %lu", typename(type),
40db58b8 907 (unsigned long)datlen) + 1;
ac47a738
SP
908 SHA1_Init(&c);
909 SHA1_Update(&c, hdr, hdrlen);
910 SHA1_Update(&c, dat, datlen);
911 SHA1_Final(sha1, &c);
6bb5b329 912 if (sha1out)
445b8599 913 hashcpy(sha1out, sha1);
ac47a738
SP
914
915 e = insert_object(sha1);
d8397168
SP
916 if (mark)
917 insert_mark(mark, e);
ac47a738 918 if (e->offset) {
6143f064 919 duplicate_count_by_type[type]++;
463acbe1 920 return 1;
a5c1780a
SP
921 } else if (find_sha1_pack(sha1, packed_git)) {
922 e->type = type;
923 e->pack_id = MAX_PACK_ID;
924 e->offset = 1; /* just not zero! */
925 duplicate_count_by_type[type]++;
926 return 1;
ac47a738 927 }
db5e523f 928
d9ee53ce 929 if (last && last->data && last->depth < max_depth) {
ac47a738 930 delta = diff_delta(last->data, last->len,
db5e523f
SP
931 dat, datlen,
932 &deltalen, 0);
d9ee53ce
SP
933 if (delta && deltalen >= datlen) {
934 free(delta);
935 delta = NULL;
936 }
937 } else
938 delta = NULL;
db5e523f
SP
939
940 memset(&s, 0, sizeof(s));
941 deflateInit(&s, zlib_compression_level);
d9ee53ce
SP
942 if (delta) {
943 s.next_in = delta;
944 s.avail_in = deltalen;
945 } else {
946 s.next_in = dat;
947 s.avail_in = datlen;
948 }
949 s.avail_out = deflateBound(&s, s.avail_in);
950 s.next_out = out = xmalloc(s.avail_out);
951 while (deflate(&s, Z_FINISH) == Z_OK)
952 /* nothing */;
953 deflateEnd(&s);
954
955 /* Determine if we should auto-checkpoint. */
e5808826 956 if ((pack_size + 60 + s.total_out) > max_packsize
d9ee53ce
SP
957 || (pack_size + 60 + s.total_out) < pack_size) {
958
959 /* This new object needs to *not* have the current pack_id. */
960 e->pack_id = pack_id + 1;
820b9310 961 cycle_packfile();
d9ee53ce
SP
962
963 /* We cannot carry a delta into the new pack. */
964 if (delta) {
965 free(delta);
966 delta = NULL;
5d6f3ef6
SP
967
968 memset(&s, 0, sizeof(s));
969 deflateInit(&s, zlib_compression_level);
970 s.next_in = dat;
971 s.avail_in = datlen;
972 s.avail_out = deflateBound(&s, s.avail_in);
973 s.next_out = out = xrealloc(out, s.avail_out);
974 while (deflate(&s, Z_FINISH) == Z_OK)
975 /* nothing */;
976 deflateEnd(&s);
d9ee53ce 977 }
d9ee53ce
SP
978 }
979
980 e->type = type;
981 e->pack_id = pack_id;
982 e->offset = pack_size;
983 object_count++;
984 object_count_by_type[type]++;
db5e523f
SP
985
986 if (delta) {
d489bc14
SP
987 unsigned long ofs = e->offset - last->offset;
988 unsigned pos = sizeof(hdr) - 1;
989
4cabf858 990 delta_count_by_type[type]++;
ac47a738 991 last->depth++;
d489bc14
SP
992
993 hdrlen = encode_header(OBJ_OFS_DELTA, deltalen, hdr);
0fcbcae7 994 write_or_die(pack_data->pack_fd, hdr, hdrlen);
d489bc14
SP
995 pack_size += hdrlen;
996
997 hdr[pos] = ofs & 127;
998 while (ofs >>= 7)
999 hdr[--pos] = 128 | (--ofs & 127);
0fcbcae7 1000 write_or_die(pack_data->pack_fd, hdr + pos, sizeof(hdr) - pos);
d489bc14 1001 pack_size += sizeof(hdr) - pos;
db5e523f 1002 } else {
463acbe1
SP
1003 if (last)
1004 last->depth = 0;
ac47a738 1005 hdrlen = encode_header(type, datlen, hdr);
0fcbcae7 1006 write_or_die(pack_data->pack_fd, hdr, hdrlen);
41e5257f 1007 pack_size += hdrlen;
db5e523f
SP
1008 }
1009
0fcbcae7 1010 write_or_die(pack_data->pack_fd, out, s.total_out);
41e5257f 1011 pack_size += s.total_out;
db5e523f
SP
1012
1013 free(out);
e7d06a4b 1014 free(delta);
463acbe1 1015 if (last) {
e7d06a4b 1016 if (!last->no_free)
463acbe1
SP
1017 free(last->data);
1018 last->data = dat;
d489bc14 1019 last->offset = e->offset;
463acbe1 1020 last->len = datlen;
463acbe1
SP
1021 }
1022 return 0;
1023}
1024
7bfe6e26
SP
1025static void *gfi_unpack_entry(
1026 struct object_entry *oe,
1027 unsigned long *sizep)
41e5257f 1028{
21666f1a 1029 enum object_type type;
7bfe6e26
SP
1030 struct packed_git *p = all_packs[oe->pack_id];
1031 if (p == pack_data)
1032 p->pack_size = pack_size + 20;
21666f1a 1033 return unpack_entry(p, oe->offset, &type, sizep);
41e5257f
SP
1034}
1035
10831c55 1036static const char *get_mode(const char *str, uint16_t *modep)
463acbe1
SP
1037{
1038 unsigned char c;
10831c55 1039 uint16_t mode = 0;
463acbe1
SP
1040
1041 while ((c = *str++) != ' ') {
1042 if (c < '0' || c > '7')
1043 return NULL;
1044 mode = (mode << 3) + (c - '0');
1045 }
1046 *modep = mode;
1047 return str;
1048}
1049
1050static void load_tree(struct tree_entry *root)
1051{
4cabf858 1052 unsigned char* sha1 = root->versions[1].sha1;
463acbe1
SP
1053 struct object_entry *myoe;
1054 struct tree_content *t;
1055 unsigned long size;
1056 char *buf;
1057 const char *c;
463acbe1
SP
1058
1059 root->tree = t = new_tree_content(8);
4cabf858 1060 if (is_null_sha1(sha1))
463acbe1
SP
1061 return;
1062
4cabf858 1063 myoe = find_object(sha1);
20f546a8 1064 if (myoe && myoe->pack_id != MAX_PACK_ID) {
41e5257f 1065 if (myoe->type != OBJ_TREE)
4cabf858 1066 die("Not a tree: %s", sha1_to_hex(sha1));
d489bc14 1067 t->delta_depth = 0;
7bfe6e26 1068 buf = gfi_unpack_entry(myoe, &size);
463acbe1 1069 } else {
21666f1a
NP
1070 enum object_type type;
1071 buf = read_sha1_file(sha1, &type, &size);
1072 if (!buf || type != OBJ_TREE)
4cabf858 1073 die("Can't load tree %s", sha1_to_hex(sha1));
463acbe1
SP
1074 }
1075
1076 c = buf;
1077 while (c != (buf + size)) {
1078 struct tree_entry *e = new_tree_entry();
1079
1080 if (t->entry_count == t->entry_capacity)
f022f85f 1081 root->tree = t = grow_tree_content(t, t->entry_count);
463acbe1
SP
1082 t->entries[t->entry_count++] = e;
1083
1084 e->tree = NULL;
4cabf858 1085 c = get_mode(c, &e->versions[1].mode);
463acbe1 1086 if (!c)
4cabf858
SP
1087 die("Corrupt mode in %s", sha1_to_hex(sha1));
1088 e->versions[0].mode = e->versions[1].mode;
061e35c5 1089 e->name = to_atom(c, strlen(c));
463acbe1 1090 c += e->name->str_len + 1;
4cabf858
SP
1091 hashcpy(e->versions[0].sha1, (unsigned char*)c);
1092 hashcpy(e->versions[1].sha1, (unsigned char*)c);
463acbe1
SP
1093 c += 20;
1094 }
1095 free(buf);
1096}
1097
4cabf858 1098static int tecmp0 (const void *_a, const void *_b)
463acbe1
SP
1099{
1100 struct tree_entry *a = *((struct tree_entry**)_a);
1101 struct tree_entry *b = *((struct tree_entry**)_b);
1102 return base_name_compare(
4cabf858
SP
1103 a->name->str_dat, a->name->str_len, a->versions[0].mode,
1104 b->name->str_dat, b->name->str_len, b->versions[0].mode);
463acbe1
SP
1105}
1106
4cabf858 1107static int tecmp1 (const void *_a, const void *_b)
463acbe1 1108{
4cabf858
SP
1109 struct tree_entry *a = *((struct tree_entry**)_a);
1110 struct tree_entry *b = *((struct tree_entry**)_b);
1111 return base_name_compare(
1112 a->name->str_dat, a->name->str_len, a->versions[1].mode,
1113 b->name->str_dat, b->name->str_len, b->versions[1].mode);
1114}
1115
e2eb469d
SP
1116static void mktree(struct tree_content *t,
1117 int v,
1118 unsigned long *szp,
1119 struct dbuf *b)
4cabf858
SP
1120{
1121 size_t maxlen = 0;
463acbe1 1122 unsigned int i;
e2eb469d 1123 char *c;
463acbe1 1124
4cabf858
SP
1125 if (!v)
1126 qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp0);
1127 else
1128 qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp1);
463acbe1 1129
463acbe1 1130 for (i = 0; i < t->entry_count; i++) {
4cabf858
SP
1131 if (t->entries[i]->versions[v].mode)
1132 maxlen += t->entries[i]->name->str_len + 34;
463acbe1
SP
1133 }
1134
243f801d 1135 size_dbuf(b, maxlen);
e2eb469d 1136 c = b->buffer;
463acbe1
SP
1137 for (i = 0; i < t->entry_count; i++) {
1138 struct tree_entry *e = t->entries[i];
4cabf858
SP
1139 if (!e->versions[v].mode)
1140 continue;
10831c55 1141 c += sprintf(c, "%o", (unsigned int)e->versions[v].mode);
463acbe1
SP
1142 *c++ = ' ';
1143 strcpy(c, e->name->str_dat);
1144 c += e->name->str_len + 1;
4cabf858 1145 hashcpy((unsigned char*)c, e->versions[v].sha1);
463acbe1
SP
1146 c += 20;
1147 }
e2eb469d 1148 *szp = c - (char*)b->buffer;
4cabf858
SP
1149}
1150
1151static void store_tree(struct tree_entry *root)
1152{
1153 struct tree_content *t = root->tree;
1154 unsigned int i, j, del;
e2eb469d 1155 unsigned long new_len;
4cabf858 1156 struct last_object lo;
d489bc14 1157 struct object_entry *le;
4cabf858
SP
1158
1159 if (!is_null_sha1(root->versions[1].sha1))
1160 return;
1161
1162 for (i = 0; i < t->entry_count; i++) {
1163 if (t->entries[i]->tree)
1164 store_tree(t->entries[i]);
1165 }
1166
d489bc14 1167 le = find_object(root->versions[0].sha1);
7bfe6e26
SP
1168 if (!S_ISDIR(root->versions[0].mode)
1169 || !le
1170 || le->pack_id != pack_id) {
4cabf858
SP
1171 lo.data = NULL;
1172 lo.depth = 0;
b259157f 1173 lo.no_free = 0;
4cabf858 1174 } else {
e2eb469d
SP
1175 mktree(t, 0, &lo.len, &old_tree);
1176 lo.data = old_tree.buffer;
d489bc14 1177 lo.offset = le->offset;
4cabf858 1178 lo.depth = t->delta_depth;
e2eb469d 1179 lo.no_free = 1;
4cabf858 1180 }
4cabf858 1181
8a8c55ea 1182 mktree(t, 1, &new_len, &new_tree);
e2eb469d 1183 store_object(OBJ_TREE, new_tree.buffer, new_len,
4cabf858 1184 &lo, root->versions[1].sha1, 0);
4cabf858
SP
1185
1186 t->delta_depth = lo.depth;
4cabf858
SP
1187 for (i = 0, j = 0, del = 0; i < t->entry_count; i++) {
1188 struct tree_entry *e = t->entries[i];
1189 if (e->versions[1].mode) {
1190 e->versions[0].mode = e->versions[1].mode;
1191 hashcpy(e->versions[0].sha1, e->versions[1].sha1);
1192 t->entries[j++] = e;
1193 } else {
1194 release_tree_entry(e);
1195 del++;
1196 }
1197 }
1198 t->entry_count -= del;
463acbe1
SP
1199}
1200
1201static int tree_content_set(
1202 struct tree_entry *root,
1203 const char *p,
1204 const unsigned char *sha1,
f39a946a
SP
1205 const uint16_t mode,
1206 struct tree_content *subtree)
463acbe1
SP
1207{
1208 struct tree_content *t = root->tree;
1209 const char *slash1;
1210 unsigned int i, n;
1211 struct tree_entry *e;
1212
1213 slash1 = strchr(p, '/');
1214 if (slash1)
1215 n = slash1 - p;
1216 else
1217 n = strlen(p);
475d1b33
SP
1218 if (!n)
1219 die("Empty path component found in input");
f39a946a
SP
1220 if (!slash1 && !S_ISDIR(mode) && subtree)
1221 die("Non-directories cannot have subtrees");
463acbe1
SP
1222
1223 for (i = 0; i < t->entry_count; i++) {
1224 e = t->entries[i];
1225 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
1226 if (!slash1) {
f39a946a
SP
1227 if (!S_ISDIR(mode)
1228 && e->versions[1].mode == mode
4cabf858 1229 && !hashcmp(e->versions[1].sha1, sha1))
463acbe1 1230 return 0;
4cabf858
SP
1231 e->versions[1].mode = mode;
1232 hashcpy(e->versions[1].sha1, sha1);
f39a946a 1233 if (e->tree)
afde8dd9 1234 release_tree_content_recursive(e->tree);
f39a946a 1235 e->tree = subtree;
4cabf858 1236 hashclr(root->versions[1].sha1);
463acbe1
SP
1237 return 1;
1238 }
4cabf858 1239 if (!S_ISDIR(e->versions[1].mode)) {
463acbe1 1240 e->tree = new_tree_content(8);
4cabf858 1241 e->versions[1].mode = S_IFDIR;
463acbe1
SP
1242 }
1243 if (!e->tree)
1244 load_tree(e);
f39a946a 1245 if (tree_content_set(e, slash1 + 1, sha1, mode, subtree)) {
4cabf858 1246 hashclr(root->versions[1].sha1);
463acbe1
SP
1247 return 1;
1248 }
1249 return 0;
1250 }
1251 }
1252
1253 if (t->entry_count == t->entry_capacity)
f022f85f 1254 root->tree = t = grow_tree_content(t, t->entry_count);
463acbe1 1255 e = new_tree_entry();
061e35c5 1256 e->name = to_atom(p, n);
4cabf858
SP
1257 e->versions[0].mode = 0;
1258 hashclr(e->versions[0].sha1);
463acbe1
SP
1259 t->entries[t->entry_count++] = e;
1260 if (slash1) {
1261 e->tree = new_tree_content(8);
4cabf858 1262 e->versions[1].mode = S_IFDIR;
f39a946a 1263 tree_content_set(e, slash1 + 1, sha1, mode, subtree);
463acbe1 1264 } else {
f39a946a 1265 e->tree = subtree;
4cabf858
SP
1266 e->versions[1].mode = mode;
1267 hashcpy(e->versions[1].sha1, sha1);
463acbe1 1268 }
4cabf858 1269 hashclr(root->versions[1].sha1);
463acbe1
SP
1270 return 1;
1271}
1272
f39a946a
SP
1273static int tree_content_remove(
1274 struct tree_entry *root,
1275 const char *p,
1276 struct tree_entry *backup_leaf)
463acbe1
SP
1277{
1278 struct tree_content *t = root->tree;
1279 const char *slash1;
1280 unsigned int i, n;
1281 struct tree_entry *e;
1282
1283 slash1 = strchr(p, '/');
1284 if (slash1)
1285 n = slash1 - p;
1286 else
1287 n = strlen(p);
1288
1289 for (i = 0; i < t->entry_count; i++) {
1290 e = t->entries[i];
1291 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
4cabf858 1292 if (!slash1 || !S_ISDIR(e->versions[1].mode))
463acbe1
SP
1293 goto del_entry;
1294 if (!e->tree)
1295 load_tree(e);
f39a946a 1296 if (tree_content_remove(e, slash1 + 1, backup_leaf)) {
b54d6422
SP
1297 for (n = 0; n < e->tree->entry_count; n++) {
1298 if (e->tree->entries[n]->versions[1].mode) {
1299 hashclr(root->versions[1].sha1);
1300 return 1;
1301 }
1302 }
f39a946a 1303 backup_leaf = NULL;
b54d6422 1304 goto del_entry;
463acbe1
SP
1305 }
1306 return 0;
1307 }
1308 }
1309 return 0;
1310
1311del_entry:
f39a946a
SP
1312 if (backup_leaf)
1313 memcpy(backup_leaf, e, sizeof(*backup_leaf));
1314 else if (e->tree)
4cabf858 1315 release_tree_content_recursive(e->tree);
f39a946a 1316 e->tree = NULL;
4cabf858
SP
1317 e->versions[1].mode = 0;
1318 hashclr(e->versions[1].sha1);
1319 hashclr(root->versions[1].sha1);
ac47a738 1320 return 1;
db5e523f
SP
1321}
1322
b6f3481b
SP
1323static int tree_content_get(
1324 struct tree_entry *root,
1325 const char *p,
1326 struct tree_entry *leaf)
1327{
1328 struct tree_content *t = root->tree;
1329 const char *slash1;
1330 unsigned int i, n;
1331 struct tree_entry *e;
1332
1333 slash1 = strchr(p, '/');
1334 if (slash1)
1335 n = slash1 - p;
1336 else
1337 n = strlen(p);
1338
1339 for (i = 0; i < t->entry_count; i++) {
1340 e = t->entries[i];
1341 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
1342 if (!slash1) {
1343 memcpy(leaf, e, sizeof(*leaf));
1344 if (e->tree && is_null_sha1(e->versions[1].sha1))
1345 leaf->tree = dup_tree_content(e->tree);
1346 else
1347 leaf->tree = NULL;
1348 return 1;
1349 }
1350 if (!S_ISDIR(e->versions[1].mode))
1351 return 0;
1352 if (!e->tree)
1353 load_tree(e);
1354 return tree_content_get(e, slash1 + 1, leaf);
1355 }
1356 }
1357 return 0;
1358}
1359
7073e69e 1360static int update_branch(struct branch *b)
463acbe1
SP
1361{
1362 static const char *msg = "fast-import";
7073e69e
SP
1363 struct ref_lock *lock;
1364 unsigned char old_sha1[20];
1365
1366 if (read_ref(b->name, old_sha1))
1367 hashclr(old_sha1);
68db31cc 1368 lock = lock_any_ref_for_update(b->name, old_sha1, 0);
7073e69e
SP
1369 if (!lock)
1370 return error("Unable to lock %s", b->name);
1371 if (!force_update && !is_null_sha1(old_sha1)) {
1372 struct commit *old_cmit, *new_cmit;
1373
1374 old_cmit = lookup_commit_reference_gently(old_sha1, 0);
1375 new_cmit = lookup_commit_reference_gently(b->sha1, 0);
1376 if (!old_cmit || !new_cmit) {
1377 unlock_ref(lock);
1378 return error("Branch %s is missing commits.", b->name);
1379 }
1380
4a164d48 1381 if (!in_merge_bases(old_cmit, &new_cmit, 1)) {
7073e69e 1382 unlock_ref(lock);
46efd2d9 1383 warning("Not updating %s"
7073e69e
SP
1384 " (new tip %s does not contain %s)",
1385 b->name, sha1_to_hex(b->sha1), sha1_to_hex(old_sha1));
1386 return -1;
1387 }
1388 }
1389 if (write_ref_sha1(lock, b->sha1, msg) < 0)
1390 return error("Unable to update %s", b->name);
1391 return 0;
1392}
1393
1394static void dump_branches(void)
1395{
463acbe1
SP
1396 unsigned int i;
1397 struct branch *b;
463acbe1
SP
1398
1399 for (i = 0; i < branch_table_sz; i++) {
7073e69e
SP
1400 for (b = branch_table[i]; b; b = b->table_next_branch)
1401 failure |= update_branch(b);
463acbe1
SP
1402 }
1403}
1404
fd99224e 1405static void dump_tags(void)
72303d44
SP
1406{
1407 static const char *msg = "fast-import";
1408 struct tag *t;
1409 struct ref_lock *lock;
7073e69e 1410 char ref_name[PATH_MAX];
72303d44
SP
1411
1412 for (t = first_tag; t; t = t->next_tag) {
7073e69e
SP
1413 sprintf(ref_name, "tags/%s", t->name);
1414 lock = lock_ref_sha1(ref_name, NULL);
72303d44 1415 if (!lock || write_ref_sha1(lock, t->sha1, msg) < 0)
7073e69e 1416 failure |= error("Unable to update %s", ref_name);
72303d44
SP
1417 }
1418}
1419
a6a1a831 1420static void dump_marks_helper(FILE *f,
0ea9f045 1421 uintmax_t base,
a6a1a831
SP
1422 struct mark_set *m)
1423{
0ea9f045 1424 uintmax_t k;
a6a1a831
SP
1425 if (m->shift) {
1426 for (k = 0; k < 1024; k++) {
1427 if (m->data.sets[k])
1428 dump_marks_helper(f, (base + k) << m->shift,
1429 m->data.sets[k]);
1430 }
1431 } else {
1432 for (k = 0; k < 1024; k++) {
1433 if (m->data.marked[k])
3efb1f34 1434 fprintf(f, ":%" PRIuMAX " %s\n", base + k,
a6a1a831
SP
1435 sha1_to_hex(m->data.marked[k]->sha1));
1436 }
1437 }
1438}
1439
fd99224e 1440static void dump_marks(void)
a6a1a831 1441{
60b9004c
SP
1442 static struct lock_file mark_lock;
1443 int mark_fd;
1444 FILE *f;
1445
1446 if (!mark_file)
1447 return;
1448
1449 mark_fd = hold_lock_file_for_update(&mark_lock, mark_file, 0);
1450 if (mark_fd < 0) {
1451 failure |= error("Unable to write marks file %s: %s",
1452 mark_file, strerror(errno));
1453 return;
a6a1a831 1454 }
60b9004c
SP
1455
1456 f = fdopen(mark_fd, "w");
1457 if (!f) {
1458 rollback_lock_file(&mark_lock);
1459 failure |= error("Unable to write marks file %s: %s",
1460 mark_file, strerror(errno));
1461 return;
a6a1a831 1462 }
60b9004c
SP
1463
1464 dump_marks_helper(f, 0, marks);
1465 fclose(f);
1466 if (commit_lock_file(&mark_lock))
1467 failure |= error("Unable to write marks file %s: %s",
1468 mark_file, strerror(errno));
a6a1a831
SP
1469}
1470
fd99224e 1471static void read_next_command(void)
c44cdc7e 1472{
401d53fa 1473 do {
1fdb649c
SP
1474 if (unread_command_buf)
1475 unread_command_buf = 0;
1476 else
1477 read_line(&command_buf, stdin, '\n');
401d53fa 1478 } while (!command_buf.eof && command_buf.buf[0] == '#');
c44cdc7e
SP
1479}
1480
2c570cde
SP
1481static void skip_optional_lf()
1482{
1483 int term_char = fgetc(stdin);
1484 if (term_char != '\n' && term_char != EOF)
1485 ungetc(term_char, stdin);
1486}
1487
fd99224e 1488static void cmd_mark(void)
c44cdc7e 1489{
599065a3 1490 if (!prefixcmp(command_buf.buf, "mark :")) {
0ea9f045 1491 next_mark = strtoumax(command_buf.buf + 6, NULL, 10);
c44cdc7e
SP
1492 read_next_command();
1493 }
1494 else
d8397168 1495 next_mark = 0;
c44cdc7e
SP
1496}
1497
e5b1444b 1498static void *cmd_data (size_t *size)
c44cdc7e 1499{
c44cdc7e 1500 size_t length;
3b4dce02 1501 char *buffer;
c44cdc7e 1502
599065a3 1503 if (prefixcmp(command_buf.buf, "data "))
c44cdc7e
SP
1504 die("Expected 'data n' command, found: %s", command_buf.buf);
1505
599065a3 1506 if (!prefixcmp(command_buf.buf + 5, "<<")) {
3b4dce02
SP
1507 char *term = xstrdup(command_buf.buf + 5 + 2);
1508 size_t sz = 8192, term_len = command_buf.len - 5 - 2;
1509 length = 0;
1510 buffer = xmalloc(sz);
1511 for (;;) {
401d53fa 1512 read_line(&command_buf, stdin, '\n');
3b4dce02
SP
1513 if (command_buf.eof)
1514 die("EOF in data (terminator '%s' not found)", term);
1515 if (term_len == command_buf.len
1516 && !strcmp(term, command_buf.buf))
1517 break;
31490074 1518 ALLOC_GROW(buffer, length + command_buf.len, sz);
3b4dce02
SP
1519 memcpy(buffer + length,
1520 command_buf.buf,
1521 command_buf.len - 1);
1522 length += command_buf.len - 1;
1523 buffer[length++] = '\n';
1524 }
1525 free(term);
1526 }
1527 else {
1528 size_t n = 0;
1529 length = strtoul(command_buf.buf + 5, NULL, 10);
1530 buffer = xmalloc(length);
1531 while (n < length) {
1532 size_t s = fread(buffer + n, 1, length - n, stdin);
1533 if (!s && feof(stdin))
40db58b8
JS
1534 die("EOF in data (%lu bytes remaining)",
1535 (unsigned long)(length - n));
3b4dce02
SP
1536 n += s;
1537 }
c44cdc7e
SP
1538 }
1539
2c570cde 1540 skip_optional_lf();
c44cdc7e
SP
1541 *size = length;
1542 return buffer;
1543}
1544
63e0c8b3
SP
1545static int validate_raw_date(const char *src, char *result, int maxlen)
1546{
1547 const char *orig_src = src;
1548 char *endp, sign;
1549
1550 strtoul(src, &endp, 10);
1551 if (endp == src || *endp != ' ')
1552 return -1;
1553
1554 src = endp + 1;
1555 if (*src != '-' && *src != '+')
1556 return -1;
1557 sign = *src;
1558
1559 strtoul(src + 1, &endp, 10);
1560 if (endp == src || *endp || (endp - orig_src) >= maxlen)
1561 return -1;
1562
1563 strcpy(result, orig_src);
1564 return 0;
1565}
1566
1567static char *parse_ident(const char *buf)
1568{
1569 const char *gt;
1570 size_t name_len;
1571 char *ident;
1572
1573 gt = strrchr(buf, '>');
1574 if (!gt)
1575 die("Missing > in ident string: %s", buf);
1576 gt++;
1577 if (*gt != ' ')
1578 die("Missing space after > in ident string: %s", buf);
1579 gt++;
1580 name_len = gt - buf;
1581 ident = xmalloc(name_len + 24);
1582 strncpy(ident, buf, name_len);
1583
1584 switch (whenspec) {
1585 case WHENSPEC_RAW:
1586 if (validate_raw_date(gt, ident + name_len, 24) < 0)
1587 die("Invalid raw date \"%s\" in ident: %s", gt, buf);
1588 break;
1589 case WHENSPEC_RFC2822:
1590 if (parse_date(gt, ident + name_len, 24) < 0)
1591 die("Invalid rfc2822 date \"%s\" in ident: %s", gt, buf);
1592 break;
1593 case WHENSPEC_NOW:
1594 if (strcmp("now", gt))
1595 die("Date in ident must be 'now': %s", buf);
1596 datestamp(ident + name_len, 24);
1597 break;
1598 }
1599
1600 return ident;
1601}
1602
fd99224e 1603static void cmd_new_blob(void)
6143f064 1604{
d8397168
SP
1605 size_t l;
1606 void *d;
c44cdc7e
SP
1607
1608 read_next_command();
1609 cmd_mark();
d8397168 1610 d = cmd_data(&l);
6143f064 1611
d8397168
SP
1612 if (store_object(OBJ_BLOB, d, l, &last_blob, NULL, next_mark))
1613 free(d);
6143f064
SP
1614}
1615
fd99224e 1616static void unload_one_branch(void)
6bb5b329 1617{
41e5257f
SP
1618 while (cur_active_branches
1619 && cur_active_branches >= max_active_branches) {
6777a59f 1620 uintmax_t min_commit = ULONG_MAX;
463acbe1
SP
1621 struct branch *e, *l = NULL, *p = NULL;
1622
1623 for (e = active_branches; e; e = e->active_next_branch) {
1624 if (e->last_commit < min_commit) {
1625 p = l;
1626 min_commit = e->last_commit;
1627 }
1628 l = e;
1629 }
1630
1631 if (p) {
1632 e = p->active_next_branch;
1633 p->active_next_branch = e->active_next_branch;
1634 } else {
1635 e = active_branches;
1636 active_branches = e->active_next_branch;
1637 }
734c91f9 1638 e->active = 0;
463acbe1
SP
1639 e->active_next_branch = NULL;
1640 if (e->branch_tree.tree) {
afde8dd9 1641 release_tree_content_recursive(e->branch_tree.tree);
463acbe1
SP
1642 e->branch_tree.tree = NULL;
1643 }
1644 cur_active_branches--;
6bb5b329 1645 }
6bb5b329
SP
1646}
1647
463acbe1 1648static void load_branch(struct branch *b)
6bb5b329 1649{
463acbe1 1650 load_tree(&b->branch_tree);
734c91f9
SP
1651 if (!b->active) {
1652 b->active = 1;
1653 b->active_next_branch = active_branches;
1654 active_branches = b;
1655 cur_active_branches++;
1656 branch_load_count++;
1657 }
6bb5b329
SP
1658}
1659
463acbe1 1660static void file_change_m(struct branch *b)
6bb5b329 1661{
c44cdc7e
SP
1662 const char *p = command_buf.buf + 2;
1663 char *p_uq;
1664 const char *endp;
10e8d688 1665 struct object_entry *oe = oe;
463acbe1 1666 unsigned char sha1[20];
10831c55 1667 uint16_t mode, inline_data = 0;
6bb5b329 1668
c44cdc7e
SP
1669 p = get_mode(p, &mode);
1670 if (!p)
1671 die("Corrupt mode: %s", command_buf.buf);
1672 switch (mode) {
1673 case S_IFREG | 0644:
1674 case S_IFREG | 0755:
ace4a9d1 1675 case S_IFLNK:
c44cdc7e
SP
1676 case 0644:
1677 case 0755:
1678 /* ok */
1679 break;
1680 default:
1681 die("Corrupt mode: %s", command_buf.buf);
1682 }
1683
d8397168
SP
1684 if (*p == ':') {
1685 char *x;
0ea9f045 1686 oe = find_mark(strtoumax(p + 1, &x, 10));
cacbdd0a 1687 hashcpy(sha1, oe->sha1);
d8397168 1688 p = x;
599065a3 1689 } else if (!prefixcmp(p, "inline")) {
b715cfbb
SP
1690 inline_data = 1;
1691 p += 6;
d8397168
SP
1692 } else {
1693 if (get_sha1_hex(p, sha1))
1694 die("Invalid SHA1: %s", command_buf.buf);
1695 oe = find_object(sha1);
1696 p += 40;
1697 }
c44cdc7e
SP
1698 if (*p++ != ' ')
1699 die("Missing space after SHA1: %s", command_buf.buf);
1700
1701 p_uq = unquote_c_style(p, &endp);
1702 if (p_uq) {
1703 if (*endp)
1704 die("Garbage after path in: %s", command_buf.buf);
1705 p = p_uq;
1706 }
6bb5b329 1707
b715cfbb
SP
1708 if (inline_data) {
1709 size_t l;
1710 void *d;
1711 if (!p_uq)
1712 p = p_uq = xstrdup(p);
1713 read_next_command();
1714 d = cmd_data(&l);
1715 if (store_object(OBJ_BLOB, d, l, &last_blob, sha1, 0))
1716 free(d);
1717 } else if (oe) {
7111feed 1718 if (oe->type != OBJ_BLOB)
c44cdc7e 1719 die("Not a blob (actually a %s): %s",
df843662 1720 command_buf.buf, typename(oe->type));
7111feed 1721 } else {
21666f1a
NP
1722 enum object_type type = sha1_object_info(sha1, NULL);
1723 if (type < 0)
c44cdc7e 1724 die("Blob not found: %s", command_buf.buf);
21666f1a 1725 if (type != OBJ_BLOB)
c44cdc7e 1726 die("Not a blob (actually a %s): %s",
21666f1a 1727 typename(type), command_buf.buf);
7111feed 1728 }
6bb5b329 1729
f39a946a 1730 tree_content_set(&b->branch_tree, p, sha1, S_IFREG | mode, NULL);
e7d06a4b 1731 free(p_uq);
463acbe1 1732}
6bb5b329 1733
463acbe1
SP
1734static void file_change_d(struct branch *b)
1735{
c44cdc7e
SP
1736 const char *p = command_buf.buf + 2;
1737 char *p_uq;
1738 const char *endp;
1739
1740 p_uq = unquote_c_style(p, &endp);
1741 if (p_uq) {
1742 if (*endp)
1743 die("Garbage after path in: %s", command_buf.buf);
1744 p = p_uq;
1745 }
f39a946a 1746 tree_content_remove(&b->branch_tree, p, NULL);
e7d06a4b 1747 free(p_uq);
6bb5b329
SP
1748}
1749
b6f3481b 1750static void file_change_cr(struct branch *b, int rename)
f39a946a
SP
1751{
1752 const char *s, *d;
1753 char *s_uq, *d_uq;
1754 const char *endp;
1755 struct tree_entry leaf;
1756
1757 s = command_buf.buf + 2;
1758 s_uq = unquote_c_style(s, &endp);
1759 if (s_uq) {
1760 if (*endp != ' ')
1761 die("Missing space after source: %s", command_buf.buf);
1762 }
1763 else {
1764 endp = strchr(s, ' ');
1765 if (!endp)
1766 die("Missing space after source: %s", command_buf.buf);
1767 s_uq = xmalloc(endp - s + 1);
1768 memcpy(s_uq, s, endp - s);
1769 s_uq[endp - s] = 0;
1770 }
1771 s = s_uq;
1772
1773 endp++;
1774 if (!*endp)
1775 die("Missing dest: %s", command_buf.buf);
1776
1777 d = endp;
1778 d_uq = unquote_c_style(d, &endp);
1779 if (d_uq) {
1780 if (*endp)
1781 die("Garbage after dest in: %s", command_buf.buf);
1782 d = d_uq;
1783 }
1784
1785 memset(&leaf, 0, sizeof(leaf));
b6f3481b
SP
1786 if (rename)
1787 tree_content_remove(&b->branch_tree, s, &leaf);
1788 else
1789 tree_content_get(&b->branch_tree, s, &leaf);
f39a946a
SP
1790 if (!leaf.versions[1].mode)
1791 die("Path %s not in branch", s);
1792 tree_content_set(&b->branch_tree, d,
1793 leaf.versions[1].sha1,
1794 leaf.versions[1].mode,
1795 leaf.tree);
1796
1797 free(s_uq);
1798 free(d_uq);
1799}
1800
825769a8
SP
1801static void file_change_deleteall(struct branch *b)
1802{
1803 release_tree_content_recursive(b->branch_tree.tree);
1804 hashclr(b->branch_tree.versions[0].sha1);
1805 hashclr(b->branch_tree.versions[1].sha1);
1806 load_tree(&b->branch_tree);
1807}
1808
654aaa37
SP
1809static void cmd_from_commit(struct branch *b, char *buf, unsigned long size)
1810{
1811 if (!buf || size < 46)
1812 die("Not a valid commit: %s", sha1_to_hex(b->sha1));
1813 if (memcmp("tree ", buf, 5)
1814 || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
1815 die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1816 hashcpy(b->branch_tree.versions[0].sha1,
1817 b->branch_tree.versions[1].sha1);
1818}
1819
1820static void cmd_from_existing(struct branch *b)
1821{
1822 if (is_null_sha1(b->sha1)) {
1823 hashclr(b->branch_tree.versions[0].sha1);
1824 hashclr(b->branch_tree.versions[1].sha1);
1825 } else {
1826 unsigned long size;
1827 char *buf;
1828
1829 buf = read_object_with_reference(b->sha1,
1830 commit_type, &size, b->sha1);
1831 cmd_from_commit(b, buf, size);
1832 free(buf);
1833 }
1834}
1835
1fdb649c 1836static int cmd_from(struct branch *b)
00e2b884 1837{
6c3aac1c 1838 const char *from;
00e2b884
SP
1839 struct branch *s;
1840
599065a3 1841 if (prefixcmp(command_buf.buf, "from "))
1fdb649c 1842 return 0;
00e2b884 1843
ea5e370a
SP
1844 if (b->branch_tree.tree) {
1845 release_tree_content_recursive(b->branch_tree.tree);
1846 b->branch_tree.tree = NULL;
1847 }
00e2b884
SP
1848
1849 from = strchr(command_buf.buf, ' ') + 1;
00e2b884
SP
1850 s = lookup_branch(from);
1851 if (b == s)
1852 die("Can't create a branch from itself: %s", b->name);
1853 else if (s) {
4cabf858 1854 unsigned char *t = s->branch_tree.versions[1].sha1;
445b8599 1855 hashcpy(b->sha1, s->sha1);
4cabf858
SP
1856 hashcpy(b->branch_tree.versions[0].sha1, t);
1857 hashcpy(b->branch_tree.versions[1].sha1, t);
00e2b884 1858 } else if (*from == ':') {
0ea9f045 1859 uintmax_t idnum = strtoumax(from + 1, NULL, 10);
00e2b884 1860 struct object_entry *oe = find_mark(idnum);
00e2b884 1861 if (oe->type != OBJ_COMMIT)
3efb1f34 1862 die("Mark :%" PRIuMAX " not a commit", idnum);
445b8599 1863 hashcpy(b->sha1, oe->sha1);
aac65ed1 1864 if (oe->pack_id != MAX_PACK_ID) {
00e2b884 1865 unsigned long size;
aac65ed1
SP
1866 char *buf = gfi_unpack_entry(oe, &size);
1867 cmd_from_commit(b, buf, size);
00e2b884 1868 free(buf);
aac65ed1
SP
1869 } else
1870 cmd_from_existing(b);
654aaa37
SP
1871 } else if (!get_sha1(from, b->sha1))
1872 cmd_from_existing(b);
1873 else
00e2b884
SP
1874 die("Invalid ref name or SHA1 expression: %s", from);
1875
1876 read_next_command();
1fdb649c 1877 return 1;
00e2b884
SP
1878}
1879
e5b1444b 1880static struct hash_list *cmd_merge(unsigned int *count)
62b6f483 1881{
10e8d688 1882 struct hash_list *list = NULL, *n, *e = e;
6c3aac1c 1883 const char *from;
62b6f483
SP
1884 struct branch *s;
1885
1886 *count = 0;
599065a3 1887 while (!prefixcmp(command_buf.buf, "merge ")) {
62b6f483 1888 from = strchr(command_buf.buf, ' ') + 1;
62b6f483
SP
1889 n = xmalloc(sizeof(*n));
1890 s = lookup_branch(from);
1891 if (s)
1892 hashcpy(n->sha1, s->sha1);
1893 else if (*from == ':') {
0ea9f045 1894 uintmax_t idnum = strtoumax(from + 1, NULL, 10);
62b6f483
SP
1895 struct object_entry *oe = find_mark(idnum);
1896 if (oe->type != OBJ_COMMIT)
3efb1f34 1897 die("Mark :%" PRIuMAX " not a commit", idnum);
62b6f483 1898 hashcpy(n->sha1, oe->sha1);
2f6dc35d
SP
1899 } else if (!get_sha1(from, n->sha1)) {
1900 unsigned long size;
1901 char *buf = read_object_with_reference(n->sha1,
6b4318e6 1902 commit_type, &size, n->sha1);
2f6dc35d
SP
1903 if (!buf || size < 46)
1904 die("Not a valid commit: %s", from);
1905 free(buf);
1906 } else
62b6f483
SP
1907 die("Invalid ref name or SHA1 expression: %s", from);
1908
1909 n->next = NULL;
1910 if (list)
1911 e->next = n;
1912 else
1913 list = n;
1914 e = n;
10e8d688 1915 (*count)++;
62b6f483
SP
1916 read_next_command();
1917 }
1918 return list;
1919}
1920
fd99224e 1921static void cmd_new_commit(void)
6bb5b329 1922{
c44cdc7e
SP
1923 struct branch *b;
1924 void *msg;
1925 size_t msglen;
c44cdc7e
SP
1926 char *sp;
1927 char *author = NULL;
1928 char *committer = NULL;
62b6f483
SP
1929 struct hash_list *merge_list = NULL;
1930 unsigned int merge_count;
c44cdc7e
SP
1931
1932 /* Obtain the branch name from the rest of our command */
1933 sp = strchr(command_buf.buf, ' ') + 1;
c44cdc7e 1934 b = lookup_branch(sp);
463acbe1 1935 if (!b)
00e2b884 1936 b = new_branch(sp);
c44cdc7e
SP
1937
1938 read_next_command();
1939 cmd_mark();
599065a3 1940 if (!prefixcmp(command_buf.buf, "author ")) {
63e0c8b3 1941 author = parse_ident(command_buf.buf + 7);
c44cdc7e
SP
1942 read_next_command();
1943 }
599065a3 1944 if (!prefixcmp(command_buf.buf, "committer ")) {
63e0c8b3 1945 committer = parse_ident(command_buf.buf + 10);
c44cdc7e
SP
1946 read_next_command();
1947 }
1948 if (!committer)
1949 die("Expected committer but didn't get one");
1950 msg = cmd_data(&msglen);
02f3389d
SP
1951 read_next_command();
1952 cmd_from(b);
62b6f483 1953 merge_list = cmd_merge(&merge_count);
c44cdc7e
SP
1954
1955 /* ensure the branch is active/loaded */
41e5257f 1956 if (!b->branch_tree.tree || !max_active_branches) {
463acbe1
SP
1957 unload_one_branch();
1958 load_branch(b);
1959 }
6bb5b329 1960
463acbe1 1961 /* file_change* */
1fdb649c
SP
1962 while (!command_buf.eof && command_buf.len > 1) {
1963 if (!prefixcmp(command_buf.buf, "M "))
463acbe1 1964 file_change_m(b);
599065a3 1965 else if (!prefixcmp(command_buf.buf, "D "))
463acbe1 1966 file_change_d(b);
f39a946a 1967 else if (!prefixcmp(command_buf.buf, "R "))
b6f3481b
SP
1968 file_change_cr(b, 1);
1969 else if (!prefixcmp(command_buf.buf, "C "))
1970 file_change_cr(b, 0);
825769a8
SP
1971 else if (!strcmp("deleteall", command_buf.buf))
1972 file_change_deleteall(b);
1fdb649c
SP
1973 else {
1974 unread_command_buf = 1;
1975 break;
1976 }
02f3389d 1977 read_next_command();
6bb5b329 1978 }
6bb5b329 1979
c44cdc7e 1980 /* build the tree and the commit */
463acbe1 1981 store_tree(&b->branch_tree);
8a8c55ea
SP
1982 hashcpy(b->branch_tree.versions[0].sha1,
1983 b->branch_tree.versions[1].sha1);
63e0c8b3 1984 size_dbuf(&new_data, 114 + msglen
62b6f483 1985 + merge_count * 49
c44cdc7e
SP
1986 + (author
1987 ? strlen(author) + strlen(committer)
1988 : 2 * strlen(committer)));
243f801d 1989 sp = new_data.buffer;
4cabf858
SP
1990 sp += sprintf(sp, "tree %s\n",
1991 sha1_to_hex(b->branch_tree.versions[1].sha1));
445b8599 1992 if (!is_null_sha1(b->sha1))
c44cdc7e 1993 sp += sprintf(sp, "parent %s\n", sha1_to_hex(b->sha1));
62b6f483
SP
1994 while (merge_list) {
1995 struct hash_list *next = merge_list->next;
1996 sp += sprintf(sp, "parent %s\n", sha1_to_hex(merge_list->sha1));
1997 free(merge_list);
1998 merge_list = next;
1999 }
63e0c8b3
SP
2000 sp += sprintf(sp, "author %s\n", author ? author : committer);
2001 sp += sprintf(sp, "committer %s\n", committer);
2002 *sp++ = '\n';
c44cdc7e
SP
2003 memcpy(sp, msg, msglen);
2004 sp += msglen;
e7d06a4b 2005 free(author);
c44cdc7e
SP
2006 free(committer);
2007 free(msg);
2008
69e74e74 2009 if (!store_object(OBJ_COMMIT,
243f801d 2010 new_data.buffer, sp - (char*)new_data.buffer,
69e74e74
SP
2011 NULL, b->sha1, next_mark))
2012 b->pack_id = pack_id;
463acbe1 2013 b->last_commit = object_count_by_type[OBJ_COMMIT];
6bb5b329
SP
2014}
2015
fd99224e 2016static void cmd_new_tag(void)
72303d44 2017{
72303d44
SP
2018 char *sp;
2019 const char *from;
2020 char *tagger;
2021 struct branch *s;
2022 void *msg;
2023 size_t msglen;
72303d44 2024 struct tag *t;
0ea9f045 2025 uintmax_t from_mark = 0;
72303d44
SP
2026 unsigned char sha1[20];
2027
2028 /* Obtain the new tag name from the rest of our command */
2029 sp = strchr(command_buf.buf, ' ') + 1;
72303d44
SP
2030 t = pool_alloc(sizeof(struct tag));
2031 t->next_tag = NULL;
2032 t->name = pool_strdup(sp);
2033 if (last_tag)
2034 last_tag->next_tag = t;
2035 else
2036 first_tag = t;
2037 last_tag = t;
72303d44
SP
2038 read_next_command();
2039
2040 /* from ... */
599065a3 2041 if (prefixcmp(command_buf.buf, "from "))
72303d44 2042 die("Expected from command, got %s", command_buf.buf);
72303d44 2043 from = strchr(command_buf.buf, ' ') + 1;
72303d44
SP
2044 s = lookup_branch(from);
2045 if (s) {
445b8599 2046 hashcpy(sha1, s->sha1);
72303d44 2047 } else if (*from == ':') {
10e8d688 2048 struct object_entry *oe;
0ea9f045 2049 from_mark = strtoumax(from + 1, NULL, 10);
10e8d688 2050 oe = find_mark(from_mark);
72303d44 2051 if (oe->type != OBJ_COMMIT)
3efb1f34 2052 die("Mark :%" PRIuMAX " not a commit", from_mark);
445b8599 2053 hashcpy(sha1, oe->sha1);
72303d44
SP
2054 } else if (!get_sha1(from, sha1)) {
2055 unsigned long size;
2056 char *buf;
2057
2058 buf = read_object_with_reference(sha1,
df843662 2059 commit_type, &size, sha1);
72303d44
SP
2060 if (!buf || size < 46)
2061 die("Not a valid commit: %s", from);
2062 free(buf);
2063 } else
2064 die("Invalid ref name or SHA1 expression: %s", from);
72303d44
SP
2065 read_next_command();
2066
2067 /* tagger ... */
599065a3 2068 if (prefixcmp(command_buf.buf, "tagger "))
72303d44 2069 die("Expected tagger command, got %s", command_buf.buf);
63e0c8b3 2070 tagger = parse_ident(command_buf.buf + 7);
72303d44
SP
2071
2072 /* tag payload/message */
2073 read_next_command();
2074 msg = cmd_data(&msglen);
2075
2076 /* build the tag object */
243f801d
SP
2077 size_dbuf(&new_data, 67+strlen(t->name)+strlen(tagger)+msglen);
2078 sp = new_data.buffer;
72303d44 2079 sp += sprintf(sp, "object %s\n", sha1_to_hex(sha1));
df843662 2080 sp += sprintf(sp, "type %s\n", commit_type);
72303d44 2081 sp += sprintf(sp, "tag %s\n", t->name);
63e0c8b3
SP
2082 sp += sprintf(sp, "tagger %s\n", tagger);
2083 *sp++ = '\n';
72303d44
SP
2084 memcpy(sp, msg, msglen);
2085 sp += msglen;
2086 free(tagger);
2087 free(msg);
2088
69e74e74
SP
2089 if (store_object(OBJ_TAG, new_data.buffer,
2090 sp - (char*)new_data.buffer,
2091 NULL, t->sha1, 0))
2092 t->pack_id = MAX_PACK_ID;
2093 else
2094 t->pack_id = pack_id;
72303d44
SP
2095}
2096
fd99224e 2097static void cmd_reset_branch(void)
5fced8dc
SP
2098{
2099 struct branch *b;
5fced8dc
SP
2100 char *sp;
2101
2102 /* Obtain the branch name from the rest of our command */
2103 sp = strchr(command_buf.buf, ' ') + 1;
5fced8dc
SP
2104 b = lookup_branch(sp);
2105 if (b) {
ea5e370a
SP
2106 hashclr(b->sha1);
2107 hashclr(b->branch_tree.versions[0].sha1);
2108 hashclr(b->branch_tree.versions[1].sha1);
5fced8dc
SP
2109 if (b->branch_tree.tree) {
2110 release_tree_content_recursive(b->branch_tree.tree);
2111 b->branch_tree.tree = NULL;
2112 }
2113 }
9938ffc5
SP
2114 else
2115 b = new_branch(sp);
9938ffc5 2116 read_next_command();
1fdb649c
SP
2117 if (!cmd_from(b) && command_buf.len > 1)
2118 unread_command_buf = 1;
5fced8dc
SP
2119}
2120
fd99224e 2121static void cmd_checkpoint(void)
7bfe6e26 2122{
820b9310
SP
2123 if (object_count) {
2124 cycle_packfile();
2125 dump_branches();
2126 dump_tags();
2127 dump_marks();
2128 }
1fdb649c 2129 skip_optional_lf();
7bfe6e26
SP
2130}
2131
ac053c02
SP
2132static void cmd_progress(void)
2133{
2134 fwrite(command_buf.buf, 1, command_buf.len - 1, stdout);
2135 fputc('\n', stdout);
2136 fflush(stdout);
2137 skip_optional_lf();
2138}
2139
e8438420
SP
2140static void import_marks(const char *input_file)
2141{
2142 char line[512];
2143 FILE *f = fopen(input_file, "r");
2144 if (!f)
2145 die("cannot read %s: %s", input_file, strerror(errno));
2146 while (fgets(line, sizeof(line), f)) {
2147 uintmax_t mark;
2148 char *end;
2149 unsigned char sha1[20];
2150 struct object_entry *e;
2151
2152 end = strchr(line, '\n');
2153 if (line[0] != ':' || !end)
2154 die("corrupt mark line: %s", line);
2155 *end = 0;
2156 mark = strtoumax(line + 1, &end, 10);
2157 if (!mark || end == line + 1
2158 || *end != ' ' || get_sha1(end + 1, sha1))
2159 die("corrupt mark line: %s", line);
2160 e = find_object(sha1);
2161 if (!e) {
2162 enum object_type type = sha1_object_info(sha1, NULL);
2163 if (type < 0)
2164 die("object not found: %s", sha1_to_hex(sha1));
2165 e = insert_object(sha1);
2166 e->type = type;
2167 e->pack_id = MAX_PACK_ID;
a5c1780a 2168 e->offset = 1; /* just not zero! */
e8438420
SP
2169 }
2170 insert_mark(mark, e);
2171 }
2172 fclose(f);
2173}
2174
d5c57b28 2175static const char fast_import_usage[] =
63e0c8b3 2176"git-fast-import [--date-format=f] [--max-pack-size=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file]";
d5c57b28 2177
8bcce301
SP
2178int main(int argc, const char **argv)
2179{
c499d768 2180 int i, show_stats = 1;
8bcce301 2181
463acbe1 2182 git_config(git_default_config);
93e72d8d
SP
2183 alloc_objects(object_entry_alloc);
2184 strbuf_init(&command_buf);
2185 atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));
2186 branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));
2187 avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
2188 marks = pool_calloc(1, sizeof(struct mark_set));
463acbe1 2189
d5c57b28
SP
2190 for (i = 1; i < argc; i++) {
2191 const char *a = argv[i];
2192
2193 if (*a != '-' || !strcmp(a, "--"))
2194 break;
cc44c765 2195 else if (!prefixcmp(a, "--date-format=")) {
63e0c8b3
SP
2196 const char *fmt = a + 14;
2197 if (!strcmp(fmt, "raw"))
2198 whenspec = WHENSPEC_RAW;
2199 else if (!strcmp(fmt, "rfc2822"))
2200 whenspec = WHENSPEC_RFC2822;
2201 else if (!strcmp(fmt, "now"))
2202 whenspec = WHENSPEC_NOW;
2203 else
2204 die("unknown --date-format argument %s", fmt);
2205 }
cc44c765 2206 else if (!prefixcmp(a, "--max-pack-size="))
0ea9f045 2207 max_packsize = strtoumax(a + 16, NULL, 0) * 1024 * 1024;
cc44c765 2208 else if (!prefixcmp(a, "--depth="))
d5c57b28 2209 max_depth = strtoul(a + 8, NULL, 0);
cc44c765 2210 else if (!prefixcmp(a, "--active-branches="))
d5c57b28 2211 max_active_branches = strtoul(a + 18, NULL, 0);
e8438420
SP
2212 else if (!prefixcmp(a, "--import-marks="))
2213 import_marks(a + 15);
cc44c765 2214 else if (!prefixcmp(a, "--export-marks="))
a6a1a831 2215 mark_file = a + 15;
cc44c765 2216 else if (!prefixcmp(a, "--export-pack-edges=")) {
bdf1c06d
SP
2217 if (pack_edges)
2218 fclose(pack_edges);
2219 pack_edges = fopen(a + 20, "a");
2220 if (!pack_edges)
2221 die("Cannot open %s: %s", a + 20, strerror(errno));
2222 } else if (!strcmp(a, "--force"))
7073e69e 2223 force_update = 1;
c499d768
SP
2224 else if (!strcmp(a, "--quiet"))
2225 show_stats = 0;
2226 else if (!strcmp(a, "--stats"))
2227 show_stats = 1;
d5c57b28
SP
2228 else
2229 die("unknown option %s", a);
2230 }
8455e484 2231 if (i != argc)
d5c57b28 2232 usage(fast_import_usage);
d5c57b28 2233
a5c1780a 2234 prepare_packed_git();
f70b6534 2235 start_packfile();
db5e523f 2236 for (;;) {
c44cdc7e
SP
2237 read_next_command();
2238 if (command_buf.eof)
db5e523f 2239 break;
c44cdc7e
SP
2240 else if (!strcmp("blob", command_buf.buf))
2241 cmd_new_blob();
599065a3 2242 else if (!prefixcmp(command_buf.buf, "commit "))
c44cdc7e 2243 cmd_new_commit();
599065a3 2244 else if (!prefixcmp(command_buf.buf, "tag "))
72303d44 2245 cmd_new_tag();
599065a3 2246 else if (!prefixcmp(command_buf.buf, "reset "))
5fced8dc 2247 cmd_reset_branch();
7bfe6e26
SP
2248 else if (!strcmp("checkpoint", command_buf.buf))
2249 cmd_checkpoint();
ac053c02
SP
2250 else if (!prefixcmp(command_buf.buf, "progress "))
2251 cmd_progress();
c44cdc7e
SP
2252 else
2253 die("Unsupported command: %s", command_buf.buf);
db5e523f 2254 }
f70b6534 2255 end_packfile();
c44cdc7e 2256
463acbe1 2257 dump_branches();
72303d44 2258 dump_tags();
8455e484 2259 unkeep_all_packs();
a6a1a831 2260 dump_marks();
8bcce301 2261
bdf1c06d
SP
2262 if (pack_edges)
2263 fclose(pack_edges);
2264
c499d768
SP
2265 if (show_stats) {
2266 uintmax_t total_count = 0, duplicate_count = 0;
2267 for (i = 0; i < ARRAY_SIZE(object_count_by_type); i++)
2268 total_count += object_count_by_type[i];
2269 for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++)
2270 duplicate_count += duplicate_count_by_type[i];
2271
2272 fprintf(stderr, "%s statistics:\n", argv[0]);
2273 fprintf(stderr, "---------------------------------------------------------------------\n");
3efb1f34
JR
2274 fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count);
2275 fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count);
2276 fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
2277 fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
2278 fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
2279 fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
c499d768 2280 fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
3efb1f34 2281 fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
c499d768 2282 fprintf(stderr, " atoms: %10u\n", atom_cnt);
3efb1f34 2283 fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
40db58b8 2284 fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)(total_allocd/1024));
3efb1f34 2285 fprintf(stderr, " objects: %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
c499d768
SP
2286 fprintf(stderr, "---------------------------------------------------------------------\n");
2287 pack_report();
2288 fprintf(stderr, "---------------------------------------------------------------------\n");
2289 fprintf(stderr, "\n");
2290 }
db5e523f 2291
7073e69e 2292 return failure ? 1 : 0;
db5e523f 2293}