allow cloning a repository "shallowly"
[git/git.git] / fetch-pack.c
CommitLineData
def88e9a 1#include "cache.h"
fb9040cc 2#include "refs.h"
def88e9a 3#include "pkt-line.h"
49bb805e
JH
4#include "commit.h"
5#include "tag.h"
da093d37
NP
6#include "exec_cmd.h"
7#include "sideband.h"
8#include <sys/wait.h>
def88e9a 9
ad897215 10static int keep_pack;
8b3d9dc0 11static int quiet;
33b83034 12static int verbose;
dfeff66e 13static int fetch_all;
016e6ccb 14static int depth;
33b83034 15static const char fetch_pack_usage[] =
016e6ccb 16"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
def88e9a
LT
17static const char *exec = "git-upload-pack";
18
0a8944dd 19#define COMPLETE (1U << 0)
23d61f83
JS
20#define COMMON (1U << 1)
21#define COMMON_REF (1U << 2)
22#define SEEN (1U << 3)
23#define POPPED (1U << 4)
24
f061e5fd
JH
25/*
26 * After sending this many "have"s if we do not get any new ACK , we
27 * give up traversing our history.
28 */
29#define MAX_IN_VAIN 256
30
96f1e58f
DR
31static struct commit_list *rev_list;
32static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
23d61f83
JS
33
34static void rev_list_push(struct commit *commit, int mark)
35{
36 if (!(commit->object.flags & mark)) {
37 commit->object.flags |= mark;
38
39 if (!(commit->object.parsed))
40 parse_commit(commit);
41
42 insert_by_date(commit, &rev_list);
43
44 if (!(commit->object.flags & COMMON))
45 non_common_revs++;
46 }
47}
48
8da19775 49static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
23d61f83 50{
9534f40b 51 struct object *o = deref_tag(parse_object(sha1), path, 0);
23d61f83 52
1974632c 53 if (o && o->type == OBJ_COMMIT)
23d61f83
JS
54 rev_list_push((struct commit *)o, SEEN);
55
56 return 0;
57}
58
59/*
60 This function marks a rev and its ancestors as common.
61 In some cases, it is desirable to mark only the ancestors (for example
62 when only the server does not yet know that they are common).
63*/
64
65static void mark_common(struct commit *commit,
66 int ancestors_only, int dont_parse)
67{
68 if (commit != NULL && !(commit->object.flags & COMMON)) {
69 struct object *o = (struct object *)commit;
70
71 if (!ancestors_only)
72 o->flags |= COMMON;
73
74 if (!(o->flags & SEEN))
75 rev_list_push(commit, SEEN);
76 else {
77 struct commit_list *parents;
78
79 if (!ancestors_only && !(o->flags & POPPED))
80 non_common_revs--;
81 if (!o->parsed && !dont_parse)
82 parse_commit(commit);
83
84 for (parents = commit->parents;
85 parents;
86 parents = parents->next)
87 mark_common(parents->item, 0, dont_parse);
88 }
89 }
90}
91
92/*
93 Get the next rev to send, ignoring the common.
94*/
95
962554c6 96static const unsigned char* get_rev(void)
23d61f83
JS
97{
98 struct commit *commit = NULL;
99
100 while (commit == NULL) {
101 unsigned int mark;
102 struct commit_list* parents;
103
104 if (rev_list == NULL || non_common_revs == 0)
105 return NULL;
106
107 commit = rev_list->item;
108 if (!(commit->object.parsed))
109 parse_commit(commit);
110 commit->object.flags |= POPPED;
111 if (!(commit->object.flags & COMMON))
112 non_common_revs--;
113
114 parents = commit->parents;
115
116 if (commit->object.flags & COMMON) {
117 /* do not send "have", and ignore ancestors */
118 commit = NULL;
119 mark = COMMON | SEEN;
120 } else if (commit->object.flags & COMMON_REF)
121 /* send "have", and ignore ancestors */
122 mark = COMMON | SEEN;
123 else
124 /* send "have", also for its ancestors */
125 mark = SEEN;
126
127 while (parents) {
128 if (!(parents->item->object.flags & SEEN))
129 rev_list_push(parents->item, mark);
130 if (mark & COMMON)
131 mark_common(parents->item, 1, 0);
132 parents = parents->next;
133 }
134
135 rev_list = rev_list->next;
136 }
137
138 return commit->object.sha1;
139}
0a8944dd 140
33b83034
JH
141static int find_common(int fd[2], unsigned char *result_sha1,
142 struct ref *refs)
def88e9a 143{
2759cbc7 144 int fetching;
23d61f83
JS
145 int count = 0, flushes = 0, retval;
146 const unsigned char *sha1;
f061e5fd
JH
147 unsigned in_vain = 0;
148 int got_continue = 0;
23d61f83 149
cb5d709f 150 for_each_ref(rev_list_insert_ref, NULL);
def88e9a 151
2759cbc7
LT
152 fetching = 0;
153 for ( ; refs ; refs = refs->next) {
33b83034 154 unsigned char *remote = refs->old_sha1;
4dab94d5 155 struct object *o;
2759cbc7 156
0a8944dd 157 /*
4dab94d5
JH
158 * If that object is complete (i.e. it is an ancestor of a
159 * local ref), we tell them we have it but do not have to
160 * tell them about its ancestors, which they already know
161 * about.
f1f0a2be
JH
162 *
163 * We use lookup_object here because we are only
164 * interested in the case we *know* the object is
165 * reachable and we have already scanned it.
4dab94d5 166 */
f1f0a2be 167 if (((o = lookup_object(remote)) != NULL) &&
1baaae5e 168 (o->flags & COMPLETE)) {
2759cbc7 169 continue;
0a8944dd 170 }
23d61f83 171
583b7ea3 172 if (!fetching)
e4fe4b8e 173 packet_write(fd[1], "want %s%s%s%s%s%s\n",
583b7ea3
JH
174 sha1_to_hex(remote),
175 (multi_ack ? " multi_ack" : ""),
d47f3db7
JH
176 (use_sideband == 2 ? " side-band-64k" : ""),
177 (use_sideband == 1 ? " side-band" : ""),
e4fe4b8e
NP
178 (use_thin_pack ? " thin-pack" : ""),
179 " ofs-delta");
583b7ea3
JH
180 else
181 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
2759cbc7 182 fetching++;
33b83034 183 }
ed09aef0
JS
184 if (is_repository_shallow())
185 write_shallow_commits(fd[1], 1);
016e6ccb
JS
186 if (depth > 0)
187 packet_write(fd[1], "deepen %d", depth);
fb9040cc 188 packet_flush(fd[1]);
2759cbc7
LT
189 if (!fetching)
190 return 1;
0a8944dd 191
016e6ccb
JS
192 if (depth > 0) {
193 char line[1024];
194 unsigned char sha1[20];
195 int len;
196
197 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
198 if (!strncmp("shallow ", line, 8)) {
199 if (get_sha1_hex(line + 8, sha1))
200 die("invalid shallow line: %s", line);
201 /* no need making it shallow if we have it already */
202 if (lookup_object(sha1))
203 continue;
204 register_shallow(sha1);
205 }
206 }
207 }
208
23d61f83 209 flushes = 0;
75bfc6c2 210 retval = -1;
23d61f83 211 while ((sha1 = get_rev())) {
def88e9a 212 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
33b83034
JH
213 if (verbose)
214 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
f061e5fd 215 in_vain++;
def88e9a 216 if (!(31 & ++count)) {
c4c86f07
JS
217 int ack;
218
def88e9a
LT
219 packet_flush(fd[1]);
220 flushes++;
221
222 /*
223 * We keep one window "ahead" of the other side, and
224 * will wait for an ACK only on the next one
225 */
226 if (count == 32)
227 continue;
c4c86f07
JS
228
229 do {
230 ack = get_ack(fd[0], result_sha1);
231 if (verbose && ack)
232 fprintf(stderr, "got ack %d %s\n", ack,
233 sha1_to_hex(result_sha1));
234 if (ack == 1) {
235 flushes = 0;
236 multi_ack = 0;
237 retval = 0;
238 goto done;
239 } else if (ack == 2) {
240 struct commit *commit =
241 lookup_commit(result_sha1);
242 mark_common(commit, 0, 1);
243 retval = 0;
f061e5fd
JH
244 in_vain = 0;
245 got_continue = 1;
c4c86f07
JS
246 }
247 } while (ack);
def88e9a 248 flushes--;
f061e5fd
JH
249 if (got_continue && MAX_IN_VAIN < in_vain) {
250 if (verbose)
251 fprintf(stderr, "giving up\n");
252 break; /* give up */
253 }
def88e9a
LT
254 }
255 }
c4c86f07 256done:
75bfc6c2 257 packet_write(fd[1], "done\n");
33b83034
JH
258 if (verbose)
259 fprintf(stderr, "done\n");
c4c86f07
JS
260 if (retval != 0) {
261 multi_ack = 0;
23d61f83 262 flushes++;
c4c86f07
JS
263 }
264 while (flushes || multi_ack) {
265 int ack = get_ack(fd[0], result_sha1);
266 if (ack) {
33b83034 267 if (verbose)
c4c86f07
JS
268 fprintf(stderr, "got ack (%d) %s\n", ack,
269 sha1_to_hex(result_sha1));
270 if (ack == 1)
271 return 0;
272 multi_ack = 1;
273 continue;
33b83034 274 }
c4c86f07 275 flushes--;
def88e9a 276 }
75bfc6c2 277 return retval;
def88e9a
LT
278}
279
96f1e58f 280static struct commit_list *complete;
49bb805e 281
8da19775 282static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
49bb805e
JH
283{
284 struct object *o = parse_object(sha1);
285
1974632c 286 while (o && o->type == OBJ_TAG) {
f1f0a2be
JH
287 struct tag *t = (struct tag *) o;
288 if (!t->tagged)
289 break; /* broken repository */
49bb805e 290 o->flags |= COMPLETE;
f1f0a2be 291 o = parse_object(t->tagged->sha1);
49bb805e 292 }
1974632c 293 if (o && o->type == OBJ_COMMIT) {
49bb805e
JH
294 struct commit *commit = (struct commit *)o;
295 commit->object.flags |= COMPLETE;
296 insert_by_date(commit, &complete);
297 }
298 return 0;
299}
300
301static void mark_recent_complete_commits(unsigned long cutoff)
302{
303 while (complete && cutoff <= complete->item->date) {
304 if (verbose)
305 fprintf(stderr, "Marking %s as complete\n",
306 sha1_to_hex(complete->item->object.sha1));
307 pop_most_recent_commit(&complete, COMPLETE);
308 }
309}
310
1baaae5e
JS
311static void filter_refs(struct ref **refs, int nr_match, char **match)
312{
9546010b
JH
313 struct ref **return_refs;
314 struct ref *newlist = NULL;
315 struct ref **newtail = &newlist;
316 struct ref *ref, *next;
317 struct ref *fastarray[32];
318
319 if (nr_match && !fetch_all) {
320 if (ARRAY_SIZE(fastarray) < nr_match)
321 return_refs = xcalloc(nr_match, sizeof(struct ref *));
322 else {
323 return_refs = fastarray;
324 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
325 }
326 }
327 else
328 return_refs = NULL;
329
330 for (ref = *refs; ref; ref = next) {
331 next = ref->next;
332 if (!memcmp(ref->name, "refs/", 5) &&
333 check_ref_format(ref->name + 5))
334 ; /* trash */
335 else if (fetch_all) {
336 *newtail = ref;
337 ref->next = NULL;
338 newtail = &ref->next;
339 continue;
340 }
341 else {
342 int order = path_match(ref->name, nr_match, match);
343 if (order) {
344 return_refs[order-1] = ref;
345 continue; /* we will link it later */
346 }
347 }
348 free(ref);
349 }
350
351 if (!fetch_all) {
352 int i;
353 for (i = 0; i < nr_match; i++) {
354 ref = return_refs[i];
355 if (ref) {
356 *newtail = ref;
357 ref->next = NULL;
358 newtail = &ref->next;
359 }
360 }
361 if (return_refs != fastarray)
362 free(return_refs);
1baaae5e 363 }
9546010b 364 *refs = newlist;
1baaae5e
JS
365}
366
367static int everything_local(struct ref **refs, int nr_match, char **match)
2759cbc7 368{
49bb805e 369 struct ref *ref;
2759cbc7 370 int retval;
49bb805e
JH
371 unsigned long cutoff = 0;
372
373 track_object_refs = 0;
374 save_commit_buffer = 0;
375
1baaae5e 376 for (ref = *refs; ref; ref = ref->next) {
49bb805e
JH
377 struct object *o;
378
379 o = parse_object(ref->old_sha1);
380 if (!o)
381 continue;
382
383 /* We already have it -- which may mean that we were
384 * in sync with the other side at some time after
385 * that (it is OK if we guess wrong here).
386 */
1974632c 387 if (o->type == OBJ_COMMIT) {
49bb805e
JH
388 struct commit *commit = (struct commit *)o;
389 if (!cutoff || cutoff < commit->date)
390 cutoff = commit->date;
391 }
392 }
393
cb5d709f 394 for_each_ref(mark_complete, NULL);
49bb805e
JH
395 if (cutoff)
396 mark_recent_complete_commits(cutoff);
2759cbc7 397
1baaae5e
JS
398 /*
399 * Mark all complete remote refs as common refs.
400 * Don't mark them common yet; the server has to be told so first.
401 */
402 for (ref = *refs; ref; ref = ref->next) {
9534f40b
JH
403 struct object *o = deref_tag(lookup_object(ref->old_sha1),
404 NULL, 0);
1baaae5e 405
1974632c 406 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
1baaae5e
JS
407 continue;
408
409 if (!(o->flags & SEEN)) {
410 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
411
412 mark_common((struct commit *)o, 1, 1);
413 }
414 }
415
416 filter_refs(refs, nr_match, match);
417
418 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
419 const unsigned char *remote = ref->old_sha1;
2759cbc7 420 unsigned char local[20];
49bb805e 421 struct object *o;
2759cbc7 422
1baaae5e 423 o = lookup_object(remote);
49bb805e 424 if (!o || !(o->flags & COMPLETE)) {
2759cbc7
LT
425 retval = 0;
426 if (!verbose)
427 continue;
428 fprintf(stderr,
429 "want %s (%s)\n", sha1_to_hex(remote),
1baaae5e 430 ref->name);
2759cbc7
LT
431 continue;
432 }
433
e702496e 434 hashcpy(ref->new_sha1, local);
2759cbc7
LT
435 if (!verbose)
436 continue;
437 fprintf(stderr,
438 "already have %s (%s)\n", sha1_to_hex(remote),
1baaae5e 439 ref->name);
2759cbc7
LT
440 }
441 return retval;
442}
443
da093d37
NP
444static pid_t setup_sideband(int fd[2], int xd[2])
445{
446 pid_t side_pid;
447
448 if (!use_sideband) {
449 fd[0] = xd[0];
450 fd[1] = xd[1];
451 return 0;
452 }
453 /* xd[] is talking with upload-pack; subprocess reads from
454 * xd[0], spits out band#2 to stderr, and feeds us band#1
455 * through our fd[0].
456 */
457 if (pipe(fd) < 0)
458 die("fetch-pack: unable to set up pipe");
459 side_pid = fork();
460 if (side_pid < 0)
461 die("fetch-pack: unable to fork off sideband demultiplexer");
462 if (!side_pid) {
463 /* subprocess */
464 close(fd[0]);
465 if (xd[0] != xd[1])
466 close(xd[1]);
467 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
468 exit(1);
469 exit(0);
470 }
471 close(xd[0]);
472 close(fd[1]);
473 fd[1] = xd[1];
474 return side_pid;
475}
476
477static int get_pack(int xd[2], const char **argv)
478{
479 int status;
480 pid_t pid, side_pid;
481 int fd[2];
482
483 side_pid = setup_sideband(fd, xd);
484 pid = fork();
485 if (pid < 0)
486 die("fetch-pack: unable to fork off %s", argv[0]);
487 if (!pid) {
488 dup2(fd[0], 0);
489 close(fd[0]);
490 close(fd[1]);
491 execv_git_cmd(argv);
492 die("%s exec failed", argv[0]);
493 }
494 close(fd[0]);
495 close(fd[1]);
496 while (waitpid(pid, &status, 0) < 0) {
497 if (errno != EINTR)
498 die("waiting for %s: %s", argv[0], strerror(errno));
499 }
500 if (WIFEXITED(status)) {
501 int code = WEXITSTATUS(status);
502 if (code)
503 die("%s died with error code %d", argv[0], code);
504 return 0;
505 }
506 if (WIFSIGNALED(status)) {
507 int sig = WTERMSIG(status);
508 die("%s died of signal %d", argv[0], sig);
509 }
510 die("%s died of unnatural causes %d", argv[0], status);
511}
512
513static int explode_rx_pack(int xd[2])
514{
515 const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
516 return get_pack(xd, argv);
517}
518
519static int keep_rx_pack(int xd[2])
520{
521 const char *argv[6];
522 char keep_arg[256];
523 int n = 0;
524
525 argv[n++] = "index-pack";
526 argv[n++] = "--stdin";
527 if (!quiet)
528 argv[n++] = "-v";
529 if (use_thin_pack)
530 argv[n++] = "--fix-thin";
531 if (keep_pack > 1) {
532 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
533 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
534 strcpy(keep_arg + s, "localhost");
535 argv[n++] = keep_arg;
536 }
537 argv[n] = NULL;
538 return get_pack(xd, argv);
539}
540
def88e9a
LT
541static int fetch_pack(int fd[2], int nr_match, char **match)
542{
d1c133f5
LT
543 struct ref *ref;
544 unsigned char sha1[20];
545 int status;
def88e9a 546
1baaae5e 547 get_remote_heads(fd[0], &ref, 0, NULL, 0);
ed09aef0
JS
548 if (is_repository_shallow() && !server_supports("shallow"))
549 die("Server does not support shallow clients");
c4c86f07
JS
550 if (server_supports("multi_ack")) {
551 if (verbose)
552 fprintf(stderr, "Server supports multi_ack\n");
553 multi_ack = 1;
554 }
d47f3db7
JH
555 if (server_supports("side-band-64k")) {
556 if (verbose)
557 fprintf(stderr, "Server supports side-band-64k\n");
558 use_sideband = 2;
559 }
560 else if (server_supports("side-band")) {
583b7ea3
JH
561 if (verbose)
562 fprintf(stderr, "Server supports side-band\n");
563 use_sideband = 1;
564 }
d1c133f5
LT
565 if (!ref) {
566 packet_flush(fd[1]);
567 die("no matching remote head");
568 }
1baaae5e 569 if (everything_local(&ref, nr_match, match)) {
2759cbc7
LT
570 packet_flush(fd[1]);
571 goto all_done;
572 }
33b83034 573 if (find_common(fd, sha1, ref) < 0)
da093d37 574 if (keep_pack != 1)
dfeff66e
JH
575 /* When cloning, it is not unusual to have
576 * no common commit.
577 */
578 fprintf(stderr, "warning: no common commits\n");
ad897215 579
da093d37 580 status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
ad897215
JH
581 if (status)
582 die("git-fetch-pack: fetch failed.");
583
584 all_done:
585 while (ref) {
586 printf("%s %s\n",
587 sha1_to_hex(ref->old_sha1), ref->name);
588 ref = ref->next;
75bfc6c2 589 }
ad897215 590 return 0;
def88e9a
LT
591}
592
593int main(int argc, char **argv)
594{
595 int i, ret, nr_heads;
596 char *dest = NULL, **heads;
597 int fd[2];
598 pid_t pid;
016e6ccb
JS
599 struct stat st;
600 struct lock_file lock;
def88e9a 601
5a327713
JH
602 setup_git_directory();
603
def88e9a
LT
604 nr_heads = 0;
605 heads = NULL;
606 for (i = 1; i < argc; i++) {
607 char *arg = argv[i];
608
609 if (*arg == '-') {
8b3d9dc0
JH
610 if (!strncmp("--exec=", arg, 7)) {
611 exec = arg + 7;
612 continue;
613 }
2247efb4 614 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
33b83034
JH
615 quiet = 1;
616 continue;
617 }
2247efb4 618 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
da093d37 619 keep_pack++;
ad897215
JH
620 continue;
621 }
b19696c2
JH
622 if (!strcmp("--thin", arg)) {
623 use_thin_pack = 1;
624 continue;
625 }
dfeff66e
JH
626 if (!strcmp("--all", arg)) {
627 fetch_all = 1;
628 continue;
629 }
33b83034
JH
630 if (!strcmp("-v", arg)) {
631 verbose = 1;
632 continue;
633 }
016e6ccb
JS
634 if (!strncmp("--depth=", arg, 8)) {
635 depth = strtol(arg + 8, NULL, 0);
636 if (stat(git_path("shallow"), &st))
637 st.st_mtime = 0;
638 continue;
639 }
def88e9a
LT
640 usage(fetch_pack_usage);
641 }
642 dest = arg;
643 heads = argv + i + 1;
644 nr_heads = argc - i - 1;
645 break;
646 }
647 if (!dest)
648 usage(fetch_pack_usage);
016e6ccb
JS
649 if (is_repository_shallow() && depth > 0)
650 die("Deepening of a shallow repository not yet supported!");
def88e9a
LT
651 pid = git_connect(fd, dest, exec);
652 if (pid < 0)
653 return 1;
654 ret = fetch_pack(fd, nr_heads, heads);
655 close(fd[0]);
656 close(fd[1]);
8a5dbef8 657 ret |= finish_connect(pid);
9e5d2b40
JH
658
659 if (!ret && nr_heads) {
660 /* If the heads to pull were given, we should have
661 * consumed all of them by matching the remote.
662 * Otherwise, 'git-fetch remote no-such-ref' would
663 * silently succeed without issuing an error.
664 */
665 for (i = 0; i < nr_heads; i++)
666 if (heads[i] && heads[i][0]) {
667 error("no such remote ref %s", heads[i]);
668 ret = 1;
669 }
670 }
671
016e6ccb
JS
672 if (!ret && depth > 0) {
673 struct cache_time mtime;
674 char *shallow = git_path("shallow");
675 int fd;
676
677 mtime.sec = st.st_mtime;
678#ifdef USE_NSEC
679 mtime.usec = st.st_mtim.usec;
680#endif
681 if (stat(shallow, &st)) {
682 if (mtime.sec)
683 die("shallow file was removed during fetch");
684 } else if (st.st_mtime != mtime.sec
685#ifdef USE_NSEC
686 || st.st_mtim.usec != mtime.usec
687#endif
688 )
689 die("shallow file was changed during fetch");
690
691 fd = hold_lock_file_for_update(&lock, shallow, 1);
692 if (!write_shallow_commits(fd, 0)) {
693 unlink(lock.filename);
694 rollback_lock_file(&lock);
695 } else {
696 close(fd);
697 commit_lock_file(&lock);
698 }
699 }
700
8a5dbef8 701 return !!ret;
def88e9a 702}