allow deepening of a shallow repository
[git/git.git] / fetch-pack.c
CommitLineData
def88e9a 1#include "cache.h"
fb9040cc 2#include "refs.h"
def88e9a 3#include "pkt-line.h"
49bb805e
JH
4#include "commit.h"
5#include "tag.h"
da093d37
NP
6#include "exec_cmd.h"
7#include "sideband.h"
8#include <sys/wait.h>
def88e9a 9
ad897215 10static int keep_pack;
8b3d9dc0 11static int quiet;
33b83034 12static int verbose;
dfeff66e 13static int fetch_all;
016e6ccb 14static int depth;
33b83034 15static const char fetch_pack_usage[] =
016e6ccb 16"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
def88e9a
LT
17static const char *exec = "git-upload-pack";
18
0a8944dd 19#define COMPLETE (1U << 0)
23d61f83
JS
20#define COMMON (1U << 1)
21#define COMMON_REF (1U << 2)
22#define SEEN (1U << 3)
23#define POPPED (1U << 4)
24
f061e5fd
JH
25/*
26 * After sending this many "have"s if we do not get any new ACK , we
27 * give up traversing our history.
28 */
29#define MAX_IN_VAIN 256
30
96f1e58f
DR
31static struct commit_list *rev_list;
32static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
23d61f83
JS
33
34static void rev_list_push(struct commit *commit, int mark)
35{
36 if (!(commit->object.flags & mark)) {
37 commit->object.flags |= mark;
38
39 if (!(commit->object.parsed))
40 parse_commit(commit);
41
42 insert_by_date(commit, &rev_list);
43
44 if (!(commit->object.flags & COMMON))
45 non_common_revs++;
46 }
47}
48
8da19775 49static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
23d61f83 50{
9534f40b 51 struct object *o = deref_tag(parse_object(sha1), path, 0);
23d61f83 52
1974632c 53 if (o && o->type == OBJ_COMMIT)
23d61f83
JS
54 rev_list_push((struct commit *)o, SEEN);
55
56 return 0;
57}
58
59/*
60 This function marks a rev and its ancestors as common.
61 In some cases, it is desirable to mark only the ancestors (for example
62 when only the server does not yet know that they are common).
63*/
64
65static void mark_common(struct commit *commit,
66 int ancestors_only, int dont_parse)
67{
68 if (commit != NULL && !(commit->object.flags & COMMON)) {
69 struct object *o = (struct object *)commit;
70
71 if (!ancestors_only)
72 o->flags |= COMMON;
73
74 if (!(o->flags & SEEN))
75 rev_list_push(commit, SEEN);
76 else {
77 struct commit_list *parents;
78
79 if (!ancestors_only && !(o->flags & POPPED))
80 non_common_revs--;
81 if (!o->parsed && !dont_parse)
82 parse_commit(commit);
83
84 for (parents = commit->parents;
85 parents;
86 parents = parents->next)
87 mark_common(parents->item, 0, dont_parse);
88 }
89 }
90}
91
92/*
93 Get the next rev to send, ignoring the common.
94*/
95
962554c6 96static const unsigned char* get_rev(void)
23d61f83
JS
97{
98 struct commit *commit = NULL;
99
100 while (commit == NULL) {
101 unsigned int mark;
102 struct commit_list* parents;
103
104 if (rev_list == NULL || non_common_revs == 0)
105 return NULL;
106
107 commit = rev_list->item;
108 if (!(commit->object.parsed))
109 parse_commit(commit);
110 commit->object.flags |= POPPED;
111 if (!(commit->object.flags & COMMON))
112 non_common_revs--;
113
114 parents = commit->parents;
115
116 if (commit->object.flags & COMMON) {
117 /* do not send "have", and ignore ancestors */
118 commit = NULL;
119 mark = COMMON | SEEN;
120 } else if (commit->object.flags & COMMON_REF)
121 /* send "have", and ignore ancestors */
122 mark = COMMON | SEEN;
123 else
124 /* send "have", also for its ancestors */
125 mark = SEEN;
126
127 while (parents) {
128 if (!(parents->item->object.flags & SEEN))
129 rev_list_push(parents->item, mark);
130 if (mark & COMMON)
131 mark_common(parents->item, 1, 0);
132 parents = parents->next;
133 }
134
135 rev_list = rev_list->next;
136 }
137
138 return commit->object.sha1;
139}
0a8944dd 140
33b83034
JH
141static int find_common(int fd[2], unsigned char *result_sha1,
142 struct ref *refs)
def88e9a 143{
2759cbc7 144 int fetching;
23d61f83
JS
145 int count = 0, flushes = 0, retval;
146 const unsigned char *sha1;
f061e5fd
JH
147 unsigned in_vain = 0;
148 int got_continue = 0;
23d61f83 149
cb5d709f 150 for_each_ref(rev_list_insert_ref, NULL);
def88e9a 151
2759cbc7
LT
152 fetching = 0;
153 for ( ; refs ; refs = refs->next) {
33b83034 154 unsigned char *remote = refs->old_sha1;
4dab94d5 155 struct object *o;
2759cbc7 156
0a8944dd 157 /*
4dab94d5
JH
158 * If that object is complete (i.e. it is an ancestor of a
159 * local ref), we tell them we have it but do not have to
160 * tell them about its ancestors, which they already know
161 * about.
f1f0a2be
JH
162 *
163 * We use lookup_object here because we are only
164 * interested in the case we *know* the object is
165 * reachable and we have already scanned it.
4dab94d5 166 */
f1f0a2be 167 if (((o = lookup_object(remote)) != NULL) &&
1baaae5e 168 (o->flags & COMPLETE)) {
2759cbc7 169 continue;
0a8944dd 170 }
23d61f83 171
583b7ea3 172 if (!fetching)
e4fe4b8e 173 packet_write(fd[1], "want %s%s%s%s%s%s\n",
583b7ea3
JH
174 sha1_to_hex(remote),
175 (multi_ack ? " multi_ack" : ""),
d47f3db7
JH
176 (use_sideband == 2 ? " side-band-64k" : ""),
177 (use_sideband == 1 ? " side-band" : ""),
e4fe4b8e
NP
178 (use_thin_pack ? " thin-pack" : ""),
179 " ofs-delta");
583b7ea3
JH
180 else
181 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
2759cbc7 182 fetching++;
33b83034 183 }
ed09aef0
JS
184 if (is_repository_shallow())
185 write_shallow_commits(fd[1], 1);
016e6ccb
JS
186 if (depth > 0)
187 packet_write(fd[1], "deepen %d", depth);
fb9040cc 188 packet_flush(fd[1]);
2759cbc7
LT
189 if (!fetching)
190 return 1;
0a8944dd 191
016e6ccb
JS
192 if (depth > 0) {
193 char line[1024];
194 unsigned char sha1[20];
195 int len;
196
197 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
198 if (!strncmp("shallow ", line, 8)) {
199 if (get_sha1_hex(line + 8, sha1))
200 die("invalid shallow line: %s", line);
201 /* no need making it shallow if we have it already */
202 if (lookup_object(sha1))
203 continue;
204 register_shallow(sha1);
f53514bc
JS
205 } else if (!strncmp("unshallow ", line, 10)) {
206 if (get_sha1_hex(line + 10, sha1))
207 die("invalid unshallow line: %s", line);
208 if (!lookup_object(sha1))
209 die("object not found: %s", line);
210 /* make sure that it is parsed as shallow */
211 parse_object(sha1);
212 if (unregister_shallow(sha1))
213 die("no shallow found: %s", line);
214 } else
215 die("expected shallow/unshallow, got %s", line);
016e6ccb
JS
216 }
217 }
218
23d61f83 219 flushes = 0;
75bfc6c2 220 retval = -1;
23d61f83 221 while ((sha1 = get_rev())) {
def88e9a 222 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
33b83034
JH
223 if (verbose)
224 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
f061e5fd 225 in_vain++;
def88e9a 226 if (!(31 & ++count)) {
c4c86f07
JS
227 int ack;
228
def88e9a
LT
229 packet_flush(fd[1]);
230 flushes++;
231
232 /*
233 * We keep one window "ahead" of the other side, and
234 * will wait for an ACK only on the next one
235 */
236 if (count == 32)
237 continue;
c4c86f07
JS
238
239 do {
240 ack = get_ack(fd[0], result_sha1);
241 if (verbose && ack)
242 fprintf(stderr, "got ack %d %s\n", ack,
243 sha1_to_hex(result_sha1));
244 if (ack == 1) {
245 flushes = 0;
246 multi_ack = 0;
247 retval = 0;
248 goto done;
249 } else if (ack == 2) {
250 struct commit *commit =
251 lookup_commit(result_sha1);
252 mark_common(commit, 0, 1);
253 retval = 0;
f061e5fd
JH
254 in_vain = 0;
255 got_continue = 1;
c4c86f07
JS
256 }
257 } while (ack);
def88e9a 258 flushes--;
f061e5fd
JH
259 if (got_continue && MAX_IN_VAIN < in_vain) {
260 if (verbose)
261 fprintf(stderr, "giving up\n");
262 break; /* give up */
263 }
def88e9a
LT
264 }
265 }
c4c86f07 266done:
75bfc6c2 267 packet_write(fd[1], "done\n");
33b83034
JH
268 if (verbose)
269 fprintf(stderr, "done\n");
c4c86f07
JS
270 if (retval != 0) {
271 multi_ack = 0;
23d61f83 272 flushes++;
c4c86f07
JS
273 }
274 while (flushes || multi_ack) {
275 int ack = get_ack(fd[0], result_sha1);
276 if (ack) {
33b83034 277 if (verbose)
c4c86f07
JS
278 fprintf(stderr, "got ack (%d) %s\n", ack,
279 sha1_to_hex(result_sha1));
280 if (ack == 1)
281 return 0;
282 multi_ack = 1;
283 continue;
33b83034 284 }
c4c86f07 285 flushes--;
def88e9a 286 }
75bfc6c2 287 return retval;
def88e9a
LT
288}
289
96f1e58f 290static struct commit_list *complete;
49bb805e 291
8da19775 292static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
49bb805e
JH
293{
294 struct object *o = parse_object(sha1);
295
1974632c 296 while (o && o->type == OBJ_TAG) {
f1f0a2be
JH
297 struct tag *t = (struct tag *) o;
298 if (!t->tagged)
299 break; /* broken repository */
49bb805e 300 o->flags |= COMPLETE;
f1f0a2be 301 o = parse_object(t->tagged->sha1);
49bb805e 302 }
1974632c 303 if (o && o->type == OBJ_COMMIT) {
49bb805e
JH
304 struct commit *commit = (struct commit *)o;
305 commit->object.flags |= COMPLETE;
306 insert_by_date(commit, &complete);
307 }
308 return 0;
309}
310
311static void mark_recent_complete_commits(unsigned long cutoff)
312{
313 while (complete && cutoff <= complete->item->date) {
314 if (verbose)
315 fprintf(stderr, "Marking %s as complete\n",
316 sha1_to_hex(complete->item->object.sha1));
317 pop_most_recent_commit(&complete, COMPLETE);
318 }
319}
320
1baaae5e
JS
321static void filter_refs(struct ref **refs, int nr_match, char **match)
322{
9546010b
JH
323 struct ref **return_refs;
324 struct ref *newlist = NULL;
325 struct ref **newtail = &newlist;
326 struct ref *ref, *next;
327 struct ref *fastarray[32];
328
329 if (nr_match && !fetch_all) {
330 if (ARRAY_SIZE(fastarray) < nr_match)
331 return_refs = xcalloc(nr_match, sizeof(struct ref *));
332 else {
333 return_refs = fastarray;
334 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
335 }
336 }
337 else
338 return_refs = NULL;
339
340 for (ref = *refs; ref; ref = next) {
341 next = ref->next;
342 if (!memcmp(ref->name, "refs/", 5) &&
343 check_ref_format(ref->name + 5))
344 ; /* trash */
345 else if (fetch_all) {
346 *newtail = ref;
347 ref->next = NULL;
348 newtail = &ref->next;
349 continue;
350 }
351 else {
352 int order = path_match(ref->name, nr_match, match);
353 if (order) {
354 return_refs[order-1] = ref;
355 continue; /* we will link it later */
356 }
357 }
358 free(ref);
359 }
360
361 if (!fetch_all) {
362 int i;
363 for (i = 0; i < nr_match; i++) {
364 ref = return_refs[i];
365 if (ref) {
366 *newtail = ref;
367 ref->next = NULL;
368 newtail = &ref->next;
369 }
370 }
371 if (return_refs != fastarray)
372 free(return_refs);
1baaae5e 373 }
9546010b 374 *refs = newlist;
1baaae5e
JS
375}
376
377static int everything_local(struct ref **refs, int nr_match, char **match)
2759cbc7 378{
49bb805e 379 struct ref *ref;
2759cbc7 380 int retval;
49bb805e
JH
381 unsigned long cutoff = 0;
382
383 track_object_refs = 0;
384 save_commit_buffer = 0;
385
1baaae5e 386 for (ref = *refs; ref; ref = ref->next) {
49bb805e
JH
387 struct object *o;
388
389 o = parse_object(ref->old_sha1);
390 if (!o)
391 continue;
392
393 /* We already have it -- which may mean that we were
394 * in sync with the other side at some time after
395 * that (it is OK if we guess wrong here).
396 */
1974632c 397 if (o->type == OBJ_COMMIT) {
49bb805e
JH
398 struct commit *commit = (struct commit *)o;
399 if (!cutoff || cutoff < commit->date)
400 cutoff = commit->date;
401 }
402 }
403
f53514bc
JS
404 if (!depth) {
405 for_each_ref(mark_complete, NULL);
406 if (cutoff)
407 mark_recent_complete_commits(cutoff);
408 }
2759cbc7 409
1baaae5e
JS
410 /*
411 * Mark all complete remote refs as common refs.
412 * Don't mark them common yet; the server has to be told so first.
413 */
414 for (ref = *refs; ref; ref = ref->next) {
9534f40b
JH
415 struct object *o = deref_tag(lookup_object(ref->old_sha1),
416 NULL, 0);
1baaae5e 417
1974632c 418 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
1baaae5e
JS
419 continue;
420
421 if (!(o->flags & SEEN)) {
422 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
423
424 mark_common((struct commit *)o, 1, 1);
425 }
426 }
427
428 filter_refs(refs, nr_match, match);
429
430 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
431 const unsigned char *remote = ref->old_sha1;
2759cbc7 432 unsigned char local[20];
49bb805e 433 struct object *o;
2759cbc7 434
1baaae5e 435 o = lookup_object(remote);
49bb805e 436 if (!o || !(o->flags & COMPLETE)) {
2759cbc7
LT
437 retval = 0;
438 if (!verbose)
439 continue;
440 fprintf(stderr,
441 "want %s (%s)\n", sha1_to_hex(remote),
1baaae5e 442 ref->name);
2759cbc7
LT
443 continue;
444 }
445
e702496e 446 hashcpy(ref->new_sha1, local);
2759cbc7
LT
447 if (!verbose)
448 continue;
449 fprintf(stderr,
450 "already have %s (%s)\n", sha1_to_hex(remote),
1baaae5e 451 ref->name);
2759cbc7
LT
452 }
453 return retval;
454}
455
da093d37
NP
456static pid_t setup_sideband(int fd[2], int xd[2])
457{
458 pid_t side_pid;
459
460 if (!use_sideband) {
461 fd[0] = xd[0];
462 fd[1] = xd[1];
463 return 0;
464 }
465 /* xd[] is talking with upload-pack; subprocess reads from
466 * xd[0], spits out band#2 to stderr, and feeds us band#1
467 * through our fd[0].
468 */
469 if (pipe(fd) < 0)
470 die("fetch-pack: unable to set up pipe");
471 side_pid = fork();
472 if (side_pid < 0)
473 die("fetch-pack: unable to fork off sideband demultiplexer");
474 if (!side_pid) {
475 /* subprocess */
476 close(fd[0]);
477 if (xd[0] != xd[1])
478 close(xd[1]);
479 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
480 exit(1);
481 exit(0);
482 }
483 close(xd[0]);
484 close(fd[1]);
485 fd[1] = xd[1];
486 return side_pid;
487}
488
489static int get_pack(int xd[2], const char **argv)
490{
491 int status;
492 pid_t pid, side_pid;
493 int fd[2];
494
495 side_pid = setup_sideband(fd, xd);
496 pid = fork();
497 if (pid < 0)
498 die("fetch-pack: unable to fork off %s", argv[0]);
499 if (!pid) {
500 dup2(fd[0], 0);
501 close(fd[0]);
502 close(fd[1]);
503 execv_git_cmd(argv);
504 die("%s exec failed", argv[0]);
505 }
506 close(fd[0]);
507 close(fd[1]);
508 while (waitpid(pid, &status, 0) < 0) {
509 if (errno != EINTR)
510 die("waiting for %s: %s", argv[0], strerror(errno));
511 }
512 if (WIFEXITED(status)) {
513 int code = WEXITSTATUS(status);
514 if (code)
515 die("%s died with error code %d", argv[0], code);
516 return 0;
517 }
518 if (WIFSIGNALED(status)) {
519 int sig = WTERMSIG(status);
520 die("%s died of signal %d", argv[0], sig);
521 }
522 die("%s died of unnatural causes %d", argv[0], status);
523}
524
525static int explode_rx_pack(int xd[2])
526{
527 const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
528 return get_pack(xd, argv);
529}
530
531static int keep_rx_pack(int xd[2])
532{
533 const char *argv[6];
534 char keep_arg[256];
535 int n = 0;
536
537 argv[n++] = "index-pack";
538 argv[n++] = "--stdin";
539 if (!quiet)
540 argv[n++] = "-v";
541 if (use_thin_pack)
542 argv[n++] = "--fix-thin";
543 if (keep_pack > 1) {
544 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
545 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
546 strcpy(keep_arg + s, "localhost");
547 argv[n++] = keep_arg;
548 }
549 argv[n] = NULL;
550 return get_pack(xd, argv);
551}
552
def88e9a
LT
553static int fetch_pack(int fd[2], int nr_match, char **match)
554{
d1c133f5
LT
555 struct ref *ref;
556 unsigned char sha1[20];
557 int status;
def88e9a 558
1baaae5e 559 get_remote_heads(fd[0], &ref, 0, NULL, 0);
ed09aef0
JS
560 if (is_repository_shallow() && !server_supports("shallow"))
561 die("Server does not support shallow clients");
c4c86f07
JS
562 if (server_supports("multi_ack")) {
563 if (verbose)
564 fprintf(stderr, "Server supports multi_ack\n");
565 multi_ack = 1;
566 }
d47f3db7
JH
567 if (server_supports("side-band-64k")) {
568 if (verbose)
569 fprintf(stderr, "Server supports side-band-64k\n");
570 use_sideband = 2;
571 }
572 else if (server_supports("side-band")) {
583b7ea3
JH
573 if (verbose)
574 fprintf(stderr, "Server supports side-band\n");
575 use_sideband = 1;
576 }
d1c133f5
LT
577 if (!ref) {
578 packet_flush(fd[1]);
579 die("no matching remote head");
580 }
1baaae5e 581 if (everything_local(&ref, nr_match, match)) {
2759cbc7
LT
582 packet_flush(fd[1]);
583 goto all_done;
584 }
33b83034 585 if (find_common(fd, sha1, ref) < 0)
da093d37 586 if (keep_pack != 1)
dfeff66e
JH
587 /* When cloning, it is not unusual to have
588 * no common commit.
589 */
590 fprintf(stderr, "warning: no common commits\n");
ad897215 591
da093d37 592 status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
ad897215
JH
593 if (status)
594 die("git-fetch-pack: fetch failed.");
595
596 all_done:
597 while (ref) {
598 printf("%s %s\n",
599 sha1_to_hex(ref->old_sha1), ref->name);
600 ref = ref->next;
75bfc6c2 601 }
ad897215 602 return 0;
def88e9a
LT
603}
604
605int main(int argc, char **argv)
606{
607 int i, ret, nr_heads;
608 char *dest = NULL, **heads;
609 int fd[2];
610 pid_t pid;
016e6ccb
JS
611 struct stat st;
612 struct lock_file lock;
def88e9a 613
5a327713
JH
614 setup_git_directory();
615
def88e9a
LT
616 nr_heads = 0;
617 heads = NULL;
618 for (i = 1; i < argc; i++) {
619 char *arg = argv[i];
620
621 if (*arg == '-') {
8b3d9dc0
JH
622 if (!strncmp("--exec=", arg, 7)) {
623 exec = arg + 7;
624 continue;
625 }
2247efb4 626 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
33b83034
JH
627 quiet = 1;
628 continue;
629 }
2247efb4 630 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
da093d37 631 keep_pack++;
ad897215
JH
632 continue;
633 }
b19696c2
JH
634 if (!strcmp("--thin", arg)) {
635 use_thin_pack = 1;
636 continue;
637 }
dfeff66e
JH
638 if (!strcmp("--all", arg)) {
639 fetch_all = 1;
640 continue;
641 }
33b83034
JH
642 if (!strcmp("-v", arg)) {
643 verbose = 1;
644 continue;
645 }
016e6ccb
JS
646 if (!strncmp("--depth=", arg, 8)) {
647 depth = strtol(arg + 8, NULL, 0);
648 if (stat(git_path("shallow"), &st))
649 st.st_mtime = 0;
650 continue;
651 }
def88e9a
LT
652 usage(fetch_pack_usage);
653 }
654 dest = arg;
655 heads = argv + i + 1;
656 nr_heads = argc - i - 1;
657 break;
658 }
659 if (!dest)
660 usage(fetch_pack_usage);
661 pid = git_connect(fd, dest, exec);
662 if (pid < 0)
663 return 1;
664 ret = fetch_pack(fd, nr_heads, heads);
665 close(fd[0]);
666 close(fd[1]);
8a5dbef8 667 ret |= finish_connect(pid);
9e5d2b40
JH
668
669 if (!ret && nr_heads) {
670 /* If the heads to pull were given, we should have
671 * consumed all of them by matching the remote.
672 * Otherwise, 'git-fetch remote no-such-ref' would
673 * silently succeed without issuing an error.
674 */
675 for (i = 0; i < nr_heads; i++)
676 if (heads[i] && heads[i][0]) {
677 error("no such remote ref %s", heads[i]);
678 ret = 1;
679 }
680 }
681
016e6ccb
JS
682 if (!ret && depth > 0) {
683 struct cache_time mtime;
684 char *shallow = git_path("shallow");
685 int fd;
686
687 mtime.sec = st.st_mtime;
688#ifdef USE_NSEC
689 mtime.usec = st.st_mtim.usec;
690#endif
691 if (stat(shallow, &st)) {
692 if (mtime.sec)
693 die("shallow file was removed during fetch");
694 } else if (st.st_mtime != mtime.sec
695#ifdef USE_NSEC
696 || st.st_mtim.usec != mtime.usec
697#endif
698 )
699 die("shallow file was changed during fetch");
700
701 fd = hold_lock_file_for_update(&lock, shallow, 1);
702 if (!write_shallow_commits(fd, 0)) {
703 unlink(lock.filename);
704 rollback_lock_file(&lock);
705 } else {
706 close(fd);
707 commit_lock_file(&lock);
708 }
709 }
710
8a5dbef8 711 return !!ret;
def88e9a 712}