support fetching into a shallow repository
[git/git.git] / fetch-pack.c
CommitLineData
def88e9a 1#include "cache.h"
fb9040cc 2#include "refs.h"
def88e9a 3#include "pkt-line.h"
49bb805e
JH
4#include "commit.h"
5#include "tag.h"
da093d37
NP
6#include "exec_cmd.h"
7#include "sideband.h"
8#include <sys/wait.h>
def88e9a 9
ad897215 10static int keep_pack;
8b3d9dc0 11static int quiet;
33b83034 12static int verbose;
dfeff66e 13static int fetch_all;
33b83034 14static const char fetch_pack_usage[] =
dfeff66e 15"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory <refs>...";
def88e9a
LT
16static const char *exec = "git-upload-pack";
17
0a8944dd 18#define COMPLETE (1U << 0)
23d61f83
JS
19#define COMMON (1U << 1)
20#define COMMON_REF (1U << 2)
21#define SEEN (1U << 3)
22#define POPPED (1U << 4)
23
f061e5fd
JH
24/*
25 * After sending this many "have"s if we do not get any new ACK , we
26 * give up traversing our history.
27 */
28#define MAX_IN_VAIN 256
29
96f1e58f
DR
30static struct commit_list *rev_list;
31static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
23d61f83
JS
32
33static void rev_list_push(struct commit *commit, int mark)
34{
35 if (!(commit->object.flags & mark)) {
36 commit->object.flags |= mark;
37
38 if (!(commit->object.parsed))
39 parse_commit(commit);
40
41 insert_by_date(commit, &rev_list);
42
43 if (!(commit->object.flags & COMMON))
44 non_common_revs++;
45 }
46}
47
8da19775 48static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
23d61f83 49{
9534f40b 50 struct object *o = deref_tag(parse_object(sha1), path, 0);
23d61f83 51
1974632c 52 if (o && o->type == OBJ_COMMIT)
23d61f83
JS
53 rev_list_push((struct commit *)o, SEEN);
54
55 return 0;
56}
57
58/*
59 This function marks a rev and its ancestors as common.
60 In some cases, it is desirable to mark only the ancestors (for example
61 when only the server does not yet know that they are common).
62*/
63
64static void mark_common(struct commit *commit,
65 int ancestors_only, int dont_parse)
66{
67 if (commit != NULL && !(commit->object.flags & COMMON)) {
68 struct object *o = (struct object *)commit;
69
70 if (!ancestors_only)
71 o->flags |= COMMON;
72
73 if (!(o->flags & SEEN))
74 rev_list_push(commit, SEEN);
75 else {
76 struct commit_list *parents;
77
78 if (!ancestors_only && !(o->flags & POPPED))
79 non_common_revs--;
80 if (!o->parsed && !dont_parse)
81 parse_commit(commit);
82
83 for (parents = commit->parents;
84 parents;
85 parents = parents->next)
86 mark_common(parents->item, 0, dont_parse);
87 }
88 }
89}
90
91/*
92 Get the next rev to send, ignoring the common.
93*/
94
962554c6 95static const unsigned char* get_rev(void)
23d61f83
JS
96{
97 struct commit *commit = NULL;
98
99 while (commit == NULL) {
100 unsigned int mark;
101 struct commit_list* parents;
102
103 if (rev_list == NULL || non_common_revs == 0)
104 return NULL;
105
106 commit = rev_list->item;
107 if (!(commit->object.parsed))
108 parse_commit(commit);
109 commit->object.flags |= POPPED;
110 if (!(commit->object.flags & COMMON))
111 non_common_revs--;
112
113 parents = commit->parents;
114
115 if (commit->object.flags & COMMON) {
116 /* do not send "have", and ignore ancestors */
117 commit = NULL;
118 mark = COMMON | SEEN;
119 } else if (commit->object.flags & COMMON_REF)
120 /* send "have", and ignore ancestors */
121 mark = COMMON | SEEN;
122 else
123 /* send "have", also for its ancestors */
124 mark = SEEN;
125
126 while (parents) {
127 if (!(parents->item->object.flags & SEEN))
128 rev_list_push(parents->item, mark);
129 if (mark & COMMON)
130 mark_common(parents->item, 1, 0);
131 parents = parents->next;
132 }
133
134 rev_list = rev_list->next;
135 }
136
137 return commit->object.sha1;
138}
0a8944dd 139
33b83034
JH
140static int find_common(int fd[2], unsigned char *result_sha1,
141 struct ref *refs)
def88e9a 142{
2759cbc7 143 int fetching;
23d61f83
JS
144 int count = 0, flushes = 0, retval;
145 const unsigned char *sha1;
f061e5fd
JH
146 unsigned in_vain = 0;
147 int got_continue = 0;
23d61f83 148
cb5d709f 149 for_each_ref(rev_list_insert_ref, NULL);
def88e9a 150
2759cbc7
LT
151 fetching = 0;
152 for ( ; refs ; refs = refs->next) {
33b83034 153 unsigned char *remote = refs->old_sha1;
4dab94d5 154 struct object *o;
2759cbc7 155
0a8944dd 156 /*
4dab94d5
JH
157 * If that object is complete (i.e. it is an ancestor of a
158 * local ref), we tell them we have it but do not have to
159 * tell them about its ancestors, which they already know
160 * about.
f1f0a2be
JH
161 *
162 * We use lookup_object here because we are only
163 * interested in the case we *know* the object is
164 * reachable and we have already scanned it.
4dab94d5 165 */
f1f0a2be 166 if (((o = lookup_object(remote)) != NULL) &&
1baaae5e 167 (o->flags & COMPLETE)) {
2759cbc7 168 continue;
0a8944dd 169 }
23d61f83 170
583b7ea3 171 if (!fetching)
e4fe4b8e 172 packet_write(fd[1], "want %s%s%s%s%s%s\n",
583b7ea3
JH
173 sha1_to_hex(remote),
174 (multi_ack ? " multi_ack" : ""),
d47f3db7
JH
175 (use_sideband == 2 ? " side-band-64k" : ""),
176 (use_sideband == 1 ? " side-band" : ""),
e4fe4b8e
NP
177 (use_thin_pack ? " thin-pack" : ""),
178 " ofs-delta");
583b7ea3
JH
179 else
180 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
2759cbc7 181 fetching++;
33b83034 182 }
ed09aef0
JS
183 if (is_repository_shallow())
184 write_shallow_commits(fd[1], 1);
fb9040cc 185 packet_flush(fd[1]);
2759cbc7
LT
186 if (!fetching)
187 return 1;
0a8944dd 188
23d61f83 189 flushes = 0;
75bfc6c2 190 retval = -1;
23d61f83 191 while ((sha1 = get_rev())) {
def88e9a 192 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
33b83034
JH
193 if (verbose)
194 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
f061e5fd 195 in_vain++;
def88e9a 196 if (!(31 & ++count)) {
c4c86f07
JS
197 int ack;
198
def88e9a
LT
199 packet_flush(fd[1]);
200 flushes++;
201
202 /*
203 * We keep one window "ahead" of the other side, and
204 * will wait for an ACK only on the next one
205 */
206 if (count == 32)
207 continue;
c4c86f07
JS
208
209 do {
210 ack = get_ack(fd[0], result_sha1);
211 if (verbose && ack)
212 fprintf(stderr, "got ack %d %s\n", ack,
213 sha1_to_hex(result_sha1));
214 if (ack == 1) {
215 flushes = 0;
216 multi_ack = 0;
217 retval = 0;
218 goto done;
219 } else if (ack == 2) {
220 struct commit *commit =
221 lookup_commit(result_sha1);
222 mark_common(commit, 0, 1);
223 retval = 0;
f061e5fd
JH
224 in_vain = 0;
225 got_continue = 1;
c4c86f07
JS
226 }
227 } while (ack);
def88e9a 228 flushes--;
f061e5fd
JH
229 if (got_continue && MAX_IN_VAIN < in_vain) {
230 if (verbose)
231 fprintf(stderr, "giving up\n");
232 break; /* give up */
233 }
def88e9a
LT
234 }
235 }
c4c86f07 236done:
75bfc6c2 237 packet_write(fd[1], "done\n");
33b83034
JH
238 if (verbose)
239 fprintf(stderr, "done\n");
c4c86f07
JS
240 if (retval != 0) {
241 multi_ack = 0;
23d61f83 242 flushes++;
c4c86f07
JS
243 }
244 while (flushes || multi_ack) {
245 int ack = get_ack(fd[0], result_sha1);
246 if (ack) {
33b83034 247 if (verbose)
c4c86f07
JS
248 fprintf(stderr, "got ack (%d) %s\n", ack,
249 sha1_to_hex(result_sha1));
250 if (ack == 1)
251 return 0;
252 multi_ack = 1;
253 continue;
33b83034 254 }
c4c86f07 255 flushes--;
def88e9a 256 }
75bfc6c2 257 return retval;
def88e9a
LT
258}
259
96f1e58f 260static struct commit_list *complete;
49bb805e 261
8da19775 262static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
49bb805e
JH
263{
264 struct object *o = parse_object(sha1);
265
1974632c 266 while (o && o->type == OBJ_TAG) {
f1f0a2be
JH
267 struct tag *t = (struct tag *) o;
268 if (!t->tagged)
269 break; /* broken repository */
49bb805e 270 o->flags |= COMPLETE;
f1f0a2be 271 o = parse_object(t->tagged->sha1);
49bb805e 272 }
1974632c 273 if (o && o->type == OBJ_COMMIT) {
49bb805e
JH
274 struct commit *commit = (struct commit *)o;
275 commit->object.flags |= COMPLETE;
276 insert_by_date(commit, &complete);
277 }
278 return 0;
279}
280
281static void mark_recent_complete_commits(unsigned long cutoff)
282{
283 while (complete && cutoff <= complete->item->date) {
284 if (verbose)
285 fprintf(stderr, "Marking %s as complete\n",
286 sha1_to_hex(complete->item->object.sha1));
287 pop_most_recent_commit(&complete, COMPLETE);
288 }
289}
290
1baaae5e
JS
291static void filter_refs(struct ref **refs, int nr_match, char **match)
292{
9546010b
JH
293 struct ref **return_refs;
294 struct ref *newlist = NULL;
295 struct ref **newtail = &newlist;
296 struct ref *ref, *next;
297 struct ref *fastarray[32];
298
299 if (nr_match && !fetch_all) {
300 if (ARRAY_SIZE(fastarray) < nr_match)
301 return_refs = xcalloc(nr_match, sizeof(struct ref *));
302 else {
303 return_refs = fastarray;
304 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
305 }
306 }
307 else
308 return_refs = NULL;
309
310 for (ref = *refs; ref; ref = next) {
311 next = ref->next;
312 if (!memcmp(ref->name, "refs/", 5) &&
313 check_ref_format(ref->name + 5))
314 ; /* trash */
315 else if (fetch_all) {
316 *newtail = ref;
317 ref->next = NULL;
318 newtail = &ref->next;
319 continue;
320 }
321 else {
322 int order = path_match(ref->name, nr_match, match);
323 if (order) {
324 return_refs[order-1] = ref;
325 continue; /* we will link it later */
326 }
327 }
328 free(ref);
329 }
330
331 if (!fetch_all) {
332 int i;
333 for (i = 0; i < nr_match; i++) {
334 ref = return_refs[i];
335 if (ref) {
336 *newtail = ref;
337 ref->next = NULL;
338 newtail = &ref->next;
339 }
340 }
341 if (return_refs != fastarray)
342 free(return_refs);
1baaae5e 343 }
9546010b 344 *refs = newlist;
1baaae5e
JS
345}
346
347static int everything_local(struct ref **refs, int nr_match, char **match)
2759cbc7 348{
49bb805e 349 struct ref *ref;
2759cbc7 350 int retval;
49bb805e
JH
351 unsigned long cutoff = 0;
352
353 track_object_refs = 0;
354 save_commit_buffer = 0;
355
1baaae5e 356 for (ref = *refs; ref; ref = ref->next) {
49bb805e
JH
357 struct object *o;
358
359 o = parse_object(ref->old_sha1);
360 if (!o)
361 continue;
362
363 /* We already have it -- which may mean that we were
364 * in sync with the other side at some time after
365 * that (it is OK if we guess wrong here).
366 */
1974632c 367 if (o->type == OBJ_COMMIT) {
49bb805e
JH
368 struct commit *commit = (struct commit *)o;
369 if (!cutoff || cutoff < commit->date)
370 cutoff = commit->date;
371 }
372 }
373
cb5d709f 374 for_each_ref(mark_complete, NULL);
49bb805e
JH
375 if (cutoff)
376 mark_recent_complete_commits(cutoff);
2759cbc7 377
1baaae5e
JS
378 /*
379 * Mark all complete remote refs as common refs.
380 * Don't mark them common yet; the server has to be told so first.
381 */
382 for (ref = *refs; ref; ref = ref->next) {
9534f40b
JH
383 struct object *o = deref_tag(lookup_object(ref->old_sha1),
384 NULL, 0);
1baaae5e 385
1974632c 386 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
1baaae5e
JS
387 continue;
388
389 if (!(o->flags & SEEN)) {
390 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
391
392 mark_common((struct commit *)o, 1, 1);
393 }
394 }
395
396 filter_refs(refs, nr_match, match);
397
398 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
399 const unsigned char *remote = ref->old_sha1;
2759cbc7 400 unsigned char local[20];
49bb805e 401 struct object *o;
2759cbc7 402
1baaae5e 403 o = lookup_object(remote);
49bb805e 404 if (!o || !(o->flags & COMPLETE)) {
2759cbc7
LT
405 retval = 0;
406 if (!verbose)
407 continue;
408 fprintf(stderr,
409 "want %s (%s)\n", sha1_to_hex(remote),
1baaae5e 410 ref->name);
2759cbc7
LT
411 continue;
412 }
413
e702496e 414 hashcpy(ref->new_sha1, local);
2759cbc7
LT
415 if (!verbose)
416 continue;
417 fprintf(stderr,
418 "already have %s (%s)\n", sha1_to_hex(remote),
1baaae5e 419 ref->name);
2759cbc7
LT
420 }
421 return retval;
422}
423
da093d37
NP
424static pid_t setup_sideband(int fd[2], int xd[2])
425{
426 pid_t side_pid;
427
428 if (!use_sideband) {
429 fd[0] = xd[0];
430 fd[1] = xd[1];
431 return 0;
432 }
433 /* xd[] is talking with upload-pack; subprocess reads from
434 * xd[0], spits out band#2 to stderr, and feeds us band#1
435 * through our fd[0].
436 */
437 if (pipe(fd) < 0)
438 die("fetch-pack: unable to set up pipe");
439 side_pid = fork();
440 if (side_pid < 0)
441 die("fetch-pack: unable to fork off sideband demultiplexer");
442 if (!side_pid) {
443 /* subprocess */
444 close(fd[0]);
445 if (xd[0] != xd[1])
446 close(xd[1]);
447 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
448 exit(1);
449 exit(0);
450 }
451 close(xd[0]);
452 close(fd[1]);
453 fd[1] = xd[1];
454 return side_pid;
455}
456
457static int get_pack(int xd[2], const char **argv)
458{
459 int status;
460 pid_t pid, side_pid;
461 int fd[2];
462
463 side_pid = setup_sideband(fd, xd);
464 pid = fork();
465 if (pid < 0)
466 die("fetch-pack: unable to fork off %s", argv[0]);
467 if (!pid) {
468 dup2(fd[0], 0);
469 close(fd[0]);
470 close(fd[1]);
471 execv_git_cmd(argv);
472 die("%s exec failed", argv[0]);
473 }
474 close(fd[0]);
475 close(fd[1]);
476 while (waitpid(pid, &status, 0) < 0) {
477 if (errno != EINTR)
478 die("waiting for %s: %s", argv[0], strerror(errno));
479 }
480 if (WIFEXITED(status)) {
481 int code = WEXITSTATUS(status);
482 if (code)
483 die("%s died with error code %d", argv[0], code);
484 return 0;
485 }
486 if (WIFSIGNALED(status)) {
487 int sig = WTERMSIG(status);
488 die("%s died of signal %d", argv[0], sig);
489 }
490 die("%s died of unnatural causes %d", argv[0], status);
491}
492
493static int explode_rx_pack(int xd[2])
494{
495 const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
496 return get_pack(xd, argv);
497}
498
499static int keep_rx_pack(int xd[2])
500{
501 const char *argv[6];
502 char keep_arg[256];
503 int n = 0;
504
505 argv[n++] = "index-pack";
506 argv[n++] = "--stdin";
507 if (!quiet)
508 argv[n++] = "-v";
509 if (use_thin_pack)
510 argv[n++] = "--fix-thin";
511 if (keep_pack > 1) {
512 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
513 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
514 strcpy(keep_arg + s, "localhost");
515 argv[n++] = keep_arg;
516 }
517 argv[n] = NULL;
518 return get_pack(xd, argv);
519}
520
def88e9a
LT
521static int fetch_pack(int fd[2], int nr_match, char **match)
522{
d1c133f5
LT
523 struct ref *ref;
524 unsigned char sha1[20];
525 int status;
def88e9a 526
1baaae5e 527 get_remote_heads(fd[0], &ref, 0, NULL, 0);
ed09aef0
JS
528 if (is_repository_shallow() && !server_supports("shallow"))
529 die("Server does not support shallow clients");
c4c86f07
JS
530 if (server_supports("multi_ack")) {
531 if (verbose)
532 fprintf(stderr, "Server supports multi_ack\n");
533 multi_ack = 1;
534 }
d47f3db7
JH
535 if (server_supports("side-band-64k")) {
536 if (verbose)
537 fprintf(stderr, "Server supports side-band-64k\n");
538 use_sideband = 2;
539 }
540 else if (server_supports("side-band")) {
583b7ea3
JH
541 if (verbose)
542 fprintf(stderr, "Server supports side-band\n");
543 use_sideband = 1;
544 }
d1c133f5
LT
545 if (!ref) {
546 packet_flush(fd[1]);
547 die("no matching remote head");
548 }
1baaae5e 549 if (everything_local(&ref, nr_match, match)) {
2759cbc7
LT
550 packet_flush(fd[1]);
551 goto all_done;
552 }
33b83034 553 if (find_common(fd, sha1, ref) < 0)
da093d37 554 if (keep_pack != 1)
dfeff66e
JH
555 /* When cloning, it is not unusual to have
556 * no common commit.
557 */
558 fprintf(stderr, "warning: no common commits\n");
ad897215 559
da093d37 560 status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
ad897215
JH
561 if (status)
562 die("git-fetch-pack: fetch failed.");
563
564 all_done:
565 while (ref) {
566 printf("%s %s\n",
567 sha1_to_hex(ref->old_sha1), ref->name);
568 ref = ref->next;
75bfc6c2 569 }
ad897215 570 return 0;
def88e9a
LT
571}
572
573int main(int argc, char **argv)
574{
575 int i, ret, nr_heads;
576 char *dest = NULL, **heads;
577 int fd[2];
578 pid_t pid;
579
5a327713
JH
580 setup_git_directory();
581
def88e9a
LT
582 nr_heads = 0;
583 heads = NULL;
584 for (i = 1; i < argc; i++) {
585 char *arg = argv[i];
586
587 if (*arg == '-') {
8b3d9dc0
JH
588 if (!strncmp("--exec=", arg, 7)) {
589 exec = arg + 7;
590 continue;
591 }
2247efb4 592 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
33b83034
JH
593 quiet = 1;
594 continue;
595 }
2247efb4 596 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
da093d37 597 keep_pack++;
ad897215
JH
598 continue;
599 }
b19696c2
JH
600 if (!strcmp("--thin", arg)) {
601 use_thin_pack = 1;
602 continue;
603 }
dfeff66e
JH
604 if (!strcmp("--all", arg)) {
605 fetch_all = 1;
606 continue;
607 }
33b83034
JH
608 if (!strcmp("-v", arg)) {
609 verbose = 1;
610 continue;
611 }
def88e9a
LT
612 usage(fetch_pack_usage);
613 }
614 dest = arg;
615 heads = argv + i + 1;
616 nr_heads = argc - i - 1;
617 break;
618 }
619 if (!dest)
620 usage(fetch_pack_usage);
621 pid = git_connect(fd, dest, exec);
622 if (pid < 0)
623 return 1;
624 ret = fetch_pack(fd, nr_heads, heads);
625 close(fd[0]);
626 close(fd[1]);
8a5dbef8 627 ret |= finish_connect(pid);
9e5d2b40
JH
628
629 if (!ret && nr_heads) {
630 /* If the heads to pull were given, we should have
631 * consumed all of them by matching the remote.
632 * Otherwise, 'git-fetch remote no-such-ref' would
633 * silently succeed without issuing an error.
634 */
635 for (i = 0; i < nr_heads; i++)
636 if (heads[i] && heads[i][0]) {
637 error("no such remote ref %s", heads[i]);
638 ret = 1;
639 }
640 }
641
8a5dbef8 642 return !!ret;
def88e9a 643}