rename --exec to --upload-pack for fetch-pack and peek-remote
[git/git.git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "sideband.h"
8
9 static int keep_pack;
10 static int quiet;
11 static int verbose;
12 static int fetch_all;
13 static int depth;
14 static const char fetch_pack_usage[] =
15 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
16 static const char *uploadpack = "git-upload-pack";
17
18 #define COMPLETE (1U << 0)
19 #define COMMON (1U << 1)
20 #define COMMON_REF (1U << 2)
21 #define SEEN (1U << 3)
22 #define POPPED (1U << 4)
23
24 /*
25 * After sending this many "have"s if we do not get any new ACK , we
26 * give up traversing our history.
27 */
28 #define MAX_IN_VAIN 256
29
30 static struct commit_list *rev_list;
31 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
32
33 static void rev_list_push(struct commit *commit, int mark)
34 {
35 if (!(commit->object.flags & mark)) {
36 commit->object.flags |= mark;
37
38 if (!(commit->object.parsed))
39 parse_commit(commit);
40
41 insert_by_date(commit, &rev_list);
42
43 if (!(commit->object.flags & COMMON))
44 non_common_revs++;
45 }
46 }
47
48 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
49 {
50 struct object *o = deref_tag(parse_object(sha1), path, 0);
51
52 if (o && o->type == OBJ_COMMIT)
53 rev_list_push((struct commit *)o, SEEN);
54
55 return 0;
56 }
57
58 /*
59 This function marks a rev and its ancestors as common.
60 In some cases, it is desirable to mark only the ancestors (for example
61 when only the server does not yet know that they are common).
62 */
63
64 static void mark_common(struct commit *commit,
65 int ancestors_only, int dont_parse)
66 {
67 if (commit != NULL && !(commit->object.flags & COMMON)) {
68 struct object *o = (struct object *)commit;
69
70 if (!ancestors_only)
71 o->flags |= COMMON;
72
73 if (!(o->flags & SEEN))
74 rev_list_push(commit, SEEN);
75 else {
76 struct commit_list *parents;
77
78 if (!ancestors_only && !(o->flags & POPPED))
79 non_common_revs--;
80 if (!o->parsed && !dont_parse)
81 parse_commit(commit);
82
83 for (parents = commit->parents;
84 parents;
85 parents = parents->next)
86 mark_common(parents->item, 0, dont_parse);
87 }
88 }
89 }
90
91 /*
92 Get the next rev to send, ignoring the common.
93 */
94
95 static const unsigned char* get_rev(void)
96 {
97 struct commit *commit = NULL;
98
99 while (commit == NULL) {
100 unsigned int mark;
101 struct commit_list* parents;
102
103 if (rev_list == NULL || non_common_revs == 0)
104 return NULL;
105
106 commit = rev_list->item;
107 if (!(commit->object.parsed))
108 parse_commit(commit);
109 commit->object.flags |= POPPED;
110 if (!(commit->object.flags & COMMON))
111 non_common_revs--;
112
113 parents = commit->parents;
114
115 if (commit->object.flags & COMMON) {
116 /* do not send "have", and ignore ancestors */
117 commit = NULL;
118 mark = COMMON | SEEN;
119 } else if (commit->object.flags & COMMON_REF)
120 /* send "have", and ignore ancestors */
121 mark = COMMON | SEEN;
122 else
123 /* send "have", also for its ancestors */
124 mark = SEEN;
125
126 while (parents) {
127 if (!(parents->item->object.flags & SEEN))
128 rev_list_push(parents->item, mark);
129 if (mark & COMMON)
130 mark_common(parents->item, 1, 0);
131 parents = parents->next;
132 }
133
134 rev_list = rev_list->next;
135 }
136
137 return commit->object.sha1;
138 }
139
140 static int find_common(int fd[2], unsigned char *result_sha1,
141 struct ref *refs)
142 {
143 int fetching;
144 int count = 0, flushes = 0, retval;
145 const unsigned char *sha1;
146 unsigned in_vain = 0;
147 int got_continue = 0;
148
149 for_each_ref(rev_list_insert_ref, NULL);
150
151 fetching = 0;
152 for ( ; refs ; refs = refs->next) {
153 unsigned char *remote = refs->old_sha1;
154 struct object *o;
155
156 /*
157 * If that object is complete (i.e. it is an ancestor of a
158 * local ref), we tell them we have it but do not have to
159 * tell them about its ancestors, which they already know
160 * about.
161 *
162 * We use lookup_object here because we are only
163 * interested in the case we *know* the object is
164 * reachable and we have already scanned it.
165 */
166 if (((o = lookup_object(remote)) != NULL) &&
167 (o->flags & COMPLETE)) {
168 continue;
169 }
170
171 if (!fetching)
172 packet_write(fd[1], "want %s%s%s%s%s%s\n",
173 sha1_to_hex(remote),
174 (multi_ack ? " multi_ack" : ""),
175 (use_sideband == 2 ? " side-band-64k" : ""),
176 (use_sideband == 1 ? " side-band" : ""),
177 (use_thin_pack ? " thin-pack" : ""),
178 " ofs-delta");
179 else
180 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
181 fetching++;
182 }
183 if (is_repository_shallow())
184 write_shallow_commits(fd[1], 1);
185 if (depth > 0)
186 packet_write(fd[1], "deepen %d", depth);
187 packet_flush(fd[1]);
188 if (!fetching)
189 return 1;
190
191 if (depth > 0) {
192 char line[1024];
193 unsigned char sha1[20];
194 int len;
195
196 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
197 if (!strncmp("shallow ", line, 8)) {
198 if (get_sha1_hex(line + 8, sha1))
199 die("invalid shallow line: %s", line);
200 register_shallow(sha1);
201 continue;
202 }
203 if (!strncmp("unshallow ", line, 10)) {
204 if (get_sha1_hex(line + 10, sha1))
205 die("invalid unshallow line: %s", line);
206 if (!lookup_object(sha1))
207 die("object not found: %s", line);
208 /* make sure that it is parsed as shallow */
209 parse_object(sha1);
210 if (unregister_shallow(sha1))
211 die("no shallow found: %s", line);
212 continue;
213 }
214 die("expected shallow/unshallow, got %s", line);
215 }
216 }
217
218 flushes = 0;
219 retval = -1;
220 while ((sha1 = get_rev())) {
221 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
222 if (verbose)
223 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
224 in_vain++;
225 if (!(31 & ++count)) {
226 int ack;
227
228 packet_flush(fd[1]);
229 flushes++;
230
231 /*
232 * We keep one window "ahead" of the other side, and
233 * will wait for an ACK only on the next one
234 */
235 if (count == 32)
236 continue;
237
238 do {
239 ack = get_ack(fd[0], result_sha1);
240 if (verbose && ack)
241 fprintf(stderr, "got ack %d %s\n", ack,
242 sha1_to_hex(result_sha1));
243 if (ack == 1) {
244 flushes = 0;
245 multi_ack = 0;
246 retval = 0;
247 goto done;
248 } else if (ack == 2) {
249 struct commit *commit =
250 lookup_commit(result_sha1);
251 mark_common(commit, 0, 1);
252 retval = 0;
253 in_vain = 0;
254 got_continue = 1;
255 }
256 } while (ack);
257 flushes--;
258 if (got_continue && MAX_IN_VAIN < in_vain) {
259 if (verbose)
260 fprintf(stderr, "giving up\n");
261 break; /* give up */
262 }
263 }
264 }
265 done:
266 packet_write(fd[1], "done\n");
267 if (verbose)
268 fprintf(stderr, "done\n");
269 if (retval != 0) {
270 multi_ack = 0;
271 flushes++;
272 }
273 while (flushes || multi_ack) {
274 int ack = get_ack(fd[0], result_sha1);
275 if (ack) {
276 if (verbose)
277 fprintf(stderr, "got ack (%d) %s\n", ack,
278 sha1_to_hex(result_sha1));
279 if (ack == 1)
280 return 0;
281 multi_ack = 1;
282 continue;
283 }
284 flushes--;
285 }
286 return retval;
287 }
288
289 static struct commit_list *complete;
290
291 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
292 {
293 struct object *o = parse_object(sha1);
294
295 while (o && o->type == OBJ_TAG) {
296 struct tag *t = (struct tag *) o;
297 if (!t->tagged)
298 break; /* broken repository */
299 o->flags |= COMPLETE;
300 o = parse_object(t->tagged->sha1);
301 }
302 if (o && o->type == OBJ_COMMIT) {
303 struct commit *commit = (struct commit *)o;
304 commit->object.flags |= COMPLETE;
305 insert_by_date(commit, &complete);
306 }
307 return 0;
308 }
309
310 static void mark_recent_complete_commits(unsigned long cutoff)
311 {
312 while (complete && cutoff <= complete->item->date) {
313 if (verbose)
314 fprintf(stderr, "Marking %s as complete\n",
315 sha1_to_hex(complete->item->object.sha1));
316 pop_most_recent_commit(&complete, COMPLETE);
317 }
318 }
319
320 static void filter_refs(struct ref **refs, int nr_match, char **match)
321 {
322 struct ref **return_refs;
323 struct ref *newlist = NULL;
324 struct ref **newtail = &newlist;
325 struct ref *ref, *next;
326 struct ref *fastarray[32];
327
328 if (nr_match && !fetch_all) {
329 if (ARRAY_SIZE(fastarray) < nr_match)
330 return_refs = xcalloc(nr_match, sizeof(struct ref *));
331 else {
332 return_refs = fastarray;
333 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
334 }
335 }
336 else
337 return_refs = NULL;
338
339 for (ref = *refs; ref; ref = next) {
340 next = ref->next;
341 if (!memcmp(ref->name, "refs/", 5) &&
342 check_ref_format(ref->name + 5))
343 ; /* trash */
344 else if (fetch_all &&
345 (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
346 *newtail = ref;
347 ref->next = NULL;
348 newtail = &ref->next;
349 continue;
350 }
351 else {
352 int order = path_match(ref->name, nr_match, match);
353 if (order) {
354 return_refs[order-1] = ref;
355 continue; /* we will link it later */
356 }
357 }
358 free(ref);
359 }
360
361 if (!fetch_all) {
362 int i;
363 for (i = 0; i < nr_match; i++) {
364 ref = return_refs[i];
365 if (ref) {
366 *newtail = ref;
367 ref->next = NULL;
368 newtail = &ref->next;
369 }
370 }
371 if (return_refs != fastarray)
372 free(return_refs);
373 }
374 *refs = newlist;
375 }
376
377 static int everything_local(struct ref **refs, int nr_match, char **match)
378 {
379 struct ref *ref;
380 int retval;
381 unsigned long cutoff = 0;
382
383 track_object_refs = 0;
384 save_commit_buffer = 0;
385
386 for (ref = *refs; ref; ref = ref->next) {
387 struct object *o;
388
389 o = parse_object(ref->old_sha1);
390 if (!o)
391 continue;
392
393 /* We already have it -- which may mean that we were
394 * in sync with the other side at some time after
395 * that (it is OK if we guess wrong here).
396 */
397 if (o->type == OBJ_COMMIT) {
398 struct commit *commit = (struct commit *)o;
399 if (!cutoff || cutoff < commit->date)
400 cutoff = commit->date;
401 }
402 }
403
404 if (!depth) {
405 for_each_ref(mark_complete, NULL);
406 if (cutoff)
407 mark_recent_complete_commits(cutoff);
408 }
409
410 /*
411 * Mark all complete remote refs as common refs.
412 * Don't mark them common yet; the server has to be told so first.
413 */
414 for (ref = *refs; ref; ref = ref->next) {
415 struct object *o = deref_tag(lookup_object(ref->old_sha1),
416 NULL, 0);
417
418 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
419 continue;
420
421 if (!(o->flags & SEEN)) {
422 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
423
424 mark_common((struct commit *)o, 1, 1);
425 }
426 }
427
428 filter_refs(refs, nr_match, match);
429
430 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
431 const unsigned char *remote = ref->old_sha1;
432 unsigned char local[20];
433 struct object *o;
434
435 o = lookup_object(remote);
436 if (!o || !(o->flags & COMPLETE)) {
437 retval = 0;
438 if (!verbose)
439 continue;
440 fprintf(stderr,
441 "want %s (%s)\n", sha1_to_hex(remote),
442 ref->name);
443 continue;
444 }
445
446 hashcpy(ref->new_sha1, local);
447 if (!verbose)
448 continue;
449 fprintf(stderr,
450 "already have %s (%s)\n", sha1_to_hex(remote),
451 ref->name);
452 }
453 return retval;
454 }
455
456 static pid_t setup_sideband(int fd[2], int xd[2])
457 {
458 pid_t side_pid;
459
460 if (!use_sideband) {
461 fd[0] = xd[0];
462 fd[1] = xd[1];
463 return 0;
464 }
465 /* xd[] is talking with upload-pack; subprocess reads from
466 * xd[0], spits out band#2 to stderr, and feeds us band#1
467 * through our fd[0].
468 */
469 if (pipe(fd) < 0)
470 die("fetch-pack: unable to set up pipe");
471 side_pid = fork();
472 if (side_pid < 0)
473 die("fetch-pack: unable to fork off sideband demultiplexer");
474 if (!side_pid) {
475 /* subprocess */
476 close(fd[0]);
477 if (xd[0] != xd[1])
478 close(xd[1]);
479 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
480 exit(1);
481 exit(0);
482 }
483 close(xd[0]);
484 close(fd[1]);
485 fd[1] = xd[1];
486 return side_pid;
487 }
488
489 static int get_pack(int xd[2], const char **argv)
490 {
491 int status;
492 pid_t pid, side_pid;
493 int fd[2];
494
495 side_pid = setup_sideband(fd, xd);
496 pid = fork();
497 if (pid < 0)
498 die("fetch-pack: unable to fork off %s", argv[0]);
499 if (!pid) {
500 dup2(fd[0], 0);
501 close(fd[0]);
502 close(fd[1]);
503 execv_git_cmd(argv);
504 die("%s exec failed", argv[0]);
505 }
506 close(fd[0]);
507 close(fd[1]);
508 while (waitpid(pid, &status, 0) < 0) {
509 if (errno != EINTR)
510 die("waiting for %s: %s", argv[0], strerror(errno));
511 }
512 if (WIFEXITED(status)) {
513 int code = WEXITSTATUS(status);
514 if (code)
515 die("%s died with error code %d", argv[0], code);
516 return 0;
517 }
518 if (WIFSIGNALED(status)) {
519 int sig = WTERMSIG(status);
520 die("%s died of signal %d", argv[0], sig);
521 }
522 die("%s died of unnatural causes %d", argv[0], status);
523 }
524
525 static int explode_rx_pack(int xd[2])
526 {
527 const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
528 return get_pack(xd, argv);
529 }
530
531 static int keep_rx_pack(int xd[2])
532 {
533 const char *argv[6];
534 char keep_arg[256];
535 int n = 0;
536
537 argv[n++] = "index-pack";
538 argv[n++] = "--stdin";
539 if (!quiet)
540 argv[n++] = "-v";
541 if (use_thin_pack)
542 argv[n++] = "--fix-thin";
543 if (keep_pack > 1) {
544 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
545 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
546 strcpy(keep_arg + s, "localhost");
547 argv[n++] = keep_arg;
548 }
549 argv[n] = NULL;
550 return get_pack(xd, argv);
551 }
552
553 static int fetch_pack(int fd[2], int nr_match, char **match)
554 {
555 struct ref *ref;
556 unsigned char sha1[20];
557 int status;
558
559 get_remote_heads(fd[0], &ref, 0, NULL, 0);
560 if (is_repository_shallow() && !server_supports("shallow"))
561 die("Server does not support shallow clients");
562 if (server_supports("multi_ack")) {
563 if (verbose)
564 fprintf(stderr, "Server supports multi_ack\n");
565 multi_ack = 1;
566 }
567 if (server_supports("side-band-64k")) {
568 if (verbose)
569 fprintf(stderr, "Server supports side-band-64k\n");
570 use_sideband = 2;
571 }
572 else if (server_supports("side-band")) {
573 if (verbose)
574 fprintf(stderr, "Server supports side-band\n");
575 use_sideband = 1;
576 }
577 if (!ref) {
578 packet_flush(fd[1]);
579 die("no matching remote head");
580 }
581 if (everything_local(&ref, nr_match, match)) {
582 packet_flush(fd[1]);
583 goto all_done;
584 }
585 if (find_common(fd, sha1, ref) < 0)
586 if (keep_pack != 1)
587 /* When cloning, it is not unusual to have
588 * no common commit.
589 */
590 fprintf(stderr, "warning: no common commits\n");
591
592 status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
593 if (status)
594 die("git-fetch-pack: fetch failed.");
595
596 all_done:
597 while (ref) {
598 printf("%s %s\n",
599 sha1_to_hex(ref->old_sha1), ref->name);
600 ref = ref->next;
601 }
602 return 0;
603 }
604
605 static int remove_duplicates(int nr_heads, char **heads)
606 {
607 int src, dst;
608
609 for (src = dst = 0; src < nr_heads; src++) {
610 /* If heads[src] is different from any of
611 * heads[0..dst], push it in.
612 */
613 int i;
614 for (i = 0; i < dst; i++) {
615 if (!strcmp(heads[i], heads[src]))
616 break;
617 }
618 if (i < dst)
619 continue;
620 if (src != dst)
621 heads[dst] = heads[src];
622 dst++;
623 }
624 heads[dst] = 0;
625 return dst;
626 }
627
628 static struct lock_file lock;
629
630 int main(int argc, char **argv)
631 {
632 int i, ret, nr_heads;
633 char *dest = NULL, **heads;
634 int fd[2];
635 pid_t pid;
636 struct stat st;
637
638 setup_git_directory();
639
640 nr_heads = 0;
641 heads = NULL;
642 for (i = 1; i < argc; i++) {
643 char *arg = argv[i];
644
645 if (*arg == '-') {
646 if (!strncmp("--upload-pack=", arg, 14)) {
647 uploadpack = arg + 14;
648 continue;
649 }
650 if (!strncmp("--exec=", arg, 7)) {
651 uploadpack = arg + 7;
652 continue;
653 }
654 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
655 quiet = 1;
656 continue;
657 }
658 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
659 keep_pack++;
660 continue;
661 }
662 if (!strcmp("--thin", arg)) {
663 use_thin_pack = 1;
664 continue;
665 }
666 if (!strcmp("--all", arg)) {
667 fetch_all = 1;
668 continue;
669 }
670 if (!strcmp("-v", arg)) {
671 verbose = 1;
672 continue;
673 }
674 if (!strncmp("--depth=", arg, 8)) {
675 depth = strtol(arg + 8, NULL, 0);
676 if (stat(git_path("shallow"), &st))
677 st.st_mtime = 0;
678 continue;
679 }
680 usage(fetch_pack_usage);
681 }
682 dest = arg;
683 heads = argv + i + 1;
684 nr_heads = argc - i - 1;
685 break;
686 }
687 if (!dest)
688 usage(fetch_pack_usage);
689 pid = git_connect(fd, dest, uploadpack);
690 if (pid < 0)
691 return 1;
692 if (heads && nr_heads)
693 nr_heads = remove_duplicates(nr_heads, heads);
694 ret = fetch_pack(fd, nr_heads, heads);
695 close(fd[0]);
696 close(fd[1]);
697 ret |= finish_connect(pid);
698
699 if (!ret && nr_heads) {
700 /* If the heads to pull were given, we should have
701 * consumed all of them by matching the remote.
702 * Otherwise, 'git-fetch remote no-such-ref' would
703 * silently succeed without issuing an error.
704 */
705 for (i = 0; i < nr_heads; i++)
706 if (heads[i] && heads[i][0]) {
707 error("no such remote ref %s", heads[i]);
708 ret = 1;
709 }
710 }
711
712 if (!ret && depth > 0) {
713 struct cache_time mtime;
714 char *shallow = git_path("shallow");
715 int fd;
716
717 mtime.sec = st.st_mtime;
718 #ifdef USE_NSEC
719 mtime.usec = st.st_mtim.usec;
720 #endif
721 if (stat(shallow, &st)) {
722 if (mtime.sec)
723 die("shallow file was removed during fetch");
724 } else if (st.st_mtime != mtime.sec
725 #ifdef USE_NSEC
726 || st.st_mtim.usec != mtime.usec
727 #endif
728 )
729 die("shallow file was changed during fetch");
730
731 fd = hold_lock_file_for_update(&lock, shallow, 1);
732 if (!write_shallow_commits(fd, 0)) {
733 unlink(shallow);
734 rollback_lock_file(&lock);
735 } else {
736 close(fd);
737 commit_lock_file(&lock);
738 }
739 }
740
741 return !!ret;
742 }