871b7042e74837b5dfd26f17f4317d4341c01100
[git/git.git] / builtin-fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "run-command.h"
11
12 static int transfer_unpack_limit = -1;
13 static int fetch_unpack_limit = -1;
14 static int unpack_limit = 100;
15 static struct fetch_pack_args args;
16
17 static const char fetch_pack_usage[] =
18 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
19 static const char *uploadpack = "git-upload-pack";
20
21 #define COMPLETE (1U << 0)
22 #define COMMON (1U << 1)
23 #define COMMON_REF (1U << 2)
24 #define SEEN (1U << 3)
25 #define POPPED (1U << 4)
26
27 /*
28 * After sending this many "have"s if we do not get any new ACK , we
29 * give up traversing our history.
30 */
31 #define MAX_IN_VAIN 256
32
33 static struct commit_list *rev_list;
34 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
35
36 static void rev_list_push(struct commit *commit, int mark)
37 {
38 if (!(commit->object.flags & mark)) {
39 commit->object.flags |= mark;
40
41 if (!(commit->object.parsed))
42 parse_commit(commit);
43
44 insert_by_date(commit, &rev_list);
45
46 if (!(commit->object.flags & COMMON))
47 non_common_revs++;
48 }
49 }
50
51 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
52 {
53 struct object *o = deref_tag(parse_object(sha1), path, 0);
54
55 if (o && o->type == OBJ_COMMIT)
56 rev_list_push((struct commit *)o, SEEN);
57
58 return 0;
59 }
60
61 /*
62 This function marks a rev and its ancestors as common.
63 In some cases, it is desirable to mark only the ancestors (for example
64 when only the server does not yet know that they are common).
65 */
66
67 static void mark_common(struct commit *commit,
68 int ancestors_only, int dont_parse)
69 {
70 if (commit != NULL && !(commit->object.flags & COMMON)) {
71 struct object *o = (struct object *)commit;
72
73 if (!ancestors_only)
74 o->flags |= COMMON;
75
76 if (!(o->flags & SEEN))
77 rev_list_push(commit, SEEN);
78 else {
79 struct commit_list *parents;
80
81 if (!ancestors_only && !(o->flags & POPPED))
82 non_common_revs--;
83 if (!o->parsed && !dont_parse)
84 parse_commit(commit);
85
86 for (parents = commit->parents;
87 parents;
88 parents = parents->next)
89 mark_common(parents->item, 0, dont_parse);
90 }
91 }
92 }
93
94 /*
95 Get the next rev to send, ignoring the common.
96 */
97
98 static const unsigned char* get_rev(void)
99 {
100 struct commit *commit = NULL;
101
102 while (commit == NULL) {
103 unsigned int mark;
104 struct commit_list* parents;
105
106 if (rev_list == NULL || non_common_revs == 0)
107 return NULL;
108
109 commit = rev_list->item;
110 if (!(commit->object.parsed))
111 parse_commit(commit);
112 commit->object.flags |= POPPED;
113 if (!(commit->object.flags & COMMON))
114 non_common_revs--;
115
116 parents = commit->parents;
117
118 if (commit->object.flags & COMMON) {
119 /* do not send "have", and ignore ancestors */
120 commit = NULL;
121 mark = COMMON | SEEN;
122 } else if (commit->object.flags & COMMON_REF)
123 /* send "have", and ignore ancestors */
124 mark = COMMON | SEEN;
125 else
126 /* send "have", also for its ancestors */
127 mark = SEEN;
128
129 while (parents) {
130 if (!(parents->item->object.flags & SEEN))
131 rev_list_push(parents->item, mark);
132 if (mark & COMMON)
133 mark_common(parents->item, 1, 0);
134 parents = parents->next;
135 }
136
137 rev_list = rev_list->next;
138 }
139
140 return commit->object.sha1;
141 }
142
143 static int find_common(int fd[2], unsigned char *result_sha1,
144 struct ref *refs)
145 {
146 int fetching;
147 int count = 0, flushes = 0, retval;
148 const unsigned char *sha1;
149 unsigned in_vain = 0;
150 int got_continue = 0;
151
152 for_each_ref(rev_list_insert_ref, NULL);
153
154 fetching = 0;
155 for ( ; refs ; refs = refs->next) {
156 unsigned char *remote = refs->old_sha1;
157 struct object *o;
158
159 /*
160 * If that object is complete (i.e. it is an ancestor of a
161 * local ref), we tell them we have it but do not have to
162 * tell them about its ancestors, which they already know
163 * about.
164 *
165 * We use lookup_object here because we are only
166 * interested in the case we *know* the object is
167 * reachable and we have already scanned it.
168 */
169 if (((o = lookup_object(remote)) != NULL) &&
170 (o->flags & COMPLETE)) {
171 continue;
172 }
173
174 if (!fetching)
175 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
176 sha1_to_hex(remote),
177 (multi_ack ? " multi_ack" : ""),
178 (use_sideband == 2 ? " side-band-64k" : ""),
179 (use_sideband == 1 ? " side-band" : ""),
180 (use_thin_pack ? " thin-pack" : ""),
181 (args.no_progress ? " no-progress" : ""),
182 " ofs-delta");
183 else
184 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
185 fetching++;
186 }
187 if (is_repository_shallow())
188 write_shallow_commits(fd[1], 1);
189 if (args.depth > 0)
190 packet_write(fd[1], "deepen %d", args.depth);
191 packet_flush(fd[1]);
192 if (!fetching)
193 return 1;
194
195 if (args.depth > 0) {
196 char line[1024];
197 unsigned char sha1[20];
198 int len;
199
200 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
201 if (!prefixcmp(line, "shallow ")) {
202 if (get_sha1_hex(line + 8, sha1))
203 die("invalid shallow line: %s", line);
204 register_shallow(sha1);
205 continue;
206 }
207 if (!prefixcmp(line, "unshallow ")) {
208 if (get_sha1_hex(line + 10, sha1))
209 die("invalid unshallow line: %s", line);
210 if (!lookup_object(sha1))
211 die("object not found: %s", line);
212 /* make sure that it is parsed as shallow */
213 parse_object(sha1);
214 if (unregister_shallow(sha1))
215 die("no shallow found: %s", line);
216 continue;
217 }
218 die("expected shallow/unshallow, got %s", line);
219 }
220 }
221
222 flushes = 0;
223 retval = -1;
224 while ((sha1 = get_rev())) {
225 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
226 if (args.verbose)
227 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
228 in_vain++;
229 if (!(31 & ++count)) {
230 int ack;
231
232 packet_flush(fd[1]);
233 flushes++;
234
235 /*
236 * We keep one window "ahead" of the other side, and
237 * will wait for an ACK only on the next one
238 */
239 if (count == 32)
240 continue;
241
242 do {
243 ack = get_ack(fd[0], result_sha1);
244 if (args.verbose && ack)
245 fprintf(stderr, "got ack %d %s\n", ack,
246 sha1_to_hex(result_sha1));
247 if (ack == 1) {
248 flushes = 0;
249 multi_ack = 0;
250 retval = 0;
251 goto done;
252 } else if (ack == 2) {
253 struct commit *commit =
254 lookup_commit(result_sha1);
255 mark_common(commit, 0, 1);
256 retval = 0;
257 in_vain = 0;
258 got_continue = 1;
259 }
260 } while (ack);
261 flushes--;
262 if (got_continue && MAX_IN_VAIN < in_vain) {
263 if (args.verbose)
264 fprintf(stderr, "giving up\n");
265 break; /* give up */
266 }
267 }
268 }
269 done:
270 packet_write(fd[1], "done\n");
271 if (args.verbose)
272 fprintf(stderr, "done\n");
273 if (retval != 0) {
274 multi_ack = 0;
275 flushes++;
276 }
277 while (flushes || multi_ack) {
278 int ack = get_ack(fd[0], result_sha1);
279 if (ack) {
280 if (args.verbose)
281 fprintf(stderr, "got ack (%d) %s\n", ack,
282 sha1_to_hex(result_sha1));
283 if (ack == 1)
284 return 0;
285 multi_ack = 1;
286 continue;
287 }
288 flushes--;
289 }
290 return retval;
291 }
292
293 static struct commit_list *complete;
294
295 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
296 {
297 struct object *o = parse_object(sha1);
298
299 while (o && o->type == OBJ_TAG) {
300 struct tag *t = (struct tag *) o;
301 if (!t->tagged)
302 break; /* broken repository */
303 o->flags |= COMPLETE;
304 o = parse_object(t->tagged->sha1);
305 }
306 if (o && o->type == OBJ_COMMIT) {
307 struct commit *commit = (struct commit *)o;
308 commit->object.flags |= COMPLETE;
309 insert_by_date(commit, &complete);
310 }
311 return 0;
312 }
313
314 static void mark_recent_complete_commits(unsigned long cutoff)
315 {
316 while (complete && cutoff <= complete->item->date) {
317 if (args.verbose)
318 fprintf(stderr, "Marking %s as complete\n",
319 sha1_to_hex(complete->item->object.sha1));
320 pop_most_recent_commit(&complete, COMPLETE);
321 }
322 }
323
324 static void filter_refs(struct ref **refs, int nr_match, char **match)
325 {
326 struct ref **return_refs;
327 struct ref *newlist = NULL;
328 struct ref **newtail = &newlist;
329 struct ref *ref, *next;
330 struct ref *fastarray[32];
331
332 if (nr_match && !args.fetch_all) {
333 if (ARRAY_SIZE(fastarray) < nr_match)
334 return_refs = xcalloc(nr_match, sizeof(struct ref *));
335 else {
336 return_refs = fastarray;
337 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
338 }
339 }
340 else
341 return_refs = NULL;
342
343 for (ref = *refs; ref; ref = next) {
344 next = ref->next;
345 if (!memcmp(ref->name, "refs/", 5) &&
346 check_ref_format(ref->name + 5))
347 ; /* trash */
348 else if (args.fetch_all &&
349 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
350 *newtail = ref;
351 ref->next = NULL;
352 newtail = &ref->next;
353 continue;
354 }
355 else {
356 int order = path_match(ref->name, nr_match, match);
357 if (order) {
358 return_refs[order-1] = ref;
359 continue; /* we will link it later */
360 }
361 }
362 free(ref);
363 }
364
365 if (!args.fetch_all) {
366 int i;
367 for (i = 0; i < nr_match; i++) {
368 ref = return_refs[i];
369 if (ref) {
370 *newtail = ref;
371 ref->next = NULL;
372 newtail = &ref->next;
373 }
374 }
375 if (return_refs != fastarray)
376 free(return_refs);
377 }
378 *refs = newlist;
379 }
380
381 static int everything_local(struct ref **refs, int nr_match, char **match)
382 {
383 struct ref *ref;
384 int retval;
385 unsigned long cutoff = 0;
386
387 track_object_refs = 0;
388 save_commit_buffer = 0;
389
390 for (ref = *refs; ref; ref = ref->next) {
391 struct object *o;
392
393 o = parse_object(ref->old_sha1);
394 if (!o)
395 continue;
396
397 /* We already have it -- which may mean that we were
398 * in sync with the other side at some time after
399 * that (it is OK if we guess wrong here).
400 */
401 if (o->type == OBJ_COMMIT) {
402 struct commit *commit = (struct commit *)o;
403 if (!cutoff || cutoff < commit->date)
404 cutoff = commit->date;
405 }
406 }
407
408 if (!args.depth) {
409 for_each_ref(mark_complete, NULL);
410 if (cutoff)
411 mark_recent_complete_commits(cutoff);
412 }
413
414 /*
415 * Mark all complete remote refs as common refs.
416 * Don't mark them common yet; the server has to be told so first.
417 */
418 for (ref = *refs; ref; ref = ref->next) {
419 struct object *o = deref_tag(lookup_object(ref->old_sha1),
420 NULL, 0);
421
422 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
423 continue;
424
425 if (!(o->flags & SEEN)) {
426 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
427
428 mark_common((struct commit *)o, 1, 1);
429 }
430 }
431
432 filter_refs(refs, nr_match, match);
433
434 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
435 const unsigned char *remote = ref->old_sha1;
436 unsigned char local[20];
437 struct object *o;
438
439 o = lookup_object(remote);
440 if (!o || !(o->flags & COMPLETE)) {
441 retval = 0;
442 if (!args.verbose)
443 continue;
444 fprintf(stderr,
445 "want %s (%s)\n", sha1_to_hex(remote),
446 ref->name);
447 continue;
448 }
449
450 hashcpy(ref->new_sha1, local);
451 if (!args.verbose)
452 continue;
453 fprintf(stderr,
454 "already have %s (%s)\n", sha1_to_hex(remote),
455 ref->name);
456 }
457 return retval;
458 }
459
460 static pid_t setup_sideband(int fd[2], int xd[2])
461 {
462 pid_t side_pid;
463
464 if (!use_sideband) {
465 fd[0] = xd[0];
466 fd[1] = xd[1];
467 return 0;
468 }
469 /* xd[] is talking with upload-pack; subprocess reads from
470 * xd[0], spits out band#2 to stderr, and feeds us band#1
471 * through our fd[0].
472 */
473 if (pipe(fd) < 0)
474 die("fetch-pack: unable to set up pipe");
475 side_pid = fork();
476 if (side_pid < 0)
477 die("fetch-pack: unable to fork off sideband demultiplexer");
478 if (!side_pid) {
479 /* subprocess */
480 close(fd[0]);
481 if (xd[0] != xd[1])
482 close(xd[1]);
483 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
484 exit(1);
485 exit(0);
486 }
487 close(xd[0]);
488 close(fd[1]);
489 fd[1] = xd[1];
490 return side_pid;
491 }
492
493 static int get_pack(int xd[2], char **pack_lockfile)
494 {
495 pid_t side_pid;
496 int fd[2];
497 const char *argv[20];
498 char keep_arg[256];
499 char hdr_arg[256];
500 const char **av;
501 int do_keep = args.keep_pack;
502 struct child_process cmd;
503
504 side_pid = setup_sideband(fd, xd);
505
506 memset(&cmd, 0, sizeof(cmd));
507 cmd.argv = argv;
508 av = argv;
509 *hdr_arg = 0;
510 if (!args.keep_pack && unpack_limit) {
511 struct pack_header header;
512
513 if (read_pack_header(fd[0], &header))
514 die("protocol error: bad pack header");
515 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
516 ntohl(header.hdr_version), ntohl(header.hdr_entries));
517 if (ntohl(header.hdr_entries) < unpack_limit)
518 do_keep = 0;
519 else
520 do_keep = 1;
521 }
522
523 if (do_keep) {
524 if (pack_lockfile)
525 cmd.out = -1;
526 *av++ = "index-pack";
527 *av++ = "--stdin";
528 if (!args.quiet && !args.no_progress)
529 *av++ = "-v";
530 if (args.use_thin_pack)
531 *av++ = "--fix-thin";
532 if (args.lock_pack || unpack_limit) {
533 int s = sprintf(keep_arg,
534 "--keep=fetch-pack %d on ", getpid());
535 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
536 strcpy(keep_arg + s, "localhost");
537 *av++ = keep_arg;
538 }
539 }
540 else {
541 *av++ = "unpack-objects";
542 if (args.quiet)
543 *av++ = "-q";
544 }
545 if (*hdr_arg)
546 *av++ = hdr_arg;
547 *av++ = NULL;
548
549 cmd.in = fd[0];
550 cmd.git_cmd = 1;
551 if (start_command(&cmd))
552 die("fetch-pack: unable to fork off %s", argv[0]);
553 close(fd[1]);
554 if (do_keep && pack_lockfile)
555 *pack_lockfile = index_pack_lockfile(cmd.out);
556
557 if (finish_command(&cmd))
558 die("%s failed", argv[0]);
559 return 0;
560 }
561
562 static struct ref *do_fetch_pack(int fd[2],
563 int nr_match,
564 char **match,
565 char **pack_lockfile)
566 {
567 struct ref *ref;
568 unsigned char sha1[20];
569
570 get_remote_heads(fd[0], &ref, 0, NULL, 0);
571 if (is_repository_shallow() && !server_supports("shallow"))
572 die("Server does not support shallow clients");
573 if (server_supports("multi_ack")) {
574 if (args.verbose)
575 fprintf(stderr, "Server supports multi_ack\n");
576 multi_ack = 1;
577 }
578 if (server_supports("side-band-64k")) {
579 if (args.verbose)
580 fprintf(stderr, "Server supports side-band-64k\n");
581 use_sideband = 2;
582 }
583 else if (server_supports("side-band")) {
584 if (args.verbose)
585 fprintf(stderr, "Server supports side-band\n");
586 use_sideband = 1;
587 }
588 if (!ref) {
589 packet_flush(fd[1]);
590 die("no matching remote head");
591 }
592 if (everything_local(&ref, nr_match, match)) {
593 packet_flush(fd[1]);
594 goto all_done;
595 }
596 if (find_common(fd, sha1, ref) < 0)
597 if (!args.keep_pack)
598 /* When cloning, it is not unusual to have
599 * no common commit.
600 */
601 fprintf(stderr, "warning: no common commits\n");
602
603 if (get_pack(fd, pack_lockfile))
604 die("git-fetch-pack: fetch failed.");
605
606 all_done:
607 return ref;
608 }
609
610 static int remove_duplicates(int nr_heads, char **heads)
611 {
612 int src, dst;
613
614 for (src = dst = 0; src < nr_heads; src++) {
615 /* If heads[src] is different from any of
616 * heads[0..dst], push it in.
617 */
618 int i;
619 for (i = 0; i < dst; i++) {
620 if (!strcmp(heads[i], heads[src]))
621 break;
622 }
623 if (i < dst)
624 continue;
625 if (src != dst)
626 heads[dst] = heads[src];
627 dst++;
628 }
629 return dst;
630 }
631
632 static int fetch_pack_config(const char *var, const char *value)
633 {
634 if (strcmp(var, "fetch.unpacklimit") == 0) {
635 fetch_unpack_limit = git_config_int(var, value);
636 return 0;
637 }
638
639 if (strcmp(var, "transfer.unpacklimit") == 0) {
640 transfer_unpack_limit = git_config_int(var, value);
641 return 0;
642 }
643
644 return git_default_config(var, value);
645 }
646
647 static struct lock_file lock;
648
649 static void fetch_pack_setup(void)
650 {
651 static int did_setup;
652 if (did_setup)
653 return;
654 git_config(fetch_pack_config);
655 if (0 <= transfer_unpack_limit)
656 unpack_limit = transfer_unpack_limit;
657 else if (0 <= fetch_unpack_limit)
658 unpack_limit = fetch_unpack_limit;
659 did_setup = 1;
660 }
661
662 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
663 {
664 int i, ret, nr_heads;
665 struct ref *ref;
666 char *dest = NULL, **heads;
667
668 nr_heads = 0;
669 heads = NULL;
670 for (i = 1; i < argc; i++) {
671 const char *arg = argv[i];
672
673 if (*arg == '-') {
674 if (!prefixcmp(arg, "--upload-pack=")) {
675 args.uploadpack = arg + 14;
676 continue;
677 }
678 if (!prefixcmp(arg, "--exec=")) {
679 args.uploadpack = arg + 7;
680 continue;
681 }
682 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
683 args.quiet = 1;
684 continue;
685 }
686 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
687 args.lock_pack = args.keep_pack;
688 args.keep_pack = 1;
689 continue;
690 }
691 if (!strcmp("--thin", arg)) {
692 args.use_thin_pack = 1;
693 continue;
694 }
695 if (!strcmp("--all", arg)) {
696 args.fetch_all = 1;
697 continue;
698 }
699 if (!strcmp("-v", arg)) {
700 args.verbose = 1;
701 continue;
702 }
703 if (!prefixcmp(arg, "--depth=")) {
704 args.depth = strtol(arg + 8, NULL, 0);
705 continue;
706 }
707 if (!strcmp("--no-progress", arg)) {
708 args.no_progress = 1;
709 continue;
710 }
711 usage(fetch_pack_usage);
712 }
713 dest = (char *)arg;
714 heads = (char **)(argv + i + 1);
715 nr_heads = argc - i - 1;
716 break;
717 }
718 if (!dest)
719 usage(fetch_pack_usage);
720
721 ref = fetch_pack(&args, dest, nr_heads, heads, NULL);
722 ret = !ref;
723
724 while (ref) {
725 printf("%s %s\n",
726 sha1_to_hex(ref->old_sha1), ref->name);
727 ref = ref->next;
728 }
729
730 return ret;
731 }
732
733 struct ref *fetch_pack(struct fetch_pack_args *my_args,
734 const char *dest,
735 int nr_heads,
736 char **heads,
737 char **pack_lockfile)
738 {
739 int i, ret;
740 int fd[2];
741 struct child_process *conn;
742 struct ref *ref;
743 struct stat st;
744
745 fetch_pack_setup();
746 memcpy(&args, my_args, sizeof(args));
747 if (args.depth > 0) {
748 if (stat(git_path("shallow"), &st))
749 st.st_mtime = 0;
750 }
751
752 conn = git_connect(fd, (char *)dest, uploadpack,
753 args.verbose ? CONNECT_VERBOSE : 0);
754 if (heads && nr_heads)
755 nr_heads = remove_duplicates(nr_heads, heads);
756 ref = do_fetch_pack(fd, nr_heads, heads, pack_lockfile);
757 close(fd[0]);
758 close(fd[1]);
759 ret = finish_connect(conn);
760
761 if (!ret && nr_heads) {
762 /* If the heads to pull were given, we should have
763 * consumed all of them by matching the remote.
764 * Otherwise, 'git-fetch remote no-such-ref' would
765 * silently succeed without issuing an error.
766 */
767 for (i = 0; i < nr_heads; i++)
768 if (heads[i] && heads[i][0]) {
769 error("no such remote ref %s", heads[i]);
770 ret = 1;
771 }
772 }
773
774 if (!ret && args.depth > 0) {
775 struct cache_time mtime;
776 char *shallow = git_path("shallow");
777 int fd;
778
779 mtime.sec = st.st_mtime;
780 #ifdef USE_NSEC
781 mtime.usec = st.st_mtim.usec;
782 #endif
783 if (stat(shallow, &st)) {
784 if (mtime.sec)
785 die("shallow file was removed during fetch");
786 } else if (st.st_mtime != mtime.sec
787 #ifdef USE_NSEC
788 || st.st_mtim.usec != mtime.usec
789 #endif
790 )
791 die("shallow file was changed during fetch");
792
793 fd = hold_lock_file_for_update(&lock, shallow, 1);
794 if (!write_shallow_commits(fd, 0)) {
795 unlink(shallow);
796 rollback_lock_file(&lock);
797 } else {
798 close(fd);
799 commit_lock_file(&lock);
800 }
801 }
802
803 if (ret)
804 ref = NULL;
805
806 return ref;
807 }