Merge branch 'maint'
[git/git.git] / builtin-fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "run-command.h"
11
12 static int transfer_unpack_limit = -1;
13 static int fetch_unpack_limit = -1;
14 static int unpack_limit = 100;
15 static struct fetch_pack_args args = {
16 /* .uploadpack = */ "git-upload-pack",
17 };
18
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
21
22 #define COMPLETE (1U << 0)
23 #define COMMON (1U << 1)
24 #define COMMON_REF (1U << 2)
25 #define SEEN (1U << 3)
26 #define POPPED (1U << 4)
27
28 /*
29 * After sending this many "have"s if we do not get any new ACK , we
30 * give up traversing our history.
31 */
32 #define MAX_IN_VAIN 256
33
34 static struct commit_list *rev_list;
35 static int non_common_revs, multi_ack, use_sideband;
36
37 static void rev_list_push(struct commit *commit, int mark)
38 {
39 if (!(commit->object.flags & mark)) {
40 commit->object.flags |= mark;
41
42 if (!(commit->object.parsed))
43 parse_commit(commit);
44
45 insert_by_date(commit, &rev_list);
46
47 if (!(commit->object.flags & COMMON))
48 non_common_revs++;
49 }
50 }
51
52 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
53 {
54 struct object *o = deref_tag(parse_object(sha1), path, 0);
55
56 if (o && o->type == OBJ_COMMIT)
57 rev_list_push((struct commit *)o, SEEN);
58
59 return 0;
60 }
61
62 /*
63 This function marks a rev and its ancestors as common.
64 In some cases, it is desirable to mark only the ancestors (for example
65 when only the server does not yet know that they are common).
66 */
67
68 static void mark_common(struct commit *commit,
69 int ancestors_only, int dont_parse)
70 {
71 if (commit != NULL && !(commit->object.flags & COMMON)) {
72 struct object *o = (struct object *)commit;
73
74 if (!ancestors_only)
75 o->flags |= COMMON;
76
77 if (!(o->flags & SEEN))
78 rev_list_push(commit, SEEN);
79 else {
80 struct commit_list *parents;
81
82 if (!ancestors_only && !(o->flags & POPPED))
83 non_common_revs--;
84 if (!o->parsed && !dont_parse)
85 parse_commit(commit);
86
87 for (parents = commit->parents;
88 parents;
89 parents = parents->next)
90 mark_common(parents->item, 0, dont_parse);
91 }
92 }
93 }
94
95 /*
96 Get the next rev to send, ignoring the common.
97 */
98
99 static const unsigned char* get_rev(void)
100 {
101 struct commit *commit = NULL;
102
103 while (commit == NULL) {
104 unsigned int mark;
105 struct commit_list* parents;
106
107 if (rev_list == NULL || non_common_revs == 0)
108 return NULL;
109
110 commit = rev_list->item;
111 if (!(commit->object.parsed))
112 parse_commit(commit);
113 commit->object.flags |= POPPED;
114 if (!(commit->object.flags & COMMON))
115 non_common_revs--;
116
117 parents = commit->parents;
118
119 if (commit->object.flags & COMMON) {
120 /* do not send "have", and ignore ancestors */
121 commit = NULL;
122 mark = COMMON | SEEN;
123 } else if (commit->object.flags & COMMON_REF)
124 /* send "have", and ignore ancestors */
125 mark = COMMON | SEEN;
126 else
127 /* send "have", also for its ancestors */
128 mark = SEEN;
129
130 while (parents) {
131 if (!(parents->item->object.flags & SEEN))
132 rev_list_push(parents->item, mark);
133 if (mark & COMMON)
134 mark_common(parents->item, 1, 0);
135 parents = parents->next;
136 }
137
138 rev_list = rev_list->next;
139 }
140
141 return commit->object.sha1;
142 }
143
144 static int find_common(int fd[2], unsigned char *result_sha1,
145 struct ref *refs)
146 {
147 int fetching;
148 int count = 0, flushes = 0, retval;
149 const unsigned char *sha1;
150 unsigned in_vain = 0;
151 int got_continue = 0;
152
153 for_each_ref(rev_list_insert_ref, NULL);
154
155 fetching = 0;
156 for ( ; refs ; refs = refs->next) {
157 unsigned char *remote = refs->old_sha1;
158 struct object *o;
159
160 /*
161 * If that object is complete (i.e. it is an ancestor of a
162 * local ref), we tell them we have it but do not have to
163 * tell them about its ancestors, which they already know
164 * about.
165 *
166 * We use lookup_object here because we are only
167 * interested in the case we *know* the object is
168 * reachable and we have already scanned it.
169 */
170 if (((o = lookup_object(remote)) != NULL) &&
171 (o->flags & COMPLETE)) {
172 continue;
173 }
174
175 if (!fetching)
176 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
177 sha1_to_hex(remote),
178 (multi_ack ? " multi_ack" : ""),
179 (use_sideband == 2 ? " side-band-64k" : ""),
180 (use_sideband == 1 ? " side-band" : ""),
181 (args.use_thin_pack ? " thin-pack" : ""),
182 (args.no_progress ? " no-progress" : ""),
183 " ofs-delta");
184 else
185 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
186 fetching++;
187 }
188 if (is_repository_shallow())
189 write_shallow_commits(fd[1], 1);
190 if (args.depth > 0)
191 packet_write(fd[1], "deepen %d", args.depth);
192 packet_flush(fd[1]);
193 if (!fetching)
194 return 1;
195
196 if (args.depth > 0) {
197 char line[1024];
198 unsigned char sha1[20];
199 int len;
200
201 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
202 if (!prefixcmp(line, "shallow ")) {
203 if (get_sha1_hex(line + 8, sha1))
204 die("invalid shallow line: %s", line);
205 register_shallow(sha1);
206 continue;
207 }
208 if (!prefixcmp(line, "unshallow ")) {
209 if (get_sha1_hex(line + 10, sha1))
210 die("invalid unshallow line: %s", line);
211 if (!lookup_object(sha1))
212 die("object not found: %s", line);
213 /* make sure that it is parsed as shallow */
214 parse_object(sha1);
215 if (unregister_shallow(sha1))
216 die("no shallow found: %s", line);
217 continue;
218 }
219 die("expected shallow/unshallow, got %s", line);
220 }
221 }
222
223 flushes = 0;
224 retval = -1;
225 while ((sha1 = get_rev())) {
226 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
227 if (args.verbose)
228 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
229 in_vain++;
230 if (!(31 & ++count)) {
231 int ack;
232
233 packet_flush(fd[1]);
234 flushes++;
235
236 /*
237 * We keep one window "ahead" of the other side, and
238 * will wait for an ACK only on the next one
239 */
240 if (count == 32)
241 continue;
242
243 do {
244 ack = get_ack(fd[0], result_sha1);
245 if (args.verbose && ack)
246 fprintf(stderr, "got ack %d %s\n", ack,
247 sha1_to_hex(result_sha1));
248 if (ack == 1) {
249 flushes = 0;
250 multi_ack = 0;
251 retval = 0;
252 goto done;
253 } else if (ack == 2) {
254 struct commit *commit =
255 lookup_commit(result_sha1);
256 mark_common(commit, 0, 1);
257 retval = 0;
258 in_vain = 0;
259 got_continue = 1;
260 }
261 } while (ack);
262 flushes--;
263 if (got_continue && MAX_IN_VAIN < in_vain) {
264 if (args.verbose)
265 fprintf(stderr, "giving up\n");
266 break; /* give up */
267 }
268 }
269 }
270 done:
271 packet_write(fd[1], "done\n");
272 if (args.verbose)
273 fprintf(stderr, "done\n");
274 if (retval != 0) {
275 multi_ack = 0;
276 flushes++;
277 }
278 while (flushes || multi_ack) {
279 int ack = get_ack(fd[0], result_sha1);
280 if (ack) {
281 if (args.verbose)
282 fprintf(stderr, "got ack (%d) %s\n", ack,
283 sha1_to_hex(result_sha1));
284 if (ack == 1)
285 return 0;
286 multi_ack = 1;
287 continue;
288 }
289 flushes--;
290 }
291 return retval;
292 }
293
294 static struct commit_list *complete;
295
296 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
297 {
298 struct object *o = parse_object(sha1);
299
300 while (o && o->type == OBJ_TAG) {
301 struct tag *t = (struct tag *) o;
302 if (!t->tagged)
303 break; /* broken repository */
304 o->flags |= COMPLETE;
305 o = parse_object(t->tagged->sha1);
306 }
307 if (o && o->type == OBJ_COMMIT) {
308 struct commit *commit = (struct commit *)o;
309 commit->object.flags |= COMPLETE;
310 insert_by_date(commit, &complete);
311 }
312 return 0;
313 }
314
315 static void mark_recent_complete_commits(unsigned long cutoff)
316 {
317 while (complete && cutoff <= complete->item->date) {
318 if (args.verbose)
319 fprintf(stderr, "Marking %s as complete\n",
320 sha1_to_hex(complete->item->object.sha1));
321 pop_most_recent_commit(&complete, COMPLETE);
322 }
323 }
324
325 static void filter_refs(struct ref **refs, int nr_match, char **match)
326 {
327 struct ref **return_refs;
328 struct ref *newlist = NULL;
329 struct ref **newtail = &newlist;
330 struct ref *ref, *next;
331 struct ref *fastarray[32];
332
333 if (nr_match && !args.fetch_all) {
334 if (ARRAY_SIZE(fastarray) < nr_match)
335 return_refs = xcalloc(nr_match, sizeof(struct ref *));
336 else {
337 return_refs = fastarray;
338 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
339 }
340 }
341 else
342 return_refs = NULL;
343
344 for (ref = *refs; ref; ref = next) {
345 next = ref->next;
346 if (!memcmp(ref->name, "refs/", 5) &&
347 check_ref_format(ref->name + 5))
348 ; /* trash */
349 else if (args.fetch_all &&
350 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
351 *newtail = ref;
352 ref->next = NULL;
353 newtail = &ref->next;
354 continue;
355 }
356 else {
357 int order = path_match(ref->name, nr_match, match);
358 if (order) {
359 return_refs[order-1] = ref;
360 continue; /* we will link it later */
361 }
362 }
363 free(ref);
364 }
365
366 if (!args.fetch_all) {
367 int i;
368 for (i = 0; i < nr_match; i++) {
369 ref = return_refs[i];
370 if (ref) {
371 *newtail = ref;
372 ref->next = NULL;
373 newtail = &ref->next;
374 }
375 }
376 if (return_refs != fastarray)
377 free(return_refs);
378 }
379 *refs = newlist;
380 }
381
382 static int everything_local(struct ref **refs, int nr_match, char **match)
383 {
384 struct ref *ref;
385 int retval;
386 unsigned long cutoff = 0;
387
388 track_object_refs = 0;
389 save_commit_buffer = 0;
390
391 for (ref = *refs; ref; ref = ref->next) {
392 struct object *o;
393
394 o = parse_object(ref->old_sha1);
395 if (!o)
396 continue;
397
398 /* We already have it -- which may mean that we were
399 * in sync with the other side at some time after
400 * that (it is OK if we guess wrong here).
401 */
402 if (o->type == OBJ_COMMIT) {
403 struct commit *commit = (struct commit *)o;
404 if (!cutoff || cutoff < commit->date)
405 cutoff = commit->date;
406 }
407 }
408
409 if (!args.depth) {
410 for_each_ref(mark_complete, NULL);
411 if (cutoff)
412 mark_recent_complete_commits(cutoff);
413 }
414
415 /*
416 * Mark all complete remote refs as common refs.
417 * Don't mark them common yet; the server has to be told so first.
418 */
419 for (ref = *refs; ref; ref = ref->next) {
420 struct object *o = deref_tag(lookup_object(ref->old_sha1),
421 NULL, 0);
422
423 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
424 continue;
425
426 if (!(o->flags & SEEN)) {
427 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
428
429 mark_common((struct commit *)o, 1, 1);
430 }
431 }
432
433 filter_refs(refs, nr_match, match);
434
435 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
436 const unsigned char *remote = ref->old_sha1;
437 unsigned char local[20];
438 struct object *o;
439
440 o = lookup_object(remote);
441 if (!o || !(o->flags & COMPLETE)) {
442 retval = 0;
443 if (!args.verbose)
444 continue;
445 fprintf(stderr,
446 "want %s (%s)\n", sha1_to_hex(remote),
447 ref->name);
448 continue;
449 }
450
451 hashcpy(ref->new_sha1, local);
452 if (!args.verbose)
453 continue;
454 fprintf(stderr,
455 "already have %s (%s)\n", sha1_to_hex(remote),
456 ref->name);
457 }
458 return retval;
459 }
460
461 static int sideband_demux(int fd, void *data)
462 {
463 int *xd = data;
464
465 close(xd[1]);
466 return recv_sideband("fetch-pack", xd[0], fd, 2);
467 }
468
469 static void setup_sideband(int fd[2], int xd[2], struct async *demux)
470 {
471 if (!use_sideband) {
472 fd[0] = xd[0];
473 fd[1] = xd[1];
474 return;
475 }
476 /* xd[] is talking with upload-pack; subprocess reads from
477 * xd[0], spits out band#2 to stderr, and feeds us band#1
478 * through demux->out.
479 */
480 demux->proc = sideband_demux;
481 demux->data = xd;
482 if (start_async(demux))
483 die("fetch-pack: unable to fork off sideband demultiplexer");
484 close(xd[0]);
485 fd[0] = demux->out;
486 fd[1] = xd[1];
487 }
488
489 static int get_pack(int xd[2], char **pack_lockfile)
490 {
491 struct async demux;
492 int fd[2];
493 const char *argv[20];
494 char keep_arg[256];
495 char hdr_arg[256];
496 const char **av;
497 int do_keep = args.keep_pack;
498 struct child_process cmd;
499
500 setup_sideband(fd, xd, &demux);
501
502 memset(&cmd, 0, sizeof(cmd));
503 cmd.argv = argv;
504 av = argv;
505 *hdr_arg = 0;
506 if (!args.keep_pack && unpack_limit) {
507 struct pack_header header;
508
509 if (read_pack_header(fd[0], &header))
510 die("protocol error: bad pack header");
511 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
512 ntohl(header.hdr_version), ntohl(header.hdr_entries));
513 if (ntohl(header.hdr_entries) < unpack_limit)
514 do_keep = 0;
515 else
516 do_keep = 1;
517 }
518
519 if (do_keep) {
520 if (pack_lockfile)
521 cmd.out = -1;
522 *av++ = "index-pack";
523 *av++ = "--stdin";
524 if (!args.quiet && !args.no_progress)
525 *av++ = "-v";
526 if (args.use_thin_pack)
527 *av++ = "--fix-thin";
528 if (args.lock_pack || unpack_limit) {
529 int s = sprintf(keep_arg,
530 "--keep=fetch-pack %d on ", getpid());
531 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
532 strcpy(keep_arg + s, "localhost");
533 *av++ = keep_arg;
534 }
535 }
536 else {
537 *av++ = "unpack-objects";
538 if (args.quiet)
539 *av++ = "-q";
540 }
541 if (*hdr_arg)
542 *av++ = hdr_arg;
543 *av++ = NULL;
544
545 cmd.in = fd[0];
546 cmd.git_cmd = 1;
547 if (start_command(&cmd))
548 die("fetch-pack: unable to fork off %s", argv[0]);
549 close(fd[1]);
550 if (do_keep && pack_lockfile)
551 *pack_lockfile = index_pack_lockfile(cmd.out);
552
553 if (finish_command(&cmd))
554 die("%s failed", argv[0]);
555 if (use_sideband && finish_async(&demux))
556 die("error in sideband demultiplexer");
557 return 0;
558 }
559
560 static struct ref *do_fetch_pack(int fd[2],
561 int nr_match,
562 char **match,
563 char **pack_lockfile)
564 {
565 struct ref *ref;
566 unsigned char sha1[20];
567
568 get_remote_heads(fd[0], &ref, 0, NULL, 0);
569 if (is_repository_shallow() && !server_supports("shallow"))
570 die("Server does not support shallow clients");
571 if (server_supports("multi_ack")) {
572 if (args.verbose)
573 fprintf(stderr, "Server supports multi_ack\n");
574 multi_ack = 1;
575 }
576 if (server_supports("side-band-64k")) {
577 if (args.verbose)
578 fprintf(stderr, "Server supports side-band-64k\n");
579 use_sideband = 2;
580 }
581 else if (server_supports("side-band")) {
582 if (args.verbose)
583 fprintf(stderr, "Server supports side-band\n");
584 use_sideband = 1;
585 }
586 if (!ref) {
587 packet_flush(fd[1]);
588 die("no matching remote head");
589 }
590 if (everything_local(&ref, nr_match, match)) {
591 packet_flush(fd[1]);
592 goto all_done;
593 }
594 if (find_common(fd, sha1, ref) < 0)
595 if (!args.keep_pack)
596 /* When cloning, it is not unusual to have
597 * no common commit.
598 */
599 fprintf(stderr, "warning: no common commits\n");
600
601 if (get_pack(fd, pack_lockfile))
602 die("git-fetch-pack: fetch failed.");
603
604 all_done:
605 return ref;
606 }
607
608 static int remove_duplicates(int nr_heads, char **heads)
609 {
610 int src, dst;
611
612 for (src = dst = 0; src < nr_heads; src++) {
613 /* If heads[src] is different from any of
614 * heads[0..dst], push it in.
615 */
616 int i;
617 for (i = 0; i < dst; i++) {
618 if (!strcmp(heads[i], heads[src]))
619 break;
620 }
621 if (i < dst)
622 continue;
623 if (src != dst)
624 heads[dst] = heads[src];
625 dst++;
626 }
627 return dst;
628 }
629
630 static int fetch_pack_config(const char *var, const char *value)
631 {
632 if (strcmp(var, "fetch.unpacklimit") == 0) {
633 fetch_unpack_limit = git_config_int(var, value);
634 return 0;
635 }
636
637 if (strcmp(var, "transfer.unpacklimit") == 0) {
638 transfer_unpack_limit = git_config_int(var, value);
639 return 0;
640 }
641
642 return git_default_config(var, value);
643 }
644
645 static struct lock_file lock;
646
647 static void fetch_pack_setup(void)
648 {
649 static int did_setup;
650 if (did_setup)
651 return;
652 git_config(fetch_pack_config);
653 if (0 <= transfer_unpack_limit)
654 unpack_limit = transfer_unpack_limit;
655 else if (0 <= fetch_unpack_limit)
656 unpack_limit = fetch_unpack_limit;
657 did_setup = 1;
658 }
659
660 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
661 {
662 int i, ret, nr_heads;
663 struct ref *ref;
664 char *dest = NULL, **heads;
665
666 nr_heads = 0;
667 heads = NULL;
668 for (i = 1; i < argc; i++) {
669 const char *arg = argv[i];
670
671 if (*arg == '-') {
672 if (!prefixcmp(arg, "--upload-pack=")) {
673 args.uploadpack = arg + 14;
674 continue;
675 }
676 if (!prefixcmp(arg, "--exec=")) {
677 args.uploadpack = arg + 7;
678 continue;
679 }
680 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
681 args.quiet = 1;
682 continue;
683 }
684 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
685 args.lock_pack = args.keep_pack;
686 args.keep_pack = 1;
687 continue;
688 }
689 if (!strcmp("--thin", arg)) {
690 args.use_thin_pack = 1;
691 continue;
692 }
693 if (!strcmp("--all", arg)) {
694 args.fetch_all = 1;
695 continue;
696 }
697 if (!strcmp("-v", arg)) {
698 args.verbose = 1;
699 continue;
700 }
701 if (!prefixcmp(arg, "--depth=")) {
702 args.depth = strtol(arg + 8, NULL, 0);
703 continue;
704 }
705 if (!strcmp("--no-progress", arg)) {
706 args.no_progress = 1;
707 continue;
708 }
709 usage(fetch_pack_usage);
710 }
711 dest = (char *)arg;
712 heads = (char **)(argv + i + 1);
713 nr_heads = argc - i - 1;
714 break;
715 }
716 if (!dest)
717 usage(fetch_pack_usage);
718
719 ref = fetch_pack(&args, dest, nr_heads, heads, NULL);
720 ret = !ref;
721
722 while (ref) {
723 printf("%s %s\n",
724 sha1_to_hex(ref->old_sha1), ref->name);
725 ref = ref->next;
726 }
727
728 return ret;
729 }
730
731 struct ref *fetch_pack(struct fetch_pack_args *my_args,
732 const char *dest,
733 int nr_heads,
734 char **heads,
735 char **pack_lockfile)
736 {
737 int i, ret;
738 int fd[2];
739 struct child_process *conn;
740 struct ref *ref;
741 struct stat st;
742
743 fetch_pack_setup();
744 memcpy(&args, my_args, sizeof(args));
745 if (args.depth > 0) {
746 if (stat(git_path("shallow"), &st))
747 st.st_mtime = 0;
748 }
749
750 conn = git_connect(fd, (char *)dest, args.uploadpack,
751 args.verbose ? CONNECT_VERBOSE : 0);
752 if (heads && nr_heads)
753 nr_heads = remove_duplicates(nr_heads, heads);
754 ref = do_fetch_pack(fd, nr_heads, heads, pack_lockfile);
755 close(fd[0]);
756 close(fd[1]);
757 ret = finish_connect(conn);
758
759 if (!ret && nr_heads) {
760 /* If the heads to pull were given, we should have
761 * consumed all of them by matching the remote.
762 * Otherwise, 'git-fetch remote no-such-ref' would
763 * silently succeed without issuing an error.
764 */
765 for (i = 0; i < nr_heads; i++)
766 if (heads[i] && heads[i][0]) {
767 error("no such remote ref %s", heads[i]);
768 ret = 1;
769 }
770 }
771
772 if (!ret && args.depth > 0) {
773 struct cache_time mtime;
774 char *shallow = git_path("shallow");
775 int fd;
776
777 mtime.sec = st.st_mtime;
778 #ifdef USE_NSEC
779 mtime.usec = st.st_mtim.usec;
780 #endif
781 if (stat(shallow, &st)) {
782 if (mtime.sec)
783 die("shallow file was removed during fetch");
784 } else if (st.st_mtime != mtime.sec
785 #ifdef USE_NSEC
786 || st.st_mtim.usec != mtime.usec
787 #endif
788 )
789 die("shallow file was changed during fetch");
790
791 fd = hold_lock_file_for_update(&lock, shallow, 1);
792 if (!write_shallow_commits(fd, 0)) {
793 unlink(shallow);
794 rollback_lock_file(&lock);
795 } else {
796 close(fd);
797 commit_lock_file(&lock);
798 }
799 }
800
801 if (ret)
802 ref = NULL;
803
804 return ref;
805 }