Merge branch 'mk/maint-parse-careful'
[git/git.git] / builtin-fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
12
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
18 };
19
20 static const char fetch_pack_usage[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
28
29 /*
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
32 */
33 #define MAX_IN_VAIN 256
34
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_sideband;
37
38 static void rev_list_push(struct commit *commit, int mark)
39 {
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
42
43 if (!(commit->object.parsed))
44 parse_commit(commit);
45
46 insert_by_date(commit, &rev_list);
47
48 if (!(commit->object.flags & COMMON))
49 non_common_revs++;
50 }
51 }
52
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
54 {
55 struct object *o = deref_tag(parse_object(sha1), path, 0);
56
57 if (o && o->type == OBJ_COMMIT)
58 rev_list_push((struct commit *)o, SEEN);
59
60 return 0;
61 }
62
63 /*
64 This function marks a rev and its ancestors as common.
65 In some cases, it is desirable to mark only the ancestors (for example
66 when only the server does not yet know that they are common).
67 */
68
69 static void mark_common(struct commit *commit,
70 int ancestors_only, int dont_parse)
71 {
72 if (commit != NULL && !(commit->object.flags & COMMON)) {
73 struct object *o = (struct object *)commit;
74
75 if (!ancestors_only)
76 o->flags |= COMMON;
77
78 if (!(o->flags & SEEN))
79 rev_list_push(commit, SEEN);
80 else {
81 struct commit_list *parents;
82
83 if (!ancestors_only && !(o->flags & POPPED))
84 non_common_revs--;
85 if (!o->parsed && !dont_parse)
86 parse_commit(commit);
87
88 for (parents = commit->parents;
89 parents;
90 parents = parents->next)
91 mark_common(parents->item, 0, dont_parse);
92 }
93 }
94 }
95
96 /*
97 Get the next rev to send, ignoring the common.
98 */
99
100 static const unsigned char* get_rev(void)
101 {
102 struct commit *commit = NULL;
103
104 while (commit == NULL) {
105 unsigned int mark;
106 struct commit_list* parents;
107
108 if (rev_list == NULL || non_common_revs == 0)
109 return NULL;
110
111 commit = rev_list->item;
112 if (!(commit->object.parsed))
113 parse_commit(commit);
114 commit->object.flags |= POPPED;
115 if (!(commit->object.flags & COMMON))
116 non_common_revs--;
117
118 parents = commit->parents;
119
120 if (commit->object.flags & COMMON) {
121 /* do not send "have", and ignore ancestors */
122 commit = NULL;
123 mark = COMMON | SEEN;
124 } else if (commit->object.flags & COMMON_REF)
125 /* send "have", and ignore ancestors */
126 mark = COMMON | SEEN;
127 else
128 /* send "have", also for its ancestors */
129 mark = SEEN;
130
131 while (parents) {
132 if (!(parents->item->object.flags & SEEN))
133 rev_list_push(parents->item, mark);
134 if (mark & COMMON)
135 mark_common(parents->item, 1, 0);
136 parents = parents->next;
137 }
138
139 rev_list = rev_list->next;
140 }
141
142 return commit->object.sha1;
143 }
144
145 static int find_common(int fd[2], unsigned char *result_sha1,
146 struct ref *refs)
147 {
148 int fetching;
149 int count = 0, flushes = 0, retval;
150 const unsigned char *sha1;
151 unsigned in_vain = 0;
152 int got_continue = 0;
153
154 for_each_ref(rev_list_insert_ref, NULL);
155
156 fetching = 0;
157 for ( ; refs ; refs = refs->next) {
158 unsigned char *remote = refs->old_sha1;
159 struct object *o;
160
161 /*
162 * If that object is complete (i.e. it is an ancestor of a
163 * local ref), we tell them we have it but do not have to
164 * tell them about its ancestors, which they already know
165 * about.
166 *
167 * We use lookup_object here because we are only
168 * interested in the case we *know* the object is
169 * reachable and we have already scanned it.
170 */
171 if (((o = lookup_object(remote)) != NULL) &&
172 (o->flags & COMPLETE)) {
173 continue;
174 }
175
176 if (!fetching)
177 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
178 sha1_to_hex(remote),
179 (multi_ack ? " multi_ack" : ""),
180 (use_sideband == 2 ? " side-band-64k" : ""),
181 (use_sideband == 1 ? " side-band" : ""),
182 (args.use_thin_pack ? " thin-pack" : ""),
183 (args.no_progress ? " no-progress" : ""),
184 " ofs-delta");
185 else
186 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
187 fetching++;
188 }
189 if (is_repository_shallow())
190 write_shallow_commits(fd[1], 1);
191 if (args.depth > 0)
192 packet_write(fd[1], "deepen %d", args.depth);
193 packet_flush(fd[1]);
194 if (!fetching)
195 return 1;
196
197 if (args.depth > 0) {
198 char line[1024];
199 unsigned char sha1[20];
200 int len;
201
202 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
203 if (!prefixcmp(line, "shallow ")) {
204 if (get_sha1_hex(line + 8, sha1))
205 die("invalid shallow line: %s", line);
206 register_shallow(sha1);
207 continue;
208 }
209 if (!prefixcmp(line, "unshallow ")) {
210 if (get_sha1_hex(line + 10, sha1))
211 die("invalid unshallow line: %s", line);
212 if (!lookup_object(sha1))
213 die("object not found: %s", line);
214 /* make sure that it is parsed as shallow */
215 parse_object(sha1);
216 if (unregister_shallow(sha1))
217 die("no shallow found: %s", line);
218 continue;
219 }
220 die("expected shallow/unshallow, got %s", line);
221 }
222 }
223
224 flushes = 0;
225 retval = -1;
226 while ((sha1 = get_rev())) {
227 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
228 if (args.verbose)
229 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
230 in_vain++;
231 if (!(31 & ++count)) {
232 int ack;
233
234 packet_flush(fd[1]);
235 flushes++;
236
237 /*
238 * We keep one window "ahead" of the other side, and
239 * will wait for an ACK only on the next one
240 */
241 if (count == 32)
242 continue;
243
244 do {
245 ack = get_ack(fd[0], result_sha1);
246 if (args.verbose && ack)
247 fprintf(stderr, "got ack %d %s\n", ack,
248 sha1_to_hex(result_sha1));
249 if (ack == 1) {
250 flushes = 0;
251 multi_ack = 0;
252 retval = 0;
253 goto done;
254 } else if (ack == 2) {
255 struct commit *commit =
256 lookup_commit(result_sha1);
257 mark_common(commit, 0, 1);
258 retval = 0;
259 in_vain = 0;
260 got_continue = 1;
261 }
262 } while (ack);
263 flushes--;
264 if (got_continue && MAX_IN_VAIN < in_vain) {
265 if (args.verbose)
266 fprintf(stderr, "giving up\n");
267 break; /* give up */
268 }
269 }
270 }
271 done:
272 packet_write(fd[1], "done\n");
273 if (args.verbose)
274 fprintf(stderr, "done\n");
275 if (retval != 0) {
276 multi_ack = 0;
277 flushes++;
278 }
279 while (flushes || multi_ack) {
280 int ack = get_ack(fd[0], result_sha1);
281 if (ack) {
282 if (args.verbose)
283 fprintf(stderr, "got ack (%d) %s\n", ack,
284 sha1_to_hex(result_sha1));
285 if (ack == 1)
286 return 0;
287 multi_ack = 1;
288 continue;
289 }
290 flushes--;
291 }
292 return retval;
293 }
294
295 static struct commit_list *complete;
296
297 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
298 {
299 struct object *o = parse_object(sha1);
300
301 while (o && o->type == OBJ_TAG) {
302 struct tag *t = (struct tag *) o;
303 if (!t->tagged)
304 break; /* broken repository */
305 o->flags |= COMPLETE;
306 o = parse_object(t->tagged->sha1);
307 }
308 if (o && o->type == OBJ_COMMIT) {
309 struct commit *commit = (struct commit *)o;
310 commit->object.flags |= COMPLETE;
311 insert_by_date(commit, &complete);
312 }
313 return 0;
314 }
315
316 static void mark_recent_complete_commits(unsigned long cutoff)
317 {
318 while (complete && cutoff <= complete->item->date) {
319 if (args.verbose)
320 fprintf(stderr, "Marking %s as complete\n",
321 sha1_to_hex(complete->item->object.sha1));
322 pop_most_recent_commit(&complete, COMPLETE);
323 }
324 }
325
326 static void filter_refs(struct ref **refs, int nr_match, char **match)
327 {
328 struct ref **return_refs;
329 struct ref *newlist = NULL;
330 struct ref **newtail = &newlist;
331 struct ref *ref, *next;
332 struct ref *fastarray[32];
333
334 if (nr_match && !args.fetch_all) {
335 if (ARRAY_SIZE(fastarray) < nr_match)
336 return_refs = xcalloc(nr_match, sizeof(struct ref *));
337 else {
338 return_refs = fastarray;
339 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
340 }
341 }
342 else
343 return_refs = NULL;
344
345 for (ref = *refs; ref; ref = next) {
346 next = ref->next;
347 if (!memcmp(ref->name, "refs/", 5) &&
348 check_ref_format(ref->name + 5))
349 ; /* trash */
350 else if (args.fetch_all &&
351 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
352 *newtail = ref;
353 ref->next = NULL;
354 newtail = &ref->next;
355 continue;
356 }
357 else {
358 int order = path_match(ref->name, nr_match, match);
359 if (order) {
360 return_refs[order-1] = ref;
361 continue; /* we will link it later */
362 }
363 }
364 free(ref);
365 }
366
367 if (!args.fetch_all) {
368 int i;
369 for (i = 0; i < nr_match; i++) {
370 ref = return_refs[i];
371 if (ref) {
372 *newtail = ref;
373 ref->next = NULL;
374 newtail = &ref->next;
375 }
376 }
377 if (return_refs != fastarray)
378 free(return_refs);
379 }
380 *refs = newlist;
381 }
382
383 static int everything_local(struct ref **refs, int nr_match, char **match)
384 {
385 struct ref *ref;
386 int retval;
387 unsigned long cutoff = 0;
388
389 save_commit_buffer = 0;
390
391 for (ref = *refs; ref; ref = ref->next) {
392 struct object *o;
393
394 o = parse_object(ref->old_sha1);
395 if (!o)
396 continue;
397
398 /* We already have it -- which may mean that we were
399 * in sync with the other side at some time after
400 * that (it is OK if we guess wrong here).
401 */
402 if (o->type == OBJ_COMMIT) {
403 struct commit *commit = (struct commit *)o;
404 if (!cutoff || cutoff < commit->date)
405 cutoff = commit->date;
406 }
407 }
408
409 if (!args.depth) {
410 for_each_ref(mark_complete, NULL);
411 if (cutoff)
412 mark_recent_complete_commits(cutoff);
413 }
414
415 /*
416 * Mark all complete remote refs as common refs.
417 * Don't mark them common yet; the server has to be told so first.
418 */
419 for (ref = *refs; ref; ref = ref->next) {
420 struct object *o = deref_tag(lookup_object(ref->old_sha1),
421 NULL, 0);
422
423 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
424 continue;
425
426 if (!(o->flags & SEEN)) {
427 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
428
429 mark_common((struct commit *)o, 1, 1);
430 }
431 }
432
433 filter_refs(refs, nr_match, match);
434
435 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
436 const unsigned char *remote = ref->old_sha1;
437 unsigned char local[20];
438 struct object *o;
439
440 o = lookup_object(remote);
441 if (!o || !(o->flags & COMPLETE)) {
442 retval = 0;
443 if (!args.verbose)
444 continue;
445 fprintf(stderr,
446 "want %s (%s)\n", sha1_to_hex(remote),
447 ref->name);
448 continue;
449 }
450
451 hashcpy(ref->new_sha1, local);
452 if (!args.verbose)
453 continue;
454 fprintf(stderr,
455 "already have %s (%s)\n", sha1_to_hex(remote),
456 ref->name);
457 }
458 return retval;
459 }
460
461 static int sideband_demux(int fd, void *data)
462 {
463 int *xd = data;
464
465 return recv_sideband("fetch-pack", xd[0], fd, 2);
466 }
467
468 static int get_pack(int xd[2], char **pack_lockfile)
469 {
470 struct async demux;
471 const char *argv[20];
472 char keep_arg[256];
473 char hdr_arg[256];
474 const char **av;
475 int do_keep = args.keep_pack;
476 struct child_process cmd;
477
478 memset(&demux, 0, sizeof(demux));
479 if (use_sideband) {
480 /* xd[] is talking with upload-pack; subprocess reads from
481 * xd[0], spits out band#2 to stderr, and feeds us band#1
482 * through demux->out.
483 */
484 demux.proc = sideband_demux;
485 demux.data = xd;
486 if (start_async(&demux))
487 die("fetch-pack: unable to fork off sideband"
488 " demultiplexer");
489 }
490 else
491 demux.out = xd[0];
492
493 memset(&cmd, 0, sizeof(cmd));
494 cmd.argv = argv;
495 av = argv;
496 *hdr_arg = 0;
497 if (!args.keep_pack && unpack_limit) {
498 struct pack_header header;
499
500 if (read_pack_header(demux.out, &header))
501 die("protocol error: bad pack header");
502 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
503 ntohl(header.hdr_version), ntohl(header.hdr_entries));
504 if (ntohl(header.hdr_entries) < unpack_limit)
505 do_keep = 0;
506 else
507 do_keep = 1;
508 }
509
510 if (do_keep) {
511 if (pack_lockfile)
512 cmd.out = -1;
513 *av++ = "index-pack";
514 *av++ = "--stdin";
515 if (!args.quiet && !args.no_progress)
516 *av++ = "-v";
517 if (args.use_thin_pack)
518 *av++ = "--fix-thin";
519 if (args.lock_pack || unpack_limit) {
520 int s = sprintf(keep_arg,
521 "--keep=fetch-pack %d on ", getpid());
522 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
523 strcpy(keep_arg + s, "localhost");
524 *av++ = keep_arg;
525 }
526 }
527 else {
528 *av++ = "unpack-objects";
529 if (args.quiet)
530 *av++ = "-q";
531 }
532 if (*hdr_arg)
533 *av++ = hdr_arg;
534 *av++ = NULL;
535
536 cmd.in = demux.out;
537 cmd.git_cmd = 1;
538 if (start_command(&cmd))
539 die("fetch-pack: unable to fork off %s", argv[0]);
540 if (do_keep && pack_lockfile) {
541 *pack_lockfile = index_pack_lockfile(cmd.out);
542 close(cmd.out);
543 }
544
545 if (finish_command(&cmd))
546 die("%s failed", argv[0]);
547 if (use_sideband && finish_async(&demux))
548 die("error in sideband demultiplexer");
549 return 0;
550 }
551
552 static struct ref *do_fetch_pack(int fd[2],
553 const struct ref *orig_ref,
554 int nr_match,
555 char **match,
556 char **pack_lockfile)
557 {
558 struct ref *ref = copy_ref_list(orig_ref);
559 unsigned char sha1[20];
560
561 if (is_repository_shallow() && !server_supports("shallow"))
562 die("Server does not support shallow clients");
563 if (server_supports("multi_ack")) {
564 if (args.verbose)
565 fprintf(stderr, "Server supports multi_ack\n");
566 multi_ack = 1;
567 }
568 if (server_supports("side-band-64k")) {
569 if (args.verbose)
570 fprintf(stderr, "Server supports side-band-64k\n");
571 use_sideband = 2;
572 }
573 else if (server_supports("side-band")) {
574 if (args.verbose)
575 fprintf(stderr, "Server supports side-band\n");
576 use_sideband = 1;
577 }
578 if (everything_local(&ref, nr_match, match)) {
579 packet_flush(fd[1]);
580 goto all_done;
581 }
582 if (find_common(fd, sha1, ref) < 0)
583 if (!args.keep_pack)
584 /* When cloning, it is not unusual to have
585 * no common commit.
586 */
587 fprintf(stderr, "warning: no common commits\n");
588
589 if (get_pack(fd, pack_lockfile))
590 die("git-fetch-pack: fetch failed.");
591
592 all_done:
593 return ref;
594 }
595
596 static int remove_duplicates(int nr_heads, char **heads)
597 {
598 int src, dst;
599
600 for (src = dst = 0; src < nr_heads; src++) {
601 /* If heads[src] is different from any of
602 * heads[0..dst], push it in.
603 */
604 int i;
605 for (i = 0; i < dst; i++) {
606 if (!strcmp(heads[i], heads[src]))
607 break;
608 }
609 if (i < dst)
610 continue;
611 if (src != dst)
612 heads[dst] = heads[src];
613 dst++;
614 }
615 return dst;
616 }
617
618 static int fetch_pack_config(const char *var, const char *value)
619 {
620 if (strcmp(var, "fetch.unpacklimit") == 0) {
621 fetch_unpack_limit = git_config_int(var, value);
622 return 0;
623 }
624
625 if (strcmp(var, "transfer.unpacklimit") == 0) {
626 transfer_unpack_limit = git_config_int(var, value);
627 return 0;
628 }
629
630 return git_default_config(var, value);
631 }
632
633 static struct lock_file lock;
634
635 static void fetch_pack_setup(void)
636 {
637 static int did_setup;
638 if (did_setup)
639 return;
640 git_config(fetch_pack_config);
641 if (0 <= transfer_unpack_limit)
642 unpack_limit = transfer_unpack_limit;
643 else if (0 <= fetch_unpack_limit)
644 unpack_limit = fetch_unpack_limit;
645 did_setup = 1;
646 }
647
648 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
649 {
650 int i, ret, nr_heads;
651 struct ref *ref = NULL;
652 char *dest = NULL, **heads;
653 int fd[2];
654 struct child_process *conn;
655
656 nr_heads = 0;
657 heads = NULL;
658 for (i = 1; i < argc; i++) {
659 const char *arg = argv[i];
660
661 if (*arg == '-') {
662 if (!prefixcmp(arg, "--upload-pack=")) {
663 args.uploadpack = arg + 14;
664 continue;
665 }
666 if (!prefixcmp(arg, "--exec=")) {
667 args.uploadpack = arg + 7;
668 continue;
669 }
670 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
671 args.quiet = 1;
672 continue;
673 }
674 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
675 args.lock_pack = args.keep_pack;
676 args.keep_pack = 1;
677 continue;
678 }
679 if (!strcmp("--thin", arg)) {
680 args.use_thin_pack = 1;
681 continue;
682 }
683 if (!strcmp("--all", arg)) {
684 args.fetch_all = 1;
685 continue;
686 }
687 if (!strcmp("-v", arg)) {
688 args.verbose = 1;
689 continue;
690 }
691 if (!prefixcmp(arg, "--depth=")) {
692 args.depth = strtol(arg + 8, NULL, 0);
693 continue;
694 }
695 if (!strcmp("--no-progress", arg)) {
696 args.no_progress = 1;
697 continue;
698 }
699 usage(fetch_pack_usage);
700 }
701 dest = (char *)arg;
702 heads = (char **)(argv + i + 1);
703 nr_heads = argc - i - 1;
704 break;
705 }
706 if (!dest)
707 usage(fetch_pack_usage);
708
709 conn = git_connect(fd, (char *)dest, args.uploadpack,
710 args.verbose ? CONNECT_VERBOSE : 0);
711 if (conn) {
712 get_remote_heads(fd[0], &ref, 0, NULL, 0);
713
714 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
715 close(fd[0]);
716 close(fd[1]);
717 if (finish_connect(conn))
718 ref = NULL;
719 } else {
720 ref = NULL;
721 }
722 ret = !ref;
723
724 if (!ret && nr_heads) {
725 /* If the heads to pull were given, we should have
726 * consumed all of them by matching the remote.
727 * Otherwise, 'git-fetch remote no-such-ref' would
728 * silently succeed without issuing an error.
729 */
730 for (i = 0; i < nr_heads; i++)
731 if (heads[i] && heads[i][0]) {
732 error("no such remote ref %s", heads[i]);
733 ret = 1;
734 }
735 }
736 while (ref) {
737 printf("%s %s\n",
738 sha1_to_hex(ref->old_sha1), ref->name);
739 ref = ref->next;
740 }
741
742 return ret;
743 }
744
745 struct ref *fetch_pack(struct fetch_pack_args *my_args,
746 int fd[], struct child_process *conn,
747 const struct ref *ref,
748 const char *dest,
749 int nr_heads,
750 char **heads,
751 char **pack_lockfile)
752 {
753 struct stat st;
754 struct ref *ref_cpy;
755
756 fetch_pack_setup();
757 memcpy(&args, my_args, sizeof(args));
758 if (args.depth > 0) {
759 if (stat(git_path("shallow"), &st))
760 st.st_mtime = 0;
761 }
762
763 if (heads && nr_heads)
764 nr_heads = remove_duplicates(nr_heads, heads);
765 if (!ref) {
766 packet_flush(fd[1]);
767 die("no matching remote head");
768 }
769 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
770
771 if (args.depth > 0) {
772 struct cache_time mtime;
773 char *shallow = git_path("shallow");
774 int fd;
775
776 mtime.sec = st.st_mtime;
777 #ifdef USE_NSEC
778 mtime.usec = st.st_mtim.usec;
779 #endif
780 if (stat(shallow, &st)) {
781 if (mtime.sec)
782 die("shallow file was removed during fetch");
783 } else if (st.st_mtime != mtime.sec
784 #ifdef USE_NSEC
785 || st.st_mtim.usec != mtime.usec
786 #endif
787 )
788 die("shallow file was changed during fetch");
789
790 fd = hold_lock_file_for_update(&lock, shallow, 1);
791 if (!write_shallow_commits(fd, 0)) {
792 unlink(shallow);
793 rollback_lock_file(&lock);
794 } else {
795 commit_lock_file(&lock);
796 }
797 }
798
799 return ref_cpy;
800 }