gitweb: Clearly distinguish regexp / exact match searches
[git/git.git] / builtin-fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
12
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
18 };
19
20 static const char fetch_pack_usage[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
28
29 /*
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
32 */
33 #define MAX_IN_VAIN 256
34
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_sideband;
37
38 static void rev_list_push(struct commit *commit, int mark)
39 {
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
42
43 if (!(commit->object.parsed))
44 parse_commit(commit);
45
46 insert_by_date(commit, &rev_list);
47
48 if (!(commit->object.flags & COMMON))
49 non_common_revs++;
50 }
51 }
52
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
54 {
55 struct object *o = deref_tag(parse_object(sha1), path, 0);
56
57 if (o && o->type == OBJ_COMMIT)
58 rev_list_push((struct commit *)o, SEEN);
59
60 return 0;
61 }
62
63 /*
64 This function marks a rev and its ancestors as common.
65 In some cases, it is desirable to mark only the ancestors (for example
66 when only the server does not yet know that they are common).
67 */
68
69 static void mark_common(struct commit *commit,
70 int ancestors_only, int dont_parse)
71 {
72 if (commit != NULL && !(commit->object.flags & COMMON)) {
73 struct object *o = (struct object *)commit;
74
75 if (!ancestors_only)
76 o->flags |= COMMON;
77
78 if (!(o->flags & SEEN))
79 rev_list_push(commit, SEEN);
80 else {
81 struct commit_list *parents;
82
83 if (!ancestors_only && !(o->flags & POPPED))
84 non_common_revs--;
85 if (!o->parsed && !dont_parse)
86 parse_commit(commit);
87
88 for (parents = commit->parents;
89 parents;
90 parents = parents->next)
91 mark_common(parents->item, 0, dont_parse);
92 }
93 }
94 }
95
96 /*
97 Get the next rev to send, ignoring the common.
98 */
99
100 static const unsigned char* get_rev(void)
101 {
102 struct commit *commit = NULL;
103
104 while (commit == NULL) {
105 unsigned int mark;
106 struct commit_list* parents;
107
108 if (rev_list == NULL || non_common_revs == 0)
109 return NULL;
110
111 commit = rev_list->item;
112 if (!(commit->object.parsed))
113 parse_commit(commit);
114 commit->object.flags |= POPPED;
115 if (!(commit->object.flags & COMMON))
116 non_common_revs--;
117
118 parents = commit->parents;
119
120 if (commit->object.flags & COMMON) {
121 /* do not send "have", and ignore ancestors */
122 commit = NULL;
123 mark = COMMON | SEEN;
124 } else if (commit->object.flags & COMMON_REF)
125 /* send "have", and ignore ancestors */
126 mark = COMMON | SEEN;
127 else
128 /* send "have", also for its ancestors */
129 mark = SEEN;
130
131 while (parents) {
132 if (!(parents->item->object.flags & SEEN))
133 rev_list_push(parents->item, mark);
134 if (mark & COMMON)
135 mark_common(parents->item, 1, 0);
136 parents = parents->next;
137 }
138
139 rev_list = rev_list->next;
140 }
141
142 return commit->object.sha1;
143 }
144
145 static int find_common(int fd[2], unsigned char *result_sha1,
146 struct ref *refs)
147 {
148 int fetching;
149 int count = 0, flushes = 0, retval;
150 const unsigned char *sha1;
151 unsigned in_vain = 0;
152 int got_continue = 0;
153
154 for_each_ref(rev_list_insert_ref, NULL);
155
156 fetching = 0;
157 for ( ; refs ; refs = refs->next) {
158 unsigned char *remote = refs->old_sha1;
159 struct object *o;
160
161 /*
162 * If that object is complete (i.e. it is an ancestor of a
163 * local ref), we tell them we have it but do not have to
164 * tell them about its ancestors, which they already know
165 * about.
166 *
167 * We use lookup_object here because we are only
168 * interested in the case we *know* the object is
169 * reachable and we have already scanned it.
170 */
171 if (((o = lookup_object(remote)) != NULL) &&
172 (o->flags & COMPLETE)) {
173 continue;
174 }
175
176 if (!fetching)
177 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
178 sha1_to_hex(remote),
179 (multi_ack ? " multi_ack" : ""),
180 (use_sideband == 2 ? " side-band-64k" : ""),
181 (use_sideband == 1 ? " side-band" : ""),
182 (args.use_thin_pack ? " thin-pack" : ""),
183 (args.no_progress ? " no-progress" : ""),
184 " ofs-delta");
185 else
186 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
187 fetching++;
188 }
189 if (is_repository_shallow())
190 write_shallow_commits(fd[1], 1);
191 if (args.depth > 0)
192 packet_write(fd[1], "deepen %d", args.depth);
193 packet_flush(fd[1]);
194 if (!fetching)
195 return 1;
196
197 if (args.depth > 0) {
198 char line[1024];
199 unsigned char sha1[20];
200 int len;
201
202 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
203 if (!prefixcmp(line, "shallow ")) {
204 if (get_sha1_hex(line + 8, sha1))
205 die("invalid shallow line: %s", line);
206 register_shallow(sha1);
207 continue;
208 }
209 if (!prefixcmp(line, "unshallow ")) {
210 if (get_sha1_hex(line + 10, sha1))
211 die("invalid unshallow line: %s", line);
212 if (!lookup_object(sha1))
213 die("object not found: %s", line);
214 /* make sure that it is parsed as shallow */
215 parse_object(sha1);
216 if (unregister_shallow(sha1))
217 die("no shallow found: %s", line);
218 continue;
219 }
220 die("expected shallow/unshallow, got %s", line);
221 }
222 }
223
224 flushes = 0;
225 retval = -1;
226 while ((sha1 = get_rev())) {
227 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
228 if (args.verbose)
229 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
230 in_vain++;
231 if (!(31 & ++count)) {
232 int ack;
233
234 packet_flush(fd[1]);
235 flushes++;
236
237 /*
238 * We keep one window "ahead" of the other side, and
239 * will wait for an ACK only on the next one
240 */
241 if (count == 32)
242 continue;
243
244 do {
245 ack = get_ack(fd[0], result_sha1);
246 if (args.verbose && ack)
247 fprintf(stderr, "got ack %d %s\n", ack,
248 sha1_to_hex(result_sha1));
249 if (ack == 1) {
250 flushes = 0;
251 multi_ack = 0;
252 retval = 0;
253 goto done;
254 } else if (ack == 2) {
255 struct commit *commit =
256 lookup_commit(result_sha1);
257 mark_common(commit, 0, 1);
258 retval = 0;
259 in_vain = 0;
260 got_continue = 1;
261 }
262 } while (ack);
263 flushes--;
264 if (got_continue && MAX_IN_VAIN < in_vain) {
265 if (args.verbose)
266 fprintf(stderr, "giving up\n");
267 break; /* give up */
268 }
269 }
270 }
271 done:
272 packet_write(fd[1], "done\n");
273 if (args.verbose)
274 fprintf(stderr, "done\n");
275 if (retval != 0) {
276 multi_ack = 0;
277 flushes++;
278 }
279 while (flushes || multi_ack) {
280 int ack = get_ack(fd[0], result_sha1);
281 if (ack) {
282 if (args.verbose)
283 fprintf(stderr, "got ack (%d) %s\n", ack,
284 sha1_to_hex(result_sha1));
285 if (ack == 1)
286 return 0;
287 multi_ack = 1;
288 continue;
289 }
290 flushes--;
291 }
292 return retval;
293 }
294
295 static struct commit_list *complete;
296
297 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
298 {
299 struct object *o = parse_object(sha1);
300
301 while (o && o->type == OBJ_TAG) {
302 struct tag *t = (struct tag *) o;
303 if (!t->tagged)
304 break; /* broken repository */
305 o->flags |= COMPLETE;
306 o = parse_object(t->tagged->sha1);
307 }
308 if (o && o->type == OBJ_COMMIT) {
309 struct commit *commit = (struct commit *)o;
310 commit->object.flags |= COMPLETE;
311 insert_by_date(commit, &complete);
312 }
313 return 0;
314 }
315
316 static void mark_recent_complete_commits(unsigned long cutoff)
317 {
318 while (complete && cutoff <= complete->item->date) {
319 if (args.verbose)
320 fprintf(stderr, "Marking %s as complete\n",
321 sha1_to_hex(complete->item->object.sha1));
322 pop_most_recent_commit(&complete, COMPLETE);
323 }
324 }
325
326 static void filter_refs(struct ref **refs, int nr_match, char **match)
327 {
328 struct ref **return_refs;
329 struct ref *newlist = NULL;
330 struct ref **newtail = &newlist;
331 struct ref *ref, *next;
332 struct ref *fastarray[32];
333
334 if (nr_match && !args.fetch_all) {
335 if (ARRAY_SIZE(fastarray) < nr_match)
336 return_refs = xcalloc(nr_match, sizeof(struct ref *));
337 else {
338 return_refs = fastarray;
339 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
340 }
341 }
342 else
343 return_refs = NULL;
344
345 for (ref = *refs; ref; ref = next) {
346 next = ref->next;
347 if (!memcmp(ref->name, "refs/", 5) &&
348 check_ref_format(ref->name + 5))
349 ; /* trash */
350 else if (args.fetch_all &&
351 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
352 *newtail = ref;
353 ref->next = NULL;
354 newtail = &ref->next;
355 continue;
356 }
357 else {
358 int order = path_match(ref->name, nr_match, match);
359 if (order) {
360 return_refs[order-1] = ref;
361 continue; /* we will link it later */
362 }
363 }
364 free(ref);
365 }
366
367 if (!args.fetch_all) {
368 int i;
369 for (i = 0; i < nr_match; i++) {
370 ref = return_refs[i];
371 if (ref) {
372 *newtail = ref;
373 ref->next = NULL;
374 newtail = &ref->next;
375 }
376 }
377 if (return_refs != fastarray)
378 free(return_refs);
379 }
380 *refs = newlist;
381 }
382
383 static int everything_local(struct ref **refs, int nr_match, char **match)
384 {
385 struct ref *ref;
386 int retval;
387 unsigned long cutoff = 0;
388
389 track_object_refs = 0;
390 save_commit_buffer = 0;
391
392 for (ref = *refs; ref; ref = ref->next) {
393 struct object *o;
394
395 o = parse_object(ref->old_sha1);
396 if (!o)
397 continue;
398
399 /* We already have it -- which may mean that we were
400 * in sync with the other side at some time after
401 * that (it is OK if we guess wrong here).
402 */
403 if (o->type == OBJ_COMMIT) {
404 struct commit *commit = (struct commit *)o;
405 if (!cutoff || cutoff < commit->date)
406 cutoff = commit->date;
407 }
408 }
409
410 if (!args.depth) {
411 for_each_ref(mark_complete, NULL);
412 if (cutoff)
413 mark_recent_complete_commits(cutoff);
414 }
415
416 /*
417 * Mark all complete remote refs as common refs.
418 * Don't mark them common yet; the server has to be told so first.
419 */
420 for (ref = *refs; ref; ref = ref->next) {
421 struct object *o = deref_tag(lookup_object(ref->old_sha1),
422 NULL, 0);
423
424 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
425 continue;
426
427 if (!(o->flags & SEEN)) {
428 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
429
430 mark_common((struct commit *)o, 1, 1);
431 }
432 }
433
434 filter_refs(refs, nr_match, match);
435
436 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
437 const unsigned char *remote = ref->old_sha1;
438 unsigned char local[20];
439 struct object *o;
440
441 o = lookup_object(remote);
442 if (!o || !(o->flags & COMPLETE)) {
443 retval = 0;
444 if (!args.verbose)
445 continue;
446 fprintf(stderr,
447 "want %s (%s)\n", sha1_to_hex(remote),
448 ref->name);
449 continue;
450 }
451
452 hashcpy(ref->new_sha1, local);
453 if (!args.verbose)
454 continue;
455 fprintf(stderr,
456 "already have %s (%s)\n", sha1_to_hex(remote),
457 ref->name);
458 }
459 return retval;
460 }
461
462 static int sideband_demux(int fd, void *data)
463 {
464 int *xd = data;
465
466 return recv_sideband("fetch-pack", xd[0], fd, 2);
467 }
468
469 static int get_pack(int xd[2], char **pack_lockfile)
470 {
471 struct async demux;
472 const char *argv[20];
473 char keep_arg[256];
474 char hdr_arg[256];
475 const char **av;
476 int do_keep = args.keep_pack;
477 struct child_process cmd;
478
479 memset(&demux, 0, sizeof(demux));
480 if (use_sideband) {
481 /* xd[] is talking with upload-pack; subprocess reads from
482 * xd[0], spits out band#2 to stderr, and feeds us band#1
483 * through demux->out.
484 */
485 demux.proc = sideband_demux;
486 demux.data = xd;
487 if (start_async(&demux))
488 die("fetch-pack: unable to fork off sideband"
489 " demultiplexer");
490 }
491 else
492 demux.out = xd[0];
493
494 memset(&cmd, 0, sizeof(cmd));
495 cmd.argv = argv;
496 av = argv;
497 *hdr_arg = 0;
498 if (!args.keep_pack && unpack_limit) {
499 struct pack_header header;
500
501 if (read_pack_header(demux.out, &header))
502 die("protocol error: bad pack header");
503 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
504 ntohl(header.hdr_version), ntohl(header.hdr_entries));
505 if (ntohl(header.hdr_entries) < unpack_limit)
506 do_keep = 0;
507 else
508 do_keep = 1;
509 }
510
511 if (do_keep) {
512 if (pack_lockfile)
513 cmd.out = -1;
514 *av++ = "index-pack";
515 *av++ = "--stdin";
516 if (!args.quiet && !args.no_progress)
517 *av++ = "-v";
518 if (args.use_thin_pack)
519 *av++ = "--fix-thin";
520 if (args.lock_pack || unpack_limit) {
521 int s = sprintf(keep_arg,
522 "--keep=fetch-pack %d on ", getpid());
523 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
524 strcpy(keep_arg + s, "localhost");
525 *av++ = keep_arg;
526 }
527 }
528 else {
529 *av++ = "unpack-objects";
530 if (args.quiet)
531 *av++ = "-q";
532 }
533 if (*hdr_arg)
534 *av++ = hdr_arg;
535 *av++ = NULL;
536
537 cmd.in = demux.out;
538 cmd.git_cmd = 1;
539 if (start_command(&cmd))
540 die("fetch-pack: unable to fork off %s", argv[0]);
541 if (do_keep && pack_lockfile)
542 *pack_lockfile = index_pack_lockfile(cmd.out);
543
544 if (finish_command(&cmd))
545 die("%s failed", argv[0]);
546 if (use_sideband && finish_async(&demux))
547 die("error in sideband demultiplexer");
548 return 0;
549 }
550
551 static struct ref *do_fetch_pack(int fd[2],
552 const struct ref *orig_ref,
553 int nr_match,
554 char **match,
555 char **pack_lockfile)
556 {
557 struct ref *ref = copy_ref_list(orig_ref);
558 unsigned char sha1[20];
559
560 if (is_repository_shallow() && !server_supports("shallow"))
561 die("Server does not support shallow clients");
562 if (server_supports("multi_ack")) {
563 if (args.verbose)
564 fprintf(stderr, "Server supports multi_ack\n");
565 multi_ack = 1;
566 }
567 if (server_supports("side-band-64k")) {
568 if (args.verbose)
569 fprintf(stderr, "Server supports side-band-64k\n");
570 use_sideband = 2;
571 }
572 else if (server_supports("side-band")) {
573 if (args.verbose)
574 fprintf(stderr, "Server supports side-band\n");
575 use_sideband = 1;
576 }
577 if (everything_local(&ref, nr_match, match)) {
578 packet_flush(fd[1]);
579 goto all_done;
580 }
581 if (find_common(fd, sha1, ref) < 0)
582 if (!args.keep_pack)
583 /* When cloning, it is not unusual to have
584 * no common commit.
585 */
586 fprintf(stderr, "warning: no common commits\n");
587
588 if (get_pack(fd, pack_lockfile))
589 die("git-fetch-pack: fetch failed.");
590
591 all_done:
592 return ref;
593 }
594
595 static int remove_duplicates(int nr_heads, char **heads)
596 {
597 int src, dst;
598
599 for (src = dst = 0; src < nr_heads; src++) {
600 /* If heads[src] is different from any of
601 * heads[0..dst], push it in.
602 */
603 int i;
604 for (i = 0; i < dst; i++) {
605 if (!strcmp(heads[i], heads[src]))
606 break;
607 }
608 if (i < dst)
609 continue;
610 if (src != dst)
611 heads[dst] = heads[src];
612 dst++;
613 }
614 return dst;
615 }
616
617 static int fetch_pack_config(const char *var, const char *value)
618 {
619 if (strcmp(var, "fetch.unpacklimit") == 0) {
620 fetch_unpack_limit = git_config_int(var, value);
621 return 0;
622 }
623
624 if (strcmp(var, "transfer.unpacklimit") == 0) {
625 transfer_unpack_limit = git_config_int(var, value);
626 return 0;
627 }
628
629 return git_default_config(var, value);
630 }
631
632 static struct lock_file lock;
633
634 static void fetch_pack_setup(void)
635 {
636 static int did_setup;
637 if (did_setup)
638 return;
639 git_config(fetch_pack_config);
640 if (0 <= transfer_unpack_limit)
641 unpack_limit = transfer_unpack_limit;
642 else if (0 <= fetch_unpack_limit)
643 unpack_limit = fetch_unpack_limit;
644 did_setup = 1;
645 }
646
647 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
648 {
649 int i, ret, nr_heads;
650 struct ref *ref = NULL;
651 char *dest = NULL, **heads;
652 int fd[2];
653 struct child_process *conn;
654
655 nr_heads = 0;
656 heads = NULL;
657 for (i = 1; i < argc; i++) {
658 const char *arg = argv[i];
659
660 if (*arg == '-') {
661 if (!prefixcmp(arg, "--upload-pack=")) {
662 args.uploadpack = arg + 14;
663 continue;
664 }
665 if (!prefixcmp(arg, "--exec=")) {
666 args.uploadpack = arg + 7;
667 continue;
668 }
669 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
670 args.quiet = 1;
671 continue;
672 }
673 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
674 args.lock_pack = args.keep_pack;
675 args.keep_pack = 1;
676 continue;
677 }
678 if (!strcmp("--thin", arg)) {
679 args.use_thin_pack = 1;
680 continue;
681 }
682 if (!strcmp("--all", arg)) {
683 args.fetch_all = 1;
684 continue;
685 }
686 if (!strcmp("-v", arg)) {
687 args.verbose = 1;
688 continue;
689 }
690 if (!prefixcmp(arg, "--depth=")) {
691 args.depth = strtol(arg + 8, NULL, 0);
692 continue;
693 }
694 if (!strcmp("--no-progress", arg)) {
695 args.no_progress = 1;
696 continue;
697 }
698 usage(fetch_pack_usage);
699 }
700 dest = (char *)arg;
701 heads = (char **)(argv + i + 1);
702 nr_heads = argc - i - 1;
703 break;
704 }
705 if (!dest)
706 usage(fetch_pack_usage);
707
708 conn = git_connect(fd, (char *)dest, args.uploadpack,
709 args.verbose ? CONNECT_VERBOSE : 0);
710 if (conn) {
711 get_remote_heads(fd[0], &ref, 0, NULL, 0);
712
713 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
714 close(fd[0]);
715 close(fd[1]);
716 if (finish_connect(conn))
717 ref = NULL;
718 } else {
719 ref = NULL;
720 }
721 ret = !ref;
722
723 if (!ret && nr_heads) {
724 /* If the heads to pull were given, we should have
725 * consumed all of them by matching the remote.
726 * Otherwise, 'git-fetch remote no-such-ref' would
727 * silently succeed without issuing an error.
728 */
729 for (i = 0; i < nr_heads; i++)
730 if (heads[i] && heads[i][0]) {
731 error("no such remote ref %s", heads[i]);
732 ret = 1;
733 }
734 }
735 while (ref) {
736 printf("%s %s\n",
737 sha1_to_hex(ref->old_sha1), ref->name);
738 ref = ref->next;
739 }
740
741 return ret;
742 }
743
744 struct ref *fetch_pack(struct fetch_pack_args *my_args,
745 int fd[], struct child_process *conn,
746 const struct ref *ref,
747 const char *dest,
748 int nr_heads,
749 char **heads,
750 char **pack_lockfile)
751 {
752 struct stat st;
753 struct ref *ref_cpy;
754
755 fetch_pack_setup();
756 memcpy(&args, my_args, sizeof(args));
757 if (args.depth > 0) {
758 if (stat(git_path("shallow"), &st))
759 st.st_mtime = 0;
760 }
761
762 if (heads && nr_heads)
763 nr_heads = remove_duplicates(nr_heads, heads);
764 if (!ref) {
765 packet_flush(fd[1]);
766 die("no matching remote head");
767 }
768 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
769
770 if (args.depth > 0) {
771 struct cache_time mtime;
772 char *shallow = git_path("shallow");
773 int fd;
774
775 mtime.sec = st.st_mtime;
776 #ifdef USE_NSEC
777 mtime.usec = st.st_mtim.usec;
778 #endif
779 if (stat(shallow, &st)) {
780 if (mtime.sec)
781 die("shallow file was removed during fetch");
782 } else if (st.st_mtime != mtime.sec
783 #ifdef USE_NSEC
784 || st.st_mtim.usec != mtime.usec
785 #endif
786 )
787 die("shallow file was changed during fetch");
788
789 fd = hold_lock_file_for_update(&lock, shallow, 1);
790 if (!write_shallow_commits(fd, 0)) {
791 unlink(shallow);
792 rollback_lock_file(&lock);
793 } else {
794 commit_lock_file(&lock);
795 }
796 }
797
798 return ref_cpy;
799 }