git-fetch: allow forcing glob pattern in refspec
[git/git.git] / fetch-pack.c
CommitLineData
def88e9a 1#include "cache.h"
fb9040cc 2#include "refs.h"
def88e9a 3#include "pkt-line.h"
49bb805e
JH
4#include "commit.h"
5#include "tag.h"
da093d37
NP
6#include "exec_cmd.h"
7#include "sideband.h"
8#include <sys/wait.h>
def88e9a 9
ad897215 10static int keep_pack;
8b3d9dc0 11static int quiet;
33b83034 12static int verbose;
dfeff66e 13static int fetch_all;
33b83034 14static const char fetch_pack_usage[] =
dfeff66e 15"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory <refs>...";
def88e9a
LT
16static const char *exec = "git-upload-pack";
17
0a8944dd 18#define COMPLETE (1U << 0)
23d61f83
JS
19#define COMMON (1U << 1)
20#define COMMON_REF (1U << 2)
21#define SEEN (1U << 3)
22#define POPPED (1U << 4)
23
f061e5fd
JH
24/*
25 * After sending this many "have"s if we do not get any new ACK , we
26 * give up traversing our history.
27 */
28#define MAX_IN_VAIN 256
29
96f1e58f
DR
30static struct commit_list *rev_list;
31static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
23d61f83
JS
32
33static void rev_list_push(struct commit *commit, int mark)
34{
35 if (!(commit->object.flags & mark)) {
36 commit->object.flags |= mark;
37
38 if (!(commit->object.parsed))
39 parse_commit(commit);
40
41 insert_by_date(commit, &rev_list);
42
43 if (!(commit->object.flags & COMMON))
44 non_common_revs++;
45 }
46}
47
8da19775 48static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
23d61f83 49{
9534f40b 50 struct object *o = deref_tag(parse_object(sha1), path, 0);
23d61f83 51
1974632c 52 if (o && o->type == OBJ_COMMIT)
23d61f83
JS
53 rev_list_push((struct commit *)o, SEEN);
54
55 return 0;
56}
57
58/*
59 This function marks a rev and its ancestors as common.
60 In some cases, it is desirable to mark only the ancestors (for example
61 when only the server does not yet know that they are common).
62*/
63
64static void mark_common(struct commit *commit,
65 int ancestors_only, int dont_parse)
66{
67 if (commit != NULL && !(commit->object.flags & COMMON)) {
68 struct object *o = (struct object *)commit;
69
70 if (!ancestors_only)
71 o->flags |= COMMON;
72
73 if (!(o->flags & SEEN))
74 rev_list_push(commit, SEEN);
75 else {
76 struct commit_list *parents;
77
78 if (!ancestors_only && !(o->flags & POPPED))
79 non_common_revs--;
80 if (!o->parsed && !dont_parse)
81 parse_commit(commit);
82
83 for (parents = commit->parents;
84 parents;
85 parents = parents->next)
86 mark_common(parents->item, 0, dont_parse);
87 }
88 }
89}
90
91/*
92 Get the next rev to send, ignoring the common.
93*/
94
962554c6 95static const unsigned char* get_rev(void)
23d61f83
JS
96{
97 struct commit *commit = NULL;
98
99 while (commit == NULL) {
100 unsigned int mark;
101 struct commit_list* parents;
102
103 if (rev_list == NULL || non_common_revs == 0)
104 return NULL;
105
106 commit = rev_list->item;
107 if (!(commit->object.parsed))
108 parse_commit(commit);
109 commit->object.flags |= POPPED;
110 if (!(commit->object.flags & COMMON))
111 non_common_revs--;
112
113 parents = commit->parents;
114
115 if (commit->object.flags & COMMON) {
116 /* do not send "have", and ignore ancestors */
117 commit = NULL;
118 mark = COMMON | SEEN;
119 } else if (commit->object.flags & COMMON_REF)
120 /* send "have", and ignore ancestors */
121 mark = COMMON | SEEN;
122 else
123 /* send "have", also for its ancestors */
124 mark = SEEN;
125
126 while (parents) {
127 if (!(parents->item->object.flags & SEEN))
128 rev_list_push(parents->item, mark);
129 if (mark & COMMON)
130 mark_common(parents->item, 1, 0);
131 parents = parents->next;
132 }
133
134 rev_list = rev_list->next;
135 }
136
137 return commit->object.sha1;
138}
0a8944dd 139
33b83034
JH
140static int find_common(int fd[2], unsigned char *result_sha1,
141 struct ref *refs)
def88e9a 142{
2759cbc7 143 int fetching;
23d61f83
JS
144 int count = 0, flushes = 0, retval;
145 const unsigned char *sha1;
f061e5fd
JH
146 unsigned in_vain = 0;
147 int got_continue = 0;
23d61f83 148
cb5d709f 149 for_each_ref(rev_list_insert_ref, NULL);
def88e9a 150
2759cbc7
LT
151 fetching = 0;
152 for ( ; refs ; refs = refs->next) {
33b83034 153 unsigned char *remote = refs->old_sha1;
4dab94d5 154 struct object *o;
2759cbc7 155
0a8944dd 156 /*
4dab94d5
JH
157 * If that object is complete (i.e. it is an ancestor of a
158 * local ref), we tell them we have it but do not have to
159 * tell them about its ancestors, which they already know
160 * about.
f1f0a2be
JH
161 *
162 * We use lookup_object here because we are only
163 * interested in the case we *know* the object is
164 * reachable and we have already scanned it.
4dab94d5 165 */
f1f0a2be 166 if (((o = lookup_object(remote)) != NULL) &&
1baaae5e 167 (o->flags & COMPLETE)) {
2759cbc7 168 continue;
0a8944dd 169 }
23d61f83 170
583b7ea3 171 if (!fetching)
e4fe4b8e 172 packet_write(fd[1], "want %s%s%s%s%s%s\n",
583b7ea3
JH
173 sha1_to_hex(remote),
174 (multi_ack ? " multi_ack" : ""),
d47f3db7
JH
175 (use_sideband == 2 ? " side-band-64k" : ""),
176 (use_sideband == 1 ? " side-band" : ""),
e4fe4b8e
NP
177 (use_thin_pack ? " thin-pack" : ""),
178 " ofs-delta");
583b7ea3
JH
179 else
180 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
2759cbc7 181 fetching++;
33b83034 182 }
fb9040cc 183 packet_flush(fd[1]);
2759cbc7
LT
184 if (!fetching)
185 return 1;
0a8944dd 186
23d61f83 187 flushes = 0;
75bfc6c2 188 retval = -1;
23d61f83 189 while ((sha1 = get_rev())) {
def88e9a 190 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
33b83034
JH
191 if (verbose)
192 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
f061e5fd 193 in_vain++;
def88e9a 194 if (!(31 & ++count)) {
c4c86f07
JS
195 int ack;
196
def88e9a
LT
197 packet_flush(fd[1]);
198 flushes++;
199
200 /*
201 * We keep one window "ahead" of the other side, and
202 * will wait for an ACK only on the next one
203 */
204 if (count == 32)
205 continue;
c4c86f07
JS
206
207 do {
208 ack = get_ack(fd[0], result_sha1);
209 if (verbose && ack)
210 fprintf(stderr, "got ack %d %s\n", ack,
211 sha1_to_hex(result_sha1));
212 if (ack == 1) {
213 flushes = 0;
214 multi_ack = 0;
215 retval = 0;
216 goto done;
217 } else if (ack == 2) {
218 struct commit *commit =
219 lookup_commit(result_sha1);
220 mark_common(commit, 0, 1);
221 retval = 0;
f061e5fd
JH
222 in_vain = 0;
223 got_continue = 1;
c4c86f07
JS
224 }
225 } while (ack);
def88e9a 226 flushes--;
f061e5fd
JH
227 if (got_continue && MAX_IN_VAIN < in_vain) {
228 if (verbose)
229 fprintf(stderr, "giving up\n");
230 break; /* give up */
231 }
def88e9a
LT
232 }
233 }
c4c86f07 234done:
75bfc6c2 235 packet_write(fd[1], "done\n");
33b83034
JH
236 if (verbose)
237 fprintf(stderr, "done\n");
c4c86f07
JS
238 if (retval != 0) {
239 multi_ack = 0;
23d61f83 240 flushes++;
c4c86f07
JS
241 }
242 while (flushes || multi_ack) {
243 int ack = get_ack(fd[0], result_sha1);
244 if (ack) {
33b83034 245 if (verbose)
c4c86f07
JS
246 fprintf(stderr, "got ack (%d) %s\n", ack,
247 sha1_to_hex(result_sha1));
248 if (ack == 1)
249 return 0;
250 multi_ack = 1;
251 continue;
33b83034 252 }
c4c86f07 253 flushes--;
def88e9a 254 }
75bfc6c2 255 return retval;
def88e9a
LT
256}
257
96f1e58f 258static struct commit_list *complete;
49bb805e 259
8da19775 260static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
49bb805e
JH
261{
262 struct object *o = parse_object(sha1);
263
1974632c 264 while (o && o->type == OBJ_TAG) {
f1f0a2be
JH
265 struct tag *t = (struct tag *) o;
266 if (!t->tagged)
267 break; /* broken repository */
49bb805e 268 o->flags |= COMPLETE;
f1f0a2be 269 o = parse_object(t->tagged->sha1);
49bb805e 270 }
1974632c 271 if (o && o->type == OBJ_COMMIT) {
49bb805e
JH
272 struct commit *commit = (struct commit *)o;
273 commit->object.flags |= COMPLETE;
274 insert_by_date(commit, &complete);
275 }
276 return 0;
277}
278
279static void mark_recent_complete_commits(unsigned long cutoff)
280{
281 while (complete && cutoff <= complete->item->date) {
282 if (verbose)
283 fprintf(stderr, "Marking %s as complete\n",
284 sha1_to_hex(complete->item->object.sha1));
285 pop_most_recent_commit(&complete, COMPLETE);
286 }
287}
288
1baaae5e
JS
289static void filter_refs(struct ref **refs, int nr_match, char **match)
290{
9546010b
JH
291 struct ref **return_refs;
292 struct ref *newlist = NULL;
293 struct ref **newtail = &newlist;
294 struct ref *ref, *next;
295 struct ref *fastarray[32];
296
297 if (nr_match && !fetch_all) {
298 if (ARRAY_SIZE(fastarray) < nr_match)
299 return_refs = xcalloc(nr_match, sizeof(struct ref *));
300 else {
301 return_refs = fastarray;
302 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
303 }
304 }
305 else
306 return_refs = NULL;
307
308 for (ref = *refs; ref; ref = next) {
309 next = ref->next;
310 if (!memcmp(ref->name, "refs/", 5) &&
311 check_ref_format(ref->name + 5))
312 ; /* trash */
313 else if (fetch_all) {
314 *newtail = ref;
315 ref->next = NULL;
316 newtail = &ref->next;
317 continue;
318 }
319 else {
320 int order = path_match(ref->name, nr_match, match);
321 if (order) {
322 return_refs[order-1] = ref;
323 continue; /* we will link it later */
324 }
325 }
326 free(ref);
327 }
328
329 if (!fetch_all) {
330 int i;
331 for (i = 0; i < nr_match; i++) {
332 ref = return_refs[i];
333 if (ref) {
334 *newtail = ref;
335 ref->next = NULL;
336 newtail = &ref->next;
337 }
338 }
339 if (return_refs != fastarray)
340 free(return_refs);
1baaae5e 341 }
9546010b 342 *refs = newlist;
1baaae5e
JS
343}
344
345static int everything_local(struct ref **refs, int nr_match, char **match)
2759cbc7 346{
49bb805e 347 struct ref *ref;
2759cbc7 348 int retval;
49bb805e
JH
349 unsigned long cutoff = 0;
350
351 track_object_refs = 0;
352 save_commit_buffer = 0;
353
1baaae5e 354 for (ref = *refs; ref; ref = ref->next) {
49bb805e
JH
355 struct object *o;
356
357 o = parse_object(ref->old_sha1);
358 if (!o)
359 continue;
360
361 /* We already have it -- which may mean that we were
362 * in sync with the other side at some time after
363 * that (it is OK if we guess wrong here).
364 */
1974632c 365 if (o->type == OBJ_COMMIT) {
49bb805e
JH
366 struct commit *commit = (struct commit *)o;
367 if (!cutoff || cutoff < commit->date)
368 cutoff = commit->date;
369 }
370 }
371
cb5d709f 372 for_each_ref(mark_complete, NULL);
49bb805e
JH
373 if (cutoff)
374 mark_recent_complete_commits(cutoff);
2759cbc7 375
1baaae5e
JS
376 /*
377 * Mark all complete remote refs as common refs.
378 * Don't mark them common yet; the server has to be told so first.
379 */
380 for (ref = *refs; ref; ref = ref->next) {
9534f40b
JH
381 struct object *o = deref_tag(lookup_object(ref->old_sha1),
382 NULL, 0);
1baaae5e 383
1974632c 384 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
1baaae5e
JS
385 continue;
386
387 if (!(o->flags & SEEN)) {
388 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
389
390 mark_common((struct commit *)o, 1, 1);
391 }
392 }
393
394 filter_refs(refs, nr_match, match);
395
396 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
397 const unsigned char *remote = ref->old_sha1;
2759cbc7 398 unsigned char local[20];
49bb805e 399 struct object *o;
2759cbc7 400
1baaae5e 401 o = lookup_object(remote);
49bb805e 402 if (!o || !(o->flags & COMPLETE)) {
2759cbc7
LT
403 retval = 0;
404 if (!verbose)
405 continue;
406 fprintf(stderr,
407 "want %s (%s)\n", sha1_to_hex(remote),
1baaae5e 408 ref->name);
2759cbc7
LT
409 continue;
410 }
411
e702496e 412 hashcpy(ref->new_sha1, local);
2759cbc7
LT
413 if (!verbose)
414 continue;
415 fprintf(stderr,
416 "already have %s (%s)\n", sha1_to_hex(remote),
1baaae5e 417 ref->name);
2759cbc7
LT
418 }
419 return retval;
420}
421
da093d37
NP
422static pid_t setup_sideband(int fd[2], int xd[2])
423{
424 pid_t side_pid;
425
426 if (!use_sideband) {
427 fd[0] = xd[0];
428 fd[1] = xd[1];
429 return 0;
430 }
431 /* xd[] is talking with upload-pack; subprocess reads from
432 * xd[0], spits out band#2 to stderr, and feeds us band#1
433 * through our fd[0].
434 */
435 if (pipe(fd) < 0)
436 die("fetch-pack: unable to set up pipe");
437 side_pid = fork();
438 if (side_pid < 0)
439 die("fetch-pack: unable to fork off sideband demultiplexer");
440 if (!side_pid) {
441 /* subprocess */
442 close(fd[0]);
443 if (xd[0] != xd[1])
444 close(xd[1]);
445 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
446 exit(1);
447 exit(0);
448 }
449 close(xd[0]);
450 close(fd[1]);
451 fd[1] = xd[1];
452 return side_pid;
453}
454
455static int get_pack(int xd[2], const char **argv)
456{
457 int status;
458 pid_t pid, side_pid;
459 int fd[2];
460
461 side_pid = setup_sideband(fd, xd);
462 pid = fork();
463 if (pid < 0)
464 die("fetch-pack: unable to fork off %s", argv[0]);
465 if (!pid) {
466 dup2(fd[0], 0);
467 close(fd[0]);
468 close(fd[1]);
469 execv_git_cmd(argv);
470 die("%s exec failed", argv[0]);
471 }
472 close(fd[0]);
473 close(fd[1]);
474 while (waitpid(pid, &status, 0) < 0) {
475 if (errno != EINTR)
476 die("waiting for %s: %s", argv[0], strerror(errno));
477 }
478 if (WIFEXITED(status)) {
479 int code = WEXITSTATUS(status);
480 if (code)
481 die("%s died with error code %d", argv[0], code);
482 return 0;
483 }
484 if (WIFSIGNALED(status)) {
485 int sig = WTERMSIG(status);
486 die("%s died of signal %d", argv[0], sig);
487 }
488 die("%s died of unnatural causes %d", argv[0], status);
489}
490
491static int explode_rx_pack(int xd[2])
492{
493 const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
494 return get_pack(xd, argv);
495}
496
497static int keep_rx_pack(int xd[2])
498{
499 const char *argv[6];
500 char keep_arg[256];
501 int n = 0;
502
503 argv[n++] = "index-pack";
504 argv[n++] = "--stdin";
505 if (!quiet)
506 argv[n++] = "-v";
507 if (use_thin_pack)
508 argv[n++] = "--fix-thin";
509 if (keep_pack > 1) {
510 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
511 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
512 strcpy(keep_arg + s, "localhost");
513 argv[n++] = keep_arg;
514 }
515 argv[n] = NULL;
516 return get_pack(xd, argv);
517}
518
def88e9a
LT
519static int fetch_pack(int fd[2], int nr_match, char **match)
520{
d1c133f5
LT
521 struct ref *ref;
522 unsigned char sha1[20];
523 int status;
def88e9a 524
1baaae5e 525 get_remote_heads(fd[0], &ref, 0, NULL, 0);
c4c86f07
JS
526 if (server_supports("multi_ack")) {
527 if (verbose)
528 fprintf(stderr, "Server supports multi_ack\n");
529 multi_ack = 1;
530 }
d47f3db7
JH
531 if (server_supports("side-band-64k")) {
532 if (verbose)
533 fprintf(stderr, "Server supports side-band-64k\n");
534 use_sideband = 2;
535 }
536 else if (server_supports("side-band")) {
583b7ea3
JH
537 if (verbose)
538 fprintf(stderr, "Server supports side-band\n");
539 use_sideband = 1;
540 }
d1c133f5
LT
541 if (!ref) {
542 packet_flush(fd[1]);
543 die("no matching remote head");
544 }
1baaae5e 545 if (everything_local(&ref, nr_match, match)) {
2759cbc7
LT
546 packet_flush(fd[1]);
547 goto all_done;
548 }
33b83034 549 if (find_common(fd, sha1, ref) < 0)
da093d37 550 if (keep_pack != 1)
dfeff66e
JH
551 /* When cloning, it is not unusual to have
552 * no common commit.
553 */
554 fprintf(stderr, "warning: no common commits\n");
ad897215 555
da093d37 556 status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
ad897215
JH
557 if (status)
558 die("git-fetch-pack: fetch failed.");
559
560 all_done:
561 while (ref) {
562 printf("%s %s\n",
563 sha1_to_hex(ref->old_sha1), ref->name);
564 ref = ref->next;
75bfc6c2 565 }
ad897215 566 return 0;
def88e9a
LT
567}
568
569int main(int argc, char **argv)
570{
571 int i, ret, nr_heads;
572 char *dest = NULL, **heads;
573 int fd[2];
574 pid_t pid;
575
5a327713
JH
576 setup_git_directory();
577
def88e9a
LT
578 nr_heads = 0;
579 heads = NULL;
580 for (i = 1; i < argc; i++) {
581 char *arg = argv[i];
582
583 if (*arg == '-') {
8b3d9dc0
JH
584 if (!strncmp("--exec=", arg, 7)) {
585 exec = arg + 7;
586 continue;
587 }
2247efb4 588 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
33b83034
JH
589 quiet = 1;
590 continue;
591 }
2247efb4 592 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
da093d37 593 keep_pack++;
ad897215
JH
594 continue;
595 }
b19696c2
JH
596 if (!strcmp("--thin", arg)) {
597 use_thin_pack = 1;
598 continue;
599 }
dfeff66e
JH
600 if (!strcmp("--all", arg)) {
601 fetch_all = 1;
602 continue;
603 }
33b83034
JH
604 if (!strcmp("-v", arg)) {
605 verbose = 1;
606 continue;
607 }
def88e9a
LT
608 usage(fetch_pack_usage);
609 }
610 dest = arg;
611 heads = argv + i + 1;
612 nr_heads = argc - i - 1;
613 break;
614 }
615 if (!dest)
616 usage(fetch_pack_usage);
617 pid = git_connect(fd, dest, exec);
618 if (pid < 0)
619 return 1;
620 ret = fetch_pack(fd, nr_heads, heads);
621 close(fd[0]);
622 close(fd[1]);
8a5dbef8 623 ret |= finish_connect(pid);
9e5d2b40
JH
624
625 if (!ret && nr_heads) {
626 /* If the heads to pull were given, we should have
627 * consumed all of them by matching the remote.
628 * Otherwise, 'git-fetch remote no-such-ref' would
629 * silently succeed without issuing an error.
630 */
631 for (i = 0; i < nr_heads; i++)
632 if (heads[i] && heads[i][0]) {
633 error("no such remote ref %s", heads[i]);
634 ret = 1;
635 }
636 }
637
8a5dbef8 638 return !!ret;
def88e9a 639}