add 'git credential' plumbing command
[git/git.git] / builtin / fetch-pack.c
CommitLineData
c2e86add 1#include "builtin.h"
fb9040cc 2#include "refs.h"
def88e9a 3#include "pkt-line.h"
49bb805e
JH
4#include "commit.h"
5#include "tag.h"
da093d37 6#include "exec_cmd.h"
9e10fd1a 7#include "pack.h"
da093d37 8#include "sideband.h"
2d4177c0 9#include "fetch-pack.h"
ba227857 10#include "remote.h"
477822c3 11#include "run-command.h"
e52d7192 12#include "transport.h"
def88e9a 13
e28714c5
JH
14static int transfer_unpack_limit = -1;
15static int fetch_unpack_limit = -1;
af7cf268 16static int unpack_limit = 100;
f04833ef 17static int prefer_ofs_delta = 1;
ca0c9764 18static int no_done;
dab76d3a
JH
19static int fetch_fsck_objects = -1;
20static int transfer_fsck_objects = -1;
09149c78
DB
21static struct fetch_pack_args args = {
22 /* .uploadpack = */ "git-upload-pack",
23};
fa740529 24
33b83034 25static const char fetch_pack_usage[] =
078b895f
IT
26"git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
27"[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
28"[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
def88e9a 29
0a8944dd 30#define COMPLETE (1U << 0)
23d61f83
JS
31#define COMMON (1U << 1)
32#define COMMON_REF (1U << 2)
33#define SEEN (1U << 3)
34#define POPPED (1U << 4)
35
420e9af4
DB
36static int marked;
37
f061e5fd
JH
38/*
39 * After sending this many "have"s if we do not get any new ACK , we
40 * give up traversing our history.
41 */
42#define MAX_IN_VAIN 256
43
96f1e58f 44static struct commit_list *rev_list;
3891f390 45static int non_common_revs, multi_ack, use_sideband;
23d61f83
JS
46
47static void rev_list_push(struct commit *commit, int mark)
48{
49 if (!(commit->object.flags & mark)) {
50 commit->object.flags |= mark;
51
52 if (!(commit->object.parsed))
f3ec5494
MK
53 if (parse_commit(commit))
54 return;
23d61f83 55
47e44ed1 56 commit_list_insert_by_date(commit, &rev_list);
23d61f83
JS
57
58 if (!(commit->object.flags & COMMON))
59 non_common_revs++;
60 }
61}
62
65385ef7 63static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
23d61f83 64{
65385ef7 65 struct object *o = deref_tag(parse_object(sha1), refname, 0);
23d61f83 66
1974632c 67 if (o && o->type == OBJ_COMMIT)
23d61f83
JS
68 rev_list_push((struct commit *)o, SEEN);
69
70 return 0;
71}
72
65385ef7 73static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
420e9af4 74{
65385ef7 75 struct object *o = deref_tag(parse_object(sha1), refname, 0);
420e9af4
DB
76
77 if (o && o->type == OBJ_COMMIT)
78 clear_commit_marks((struct commit *)o,
79 COMMON | COMMON_REF | SEEN | POPPED);
80 return 0;
81}
82
23d61f83
JS
83/*
84 This function marks a rev and its ancestors as common.
85 In some cases, it is desirable to mark only the ancestors (for example
86 when only the server does not yet know that they are common).
87*/
88
89static void mark_common(struct commit *commit,
90 int ancestors_only, int dont_parse)
91{
92 if (commit != NULL && !(commit->object.flags & COMMON)) {
93 struct object *o = (struct object *)commit;
94
95 if (!ancestors_only)
96 o->flags |= COMMON;
97
98 if (!(o->flags & SEEN))
99 rev_list_push(commit, SEEN);
100 else {
101 struct commit_list *parents;
102
103 if (!ancestors_only && !(o->flags & POPPED))
104 non_common_revs--;
105 if (!o->parsed && !dont_parse)
f3ec5494
MK
106 if (parse_commit(commit))
107 return;
23d61f83
JS
108
109 for (parents = commit->parents;
110 parents;
111 parents = parents->next)
112 mark_common(parents->item, 0, dont_parse);
113 }
114 }
115}
116
117/*
118 Get the next rev to send, ignoring the common.
119*/
120
4b25d091 121static const unsigned char *get_rev(void)
23d61f83
JS
122{
123 struct commit *commit = NULL;
124
125 while (commit == NULL) {
126 unsigned int mark;
72269ad9 127 struct commit_list *parents;
23d61f83
JS
128
129 if (rev_list == NULL || non_common_revs == 0)
130 return NULL;
131
132 commit = rev_list->item;
2d8bed96 133 if (!commit->object.parsed)
72269ad9
LT
134 parse_commit(commit);
135 parents = commit->parents;
f3ec5494 136
23d61f83
JS
137 commit->object.flags |= POPPED;
138 if (!(commit->object.flags & COMMON))
139 non_common_revs--;
a6080a0a 140
23d61f83
JS
141 if (commit->object.flags & COMMON) {
142 /* do not send "have", and ignore ancestors */
143 commit = NULL;
144 mark = COMMON | SEEN;
145 } else if (commit->object.flags & COMMON_REF)
146 /* send "have", and ignore ancestors */
147 mark = COMMON | SEEN;
148 else
149 /* send "have", also for its ancestors */
150 mark = SEEN;
151
152 while (parents) {
153 if (!(parents->item->object.flags & SEEN))
154 rev_list_push(parents->item, mark);
155 if (mark & COMMON)
156 mark_common(parents->item, 1, 0);
157 parents = parents->next;
158 }
159
160 rev_list = rev_list->next;
161 }
162
163 return commit->object.sha1;
164}
0a8944dd 165
78affc49
SP
166enum ack_type {
167 NAK = 0,
168 ACK,
169 ACK_continue,
170 ACK_common,
171 ACK_ready
172};
173
249b2004
SP
174static void consume_shallow_list(int fd)
175{
176 if (args.stateless_rpc && args.depth > 0) {
177 /* If we sent a depth we will get back "duplicate"
178 * shallow and unshallow commands every time there
179 * is a block of have lines exchanged.
180 */
181 char line[1000];
182 while (packet_read_line(fd, line, sizeof(line))) {
183 if (!prefixcmp(line, "shallow "))
184 continue;
185 if (!prefixcmp(line, "unshallow "))
186 continue;
187 die("git fetch-pack: expected shallow list");
188 }
189 }
190}
191
294e15fc
NTND
192struct write_shallow_data {
193 struct strbuf *out;
194 int use_pack_protocol;
195 int count;
196};
197
198static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
199{
200 struct write_shallow_data *data = cb_data;
201 const char *hex = sha1_to_hex(graft->sha1);
202 data->count++;
203 if (data->use_pack_protocol)
204 packet_buf_write(data->out, "shallow %s", hex);
205 else {
206 strbuf_addstr(data->out, hex);
207 strbuf_addch(data->out, '\n');
208 }
209 return 0;
210}
211
212static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
213{
214 struct write_shallow_data data;
215 data.out = out;
216 data.use_pack_protocol = use_pack_protocol;
217 data.count = 0;
218 for_each_commit_graft(write_one_shallow, &data);
219 return data.count;
220}
221
78affc49 222static enum ack_type get_ack(int fd, unsigned char *result_sha1)
28754ab5
SP
223{
224 static char line[1000];
225 int len = packet_read_line(fd, line, sizeof(line));
226
227 if (!len)
228 die("git fetch-pack: expected ACK/NAK, got EOF");
229 if (line[len-1] == '\n')
230 line[--len] = 0;
231 if (!strcmp(line, "NAK"))
78affc49 232 return NAK;
28754ab5
SP
233 if (!prefixcmp(line, "ACK ")) {
234 if (!get_sha1_hex(line+4, result_sha1)) {
235 if (strstr(line+45, "continue"))
78affc49
SP
236 return ACK_continue;
237 if (strstr(line+45, "common"))
238 return ACK_common;
239 if (strstr(line+45, "ready"))
240 return ACK_ready;
241 return ACK;
28754ab5
SP
242 }
243 }
244 die("git fetch_pack: expected ACK/NAK, got '%s'", line);
245}
246
249b2004
SP
247static void send_request(int fd, struct strbuf *buf)
248{
249 if (args.stateless_rpc) {
250 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
251 packet_flush(fd);
252 } else
253 safe_write(fd, buf->buf, buf->len);
254}
255
e52d7192
JH
256static void insert_one_alternate_ref(const struct ref *ref, void *unused)
257{
258 rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
259}
260
066bf4c2 261#define INITIAL_FLUSH 16
44d8dc54 262#define PIPESAFE_FLUSH 32
6afca450 263#define LARGE_FLUSH 1024
c12f5917
JH
264
265static int next_flush(int count)
266{
44d8dc54
JH
267 int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
268
269 if (count < flush_limit)
6afca450
JH
270 count <<= 1;
271 else
44d8dc54 272 count += flush_limit;
6afca450 273 return count;
c12f5917
JH
274}
275
33b83034
JH
276static int find_common(int fd[2], unsigned char *result_sha1,
277 struct ref *refs)
def88e9a 278{
2759cbc7 279 int fetching;
c12f5917 280 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
23d61f83 281 const unsigned char *sha1;
f061e5fd
JH
282 unsigned in_vain = 0;
283 int got_continue = 0;
4e10cf9a 284 int got_ready = 0;
edace6f0 285 struct strbuf req_buf = STRBUF_INIT;
249b2004 286 size_t state_len = 0;
23d61f83 287
249b2004
SP
288 if (args.stateless_rpc && multi_ack == 1)
289 die("--stateless-rpc requires multi_ack_detailed");
420e9af4
DB
290 if (marked)
291 for_each_ref(clear_marks, NULL);
292 marked = 1;
293
cb5d709f 294 for_each_ref(rev_list_insert_ref, NULL);
c41a802f 295 for_each_alternate_ref(insert_one_alternate_ref, NULL);
def88e9a 296
2759cbc7
LT
297 fetching = 0;
298 for ( ; refs ; refs = refs->next) {
33b83034 299 unsigned char *remote = refs->old_sha1;
edace6f0 300 const char *remote_hex;
4dab94d5 301 struct object *o;
2759cbc7 302
0a8944dd 303 /*
4dab94d5
JH
304 * If that object is complete (i.e. it is an ancestor of a
305 * local ref), we tell them we have it but do not have to
306 * tell them about its ancestors, which they already know
307 * about.
f1f0a2be
JH
308 *
309 * We use lookup_object here because we are only
310 * interested in the case we *know* the object is
311 * reachable and we have already scanned it.
4dab94d5 312 */
f1f0a2be 313 if (((o = lookup_object(remote)) != NULL) &&
1baaae5e 314 (o->flags & COMPLETE)) {
2759cbc7 315 continue;
0a8944dd 316 }
23d61f83 317
edace6f0
SP
318 remote_hex = sha1_to_hex(remote);
319 if (!fetching) {
320 struct strbuf c = STRBUF_INIT;
78affc49
SP
321 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
322 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
4e10cf9a 323 if (no_done) strbuf_addstr(&c, " no-done");
edace6f0
SP
324 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
325 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
326 if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
327 if (args.no_progress) strbuf_addstr(&c, " no-progress");
328 if (args.include_tag) strbuf_addstr(&c, " include-tag");
329 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
330 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
331 strbuf_release(&c);
332 } else
333 packet_buf_write(&req_buf, "want %s\n", remote_hex);
2759cbc7 334 fetching++;
33b83034 335 }
edace6f0
SP
336
337 if (!fetching) {
338 strbuf_release(&req_buf);
339 packet_flush(fd[1]);
340 return 1;
341 }
342
ed09aef0 343 if (is_repository_shallow())
edace6f0 344 write_shallow_commits(&req_buf, 1);
fa740529 345 if (args.depth > 0)
edace6f0
SP
346 packet_buf_write(&req_buf, "deepen %d", args.depth);
347 packet_buf_flush(&req_buf);
249b2004 348 state_len = req_buf.len;
0a8944dd 349
fa740529 350 if (args.depth > 0) {
016e6ccb
JS
351 char line[1024];
352 unsigned char sha1[20];
016e6ccb 353
249b2004 354 send_request(fd[1], &req_buf);
eb3a9dd3 355 while (packet_read_line(fd[0], line, sizeof(line))) {
599065a3 356 if (!prefixcmp(line, "shallow ")) {
016e6ccb
JS
357 if (get_sha1_hex(line + 8, sha1))
358 die("invalid shallow line: %s", line);
016e6ccb 359 register_shallow(sha1);
cf01bd52
JH
360 continue;
361 }
599065a3 362 if (!prefixcmp(line, "unshallow ")) {
f53514bc
JS
363 if (get_sha1_hex(line + 10, sha1))
364 die("invalid unshallow line: %s", line);
365 if (!lookup_object(sha1))
366 die("object not found: %s", line);
367 /* make sure that it is parsed as shallow */
f3ec5494
MK
368 if (!parse_object(sha1))
369 die("error in object: %s", line);
f53514bc
JS
370 if (unregister_shallow(sha1))
371 die("no shallow found: %s", line);
cf01bd52
JH
372 continue;
373 }
374 die("expected shallow/unshallow, got %s", line);
016e6ccb 375 }
249b2004
SP
376 } else if (!args.stateless_rpc)
377 send_request(fd[1], &req_buf);
378
379 if (!args.stateless_rpc) {
380 /* If we aren't using the stateless-rpc interface
381 * we don't need to retain the headers.
382 */
383 strbuf_setlen(&req_buf, 0);
384 state_len = 0;
016e6ccb
JS
385 }
386
23d61f83 387 flushes = 0;
75bfc6c2 388 retval = -1;
23d61f83 389 while ((sha1 = get_rev())) {
249b2004 390 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
fa740529 391 if (args.verbose)
33b83034 392 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
f061e5fd 393 in_vain++;
c12f5917 394 if (flush_at <= ++count) {
c4c86f07
JS
395 int ack;
396
249b2004
SP
397 packet_buf_flush(&req_buf);
398 send_request(fd[1], &req_buf);
399 strbuf_setlen(&req_buf, state_len);
def88e9a 400 flushes++;
c12f5917 401 flush_at = next_flush(count);
def88e9a
LT
402
403 /*
404 * We keep one window "ahead" of the other side, and
405 * will wait for an ACK only on the next one
406 */
c12f5917 407 if (!args.stateless_rpc && count == INITIAL_FLUSH)
def88e9a 408 continue;
c4c86f07 409
249b2004 410 consume_shallow_list(fd[0]);
c4c86f07
JS
411 do {
412 ack = get_ack(fd[0], result_sha1);
fa740529 413 if (args.verbose && ack)
c4c86f07
JS
414 fprintf(stderr, "got ack %d %s\n", ack,
415 sha1_to_hex(result_sha1));
78affc49
SP
416 switch (ack) {
417 case ACK:
c4c86f07
JS
418 flushes = 0;
419 multi_ack = 0;
420 retval = 0;
421 goto done;
78affc49
SP
422 case ACK_common:
423 case ACK_ready:
424 case ACK_continue: {
c4c86f07
JS
425 struct commit *commit =
426 lookup_commit(result_sha1);
ec099546
NTND
427 if (!commit)
428 die("invalid commit %s", sha1_to_hex(result_sha1));
249b2004
SP
429 if (args.stateless_rpc
430 && ack == ACK_common
431 && !(commit->object.flags & COMMON)) {
432 /* We need to replay the have for this object
433 * on the next RPC request so the peer knows
434 * it is in common with us.
435 */
436 const char *hex = sha1_to_hex(result_sha1);
437 packet_buf_write(&req_buf, "have %s\n", hex);
438 state_len = req_buf.len;
439 }
c4c86f07
JS
440 mark_common(commit, 0, 1);
441 retval = 0;
f061e5fd
JH
442 in_vain = 0;
443 got_continue = 1;
4e10cf9a 444 if (ack == ACK_ready) {
f2cba929 445 rev_list = NULL;
4e10cf9a
JH
446 got_ready = 1;
447 }
78affc49
SP
448 break;
449 }
c4c86f07
JS
450 }
451 } while (ack);
def88e9a 452 flushes--;
f061e5fd 453 if (got_continue && MAX_IN_VAIN < in_vain) {
fa740529 454 if (args.verbose)
f061e5fd
JH
455 fprintf(stderr, "giving up\n");
456 break; /* give up */
457 }
def88e9a
LT
458 }
459 }
c4c86f07 460done:
4e10cf9a
JH
461 if (!got_ready || !no_done) {
462 packet_buf_write(&req_buf, "done\n");
463 send_request(fd[1], &req_buf);
464 }
fa740529 465 if (args.verbose)
33b83034 466 fprintf(stderr, "done\n");
c4c86f07
JS
467 if (retval != 0) {
468 multi_ack = 0;
23d61f83 469 flushes++;
c4c86f07 470 }
edace6f0
SP
471 strbuf_release(&req_buf);
472
249b2004 473 consume_shallow_list(fd[0]);
c4c86f07
JS
474 while (flushes || multi_ack) {
475 int ack = get_ack(fd[0], result_sha1);
476 if (ack) {
fa740529 477 if (args.verbose)
c4c86f07
JS
478 fprintf(stderr, "got ack (%d) %s\n", ack,
479 sha1_to_hex(result_sha1));
78affc49 480 if (ack == ACK)
c4c86f07
JS
481 return 0;
482 multi_ack = 1;
483 continue;
33b83034 484 }
c4c86f07 485 flushes--;
def88e9a 486 }
8cb560fc
JS
487 /* it is no error to fetch into a completely empty repo */
488 return count ? retval : 0;
def88e9a
LT
489}
490
96f1e58f 491static struct commit_list *complete;
49bb805e 492
65385ef7 493static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
49bb805e
JH
494{
495 struct object *o = parse_object(sha1);
496
1974632c 497 while (o && o->type == OBJ_TAG) {
f1f0a2be
JH
498 struct tag *t = (struct tag *) o;
499 if (!t->tagged)
500 break; /* broken repository */
49bb805e 501 o->flags |= COMPLETE;
f1f0a2be 502 o = parse_object(t->tagged->sha1);
49bb805e 503 }
1974632c 504 if (o && o->type == OBJ_COMMIT) {
49bb805e 505 struct commit *commit = (struct commit *)o;
ea5f2208
JK
506 if (!(commit->object.flags & COMPLETE)) {
507 commit->object.flags |= COMPLETE;
508 commit_list_insert_by_date(commit, &complete);
509 }
49bb805e
JH
510 }
511 return 0;
512}
513
514static void mark_recent_complete_commits(unsigned long cutoff)
515{
516 while (complete && cutoff <= complete->item->date) {
fa740529 517 if (args.verbose)
49bb805e
JH
518 fprintf(stderr, "Marking %s as complete\n",
519 sha1_to_hex(complete->item->object.sha1));
520 pop_most_recent_commit(&complete, COMPLETE);
521 }
522}
523
1baaae5e
JS
524static void filter_refs(struct ref **refs, int nr_match, char **match)
525{
9546010b
JH
526 struct ref **return_refs;
527 struct ref *newlist = NULL;
528 struct ref **newtail = &newlist;
529 struct ref *ref, *next;
530 struct ref *fastarray[32];
a0de2880 531 int match_pos;
9546010b 532
fa740529 533 if (nr_match && !args.fetch_all) {
9546010b
JH
534 if (ARRAY_SIZE(fastarray) < nr_match)
535 return_refs = xcalloc(nr_match, sizeof(struct ref *));
536 else {
537 return_refs = fastarray;
538 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
539 }
540 }
541 else
542 return_refs = NULL;
543
a0de2880 544 match_pos = 0;
9546010b
JH
545 for (ref = *refs; ref; ref = next) {
546 next = ref->next;
547 if (!memcmp(ref->name, "refs/", 5) &&
8d9c5010 548 check_refname_format(ref->name + 5, 0))
9546010b 549 ; /* trash */
fa740529
SP
550 else if (args.fetch_all &&
551 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
9546010b
JH
552 *newtail = ref;
553 ref->next = NULL;
554 newtail = &ref->next;
555 continue;
556 }
557 else {
a0de2880
JK
558 int cmp = -1;
559 while (match_pos < nr_match) {
560 cmp = strcmp(ref->name, match[match_pos]);
561 if (cmp < 0) /* definitely do not have it */
562 break;
563 else if (cmp == 0) { /* definitely have it */
564 match[match_pos][0] = '\0';
565 return_refs[match_pos] = ref;
1e7ba0f9
JK
566 break;
567 }
a0de2880
JK
568 else /* might have it; keep looking */
569 match_pos++;
9546010b 570 }
a0de2880 571 if (!cmp)
1e7ba0f9 572 continue; /* we will link it later */
9546010b
JH
573 }
574 free(ref);
575 }
576
fa740529 577 if (!args.fetch_all) {
9546010b
JH
578 int i;
579 for (i = 0; i < nr_match; i++) {
580 ref = return_refs[i];
581 if (ref) {
582 *newtail = ref;
583 ref->next = NULL;
584 newtail = &ref->next;
585 }
586 }
587 if (return_refs != fastarray)
588 free(return_refs);
1baaae5e 589 }
9546010b 590 *refs = newlist;
1baaae5e
JS
591}
592
f2576591
MH
593static void mark_alternate_complete(const struct ref *ref, void *unused)
594{
595 mark_complete(NULL, ref->old_sha1, 0, NULL);
596}
597
1baaae5e 598static int everything_local(struct ref **refs, int nr_match, char **match)
2759cbc7 599{
49bb805e 600 struct ref *ref;
2759cbc7 601 int retval;
49bb805e
JH
602 unsigned long cutoff = 0;
603
49bb805e
JH
604 save_commit_buffer = 0;
605
1baaae5e 606 for (ref = *refs; ref; ref = ref->next) {
49bb805e
JH
607 struct object *o;
608
609 o = parse_object(ref->old_sha1);
610 if (!o)
611 continue;
612
613 /* We already have it -- which may mean that we were
614 * in sync with the other side at some time after
615 * that (it is OK if we guess wrong here).
616 */
1974632c 617 if (o->type == OBJ_COMMIT) {
49bb805e
JH
618 struct commit *commit = (struct commit *)o;
619 if (!cutoff || cutoff < commit->date)
620 cutoff = commit->date;
621 }
622 }
623
fa740529 624 if (!args.depth) {
f53514bc 625 for_each_ref(mark_complete, NULL);
f2576591 626 for_each_alternate_ref(mark_alternate_complete, NULL);
f53514bc
JS
627 if (cutoff)
628 mark_recent_complete_commits(cutoff);
629 }
2759cbc7 630
1baaae5e
JS
631 /*
632 * Mark all complete remote refs as common refs.
633 * Don't mark them common yet; the server has to be told so first.
634 */
635 for (ref = *refs; ref; ref = ref->next) {
9534f40b
JH
636 struct object *o = deref_tag(lookup_object(ref->old_sha1),
637 NULL, 0);
1baaae5e 638
1974632c 639 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
1baaae5e
JS
640 continue;
641
642 if (!(o->flags & SEEN)) {
643 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
644
645 mark_common((struct commit *)o, 1, 1);
646 }
647 }
648
649 filter_refs(refs, nr_match, match);
650
651 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
652 const unsigned char *remote = ref->old_sha1;
2759cbc7 653 unsigned char local[20];
49bb805e 654 struct object *o;
2759cbc7 655
1baaae5e 656 o = lookup_object(remote);
49bb805e 657 if (!o || !(o->flags & COMPLETE)) {
2759cbc7 658 retval = 0;
fa740529 659 if (!args.verbose)
2759cbc7
LT
660 continue;
661 fprintf(stderr,
662 "want %s (%s)\n", sha1_to_hex(remote),
1baaae5e 663 ref->name);
2759cbc7
LT
664 continue;
665 }
666
e702496e 667 hashcpy(ref->new_sha1, local);
fa740529 668 if (!args.verbose)
2759cbc7
LT
669 continue;
670 fprintf(stderr,
671 "already have %s (%s)\n", sha1_to_hex(remote),
1baaae5e 672 ref->name);
2759cbc7
LT
673 }
674 return retval;
675}
676
ae6a5609 677static int sideband_demux(int in, int out, void *data)
da093d37 678{
088fab5f 679 int *xd = data;
da093d37 680
ae6a5609
EFL
681 int ret = recv_sideband("fetch-pack", xd[0], out);
682 close(out);
3ef67cf2 683 return ret;
088fab5f
JS
684}
685
1788c39c 686static int get_pack(int xd[2], char **pack_lockfile)
da093d37 687{
088fab5f 688 struct async demux;
9e10fd1a
JH
689 const char *argv[20];
690 char keep_arg[256];
691 char hdr_arg[256];
692 const char **av;
fa740529 693 int do_keep = args.keep_pack;
477822c3 694 struct child_process cmd;
da093d37 695
1f759eee
JS
696 memset(&demux, 0, sizeof(demux));
697 if (use_sideband) {
698 /* xd[] is talking with upload-pack; subprocess reads from
699 * xd[0], spits out band#2 to stderr, and feeds us band#1
700 * through demux->out.
701 */
702 demux.proc = sideband_demux;
703 demux.data = xd;
ae6a5609 704 demux.out = -1;
1f759eee
JS
705 if (start_async(&demux))
706 die("fetch-pack: unable to fork off sideband"
707 " demultiplexer");
708 }
709 else
710 demux.out = xd[0];
9e10fd1a 711
477822c3
JS
712 memset(&cmd, 0, sizeof(cmd));
713 cmd.argv = argv;
9e10fd1a
JH
714 av = argv;
715 *hdr_arg = 0;
fa740529 716 if (!args.keep_pack && unpack_limit) {
9e10fd1a
JH
717 struct pack_header header;
718
1f759eee 719 if (read_pack_header(demux.out, &header))
9e10fd1a 720 die("protocol error: bad pack header");
6e1c2344
RJ
721 snprintf(hdr_arg, sizeof(hdr_arg),
722 "--pack_header=%"PRIu32",%"PRIu32,
9e10fd1a 723 ntohl(header.hdr_version), ntohl(header.hdr_entries));
af7cf268 724 if (ntohl(header.hdr_entries) < unpack_limit)
9e10fd1a
JH
725 do_keep = 0;
726 else
727 do_keep = 1;
728 }
729
730 if (do_keep) {
477822c3
JS
731 if (pack_lockfile)
732 cmd.out = -1;
9e10fd1a
JH
733 *av++ = "index-pack";
734 *av++ = "--stdin";
fa740529 735 if (!args.quiet && !args.no_progress)
9e10fd1a 736 *av++ = "-v";
fa740529 737 if (args.use_thin_pack)
9e10fd1a 738 *av++ = "--fix-thin";
fa740529 739 if (args.lock_pack || unpack_limit) {
9e10fd1a 740 int s = sprintf(keep_arg,
85e72830 741 "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
9e10fd1a
JH
742 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
743 strcpy(keep_arg + s, "localhost");
744 *av++ = keep_arg;
745 }
746 }
747 else {
748 *av++ = "unpack-objects";
01fdc21f 749 if (args.quiet || args.no_progress)
9e10fd1a
JH
750 *av++ = "-q";
751 }
752 if (*hdr_arg)
753 *av++ = hdr_arg;
dab76d3a
JH
754 if (fetch_fsck_objects >= 0
755 ? fetch_fsck_objects
756 : transfer_fsck_objects >= 0
757 ? transfer_fsck_objects
758 : 0)
5e838ea7 759 *av++ = "--strict";
9e10fd1a
JH
760 *av++ = NULL;
761
1f759eee 762 cmd.in = demux.out;
477822c3
JS
763 cmd.git_cmd = 1;
764 if (start_command(&cmd))
da093d37 765 die("fetch-pack: unable to fork off %s", argv[0]);
e72ae288 766 if (do_keep && pack_lockfile) {
477822c3 767 *pack_lockfile = index_pack_lockfile(cmd.out);
e72ae288
JS
768 close(cmd.out);
769 }
477822c3
JS
770
771 if (finish_command(&cmd))
772 die("%s failed", argv[0]);
088fab5f
JS
773 if (use_sideband && finish_async(&demux))
774 die("error in sideband demultiplexer");
477822c3 775 return 0;
da093d37
NP
776}
777
1788c39c 778static struct ref *do_fetch_pack(int fd[2],
ba227857 779 const struct ref *orig_ref,
1788c39c
SP
780 int nr_match,
781 char **match,
782 char **pack_lockfile)
def88e9a 783{
ba227857 784 struct ref *ref = copy_ref_list(orig_ref);
d1c133f5 785 unsigned char sha1[20];
def88e9a 786
9e8e704f
JK
787 sort_ref_list(&ref, ref_compare_name);
788
ed09aef0
JS
789 if (is_repository_shallow() && !server_supports("shallow"))
790 die("Server does not support shallow clients");
78affc49
SP
791 if (server_supports("multi_ack_detailed")) {
792 if (args.verbose)
793 fprintf(stderr, "Server supports multi_ack_detailed\n");
794 multi_ack = 2;
4e10cf9a
JH
795 if (server_supports("no-done")) {
796 if (args.verbose)
797 fprintf(stderr, "Server supports no-done\n");
8e9182e0
JH
798 if (args.stateless_rpc)
799 no_done = 1;
4e10cf9a 800 }
78affc49
SP
801 }
802 else if (server_supports("multi_ack")) {
fa740529 803 if (args.verbose)
c4c86f07
JS
804 fprintf(stderr, "Server supports multi_ack\n");
805 multi_ack = 1;
806 }
d47f3db7 807 if (server_supports("side-band-64k")) {
fa740529 808 if (args.verbose)
d47f3db7
JH
809 fprintf(stderr, "Server supports side-band-64k\n");
810 use_sideband = 2;
811 }
812 else if (server_supports("side-band")) {
fa740529 813 if (args.verbose)
583b7ea3
JH
814 fprintf(stderr, "Server supports side-band\n");
815 use_sideband = 1;
816 }
f04833ef
NP
817 if (server_supports("ofs-delta")) {
818 if (args.verbose)
819 fprintf(stderr, "Server supports ofs-delta\n");
820 } else
821 prefer_ofs_delta = 0;
1baaae5e 822 if (everything_local(&ref, nr_match, match)) {
2759cbc7
LT
823 packet_flush(fd[1]);
824 goto all_done;
825 }
33b83034 826 if (find_common(fd, sha1, ref) < 0)
fa740529 827 if (!args.keep_pack)
dfeff66e
JH
828 /* When cloning, it is not unusual to have
829 * no common commit.
830 */
78509d21 831 warning("no common commits");
ad897215 832
249b2004
SP
833 if (args.stateless_rpc)
834 packet_flush(fd[1]);
1788c39c 835 if (get_pack(fd, pack_lockfile))
7e44c935 836 die("git fetch-pack: fetch failed.");
ad897215
JH
837
838 all_done:
2d4177c0 839 return ref;
def88e9a
LT
840}
841
310b86d4
JH
842static int remove_duplicates(int nr_heads, char **heads)
843{
844 int src, dst;
845
7db8d537
JK
846 if (!nr_heads)
847 return 0;
848
849 for (src = dst = 1; src < nr_heads; src++)
850 if (strcmp(heads[src], heads[dst-1]))
851 heads[dst++] = heads[src];
310b86d4
JH
852 return dst;
853}
854
ef90d6d4 855static int fetch_pack_config(const char *var, const char *value, void *cb)
af7cf268
JH
856{
857 if (strcmp(var, "fetch.unpacklimit") == 0) {
e28714c5
JH
858 fetch_unpack_limit = git_config_int(var, value);
859 return 0;
860 }
861
862 if (strcmp(var, "transfer.unpacklimit") == 0) {
863 transfer_unpack_limit = git_config_int(var, value);
af7cf268
JH
864 return 0;
865 }
866
f04833ef
NP
867 if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
868 prefer_ofs_delta = git_config_bool(var, value);
869 return 0;
870 }
871
5e838ea7
JH
872 if (!strcmp(var, "fetch.fsckobjects")) {
873 fetch_fsck_objects = git_config_bool(var, value);
874 return 0;
875 }
876
dab76d3a
JH
877 if (!strcmp(var, "transfer.fsckobjects")) {
878 transfer_fsck_objects = git_config_bool(var, value);
879 return 0;
880 }
881
ef90d6d4 882 return git_default_config(var, value, cb);
af7cf268
JH
883}
884
54b9e022
JH
885static struct lock_file lock;
886
50ab5fd3 887static void fetch_pack_setup(void)
def88e9a 888{
50ab5fd3
SP
889 static int did_setup;
890 if (did_setup)
891 return;
ef90d6d4 892 git_config(fetch_pack_config, NULL);
e28714c5
JH
893 if (0 <= transfer_unpack_limit)
894 unpack_limit = transfer_unpack_limit;
895 else if (0 <= fetch_unpack_limit)
896 unpack_limit = fetch_unpack_limit;
50ab5fd3
SP
897 did_setup = 1;
898}
899
900int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
901{
57e6fc69 902 int i, ret;
ba227857 903 struct ref *ref = NULL;
9d19c6ea 904 const char *dest = NULL;
57e6fc69
MH
905 int alloc_heads = 0, nr_heads = 0;
906 char **heads = NULL;
ba227857 907 int fd[2];
249b2004
SP
908 char *pack_lockfile = NULL;
909 char **pack_lockfile_ptr = NULL;
ba227857 910 struct child_process *conn;
e28714c5 911
bbc30f99
JK
912 packet_trace_identity("fetch-pack");
913
ff22ff99 914 for (i = 1; i < argc && *argv[i] == '-'; i++) {
2d4177c0 915 const char *arg = argv[i];
def88e9a 916
ff22ff99
MH
917 if (!prefixcmp(arg, "--upload-pack=")) {
918 args.uploadpack = arg + 14;
919 continue;
920 }
921 if (!prefixcmp(arg, "--exec=")) {
922 args.uploadpack = arg + 7;
923 continue;
def88e9a 924 }
ff22ff99
MH
925 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
926 args.quiet = 1;
927 continue;
928 }
929 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
930 args.lock_pack = args.keep_pack;
931 args.keep_pack = 1;
932 continue;
933 }
934 if (!strcmp("--thin", arg)) {
935 args.use_thin_pack = 1;
936 continue;
937 }
938 if (!strcmp("--include-tag", arg)) {
939 args.include_tag = 1;
940 continue;
941 }
942 if (!strcmp("--all", arg)) {
943 args.fetch_all = 1;
944 continue;
def88e9a 945 }
ff22ff99
MH
946 if (!strcmp("--stdin", arg)) {
947 args.stdin_refs = 1;
948 continue;
949 }
950 if (!strcmp("-v", arg)) {
951 args.verbose = 1;
952 continue;
953 }
954 if (!prefixcmp(arg, "--depth=")) {
955 args.depth = strtol(arg + 8, NULL, 0);
956 continue;
957 }
958 if (!strcmp("--no-progress", arg)) {
959 args.no_progress = 1;
960 continue;
961 }
962 if (!strcmp("--stateless-rpc", arg)) {
963 args.stateless_rpc = 1;
964 continue;
965 }
966 if (!strcmp("--lock-pack", arg)) {
967 args.lock_pack = 1;
968 pack_lockfile_ptr = &pack_lockfile;
969 continue;
970 }
971 usage(fetch_pack_usage);
def88e9a 972 }
4cc00fcf
MH
973
974 if (i < argc)
975 dest = argv[i++];
976 else
def88e9a 977 usage(fetch_pack_usage);
2d4177c0 978
57e6fc69
MH
979 /*
980 * Copy refs from cmdline to growable list, then append any
981 * refs from the standard input:
982 */
983 ALLOC_GROW(heads, argc - i, alloc_heads);
984 for (; i < argc; i++)
985 heads[nr_heads++] = xstrdup(argv[i]);
078b895f 986 if (args.stdin_refs) {
078b895f
IT
987 if (args.stateless_rpc) {
988 /* in stateless RPC mode we use pkt-line to read
989 * from stdin, until we get a flush packet
990 */
991 static char line[1000];
992 for (;;) {
993 int n = packet_read_line(0, line, sizeof(line));
994 if (!n)
995 break;
996 if (line[n-1] == '\n')
997 n--;
998 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
999 heads[nr_heads++] = xmemdupz(line, n);
1000 }
1001 }
1002 else {
1003 /* read from stdin one ref per line, until EOF */
1004 struct strbuf line = STRBUF_INIT;
1005 while (strbuf_getline(&line, stdin, '\n') != EOF) {
1006 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1007 heads[nr_heads++] = strbuf_detach(&line, NULL);
1008 }
1009 strbuf_release(&line);
1010 }
1011 }
1012
249b2004
SP
1013 if (args.stateless_rpc) {
1014 conn = NULL;
1015 fd[0] = 0;
1016 fd[1] = 1;
ba227857 1017 } else {
9d19c6ea 1018 conn = git_connect(fd, dest, args.uploadpack,
249b2004
SP
1019 args.verbose ? CONNECT_VERBOSE : 0);
1020 }
1021
afe7c5ff 1022 get_remote_heads(fd[0], &ref, 0, NULL);
249b2004
SP
1023
1024 ref = fetch_pack(&args, fd, conn, ref, dest,
1025 nr_heads, heads, pack_lockfile_ptr);
1026 if (pack_lockfile) {
1027 printf("lock %s\n", pack_lockfile);
1028 fflush(stdout);
ba227857 1029 }
249b2004
SP
1030 close(fd[0]);
1031 close(fd[1]);
1032 if (finish_connect(conn))
1033 ref = NULL;
2d4177c0
DB
1034 ret = !ref;
1035
ba227857
DB
1036 if (!ret && nr_heads) {
1037 /* If the heads to pull were given, we should have
1038 * consumed all of them by matching the remote.
05207a28 1039 * Otherwise, 'git fetch remote no-such-ref' would
ba227857
DB
1040 * silently succeed without issuing an error.
1041 */
1042 for (i = 0; i < nr_heads; i++)
1043 if (heads[i] && heads[i][0]) {
1044 error("no such remote ref %s", heads[i]);
1045 ret = 1;
1046 }
1047 }
2d4177c0
DB
1048 while (ref) {
1049 printf("%s %s\n",
1050 sha1_to_hex(ref->old_sha1), ref->name);
1051 ref = ref->next;
1052 }
1053
1054 return ret;
1055}
1056
44359685
JK
1057static int compare_heads(const void *a, const void *b)
1058{
1059 return strcmp(*(const char **)a, *(const char **)b);
1060}
1061
fa740529 1062struct ref *fetch_pack(struct fetch_pack_args *my_args,
ba227857
DB
1063 int fd[], struct child_process *conn,
1064 const struct ref *ref,
fa740529 1065 const char *dest,
1788c39c
SP
1066 int nr_heads,
1067 char **heads,
1068 char **pack_lockfile)
2d4177c0 1069{
2d4177c0 1070 struct stat st;
ba227857 1071 struct ref *ref_cpy;
2d4177c0 1072
50ab5fd3 1073 fetch_pack_setup();
d551bbaf
TR
1074 if (&args != my_args)
1075 memcpy(&args, my_args, sizeof(args));
fa740529 1076 if (args.depth > 0) {
2d4177c0
DB
1077 if (stat(git_path("shallow"), &st))
1078 st.st_mtime = 0;
1079 }
1080
44359685 1081 if (heads && nr_heads) {
44359685 1082 qsort(heads, nr_heads, sizeof(*heads), compare_heads);
310b86d4 1083 nr_heads = remove_duplicates(nr_heads, heads);
44359685
JK
1084 }
1085
ba227857
DB
1086 if (!ref) {
1087 packet_flush(fd[1]);
1088 die("no matching remote head");
9e5d2b40 1089 }
ba227857 1090 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
9e5d2b40 1091
ba227857 1092 if (args.depth > 0) {
016e6ccb 1093 struct cache_time mtime;
edace6f0 1094 struct strbuf sb = STRBUF_INIT;
016e6ccb
JS
1095 char *shallow = git_path("shallow");
1096 int fd;
1097
1098 mtime.sec = st.st_mtime;
c06ff490 1099 mtime.nsec = ST_MTIME_NSEC(st);
016e6ccb
JS
1100 if (stat(shallow, &st)) {
1101 if (mtime.sec)
1102 die("shallow file was removed during fetch");
1103 } else if (st.st_mtime != mtime.sec
1104#ifdef USE_NSEC
5bcf109c 1105 || ST_MTIME_NSEC(st) != mtime.nsec
016e6ccb
JS
1106#endif
1107 )
1108 die("shallow file was changed during fetch");
1109
acd3b9ec
JH
1110 fd = hold_lock_file_for_update(&lock, shallow,
1111 LOCK_DIE_ON_ERROR);
edace6f0
SP
1112 if (!write_shallow_commits(&sb, 0)
1113 || write_in_full(fd, sb.buf, sb.len) != sb.len) {
691f1a28 1114 unlink_or_warn(shallow);
016e6ccb
JS
1115 rollback_lock_file(&lock);
1116 } else {
016e6ccb
JS
1117 commit_lock_file(&lock);
1118 }
edace6f0 1119 strbuf_release(&sb);
016e6ccb
JS
1120 }
1121
48ec3e5c 1122 reprepare_packed_git();
ba227857 1123 return ref_cpy;
def88e9a 1124}