include agent identifier in capability string
[git/git.git] / builtin / fetch-pack.c
CommitLineData
c2e86add 1#include "builtin.h"
fb9040cc 2#include "refs.h"
def88e9a 3#include "pkt-line.h"
49bb805e
JH
4#include "commit.h"
5#include "tag.h"
da093d37 6#include "exec_cmd.h"
9e10fd1a 7#include "pack.h"
da093d37 8#include "sideband.h"
2d4177c0 9#include "fetch-pack.h"
ba227857 10#include "remote.h"
477822c3 11#include "run-command.h"
e52d7192 12#include "transport.h"
ff5effdf 13#include "version.h"
def88e9a 14
e28714c5
JH
15static int transfer_unpack_limit = -1;
16static int fetch_unpack_limit = -1;
af7cf268 17static int unpack_limit = 100;
f04833ef 18static int prefer_ofs_delta = 1;
ca0c9764 19static int no_done;
dab76d3a
JH
20static int fetch_fsck_objects = -1;
21static int transfer_fsck_objects = -1;
09149c78
DB
22static struct fetch_pack_args args = {
23 /* .uploadpack = */ "git-upload-pack",
24};
fa740529 25
33b83034 26static const char fetch_pack_usage[] =
078b895f
IT
27"git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
28"[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
29"[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
def88e9a 30
0a8944dd 31#define COMPLETE (1U << 0)
23d61f83
JS
32#define COMMON (1U << 1)
33#define COMMON_REF (1U << 2)
34#define SEEN (1U << 3)
35#define POPPED (1U << 4)
36
420e9af4
DB
37static int marked;
38
f061e5fd
JH
39/*
40 * After sending this many "have"s if we do not get any new ACK , we
41 * give up traversing our history.
42 */
43#define MAX_IN_VAIN 256
44
96f1e58f 45static struct commit_list *rev_list;
3891f390 46static int non_common_revs, multi_ack, use_sideband;
23d61f83
JS
47
48static void rev_list_push(struct commit *commit, int mark)
49{
50 if (!(commit->object.flags & mark)) {
51 commit->object.flags |= mark;
52
53 if (!(commit->object.parsed))
f3ec5494
MK
54 if (parse_commit(commit))
55 return;
23d61f83 56
47e44ed1 57 commit_list_insert_by_date(commit, &rev_list);
23d61f83
JS
58
59 if (!(commit->object.flags & COMMON))
60 non_common_revs++;
61 }
62}
63
65385ef7 64static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
23d61f83 65{
65385ef7 66 struct object *o = deref_tag(parse_object(sha1), refname, 0);
23d61f83 67
1974632c 68 if (o && o->type == OBJ_COMMIT)
23d61f83
JS
69 rev_list_push((struct commit *)o, SEEN);
70
71 return 0;
72}
73
65385ef7 74static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
420e9af4 75{
65385ef7 76 struct object *o = deref_tag(parse_object(sha1), refname, 0);
420e9af4
DB
77
78 if (o && o->type == OBJ_COMMIT)
79 clear_commit_marks((struct commit *)o,
80 COMMON | COMMON_REF | SEEN | POPPED);
81 return 0;
82}
83
23d61f83
JS
84/*
85 This function marks a rev and its ancestors as common.
86 In some cases, it is desirable to mark only the ancestors (for example
87 when only the server does not yet know that they are common).
88*/
89
90static void mark_common(struct commit *commit,
91 int ancestors_only, int dont_parse)
92{
93 if (commit != NULL && !(commit->object.flags & COMMON)) {
94 struct object *o = (struct object *)commit;
95
96 if (!ancestors_only)
97 o->flags |= COMMON;
98
99 if (!(o->flags & SEEN))
100 rev_list_push(commit, SEEN);
101 else {
102 struct commit_list *parents;
103
104 if (!ancestors_only && !(o->flags & POPPED))
105 non_common_revs--;
106 if (!o->parsed && !dont_parse)
f3ec5494
MK
107 if (parse_commit(commit))
108 return;
23d61f83
JS
109
110 for (parents = commit->parents;
111 parents;
112 parents = parents->next)
113 mark_common(parents->item, 0, dont_parse);
114 }
115 }
116}
117
118/*
119 Get the next rev to send, ignoring the common.
120*/
121
4b25d091 122static const unsigned char *get_rev(void)
23d61f83
JS
123{
124 struct commit *commit = NULL;
125
126 while (commit == NULL) {
127 unsigned int mark;
72269ad9 128 struct commit_list *parents;
23d61f83
JS
129
130 if (rev_list == NULL || non_common_revs == 0)
131 return NULL;
132
133 commit = rev_list->item;
2d8bed96 134 if (!commit->object.parsed)
72269ad9
LT
135 parse_commit(commit);
136 parents = commit->parents;
f3ec5494 137
23d61f83
JS
138 commit->object.flags |= POPPED;
139 if (!(commit->object.flags & COMMON))
140 non_common_revs--;
a6080a0a 141
23d61f83
JS
142 if (commit->object.flags & COMMON) {
143 /* do not send "have", and ignore ancestors */
144 commit = NULL;
145 mark = COMMON | SEEN;
146 } else if (commit->object.flags & COMMON_REF)
147 /* send "have", and ignore ancestors */
148 mark = COMMON | SEEN;
149 else
150 /* send "have", also for its ancestors */
151 mark = SEEN;
152
153 while (parents) {
154 if (!(parents->item->object.flags & SEEN))
155 rev_list_push(parents->item, mark);
156 if (mark & COMMON)
157 mark_common(parents->item, 1, 0);
158 parents = parents->next;
159 }
160
161 rev_list = rev_list->next;
162 }
163
164 return commit->object.sha1;
165}
0a8944dd 166
78affc49
SP
167enum ack_type {
168 NAK = 0,
169 ACK,
170 ACK_continue,
171 ACK_common,
172 ACK_ready
173};
174
249b2004
SP
175static void consume_shallow_list(int fd)
176{
177 if (args.stateless_rpc && args.depth > 0) {
178 /* If we sent a depth we will get back "duplicate"
179 * shallow and unshallow commands every time there
180 * is a block of have lines exchanged.
181 */
182 char line[1000];
183 while (packet_read_line(fd, line, sizeof(line))) {
184 if (!prefixcmp(line, "shallow "))
185 continue;
186 if (!prefixcmp(line, "unshallow "))
187 continue;
188 die("git fetch-pack: expected shallow list");
189 }
190 }
191}
192
294e15fc
NTND
193struct write_shallow_data {
194 struct strbuf *out;
195 int use_pack_protocol;
196 int count;
197};
198
199static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
200{
201 struct write_shallow_data *data = cb_data;
202 const char *hex = sha1_to_hex(graft->sha1);
203 data->count++;
204 if (data->use_pack_protocol)
205 packet_buf_write(data->out, "shallow %s", hex);
206 else {
207 strbuf_addstr(data->out, hex);
208 strbuf_addch(data->out, '\n');
209 }
210 return 0;
211}
212
213static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
214{
215 struct write_shallow_data data;
216 data.out = out;
217 data.use_pack_protocol = use_pack_protocol;
218 data.count = 0;
219 for_each_commit_graft(write_one_shallow, &data);
220 return data.count;
221}
222
78affc49 223static enum ack_type get_ack(int fd, unsigned char *result_sha1)
28754ab5
SP
224{
225 static char line[1000];
226 int len = packet_read_line(fd, line, sizeof(line));
227
228 if (!len)
229 die("git fetch-pack: expected ACK/NAK, got EOF");
230 if (line[len-1] == '\n')
231 line[--len] = 0;
232 if (!strcmp(line, "NAK"))
78affc49 233 return NAK;
28754ab5
SP
234 if (!prefixcmp(line, "ACK ")) {
235 if (!get_sha1_hex(line+4, result_sha1)) {
236 if (strstr(line+45, "continue"))
78affc49
SP
237 return ACK_continue;
238 if (strstr(line+45, "common"))
239 return ACK_common;
240 if (strstr(line+45, "ready"))
241 return ACK_ready;
242 return ACK;
28754ab5
SP
243 }
244 }
245 die("git fetch_pack: expected ACK/NAK, got '%s'", line);
246}
247
249b2004
SP
248static void send_request(int fd, struct strbuf *buf)
249{
250 if (args.stateless_rpc) {
251 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
252 packet_flush(fd);
253 } else
254 safe_write(fd, buf->buf, buf->len);
255}
256
e52d7192
JH
257static void insert_one_alternate_ref(const struct ref *ref, void *unused)
258{
259 rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
260}
261
066bf4c2 262#define INITIAL_FLUSH 16
44d8dc54 263#define PIPESAFE_FLUSH 32
6afca450 264#define LARGE_FLUSH 1024
c12f5917
JH
265
266static int next_flush(int count)
267{
44d8dc54
JH
268 int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
269
270 if (count < flush_limit)
6afca450
JH
271 count <<= 1;
272 else
44d8dc54 273 count += flush_limit;
6afca450 274 return count;
c12f5917
JH
275}
276
33b83034
JH
277static int find_common(int fd[2], unsigned char *result_sha1,
278 struct ref *refs)
def88e9a 279{
2759cbc7 280 int fetching;
c12f5917 281 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
23d61f83 282 const unsigned char *sha1;
f061e5fd
JH
283 unsigned in_vain = 0;
284 int got_continue = 0;
4e10cf9a 285 int got_ready = 0;
edace6f0 286 struct strbuf req_buf = STRBUF_INIT;
249b2004 287 size_t state_len = 0;
23d61f83 288
249b2004
SP
289 if (args.stateless_rpc && multi_ack == 1)
290 die("--stateless-rpc requires multi_ack_detailed");
420e9af4
DB
291 if (marked)
292 for_each_ref(clear_marks, NULL);
293 marked = 1;
294
cb5d709f 295 for_each_ref(rev_list_insert_ref, NULL);
c41a802f 296 for_each_alternate_ref(insert_one_alternate_ref, NULL);
def88e9a 297
2759cbc7
LT
298 fetching = 0;
299 for ( ; refs ; refs = refs->next) {
33b83034 300 unsigned char *remote = refs->old_sha1;
edace6f0 301 const char *remote_hex;
4dab94d5 302 struct object *o;
2759cbc7 303
0a8944dd 304 /*
4dab94d5
JH
305 * If that object is complete (i.e. it is an ancestor of a
306 * local ref), we tell them we have it but do not have to
307 * tell them about its ancestors, which they already know
308 * about.
f1f0a2be
JH
309 *
310 * We use lookup_object here because we are only
311 * interested in the case we *know* the object is
312 * reachable and we have already scanned it.
4dab94d5 313 */
f1f0a2be 314 if (((o = lookup_object(remote)) != NULL) &&
1baaae5e 315 (o->flags & COMPLETE)) {
2759cbc7 316 continue;
0a8944dd 317 }
23d61f83 318
edace6f0
SP
319 remote_hex = sha1_to_hex(remote);
320 if (!fetching) {
321 struct strbuf c = STRBUF_INIT;
78affc49
SP
322 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
323 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
4e10cf9a 324 if (no_done) strbuf_addstr(&c, " no-done");
edace6f0
SP
325 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
326 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
327 if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
328 if (args.no_progress) strbuf_addstr(&c, " no-progress");
329 if (args.include_tag) strbuf_addstr(&c, " include-tag");
330 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
ff5effdf 331 strbuf_addf(&c, " agent=%s", git_user_agent_sanitized());
edace6f0
SP
332 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
333 strbuf_release(&c);
334 } else
335 packet_buf_write(&req_buf, "want %s\n", remote_hex);
2759cbc7 336 fetching++;
33b83034 337 }
edace6f0
SP
338
339 if (!fetching) {
340 strbuf_release(&req_buf);
341 packet_flush(fd[1]);
342 return 1;
343 }
344
ed09aef0 345 if (is_repository_shallow())
edace6f0 346 write_shallow_commits(&req_buf, 1);
fa740529 347 if (args.depth > 0)
edace6f0
SP
348 packet_buf_write(&req_buf, "deepen %d", args.depth);
349 packet_buf_flush(&req_buf);
249b2004 350 state_len = req_buf.len;
0a8944dd 351
fa740529 352 if (args.depth > 0) {
016e6ccb
JS
353 char line[1024];
354 unsigned char sha1[20];
016e6ccb 355
249b2004 356 send_request(fd[1], &req_buf);
eb3a9dd3 357 while (packet_read_line(fd[0], line, sizeof(line))) {
599065a3 358 if (!prefixcmp(line, "shallow ")) {
016e6ccb
JS
359 if (get_sha1_hex(line + 8, sha1))
360 die("invalid shallow line: %s", line);
016e6ccb 361 register_shallow(sha1);
cf01bd52
JH
362 continue;
363 }
599065a3 364 if (!prefixcmp(line, "unshallow ")) {
f53514bc
JS
365 if (get_sha1_hex(line + 10, sha1))
366 die("invalid unshallow line: %s", line);
367 if (!lookup_object(sha1))
368 die("object not found: %s", line);
369 /* make sure that it is parsed as shallow */
f3ec5494
MK
370 if (!parse_object(sha1))
371 die("error in object: %s", line);
f53514bc
JS
372 if (unregister_shallow(sha1))
373 die("no shallow found: %s", line);
cf01bd52
JH
374 continue;
375 }
376 die("expected shallow/unshallow, got %s", line);
016e6ccb 377 }
249b2004
SP
378 } else if (!args.stateless_rpc)
379 send_request(fd[1], &req_buf);
380
381 if (!args.stateless_rpc) {
382 /* If we aren't using the stateless-rpc interface
383 * we don't need to retain the headers.
384 */
385 strbuf_setlen(&req_buf, 0);
386 state_len = 0;
016e6ccb
JS
387 }
388
23d61f83 389 flushes = 0;
75bfc6c2 390 retval = -1;
23d61f83 391 while ((sha1 = get_rev())) {
249b2004 392 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
fa740529 393 if (args.verbose)
33b83034 394 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
f061e5fd 395 in_vain++;
c12f5917 396 if (flush_at <= ++count) {
c4c86f07
JS
397 int ack;
398
249b2004
SP
399 packet_buf_flush(&req_buf);
400 send_request(fd[1], &req_buf);
401 strbuf_setlen(&req_buf, state_len);
def88e9a 402 flushes++;
c12f5917 403 flush_at = next_flush(count);
def88e9a
LT
404
405 /*
406 * We keep one window "ahead" of the other side, and
407 * will wait for an ACK only on the next one
408 */
c12f5917 409 if (!args.stateless_rpc && count == INITIAL_FLUSH)
def88e9a 410 continue;
c4c86f07 411
249b2004 412 consume_shallow_list(fd[0]);
c4c86f07
JS
413 do {
414 ack = get_ack(fd[0], result_sha1);
fa740529 415 if (args.verbose && ack)
c4c86f07
JS
416 fprintf(stderr, "got ack %d %s\n", ack,
417 sha1_to_hex(result_sha1));
78affc49
SP
418 switch (ack) {
419 case ACK:
c4c86f07
JS
420 flushes = 0;
421 multi_ack = 0;
422 retval = 0;
423 goto done;
78affc49
SP
424 case ACK_common:
425 case ACK_ready:
426 case ACK_continue: {
c4c86f07
JS
427 struct commit *commit =
428 lookup_commit(result_sha1);
ec099546
NTND
429 if (!commit)
430 die("invalid commit %s", sha1_to_hex(result_sha1));
249b2004
SP
431 if (args.stateless_rpc
432 && ack == ACK_common
433 && !(commit->object.flags & COMMON)) {
434 /* We need to replay the have for this object
435 * on the next RPC request so the peer knows
436 * it is in common with us.
437 */
438 const char *hex = sha1_to_hex(result_sha1);
439 packet_buf_write(&req_buf, "have %s\n", hex);
440 state_len = req_buf.len;
441 }
c4c86f07
JS
442 mark_common(commit, 0, 1);
443 retval = 0;
f061e5fd
JH
444 in_vain = 0;
445 got_continue = 1;
4e10cf9a 446 if (ack == ACK_ready) {
f2cba929 447 rev_list = NULL;
4e10cf9a
JH
448 got_ready = 1;
449 }
78affc49
SP
450 break;
451 }
c4c86f07
JS
452 }
453 } while (ack);
def88e9a 454 flushes--;
f061e5fd 455 if (got_continue && MAX_IN_VAIN < in_vain) {
fa740529 456 if (args.verbose)
f061e5fd
JH
457 fprintf(stderr, "giving up\n");
458 break; /* give up */
459 }
def88e9a
LT
460 }
461 }
c4c86f07 462done:
4e10cf9a
JH
463 if (!got_ready || !no_done) {
464 packet_buf_write(&req_buf, "done\n");
465 send_request(fd[1], &req_buf);
466 }
fa740529 467 if (args.verbose)
33b83034 468 fprintf(stderr, "done\n");
c4c86f07
JS
469 if (retval != 0) {
470 multi_ack = 0;
23d61f83 471 flushes++;
c4c86f07 472 }
edace6f0
SP
473 strbuf_release(&req_buf);
474
249b2004 475 consume_shallow_list(fd[0]);
c4c86f07
JS
476 while (flushes || multi_ack) {
477 int ack = get_ack(fd[0], result_sha1);
478 if (ack) {
fa740529 479 if (args.verbose)
c4c86f07
JS
480 fprintf(stderr, "got ack (%d) %s\n", ack,
481 sha1_to_hex(result_sha1));
78affc49 482 if (ack == ACK)
c4c86f07
JS
483 return 0;
484 multi_ack = 1;
485 continue;
33b83034 486 }
c4c86f07 487 flushes--;
def88e9a 488 }
8cb560fc
JS
489 /* it is no error to fetch into a completely empty repo */
490 return count ? retval : 0;
def88e9a
LT
491}
492
96f1e58f 493static struct commit_list *complete;
49bb805e 494
65385ef7 495static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
49bb805e
JH
496{
497 struct object *o = parse_object(sha1);
498
1974632c 499 while (o && o->type == OBJ_TAG) {
f1f0a2be
JH
500 struct tag *t = (struct tag *) o;
501 if (!t->tagged)
502 break; /* broken repository */
49bb805e 503 o->flags |= COMPLETE;
f1f0a2be 504 o = parse_object(t->tagged->sha1);
49bb805e 505 }
1974632c 506 if (o && o->type == OBJ_COMMIT) {
49bb805e 507 struct commit *commit = (struct commit *)o;
ea5f2208
JK
508 if (!(commit->object.flags & COMPLETE)) {
509 commit->object.flags |= COMPLETE;
510 commit_list_insert_by_date(commit, &complete);
511 }
49bb805e
JH
512 }
513 return 0;
514}
515
516static void mark_recent_complete_commits(unsigned long cutoff)
517{
518 while (complete && cutoff <= complete->item->date) {
fa740529 519 if (args.verbose)
49bb805e
JH
520 fprintf(stderr, "Marking %s as complete\n",
521 sha1_to_hex(complete->item->object.sha1));
522 pop_most_recent_commit(&complete, COMPLETE);
523 }
524}
525
1baaae5e
JS
526static void filter_refs(struct ref **refs, int nr_match, char **match)
527{
9546010b
JH
528 struct ref **return_refs;
529 struct ref *newlist = NULL;
530 struct ref **newtail = &newlist;
531 struct ref *ref, *next;
532 struct ref *fastarray[32];
a0de2880 533 int match_pos;
9546010b 534
fa740529 535 if (nr_match && !args.fetch_all) {
9546010b
JH
536 if (ARRAY_SIZE(fastarray) < nr_match)
537 return_refs = xcalloc(nr_match, sizeof(struct ref *));
538 else {
539 return_refs = fastarray;
540 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
541 }
542 }
543 else
544 return_refs = NULL;
545
a0de2880 546 match_pos = 0;
9546010b
JH
547 for (ref = *refs; ref; ref = next) {
548 next = ref->next;
549 if (!memcmp(ref->name, "refs/", 5) &&
8d9c5010 550 check_refname_format(ref->name + 5, 0))
9546010b 551 ; /* trash */
fa740529
SP
552 else if (args.fetch_all &&
553 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
9546010b
JH
554 *newtail = ref;
555 ref->next = NULL;
556 newtail = &ref->next;
557 continue;
558 }
559 else {
a0de2880
JK
560 int cmp = -1;
561 while (match_pos < nr_match) {
562 cmp = strcmp(ref->name, match[match_pos]);
563 if (cmp < 0) /* definitely do not have it */
564 break;
565 else if (cmp == 0) { /* definitely have it */
566 match[match_pos][0] = '\0';
567 return_refs[match_pos] = ref;
1e7ba0f9
JK
568 break;
569 }
a0de2880
JK
570 else /* might have it; keep looking */
571 match_pos++;
9546010b 572 }
a0de2880 573 if (!cmp)
1e7ba0f9 574 continue; /* we will link it later */
9546010b
JH
575 }
576 free(ref);
577 }
578
fa740529 579 if (!args.fetch_all) {
9546010b
JH
580 int i;
581 for (i = 0; i < nr_match; i++) {
582 ref = return_refs[i];
583 if (ref) {
584 *newtail = ref;
585 ref->next = NULL;
586 newtail = &ref->next;
587 }
588 }
589 if (return_refs != fastarray)
590 free(return_refs);
1baaae5e 591 }
9546010b 592 *refs = newlist;
1baaae5e
JS
593}
594
f2576591
MH
595static void mark_alternate_complete(const struct ref *ref, void *unused)
596{
597 mark_complete(NULL, ref->old_sha1, 0, NULL);
598}
599
1baaae5e 600static int everything_local(struct ref **refs, int nr_match, char **match)
2759cbc7 601{
49bb805e 602 struct ref *ref;
2759cbc7 603 int retval;
49bb805e
JH
604 unsigned long cutoff = 0;
605
49bb805e
JH
606 save_commit_buffer = 0;
607
1baaae5e 608 for (ref = *refs; ref; ref = ref->next) {
49bb805e
JH
609 struct object *o;
610
611 o = parse_object(ref->old_sha1);
612 if (!o)
613 continue;
614
615 /* We already have it -- which may mean that we were
616 * in sync with the other side at some time after
617 * that (it is OK if we guess wrong here).
618 */
1974632c 619 if (o->type == OBJ_COMMIT) {
49bb805e
JH
620 struct commit *commit = (struct commit *)o;
621 if (!cutoff || cutoff < commit->date)
622 cutoff = commit->date;
623 }
624 }
625
fa740529 626 if (!args.depth) {
f53514bc 627 for_each_ref(mark_complete, NULL);
f2576591 628 for_each_alternate_ref(mark_alternate_complete, NULL);
f53514bc
JS
629 if (cutoff)
630 mark_recent_complete_commits(cutoff);
631 }
2759cbc7 632
1baaae5e
JS
633 /*
634 * Mark all complete remote refs as common refs.
635 * Don't mark them common yet; the server has to be told so first.
636 */
637 for (ref = *refs; ref; ref = ref->next) {
9534f40b
JH
638 struct object *o = deref_tag(lookup_object(ref->old_sha1),
639 NULL, 0);
1baaae5e 640
1974632c 641 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
1baaae5e
JS
642 continue;
643
644 if (!(o->flags & SEEN)) {
645 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
646
647 mark_common((struct commit *)o, 1, 1);
648 }
649 }
650
651 filter_refs(refs, nr_match, match);
652
653 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
654 const unsigned char *remote = ref->old_sha1;
2759cbc7 655 unsigned char local[20];
49bb805e 656 struct object *o;
2759cbc7 657
1baaae5e 658 o = lookup_object(remote);
49bb805e 659 if (!o || !(o->flags & COMPLETE)) {
2759cbc7 660 retval = 0;
fa740529 661 if (!args.verbose)
2759cbc7
LT
662 continue;
663 fprintf(stderr,
664 "want %s (%s)\n", sha1_to_hex(remote),
1baaae5e 665 ref->name);
2759cbc7
LT
666 continue;
667 }
668
e702496e 669 hashcpy(ref->new_sha1, local);
fa740529 670 if (!args.verbose)
2759cbc7
LT
671 continue;
672 fprintf(stderr,
673 "already have %s (%s)\n", sha1_to_hex(remote),
1baaae5e 674 ref->name);
2759cbc7
LT
675 }
676 return retval;
677}
678
ae6a5609 679static int sideband_demux(int in, int out, void *data)
da093d37 680{
088fab5f 681 int *xd = data;
da093d37 682
ae6a5609
EFL
683 int ret = recv_sideband("fetch-pack", xd[0], out);
684 close(out);
3ef67cf2 685 return ret;
088fab5f
JS
686}
687
1788c39c 688static int get_pack(int xd[2], char **pack_lockfile)
da093d37 689{
088fab5f 690 struct async demux;
9e10fd1a
JH
691 const char *argv[20];
692 char keep_arg[256];
693 char hdr_arg[256];
694 const char **av;
fa740529 695 int do_keep = args.keep_pack;
477822c3 696 struct child_process cmd;
da093d37 697
1f759eee
JS
698 memset(&demux, 0, sizeof(demux));
699 if (use_sideband) {
700 /* xd[] is talking with upload-pack; subprocess reads from
701 * xd[0], spits out band#2 to stderr, and feeds us band#1
702 * through demux->out.
703 */
704 demux.proc = sideband_demux;
705 demux.data = xd;
ae6a5609 706 demux.out = -1;
1f759eee
JS
707 if (start_async(&demux))
708 die("fetch-pack: unable to fork off sideband"
709 " demultiplexer");
710 }
711 else
712 demux.out = xd[0];
9e10fd1a 713
477822c3
JS
714 memset(&cmd, 0, sizeof(cmd));
715 cmd.argv = argv;
9e10fd1a
JH
716 av = argv;
717 *hdr_arg = 0;
fa740529 718 if (!args.keep_pack && unpack_limit) {
9e10fd1a
JH
719 struct pack_header header;
720
1f759eee 721 if (read_pack_header(demux.out, &header))
9e10fd1a 722 die("protocol error: bad pack header");
6e1c2344
RJ
723 snprintf(hdr_arg, sizeof(hdr_arg),
724 "--pack_header=%"PRIu32",%"PRIu32,
9e10fd1a 725 ntohl(header.hdr_version), ntohl(header.hdr_entries));
af7cf268 726 if (ntohl(header.hdr_entries) < unpack_limit)
9e10fd1a
JH
727 do_keep = 0;
728 else
729 do_keep = 1;
730 }
731
732 if (do_keep) {
477822c3
JS
733 if (pack_lockfile)
734 cmd.out = -1;
9e10fd1a
JH
735 *av++ = "index-pack";
736 *av++ = "--stdin";
fa740529 737 if (!args.quiet && !args.no_progress)
9e10fd1a 738 *av++ = "-v";
fa740529 739 if (args.use_thin_pack)
9e10fd1a 740 *av++ = "--fix-thin";
fa740529 741 if (args.lock_pack || unpack_limit) {
9e10fd1a 742 int s = sprintf(keep_arg,
85e72830 743 "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
9e10fd1a
JH
744 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
745 strcpy(keep_arg + s, "localhost");
746 *av++ = keep_arg;
747 }
748 }
749 else {
750 *av++ = "unpack-objects";
01fdc21f 751 if (args.quiet || args.no_progress)
9e10fd1a
JH
752 *av++ = "-q";
753 }
754 if (*hdr_arg)
755 *av++ = hdr_arg;
dab76d3a
JH
756 if (fetch_fsck_objects >= 0
757 ? fetch_fsck_objects
758 : transfer_fsck_objects >= 0
759 ? transfer_fsck_objects
760 : 0)
5e838ea7 761 *av++ = "--strict";
9e10fd1a
JH
762 *av++ = NULL;
763
1f759eee 764 cmd.in = demux.out;
477822c3
JS
765 cmd.git_cmd = 1;
766 if (start_command(&cmd))
da093d37 767 die("fetch-pack: unable to fork off %s", argv[0]);
e72ae288 768 if (do_keep && pack_lockfile) {
477822c3 769 *pack_lockfile = index_pack_lockfile(cmd.out);
e72ae288
JS
770 close(cmd.out);
771 }
477822c3
JS
772
773 if (finish_command(&cmd))
774 die("%s failed", argv[0]);
088fab5f
JS
775 if (use_sideband && finish_async(&demux))
776 die("error in sideband demultiplexer");
477822c3 777 return 0;
da093d37
NP
778}
779
1788c39c 780static struct ref *do_fetch_pack(int fd[2],
ba227857 781 const struct ref *orig_ref,
1788c39c
SP
782 int nr_match,
783 char **match,
784 char **pack_lockfile)
def88e9a 785{
ba227857 786 struct ref *ref = copy_ref_list(orig_ref);
d1c133f5 787 unsigned char sha1[20];
def88e9a 788
9e8e704f
JK
789 sort_ref_list(&ref, ref_compare_name);
790
ed09aef0
JS
791 if (is_repository_shallow() && !server_supports("shallow"))
792 die("Server does not support shallow clients");
78affc49
SP
793 if (server_supports("multi_ack_detailed")) {
794 if (args.verbose)
795 fprintf(stderr, "Server supports multi_ack_detailed\n");
796 multi_ack = 2;
4e10cf9a
JH
797 if (server_supports("no-done")) {
798 if (args.verbose)
799 fprintf(stderr, "Server supports no-done\n");
8e9182e0
JH
800 if (args.stateless_rpc)
801 no_done = 1;
4e10cf9a 802 }
78affc49
SP
803 }
804 else if (server_supports("multi_ack")) {
fa740529 805 if (args.verbose)
c4c86f07
JS
806 fprintf(stderr, "Server supports multi_ack\n");
807 multi_ack = 1;
808 }
d47f3db7 809 if (server_supports("side-band-64k")) {
fa740529 810 if (args.verbose)
d47f3db7
JH
811 fprintf(stderr, "Server supports side-band-64k\n");
812 use_sideband = 2;
813 }
814 else if (server_supports("side-band")) {
fa740529 815 if (args.verbose)
583b7ea3
JH
816 fprintf(stderr, "Server supports side-band\n");
817 use_sideband = 1;
818 }
f04833ef
NP
819 if (server_supports("ofs-delta")) {
820 if (args.verbose)
821 fprintf(stderr, "Server supports ofs-delta\n");
822 } else
823 prefer_ofs_delta = 0;
1baaae5e 824 if (everything_local(&ref, nr_match, match)) {
2759cbc7
LT
825 packet_flush(fd[1]);
826 goto all_done;
827 }
33b83034 828 if (find_common(fd, sha1, ref) < 0)
fa740529 829 if (!args.keep_pack)
dfeff66e
JH
830 /* When cloning, it is not unusual to have
831 * no common commit.
832 */
78509d21 833 warning("no common commits");
ad897215 834
249b2004
SP
835 if (args.stateless_rpc)
836 packet_flush(fd[1]);
1788c39c 837 if (get_pack(fd, pack_lockfile))
7e44c935 838 die("git fetch-pack: fetch failed.");
ad897215
JH
839
840 all_done:
2d4177c0 841 return ref;
def88e9a
LT
842}
843
310b86d4
JH
844static int remove_duplicates(int nr_heads, char **heads)
845{
846 int src, dst;
847
7db8d537
JK
848 if (!nr_heads)
849 return 0;
850
851 for (src = dst = 1; src < nr_heads; src++)
852 if (strcmp(heads[src], heads[dst-1]))
853 heads[dst++] = heads[src];
310b86d4
JH
854 return dst;
855}
856
ef90d6d4 857static int fetch_pack_config(const char *var, const char *value, void *cb)
af7cf268
JH
858{
859 if (strcmp(var, "fetch.unpacklimit") == 0) {
e28714c5
JH
860 fetch_unpack_limit = git_config_int(var, value);
861 return 0;
862 }
863
864 if (strcmp(var, "transfer.unpacklimit") == 0) {
865 transfer_unpack_limit = git_config_int(var, value);
af7cf268
JH
866 return 0;
867 }
868
f04833ef
NP
869 if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
870 prefer_ofs_delta = git_config_bool(var, value);
871 return 0;
872 }
873
5e838ea7
JH
874 if (!strcmp(var, "fetch.fsckobjects")) {
875 fetch_fsck_objects = git_config_bool(var, value);
876 return 0;
877 }
878
dab76d3a
JH
879 if (!strcmp(var, "transfer.fsckobjects")) {
880 transfer_fsck_objects = git_config_bool(var, value);
881 return 0;
882 }
883
ef90d6d4 884 return git_default_config(var, value, cb);
af7cf268
JH
885}
886
54b9e022
JH
887static struct lock_file lock;
888
50ab5fd3 889static void fetch_pack_setup(void)
def88e9a 890{
50ab5fd3
SP
891 static int did_setup;
892 if (did_setup)
893 return;
ef90d6d4 894 git_config(fetch_pack_config, NULL);
e28714c5
JH
895 if (0 <= transfer_unpack_limit)
896 unpack_limit = transfer_unpack_limit;
897 else if (0 <= fetch_unpack_limit)
898 unpack_limit = fetch_unpack_limit;
50ab5fd3
SP
899 did_setup = 1;
900}
901
902int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
903{
57e6fc69 904 int i, ret;
ba227857 905 struct ref *ref = NULL;
9d19c6ea 906 const char *dest = NULL;
57e6fc69
MH
907 int alloc_heads = 0, nr_heads = 0;
908 char **heads = NULL;
ba227857 909 int fd[2];
249b2004
SP
910 char *pack_lockfile = NULL;
911 char **pack_lockfile_ptr = NULL;
ba227857 912 struct child_process *conn;
e28714c5 913
bbc30f99
JK
914 packet_trace_identity("fetch-pack");
915
ff22ff99 916 for (i = 1; i < argc && *argv[i] == '-'; i++) {
2d4177c0 917 const char *arg = argv[i];
def88e9a 918
ff22ff99
MH
919 if (!prefixcmp(arg, "--upload-pack=")) {
920 args.uploadpack = arg + 14;
921 continue;
922 }
923 if (!prefixcmp(arg, "--exec=")) {
924 args.uploadpack = arg + 7;
925 continue;
def88e9a 926 }
ff22ff99
MH
927 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
928 args.quiet = 1;
929 continue;
930 }
931 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
932 args.lock_pack = args.keep_pack;
933 args.keep_pack = 1;
934 continue;
935 }
936 if (!strcmp("--thin", arg)) {
937 args.use_thin_pack = 1;
938 continue;
939 }
940 if (!strcmp("--include-tag", arg)) {
941 args.include_tag = 1;
942 continue;
943 }
944 if (!strcmp("--all", arg)) {
945 args.fetch_all = 1;
946 continue;
def88e9a 947 }
ff22ff99
MH
948 if (!strcmp("--stdin", arg)) {
949 args.stdin_refs = 1;
950 continue;
951 }
952 if (!strcmp("-v", arg)) {
953 args.verbose = 1;
954 continue;
955 }
956 if (!prefixcmp(arg, "--depth=")) {
957 args.depth = strtol(arg + 8, NULL, 0);
958 continue;
959 }
960 if (!strcmp("--no-progress", arg)) {
961 args.no_progress = 1;
962 continue;
963 }
964 if (!strcmp("--stateless-rpc", arg)) {
965 args.stateless_rpc = 1;
966 continue;
967 }
968 if (!strcmp("--lock-pack", arg)) {
969 args.lock_pack = 1;
970 pack_lockfile_ptr = &pack_lockfile;
971 continue;
972 }
973 usage(fetch_pack_usage);
def88e9a 974 }
4cc00fcf
MH
975
976 if (i < argc)
977 dest = argv[i++];
978 else
def88e9a 979 usage(fetch_pack_usage);
2d4177c0 980
57e6fc69
MH
981 /*
982 * Copy refs from cmdline to growable list, then append any
983 * refs from the standard input:
984 */
985 ALLOC_GROW(heads, argc - i, alloc_heads);
986 for (; i < argc; i++)
987 heads[nr_heads++] = xstrdup(argv[i]);
078b895f 988 if (args.stdin_refs) {
078b895f
IT
989 if (args.stateless_rpc) {
990 /* in stateless RPC mode we use pkt-line to read
991 * from stdin, until we get a flush packet
992 */
993 static char line[1000];
994 for (;;) {
995 int n = packet_read_line(0, line, sizeof(line));
996 if (!n)
997 break;
998 if (line[n-1] == '\n')
999 n--;
1000 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1001 heads[nr_heads++] = xmemdupz(line, n);
1002 }
1003 }
1004 else {
1005 /* read from stdin one ref per line, until EOF */
1006 struct strbuf line = STRBUF_INIT;
1007 while (strbuf_getline(&line, stdin, '\n') != EOF) {
1008 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1009 heads[nr_heads++] = strbuf_detach(&line, NULL);
1010 }
1011 strbuf_release(&line);
1012 }
1013 }
1014
249b2004
SP
1015 if (args.stateless_rpc) {
1016 conn = NULL;
1017 fd[0] = 0;
1018 fd[1] = 1;
ba227857 1019 } else {
9d19c6ea 1020 conn = git_connect(fd, dest, args.uploadpack,
249b2004
SP
1021 args.verbose ? CONNECT_VERBOSE : 0);
1022 }
1023
afe7c5ff 1024 get_remote_heads(fd[0], &ref, 0, NULL);
249b2004
SP
1025
1026 ref = fetch_pack(&args, fd, conn, ref, dest,
1027 nr_heads, heads, pack_lockfile_ptr);
1028 if (pack_lockfile) {
1029 printf("lock %s\n", pack_lockfile);
1030 fflush(stdout);
ba227857 1031 }
249b2004
SP
1032 close(fd[0]);
1033 close(fd[1]);
1034 if (finish_connect(conn))
1035 ref = NULL;
2d4177c0
DB
1036 ret = !ref;
1037
ba227857
DB
1038 if (!ret && nr_heads) {
1039 /* If the heads to pull were given, we should have
1040 * consumed all of them by matching the remote.
05207a28 1041 * Otherwise, 'git fetch remote no-such-ref' would
ba227857
DB
1042 * silently succeed without issuing an error.
1043 */
1044 for (i = 0; i < nr_heads; i++)
1045 if (heads[i] && heads[i][0]) {
1046 error("no such remote ref %s", heads[i]);
1047 ret = 1;
1048 }
1049 }
2d4177c0
DB
1050 while (ref) {
1051 printf("%s %s\n",
1052 sha1_to_hex(ref->old_sha1), ref->name);
1053 ref = ref->next;
1054 }
1055
1056 return ret;
1057}
1058
44359685
JK
1059static int compare_heads(const void *a, const void *b)
1060{
1061 return strcmp(*(const char **)a, *(const char **)b);
1062}
1063
fa740529 1064struct ref *fetch_pack(struct fetch_pack_args *my_args,
ba227857
DB
1065 int fd[], struct child_process *conn,
1066 const struct ref *ref,
fa740529 1067 const char *dest,
1788c39c
SP
1068 int nr_heads,
1069 char **heads,
1070 char **pack_lockfile)
2d4177c0 1071{
2d4177c0 1072 struct stat st;
ba227857 1073 struct ref *ref_cpy;
2d4177c0 1074
50ab5fd3 1075 fetch_pack_setup();
d551bbaf
TR
1076 if (&args != my_args)
1077 memcpy(&args, my_args, sizeof(args));
fa740529 1078 if (args.depth > 0) {
2d4177c0
DB
1079 if (stat(git_path("shallow"), &st))
1080 st.st_mtime = 0;
1081 }
1082
44359685 1083 if (heads && nr_heads) {
44359685 1084 qsort(heads, nr_heads, sizeof(*heads), compare_heads);
310b86d4 1085 nr_heads = remove_duplicates(nr_heads, heads);
44359685
JK
1086 }
1087
ba227857
DB
1088 if (!ref) {
1089 packet_flush(fd[1]);
1090 die("no matching remote head");
9e5d2b40 1091 }
ba227857 1092 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
9e5d2b40 1093
ba227857 1094 if (args.depth > 0) {
016e6ccb 1095 struct cache_time mtime;
edace6f0 1096 struct strbuf sb = STRBUF_INIT;
016e6ccb
JS
1097 char *shallow = git_path("shallow");
1098 int fd;
1099
1100 mtime.sec = st.st_mtime;
c06ff490 1101 mtime.nsec = ST_MTIME_NSEC(st);
016e6ccb
JS
1102 if (stat(shallow, &st)) {
1103 if (mtime.sec)
1104 die("shallow file was removed during fetch");
1105 } else if (st.st_mtime != mtime.sec
1106#ifdef USE_NSEC
5bcf109c 1107 || ST_MTIME_NSEC(st) != mtime.nsec
016e6ccb
JS
1108#endif
1109 )
1110 die("shallow file was changed during fetch");
1111
acd3b9ec
JH
1112 fd = hold_lock_file_for_update(&lock, shallow,
1113 LOCK_DIE_ON_ERROR);
edace6f0
SP
1114 if (!write_shallow_commits(&sb, 0)
1115 || write_in_full(fd, sb.buf, sb.len) != sb.len) {
691f1a28 1116 unlink_or_warn(shallow);
016e6ccb
JS
1117 rollback_lock_file(&lock);
1118 } else {
016e6ccb
JS
1119 commit_lock_file(&lock);
1120 }
edace6f0 1121 strbuf_release(&sb);
016e6ccb
JS
1122 }
1123
48ec3e5c 1124 reprepare_packed_git();
ba227857 1125 return ref_cpy;
def88e9a 1126}