send-pack: move core code to libgit.a
[git/git.git] / builtin / fetch-pack.c
CommitLineData
c2e86add 1#include "builtin.h"
fb9040cc 2#include "refs.h"
def88e9a 3#include "pkt-line.h"
49bb805e
JH
4#include "commit.h"
5#include "tag.h"
da093d37 6#include "exec_cmd.h"
9e10fd1a 7#include "pack.h"
da093d37 8#include "sideband.h"
2d4177c0 9#include "fetch-pack.h"
ba227857 10#include "remote.h"
477822c3 11#include "run-command.h"
e52d7192 12#include "transport.h"
ff5effdf 13#include "version.h"
def88e9a 14
e28714c5
JH
15static int transfer_unpack_limit = -1;
16static int fetch_unpack_limit = -1;
af7cf268 17static int unpack_limit = 100;
f04833ef 18static int prefer_ofs_delta = 1;
ca0c9764 19static int no_done;
dab76d3a
JH
20static int fetch_fsck_objects = -1;
21static int transfer_fsck_objects = -1;
d50c3871 22static int agent_supported;
09149c78
DB
23static struct fetch_pack_args args = {
24 /* .uploadpack = */ "git-upload-pack",
25};
fa740529 26
33b83034 27static const char fetch_pack_usage[] =
078b895f
IT
28"git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
29"[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
30"[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
def88e9a 31
0a8944dd 32#define COMPLETE (1U << 0)
23d61f83
JS
33#define COMMON (1U << 1)
34#define COMMON_REF (1U << 2)
35#define SEEN (1U << 3)
36#define POPPED (1U << 4)
37
420e9af4
DB
38static int marked;
39
f061e5fd
JH
40/*
41 * After sending this many "have"s if we do not get any new ACK , we
42 * give up traversing our history.
43 */
44#define MAX_IN_VAIN 256
45
96f1e58f 46static struct commit_list *rev_list;
3891f390 47static int non_common_revs, multi_ack, use_sideband;
23d61f83
JS
48
49static void rev_list_push(struct commit *commit, int mark)
50{
51 if (!(commit->object.flags & mark)) {
52 commit->object.flags |= mark;
53
54 if (!(commit->object.parsed))
f3ec5494
MK
55 if (parse_commit(commit))
56 return;
23d61f83 57
47e44ed1 58 commit_list_insert_by_date(commit, &rev_list);
23d61f83
JS
59
60 if (!(commit->object.flags & COMMON))
61 non_common_revs++;
62 }
63}
64
65385ef7 65static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
23d61f83 66{
65385ef7 67 struct object *o = deref_tag(parse_object(sha1), refname, 0);
23d61f83 68
1974632c 69 if (o && o->type == OBJ_COMMIT)
23d61f83
JS
70 rev_list_push((struct commit *)o, SEEN);
71
72 return 0;
73}
74
65385ef7 75static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
420e9af4 76{
65385ef7 77 struct object *o = deref_tag(parse_object(sha1), refname, 0);
420e9af4
DB
78
79 if (o && o->type == OBJ_COMMIT)
80 clear_commit_marks((struct commit *)o,
81 COMMON | COMMON_REF | SEEN | POPPED);
82 return 0;
83}
84
23d61f83
JS
85/*
86 This function marks a rev and its ancestors as common.
87 In some cases, it is desirable to mark only the ancestors (for example
88 when only the server does not yet know that they are common).
89*/
90
91static void mark_common(struct commit *commit,
92 int ancestors_only, int dont_parse)
93{
94 if (commit != NULL && !(commit->object.flags & COMMON)) {
95 struct object *o = (struct object *)commit;
96
97 if (!ancestors_only)
98 o->flags |= COMMON;
99
100 if (!(o->flags & SEEN))
101 rev_list_push(commit, SEEN);
102 else {
103 struct commit_list *parents;
104
105 if (!ancestors_only && !(o->flags & POPPED))
106 non_common_revs--;
107 if (!o->parsed && !dont_parse)
f3ec5494
MK
108 if (parse_commit(commit))
109 return;
23d61f83
JS
110
111 for (parents = commit->parents;
112 parents;
113 parents = parents->next)
114 mark_common(parents->item, 0, dont_parse);
115 }
116 }
117}
118
119/*
120 Get the next rev to send, ignoring the common.
121*/
122
4b25d091 123static const unsigned char *get_rev(void)
23d61f83
JS
124{
125 struct commit *commit = NULL;
126
127 while (commit == NULL) {
128 unsigned int mark;
72269ad9 129 struct commit_list *parents;
23d61f83
JS
130
131 if (rev_list == NULL || non_common_revs == 0)
132 return NULL;
133
134 commit = rev_list->item;
2d8bed96 135 if (!commit->object.parsed)
72269ad9
LT
136 parse_commit(commit);
137 parents = commit->parents;
f3ec5494 138
23d61f83
JS
139 commit->object.flags |= POPPED;
140 if (!(commit->object.flags & COMMON))
141 non_common_revs--;
a6080a0a 142
23d61f83
JS
143 if (commit->object.flags & COMMON) {
144 /* do not send "have", and ignore ancestors */
145 commit = NULL;
146 mark = COMMON | SEEN;
147 } else if (commit->object.flags & COMMON_REF)
148 /* send "have", and ignore ancestors */
149 mark = COMMON | SEEN;
150 else
151 /* send "have", also for its ancestors */
152 mark = SEEN;
153
154 while (parents) {
155 if (!(parents->item->object.flags & SEEN))
156 rev_list_push(parents->item, mark);
157 if (mark & COMMON)
158 mark_common(parents->item, 1, 0);
159 parents = parents->next;
160 }
161
162 rev_list = rev_list->next;
163 }
164
165 return commit->object.sha1;
166}
0a8944dd 167
78affc49
SP
168enum ack_type {
169 NAK = 0,
170 ACK,
171 ACK_continue,
172 ACK_common,
173 ACK_ready
174};
175
249b2004
SP
176static void consume_shallow_list(int fd)
177{
178 if (args.stateless_rpc && args.depth > 0) {
179 /* If we sent a depth we will get back "duplicate"
180 * shallow and unshallow commands every time there
181 * is a block of have lines exchanged.
182 */
183 char line[1000];
184 while (packet_read_line(fd, line, sizeof(line))) {
185 if (!prefixcmp(line, "shallow "))
186 continue;
187 if (!prefixcmp(line, "unshallow "))
188 continue;
189 die("git fetch-pack: expected shallow list");
190 }
191 }
192}
193
294e15fc
NTND
194struct write_shallow_data {
195 struct strbuf *out;
196 int use_pack_protocol;
197 int count;
198};
199
200static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
201{
202 struct write_shallow_data *data = cb_data;
203 const char *hex = sha1_to_hex(graft->sha1);
204 data->count++;
205 if (data->use_pack_protocol)
206 packet_buf_write(data->out, "shallow %s", hex);
207 else {
208 strbuf_addstr(data->out, hex);
209 strbuf_addch(data->out, '\n');
210 }
211 return 0;
212}
213
214static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
215{
216 struct write_shallow_data data;
217 data.out = out;
218 data.use_pack_protocol = use_pack_protocol;
219 data.count = 0;
220 for_each_commit_graft(write_one_shallow, &data);
221 return data.count;
222}
223
78affc49 224static enum ack_type get_ack(int fd, unsigned char *result_sha1)
28754ab5
SP
225{
226 static char line[1000];
227 int len = packet_read_line(fd, line, sizeof(line));
228
229 if (!len)
230 die("git fetch-pack: expected ACK/NAK, got EOF");
231 if (line[len-1] == '\n')
232 line[--len] = 0;
233 if (!strcmp(line, "NAK"))
78affc49 234 return NAK;
28754ab5
SP
235 if (!prefixcmp(line, "ACK ")) {
236 if (!get_sha1_hex(line+4, result_sha1)) {
237 if (strstr(line+45, "continue"))
78affc49
SP
238 return ACK_continue;
239 if (strstr(line+45, "common"))
240 return ACK_common;
241 if (strstr(line+45, "ready"))
242 return ACK_ready;
243 return ACK;
28754ab5
SP
244 }
245 }
246 die("git fetch_pack: expected ACK/NAK, got '%s'", line);
247}
248
249b2004
SP
249static void send_request(int fd, struct strbuf *buf)
250{
251 if (args.stateless_rpc) {
252 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
253 packet_flush(fd);
254 } else
255 safe_write(fd, buf->buf, buf->len);
256}
257
e52d7192
JH
258static void insert_one_alternate_ref(const struct ref *ref, void *unused)
259{
260 rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
261}
262
066bf4c2 263#define INITIAL_FLUSH 16
44d8dc54 264#define PIPESAFE_FLUSH 32
6afca450 265#define LARGE_FLUSH 1024
c12f5917
JH
266
267static int next_flush(int count)
268{
44d8dc54
JH
269 int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
270
271 if (count < flush_limit)
6afca450
JH
272 count <<= 1;
273 else
44d8dc54 274 count += flush_limit;
6afca450 275 return count;
c12f5917
JH
276}
277
33b83034
JH
278static int find_common(int fd[2], unsigned char *result_sha1,
279 struct ref *refs)
def88e9a 280{
2759cbc7 281 int fetching;
c12f5917 282 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
23d61f83 283 const unsigned char *sha1;
f061e5fd
JH
284 unsigned in_vain = 0;
285 int got_continue = 0;
4e10cf9a 286 int got_ready = 0;
edace6f0 287 struct strbuf req_buf = STRBUF_INIT;
249b2004 288 size_t state_len = 0;
23d61f83 289
249b2004
SP
290 if (args.stateless_rpc && multi_ack == 1)
291 die("--stateless-rpc requires multi_ack_detailed");
420e9af4
DB
292 if (marked)
293 for_each_ref(clear_marks, NULL);
294 marked = 1;
295
cb5d709f 296 for_each_ref(rev_list_insert_ref, NULL);
c41a802f 297 for_each_alternate_ref(insert_one_alternate_ref, NULL);
def88e9a 298
2759cbc7
LT
299 fetching = 0;
300 for ( ; refs ; refs = refs->next) {
33b83034 301 unsigned char *remote = refs->old_sha1;
edace6f0 302 const char *remote_hex;
4dab94d5 303 struct object *o;
2759cbc7 304
0a8944dd 305 /*
4dab94d5
JH
306 * If that object is complete (i.e. it is an ancestor of a
307 * local ref), we tell them we have it but do not have to
308 * tell them about its ancestors, which they already know
309 * about.
f1f0a2be
JH
310 *
311 * We use lookup_object here because we are only
312 * interested in the case we *know* the object is
313 * reachable and we have already scanned it.
4dab94d5 314 */
f1f0a2be 315 if (((o = lookup_object(remote)) != NULL) &&
1baaae5e 316 (o->flags & COMPLETE)) {
2759cbc7 317 continue;
0a8944dd 318 }
23d61f83 319
edace6f0
SP
320 remote_hex = sha1_to_hex(remote);
321 if (!fetching) {
322 struct strbuf c = STRBUF_INIT;
78affc49
SP
323 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
324 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
4e10cf9a 325 if (no_done) strbuf_addstr(&c, " no-done");
edace6f0
SP
326 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
327 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
328 if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
329 if (args.no_progress) strbuf_addstr(&c, " no-progress");
330 if (args.include_tag) strbuf_addstr(&c, " include-tag");
331 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
d50c3871
JK
332 if (agent_supported) strbuf_addf(&c, " agent=%s",
333 git_user_agent_sanitized());
edace6f0
SP
334 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
335 strbuf_release(&c);
336 } else
337 packet_buf_write(&req_buf, "want %s\n", remote_hex);
2759cbc7 338 fetching++;
33b83034 339 }
edace6f0
SP
340
341 if (!fetching) {
342 strbuf_release(&req_buf);
343 packet_flush(fd[1]);
344 return 1;
345 }
346
ed09aef0 347 if (is_repository_shallow())
edace6f0 348 write_shallow_commits(&req_buf, 1);
fa740529 349 if (args.depth > 0)
edace6f0
SP
350 packet_buf_write(&req_buf, "deepen %d", args.depth);
351 packet_buf_flush(&req_buf);
249b2004 352 state_len = req_buf.len;
0a8944dd 353
fa740529 354 if (args.depth > 0) {
016e6ccb
JS
355 char line[1024];
356 unsigned char sha1[20];
016e6ccb 357
249b2004 358 send_request(fd[1], &req_buf);
eb3a9dd3 359 while (packet_read_line(fd[0], line, sizeof(line))) {
599065a3 360 if (!prefixcmp(line, "shallow ")) {
016e6ccb
JS
361 if (get_sha1_hex(line + 8, sha1))
362 die("invalid shallow line: %s", line);
016e6ccb 363 register_shallow(sha1);
cf01bd52
JH
364 continue;
365 }
599065a3 366 if (!prefixcmp(line, "unshallow ")) {
f53514bc
JS
367 if (get_sha1_hex(line + 10, sha1))
368 die("invalid unshallow line: %s", line);
369 if (!lookup_object(sha1))
370 die("object not found: %s", line);
371 /* make sure that it is parsed as shallow */
f3ec5494
MK
372 if (!parse_object(sha1))
373 die("error in object: %s", line);
f53514bc
JS
374 if (unregister_shallow(sha1))
375 die("no shallow found: %s", line);
cf01bd52
JH
376 continue;
377 }
378 die("expected shallow/unshallow, got %s", line);
016e6ccb 379 }
249b2004
SP
380 } else if (!args.stateless_rpc)
381 send_request(fd[1], &req_buf);
382
383 if (!args.stateless_rpc) {
384 /* If we aren't using the stateless-rpc interface
385 * we don't need to retain the headers.
386 */
387 strbuf_setlen(&req_buf, 0);
388 state_len = 0;
016e6ccb
JS
389 }
390
23d61f83 391 flushes = 0;
75bfc6c2 392 retval = -1;
23d61f83 393 while ((sha1 = get_rev())) {
249b2004 394 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
fa740529 395 if (args.verbose)
33b83034 396 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
f061e5fd 397 in_vain++;
c12f5917 398 if (flush_at <= ++count) {
c4c86f07
JS
399 int ack;
400
249b2004
SP
401 packet_buf_flush(&req_buf);
402 send_request(fd[1], &req_buf);
403 strbuf_setlen(&req_buf, state_len);
def88e9a 404 flushes++;
c12f5917 405 flush_at = next_flush(count);
def88e9a
LT
406
407 /*
408 * We keep one window "ahead" of the other side, and
409 * will wait for an ACK only on the next one
410 */
c12f5917 411 if (!args.stateless_rpc && count == INITIAL_FLUSH)
def88e9a 412 continue;
c4c86f07 413
249b2004 414 consume_shallow_list(fd[0]);
c4c86f07
JS
415 do {
416 ack = get_ack(fd[0], result_sha1);
fa740529 417 if (args.verbose && ack)
c4c86f07
JS
418 fprintf(stderr, "got ack %d %s\n", ack,
419 sha1_to_hex(result_sha1));
78affc49
SP
420 switch (ack) {
421 case ACK:
c4c86f07
JS
422 flushes = 0;
423 multi_ack = 0;
424 retval = 0;
425 goto done;
78affc49
SP
426 case ACK_common:
427 case ACK_ready:
428 case ACK_continue: {
c4c86f07
JS
429 struct commit *commit =
430 lookup_commit(result_sha1);
ec099546
NTND
431 if (!commit)
432 die("invalid commit %s", sha1_to_hex(result_sha1));
249b2004
SP
433 if (args.stateless_rpc
434 && ack == ACK_common
435 && !(commit->object.flags & COMMON)) {
436 /* We need to replay the have for this object
437 * on the next RPC request so the peer knows
438 * it is in common with us.
439 */
440 const char *hex = sha1_to_hex(result_sha1);
441 packet_buf_write(&req_buf, "have %s\n", hex);
442 state_len = req_buf.len;
443 }
c4c86f07
JS
444 mark_common(commit, 0, 1);
445 retval = 0;
f061e5fd
JH
446 in_vain = 0;
447 got_continue = 1;
4e10cf9a 448 if (ack == ACK_ready) {
f2cba929 449 rev_list = NULL;
4e10cf9a
JH
450 got_ready = 1;
451 }
78affc49
SP
452 break;
453 }
c4c86f07
JS
454 }
455 } while (ack);
def88e9a 456 flushes--;
f061e5fd 457 if (got_continue && MAX_IN_VAIN < in_vain) {
fa740529 458 if (args.verbose)
f061e5fd
JH
459 fprintf(stderr, "giving up\n");
460 break; /* give up */
461 }
def88e9a
LT
462 }
463 }
c4c86f07 464done:
4e10cf9a
JH
465 if (!got_ready || !no_done) {
466 packet_buf_write(&req_buf, "done\n");
467 send_request(fd[1], &req_buf);
468 }
fa740529 469 if (args.verbose)
33b83034 470 fprintf(stderr, "done\n");
c4c86f07
JS
471 if (retval != 0) {
472 multi_ack = 0;
23d61f83 473 flushes++;
c4c86f07 474 }
edace6f0
SP
475 strbuf_release(&req_buf);
476
249b2004 477 consume_shallow_list(fd[0]);
c4c86f07
JS
478 while (flushes || multi_ack) {
479 int ack = get_ack(fd[0], result_sha1);
480 if (ack) {
fa740529 481 if (args.verbose)
c4c86f07
JS
482 fprintf(stderr, "got ack (%d) %s\n", ack,
483 sha1_to_hex(result_sha1));
78affc49 484 if (ack == ACK)
c4c86f07
JS
485 return 0;
486 multi_ack = 1;
487 continue;
33b83034 488 }
c4c86f07 489 flushes--;
def88e9a 490 }
8cb560fc
JS
491 /* it is no error to fetch into a completely empty repo */
492 return count ? retval : 0;
def88e9a
LT
493}
494
96f1e58f 495static struct commit_list *complete;
49bb805e 496
65385ef7 497static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
49bb805e
JH
498{
499 struct object *o = parse_object(sha1);
500
1974632c 501 while (o && o->type == OBJ_TAG) {
f1f0a2be
JH
502 struct tag *t = (struct tag *) o;
503 if (!t->tagged)
504 break; /* broken repository */
49bb805e 505 o->flags |= COMPLETE;
f1f0a2be 506 o = parse_object(t->tagged->sha1);
49bb805e 507 }
1974632c 508 if (o && o->type == OBJ_COMMIT) {
49bb805e 509 struct commit *commit = (struct commit *)o;
ea5f2208
JK
510 if (!(commit->object.flags & COMPLETE)) {
511 commit->object.flags |= COMPLETE;
512 commit_list_insert_by_date(commit, &complete);
513 }
49bb805e
JH
514 }
515 return 0;
516}
517
518static void mark_recent_complete_commits(unsigned long cutoff)
519{
520 while (complete && cutoff <= complete->item->date) {
fa740529 521 if (args.verbose)
49bb805e
JH
522 fprintf(stderr, "Marking %s as complete\n",
523 sha1_to_hex(complete->item->object.sha1));
524 pop_most_recent_commit(&complete, COMPLETE);
525 }
526}
527
4ba15999
MH
528static int non_matching_ref(struct string_list_item *item, void *unused)
529{
530 if (item->util) {
531 item->util = NULL;
532 return 0;
533 }
534 else
535 return 1;
536}
537
8bee93dd 538static void filter_refs(struct ref **refs, struct string_list *sought)
1baaae5e 539{
9546010b
JH
540 struct ref *newlist = NULL;
541 struct ref **newtail = &newlist;
542 struct ref *ref, *next;
8bee93dd 543 int sought_pos;
9546010b 544
8bee93dd 545 sought_pos = 0;
9546010b 546 for (ref = *refs; ref; ref = next) {
f537cfa7 547 int keep = 0;
9546010b
JH
548 next = ref->next;
549 if (!memcmp(ref->name, "refs/", 5) &&
8d9c5010 550 check_refname_format(ref->name + 5, 0))
9546010b 551 ; /* trash */
9546010b 552 else {
8bee93dd 553 while (sought_pos < sought->nr) {
f537cfa7
MH
554 int cmp = strcmp(ref->name, sought->items[sought_pos].string);
555 if (cmp < 0)
556 break; /* definitely do not have it */
557 else if (cmp == 0) {
558 keep = 1; /* definitely have it */
4ba15999 559 sought->items[sought_pos++].util = "matched";
1e7ba0f9
JK
560 break;
561 }
f537cfa7
MH
562 else
563 sought_pos++; /* might have it; keep looking */
9546010b
JH
564 }
565 }
f537cfa7 566
5f0fc645
MH
567 if (! keep && args.fetch_all &&
568 (!args.depth || prefixcmp(ref->name, "refs/tags/")))
569 keep = 1;
570
f537cfa7
MH
571 if (keep) {
572 *newtail = ref;
573 ref->next = NULL;
574 newtail = &ref->next;
575 } else {
576 free(ref);
577 }
9546010b
JH
578 }
579
5f0fc645 580 filter_string_list(sought, 0, non_matching_ref, NULL);
9546010b 581 *refs = newlist;
1baaae5e
JS
582}
583
f2576591
MH
584static void mark_alternate_complete(const struct ref *ref, void *unused)
585{
586 mark_complete(NULL, ref->old_sha1, 0, NULL);
587}
588
8bee93dd 589static int everything_local(struct ref **refs, struct string_list *sought)
2759cbc7 590{
49bb805e 591 struct ref *ref;
2759cbc7 592 int retval;
49bb805e
JH
593 unsigned long cutoff = 0;
594
49bb805e
JH
595 save_commit_buffer = 0;
596
1baaae5e 597 for (ref = *refs; ref; ref = ref->next) {
49bb805e
JH
598 struct object *o;
599
600 o = parse_object(ref->old_sha1);
601 if (!o)
602 continue;
603
604 /* We already have it -- which may mean that we were
605 * in sync with the other side at some time after
606 * that (it is OK if we guess wrong here).
607 */
1974632c 608 if (o->type == OBJ_COMMIT) {
49bb805e
JH
609 struct commit *commit = (struct commit *)o;
610 if (!cutoff || cutoff < commit->date)
611 cutoff = commit->date;
612 }
613 }
614
fa740529 615 if (!args.depth) {
f53514bc 616 for_each_ref(mark_complete, NULL);
f2576591 617 for_each_alternate_ref(mark_alternate_complete, NULL);
f53514bc
JS
618 if (cutoff)
619 mark_recent_complete_commits(cutoff);
620 }
2759cbc7 621
1baaae5e
JS
622 /*
623 * Mark all complete remote refs as common refs.
624 * Don't mark them common yet; the server has to be told so first.
625 */
626 for (ref = *refs; ref; ref = ref->next) {
9534f40b
JH
627 struct object *o = deref_tag(lookup_object(ref->old_sha1),
628 NULL, 0);
1baaae5e 629
1974632c 630 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
1baaae5e
JS
631 continue;
632
633 if (!(o->flags & SEEN)) {
634 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
635
636 mark_common((struct commit *)o, 1, 1);
637 }
638 }
639
8bee93dd 640 filter_refs(refs, sought);
1baaae5e
JS
641
642 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
643 const unsigned char *remote = ref->old_sha1;
2759cbc7 644 unsigned char local[20];
49bb805e 645 struct object *o;
2759cbc7 646
1baaae5e 647 o = lookup_object(remote);
49bb805e 648 if (!o || !(o->flags & COMPLETE)) {
2759cbc7 649 retval = 0;
fa740529 650 if (!args.verbose)
2759cbc7
LT
651 continue;
652 fprintf(stderr,
653 "want %s (%s)\n", sha1_to_hex(remote),
1baaae5e 654 ref->name);
2759cbc7
LT
655 continue;
656 }
657
e702496e 658 hashcpy(ref->new_sha1, local);
fa740529 659 if (!args.verbose)
2759cbc7
LT
660 continue;
661 fprintf(stderr,
662 "already have %s (%s)\n", sha1_to_hex(remote),
1baaae5e 663 ref->name);
2759cbc7
LT
664 }
665 return retval;
666}
667
ae6a5609 668static int sideband_demux(int in, int out, void *data)
da093d37 669{
088fab5f 670 int *xd = data;
da093d37 671
ae6a5609
EFL
672 int ret = recv_sideband("fetch-pack", xd[0], out);
673 close(out);
3ef67cf2 674 return ret;
088fab5f
JS
675}
676
1788c39c 677static int get_pack(int xd[2], char **pack_lockfile)
da093d37 678{
088fab5f 679 struct async demux;
9e10fd1a
JH
680 const char *argv[20];
681 char keep_arg[256];
682 char hdr_arg[256];
683 const char **av;
fa740529 684 int do_keep = args.keep_pack;
477822c3 685 struct child_process cmd;
da093d37 686
1f759eee
JS
687 memset(&demux, 0, sizeof(demux));
688 if (use_sideband) {
689 /* xd[] is talking with upload-pack; subprocess reads from
690 * xd[0], spits out band#2 to stderr, and feeds us band#1
691 * through demux->out.
692 */
693 demux.proc = sideband_demux;
694 demux.data = xd;
ae6a5609 695 demux.out = -1;
1f759eee
JS
696 if (start_async(&demux))
697 die("fetch-pack: unable to fork off sideband"
698 " demultiplexer");
699 }
700 else
701 demux.out = xd[0];
9e10fd1a 702
477822c3
JS
703 memset(&cmd, 0, sizeof(cmd));
704 cmd.argv = argv;
9e10fd1a
JH
705 av = argv;
706 *hdr_arg = 0;
fa740529 707 if (!args.keep_pack && unpack_limit) {
9e10fd1a
JH
708 struct pack_header header;
709
1f759eee 710 if (read_pack_header(demux.out, &header))
9e10fd1a 711 die("protocol error: bad pack header");
6e1c2344
RJ
712 snprintf(hdr_arg, sizeof(hdr_arg),
713 "--pack_header=%"PRIu32",%"PRIu32,
9e10fd1a 714 ntohl(header.hdr_version), ntohl(header.hdr_entries));
af7cf268 715 if (ntohl(header.hdr_entries) < unpack_limit)
9e10fd1a
JH
716 do_keep = 0;
717 else
718 do_keep = 1;
719 }
720
721 if (do_keep) {
477822c3
JS
722 if (pack_lockfile)
723 cmd.out = -1;
9e10fd1a
JH
724 *av++ = "index-pack";
725 *av++ = "--stdin";
fa740529 726 if (!args.quiet && !args.no_progress)
9e10fd1a 727 *av++ = "-v";
fa740529 728 if (args.use_thin_pack)
9e10fd1a 729 *av++ = "--fix-thin";
fa740529 730 if (args.lock_pack || unpack_limit) {
9e10fd1a 731 int s = sprintf(keep_arg,
85e72830 732 "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
9e10fd1a
JH
733 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
734 strcpy(keep_arg + s, "localhost");
735 *av++ = keep_arg;
736 }
737 }
738 else {
739 *av++ = "unpack-objects";
01fdc21f 740 if (args.quiet || args.no_progress)
9e10fd1a
JH
741 *av++ = "-q";
742 }
743 if (*hdr_arg)
744 *av++ = hdr_arg;
dab76d3a
JH
745 if (fetch_fsck_objects >= 0
746 ? fetch_fsck_objects
747 : transfer_fsck_objects >= 0
748 ? transfer_fsck_objects
749 : 0)
5e838ea7 750 *av++ = "--strict";
9e10fd1a
JH
751 *av++ = NULL;
752
1f759eee 753 cmd.in = demux.out;
477822c3
JS
754 cmd.git_cmd = 1;
755 if (start_command(&cmd))
da093d37 756 die("fetch-pack: unable to fork off %s", argv[0]);
e72ae288 757 if (do_keep && pack_lockfile) {
477822c3 758 *pack_lockfile = index_pack_lockfile(cmd.out);
e72ae288
JS
759 close(cmd.out);
760 }
477822c3
JS
761
762 if (finish_command(&cmd))
763 die("%s failed", argv[0]);
088fab5f
JS
764 if (use_sideband && finish_async(&demux))
765 die("error in sideband demultiplexer");
477822c3 766 return 0;
da093d37
NP
767}
768
1788c39c 769static struct ref *do_fetch_pack(int fd[2],
ba227857 770 const struct ref *orig_ref,
8bee93dd 771 struct string_list *sought,
1788c39c 772 char **pack_lockfile)
def88e9a 773{
ba227857 774 struct ref *ref = copy_ref_list(orig_ref);
d1c133f5 775 unsigned char sha1[20];
36c60f7a
JK
776 const char *agent_feature;
777 int agent_len;
def88e9a 778
9e8e704f
JK
779 sort_ref_list(&ref, ref_compare_name);
780
ed09aef0
JS
781 if (is_repository_shallow() && !server_supports("shallow"))
782 die("Server does not support shallow clients");
78affc49
SP
783 if (server_supports("multi_ack_detailed")) {
784 if (args.verbose)
785 fprintf(stderr, "Server supports multi_ack_detailed\n");
786 multi_ack = 2;
4e10cf9a
JH
787 if (server_supports("no-done")) {
788 if (args.verbose)
789 fprintf(stderr, "Server supports no-done\n");
8e9182e0
JH
790 if (args.stateless_rpc)
791 no_done = 1;
4e10cf9a 792 }
78affc49
SP
793 }
794 else if (server_supports("multi_ack")) {
fa740529 795 if (args.verbose)
c4c86f07
JS
796 fprintf(stderr, "Server supports multi_ack\n");
797 multi_ack = 1;
798 }
d47f3db7 799 if (server_supports("side-band-64k")) {
fa740529 800 if (args.verbose)
d47f3db7
JH
801 fprintf(stderr, "Server supports side-band-64k\n");
802 use_sideband = 2;
803 }
804 else if (server_supports("side-band")) {
fa740529 805 if (args.verbose)
583b7ea3
JH
806 fprintf(stderr, "Server supports side-band\n");
807 use_sideband = 1;
808 }
74991a98
JH
809 if (!server_supports("thin-pack"))
810 args.use_thin_pack = 0;
811 if (!server_supports("no-progress"))
812 args.no_progress = 0;
813 if (!server_supports("include-tag"))
814 args.include_tag = 0;
f04833ef
NP
815 if (server_supports("ofs-delta")) {
816 if (args.verbose)
817 fprintf(stderr, "Server supports ofs-delta\n");
818 } else
819 prefer_ofs_delta = 0;
36c60f7a
JK
820
821 if ((agent_feature = server_feature_value("agent", &agent_len))) {
d50c3871 822 agent_supported = 1;
36c60f7a
JK
823 if (args.verbose && agent_len)
824 fprintf(stderr, "Server version is %.*s\n",
825 agent_len, agent_feature);
826 }
d50c3871 827
8bee93dd 828 if (everything_local(&ref, sought)) {
2759cbc7
LT
829 packet_flush(fd[1]);
830 goto all_done;
831 }
33b83034 832 if (find_common(fd, sha1, ref) < 0)
fa740529 833 if (!args.keep_pack)
dfeff66e
JH
834 /* When cloning, it is not unusual to have
835 * no common commit.
836 */
78509d21 837 warning("no common commits");
ad897215 838
249b2004
SP
839 if (args.stateless_rpc)
840 packet_flush(fd[1]);
1788c39c 841 if (get_pack(fd, pack_lockfile))
7e44c935 842 die("git fetch-pack: fetch failed.");
ad897215
JH
843
844 all_done:
2d4177c0 845 return ref;
def88e9a
LT
846}
847
ef90d6d4 848static int fetch_pack_config(const char *var, const char *value, void *cb)
af7cf268
JH
849{
850 if (strcmp(var, "fetch.unpacklimit") == 0) {
e28714c5
JH
851 fetch_unpack_limit = git_config_int(var, value);
852 return 0;
853 }
854
855 if (strcmp(var, "transfer.unpacklimit") == 0) {
856 transfer_unpack_limit = git_config_int(var, value);
af7cf268
JH
857 return 0;
858 }
859
f04833ef
NP
860 if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
861 prefer_ofs_delta = git_config_bool(var, value);
862 return 0;
863 }
864
5e838ea7
JH
865 if (!strcmp(var, "fetch.fsckobjects")) {
866 fetch_fsck_objects = git_config_bool(var, value);
867 return 0;
868 }
869
dab76d3a
JH
870 if (!strcmp(var, "transfer.fsckobjects")) {
871 transfer_fsck_objects = git_config_bool(var, value);
872 return 0;
873 }
874
ef90d6d4 875 return git_default_config(var, value, cb);
af7cf268
JH
876}
877
54b9e022
JH
878static struct lock_file lock;
879
50ab5fd3 880static void fetch_pack_setup(void)
def88e9a 881{
50ab5fd3
SP
882 static int did_setup;
883 if (did_setup)
884 return;
ef90d6d4 885 git_config(fetch_pack_config, NULL);
e28714c5
JH
886 if (0 <= transfer_unpack_limit)
887 unpack_limit = transfer_unpack_limit;
888 else if (0 <= fetch_unpack_limit)
889 unpack_limit = fetch_unpack_limit;
50ab5fd3
SP
890 did_setup = 1;
891}
892
893int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
894{
57e6fc69 895 int i, ret;
ba227857 896 struct ref *ref = NULL;
9d19c6ea 897 const char *dest = NULL;
8bee93dd 898 struct string_list sought = STRING_LIST_INIT_DUP;
ba227857 899 int fd[2];
249b2004
SP
900 char *pack_lockfile = NULL;
901 char **pack_lockfile_ptr = NULL;
ba227857 902 struct child_process *conn;
e28714c5 903
bbc30f99
JK
904 packet_trace_identity("fetch-pack");
905
ff22ff99 906 for (i = 1; i < argc && *argv[i] == '-'; i++) {
2d4177c0 907 const char *arg = argv[i];
def88e9a 908
ff22ff99
MH
909 if (!prefixcmp(arg, "--upload-pack=")) {
910 args.uploadpack = arg + 14;
911 continue;
912 }
913 if (!prefixcmp(arg, "--exec=")) {
914 args.uploadpack = arg + 7;
915 continue;
def88e9a 916 }
ff22ff99
MH
917 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
918 args.quiet = 1;
919 continue;
920 }
921 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
922 args.lock_pack = args.keep_pack;
923 args.keep_pack = 1;
924 continue;
925 }
926 if (!strcmp("--thin", arg)) {
927 args.use_thin_pack = 1;
928 continue;
929 }
930 if (!strcmp("--include-tag", arg)) {
931 args.include_tag = 1;
932 continue;
933 }
934 if (!strcmp("--all", arg)) {
935 args.fetch_all = 1;
936 continue;
def88e9a 937 }
ff22ff99
MH
938 if (!strcmp("--stdin", arg)) {
939 args.stdin_refs = 1;
940 continue;
941 }
942 if (!strcmp("-v", arg)) {
943 args.verbose = 1;
944 continue;
945 }
946 if (!prefixcmp(arg, "--depth=")) {
947 args.depth = strtol(arg + 8, NULL, 0);
948 continue;
949 }
950 if (!strcmp("--no-progress", arg)) {
951 args.no_progress = 1;
952 continue;
953 }
954 if (!strcmp("--stateless-rpc", arg)) {
955 args.stateless_rpc = 1;
956 continue;
957 }
958 if (!strcmp("--lock-pack", arg)) {
959 args.lock_pack = 1;
960 pack_lockfile_ptr = &pack_lockfile;
961 continue;
962 }
963 usage(fetch_pack_usage);
def88e9a 964 }
4cc00fcf
MH
965
966 if (i < argc)
967 dest = argv[i++];
968 else
def88e9a 969 usage(fetch_pack_usage);
2d4177c0 970
57e6fc69
MH
971 /*
972 * Copy refs from cmdline to growable list, then append any
973 * refs from the standard input:
974 */
57e6fc69 975 for (; i < argc; i++)
8bee93dd 976 string_list_append(&sought, xstrdup(argv[i]));
078b895f 977 if (args.stdin_refs) {
078b895f
IT
978 if (args.stateless_rpc) {
979 /* in stateless RPC mode we use pkt-line to read
980 * from stdin, until we get a flush packet
981 */
982 static char line[1000];
983 for (;;) {
984 int n = packet_read_line(0, line, sizeof(line));
985 if (!n)
986 break;
987 if (line[n-1] == '\n')
988 n--;
8bee93dd 989 string_list_append(&sought, xmemdupz(line, n));
078b895f
IT
990 }
991 }
992 else {
993 /* read from stdin one ref per line, until EOF */
994 struct strbuf line = STRBUF_INIT;
8bee93dd
MH
995 while (strbuf_getline(&line, stdin, '\n') != EOF)
996 string_list_append(&sought, strbuf_detach(&line, NULL));
078b895f
IT
997 strbuf_release(&line);
998 }
999 }
1000
249b2004
SP
1001 if (args.stateless_rpc) {
1002 conn = NULL;
1003 fd[0] = 0;
1004 fd[1] = 1;
ba227857 1005 } else {
9d19c6ea 1006 conn = git_connect(fd, dest, args.uploadpack,
249b2004
SP
1007 args.verbose ? CONNECT_VERBOSE : 0);
1008 }
1009
afe7c5ff 1010 get_remote_heads(fd[0], &ref, 0, NULL);
249b2004
SP
1011
1012 ref = fetch_pack(&args, fd, conn, ref, dest,
8bee93dd 1013 &sought, pack_lockfile_ptr);
249b2004
SP
1014 if (pack_lockfile) {
1015 printf("lock %s\n", pack_lockfile);
1016 fflush(stdout);
ba227857 1017 }
249b2004
SP
1018 close(fd[0]);
1019 close(fd[1]);
1020 if (finish_connect(conn))
7418f1a0 1021 return 1;
2d4177c0 1022
b285668d
MH
1023 ret = !ref || sought.nr;
1024
1025 /*
1026 * If the heads to pull were given, we should have consumed
1027 * all of them by matching the remote. Otherwise, 'git fetch
1028 * remote no-such-ref' would silently succeed without issuing
1029 * an error.
1030 */
1031 for (i = 0; i < sought.nr; i++)
1032 error("no such remote ref %s", sought.items[i].string);
2d4177c0
DB
1033 while (ref) {
1034 printf("%s %s\n",
1035 sha1_to_hex(ref->old_sha1), ref->name);
1036 ref = ref->next;
1037 }
1038
1039 return ret;
1040}
1041
fa740529 1042struct ref *fetch_pack(struct fetch_pack_args *my_args,
ba227857
DB
1043 int fd[], struct child_process *conn,
1044 const struct ref *ref,
63c69453 1045 const char *dest,
8bee93dd 1046 struct string_list *sought,
63c69453 1047 char **pack_lockfile)
2d4177c0 1048{
2d4177c0 1049 struct stat st;
ba227857 1050 struct ref *ref_cpy;
2d4177c0 1051
50ab5fd3 1052 fetch_pack_setup();
d551bbaf
TR
1053 if (&args != my_args)
1054 memcpy(&args, my_args, sizeof(args));
fa740529 1055 if (args.depth > 0) {
2d4177c0
DB
1056 if (stat(git_path("shallow"), &st))
1057 st.st_mtime = 0;
1058 }
1059
8bee93dd
MH
1060 if (sought->nr) {
1061 sort_string_list(sought);
4c58f13b 1062 string_list_remove_duplicates(sought, 0);
44359685
JK
1063 }
1064
ba227857
DB
1065 if (!ref) {
1066 packet_flush(fd[1]);
1067 die("no matching remote head");
9e5d2b40 1068 }
8bee93dd 1069 ref_cpy = do_fetch_pack(fd, ref, sought, pack_lockfile);
9e5d2b40 1070
ba227857 1071 if (args.depth > 0) {
016e6ccb 1072 struct cache_time mtime;
edace6f0 1073 struct strbuf sb = STRBUF_INIT;
016e6ccb
JS
1074 char *shallow = git_path("shallow");
1075 int fd;
1076
1077 mtime.sec = st.st_mtime;
c06ff490 1078 mtime.nsec = ST_MTIME_NSEC(st);
016e6ccb
JS
1079 if (stat(shallow, &st)) {
1080 if (mtime.sec)
1081 die("shallow file was removed during fetch");
1082 } else if (st.st_mtime != mtime.sec
1083#ifdef USE_NSEC
5bcf109c 1084 || ST_MTIME_NSEC(st) != mtime.nsec
016e6ccb
JS
1085#endif
1086 )
1087 die("shallow file was changed during fetch");
1088
acd3b9ec
JH
1089 fd = hold_lock_file_for_update(&lock, shallow,
1090 LOCK_DIE_ON_ERROR);
edace6f0
SP
1091 if (!write_shallow_commits(&sb, 0)
1092 || write_in_full(fd, sb.buf, sb.len) != sb.len) {
691f1a28 1093 unlink_or_warn(shallow);
016e6ccb
JS
1094 rollback_lock_file(&lock);
1095 } else {
016e6ccb
JS
1096 commit_lock_file(&lock);
1097 }
edace6f0 1098 strbuf_release(&sb);
016e6ccb
JS
1099 }
1100
48ec3e5c 1101 reprepare_packed_git();
ba227857 1102 return ref_cpy;
def88e9a 1103}