upload-pack: support define shallow boundary by excluding revisions
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
697cc8ef 2#include "lockfile.h"
745f7a8c
NTND
3#include "refs.h"
4#include "pkt-line.h"
5#include "commit.h"
6#include "tag.h"
7#include "exec_cmd.h"
8#include "pack.h"
9#include "sideband.h"
10#include "fetch-pack.h"
11#include "remote.h"
12#include "run-command.h"
47a59185 13#include "connect.h"
745f7a8c
NTND
14#include "transport.h"
15#include "version.h"
099327b5 16#include "prio-queue.h"
beea4152 17#include "sha1-array.h"
745f7a8c
NTND
18
19static int transfer_unpack_limit = -1;
20static int fetch_unpack_limit = -1;
21static int unpack_limit = 100;
22static int prefer_ofs_delta = 1;
23static int no_done;
508ea882 24static int deepen_since_ok;
745f7a8c
NTND
25static int fetch_fsck_objects = -1;
26static int transfer_fsck_objects = -1;
27static int agent_supported;
6035d6aa
NTND
28static struct lock_file shallow_lock;
29static const char *alternate_shallow_file;
745f7a8c 30
208acbfb 31/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
32#define COMPLETE (1U << 0)
33#define COMMON (1U << 1)
34#define COMMON_REF (1U << 2)
35#define SEEN (1U << 3)
36#define POPPED (1U << 4)
37
38static int marked;
39
40/*
41 * After sending this many "have"s if we do not get any new ACK , we
42 * give up traversing our history.
43 */
44#define MAX_IN_VAIN 256
45
099327b5 46static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
47static int non_common_revs, multi_ack, use_sideband;
48/* Allow specifying sha1 if it is a ref tip. */
49#define ALLOW_TIP_SHA1 01
68ee6289
FM
50/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
51#define ALLOW_REACHABLE_SHA1 02
7199c093 52static unsigned int allow_unadvertised_object_request;
745f7a8c 53
0d789a5b
NTND
54__attribute__((format (printf, 2, 3)))
55static inline void print_verbose(const struct fetch_pack_args *args,
56 const char *fmt, ...)
57{
58 va_list params;
59
60 if (!args->verbose)
61 return;
62
63 va_start(params, fmt);
64 vfprintf(stderr, fmt, params);
65 va_end(params);
66 fputc('\n', stderr);
67}
68
745f7a8c
NTND
69static void rev_list_push(struct commit *commit, int mark)
70{
71 if (!(commit->object.flags & mark)) {
72 commit->object.flags |= mark;
73
0064053b
JK
74 if (parse_commit(commit))
75 return;
745f7a8c 76
099327b5 77 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
78
79 if (!(commit->object.flags & COMMON))
80 non_common_revs++;
81 }
82}
83
c38cd1c8 84static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
745f7a8c
NTND
85{
86 struct object *o = deref_tag(parse_object(sha1), refname, 0);
87
88 if (o && o->type == OBJ_COMMIT)
89 rev_list_push((struct commit *)o, SEEN);
90
91 return 0;
92}
93
b1b49c6e
MH
94static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
95 int flag, void *cb_data)
745f7a8c 96{
c38cd1c8 97 return rev_list_insert_ref(refname, oid->hash);
b1b49c6e
MH
98}
99
c50fb6ce
MH
100static int clear_marks(const char *refname, const struct object_id *oid,
101 int flag, void *cb_data)
745f7a8c 102{
c50fb6ce 103 struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
745f7a8c
NTND
104
105 if (o && o->type == OBJ_COMMIT)
106 clear_commit_marks((struct commit *)o,
107 COMMON | COMMON_REF | SEEN | POPPED);
108 return 0;
109}
110
111/*
112 This function marks a rev and its ancestors as common.
113 In some cases, it is desirable to mark only the ancestors (for example
114 when only the server does not yet know that they are common).
115*/
116
117static void mark_common(struct commit *commit,
118 int ancestors_only, int dont_parse)
119{
120 if (commit != NULL && !(commit->object.flags & COMMON)) {
121 struct object *o = (struct object *)commit;
122
123 if (!ancestors_only)
124 o->flags |= COMMON;
125
126 if (!(o->flags & SEEN))
127 rev_list_push(commit, SEEN);
128 else {
129 struct commit_list *parents;
130
131 if (!ancestors_only && !(o->flags & POPPED))
132 non_common_revs--;
133 if (!o->parsed && !dont_parse)
134 if (parse_commit(commit))
135 return;
136
137 for (parents = commit->parents;
138 parents;
139 parents = parents->next)
140 mark_common(parents->item, 0, dont_parse);
141 }
142 }
143}
144
145/*
146 Get the next rev to send, ignoring the common.
147*/
148
149static const unsigned char *get_rev(void)
150{
151 struct commit *commit = NULL;
152
153 while (commit == NULL) {
154 unsigned int mark;
155 struct commit_list *parents;
156
099327b5 157 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
158 return NULL;
159
099327b5 160 commit = prio_queue_get(&rev_list);
0064053b 161 parse_commit(commit);
745f7a8c
NTND
162 parents = commit->parents;
163
164 commit->object.flags |= POPPED;
165 if (!(commit->object.flags & COMMON))
166 non_common_revs--;
167
168 if (commit->object.flags & COMMON) {
169 /* do not send "have", and ignore ancestors */
170 commit = NULL;
171 mark = COMMON | SEEN;
172 } else if (commit->object.flags & COMMON_REF)
173 /* send "have", and ignore ancestors */
174 mark = COMMON | SEEN;
175 else
176 /* send "have", also for its ancestors */
177 mark = SEEN;
178
179 while (parents) {
180 if (!(parents->item->object.flags & SEEN))
181 rev_list_push(parents->item, mark);
182 if (mark & COMMON)
183 mark_common(parents->item, 1, 0);
184 parents = parents->next;
185 }
745f7a8c
NTND
186 }
187
ed1c9977 188 return commit->object.oid.hash;
745f7a8c
NTND
189}
190
191enum ack_type {
192 NAK = 0,
193 ACK,
194 ACK_continue,
195 ACK_common,
196 ACK_ready
197};
198
199static void consume_shallow_list(struct fetch_pack_args *args, int fd)
200{
79891cb9 201 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
202 /* If we sent a depth we will get back "duplicate"
203 * shallow and unshallow commands every time there
204 * is a block of have lines exchanged.
205 */
74543a04
JK
206 char *line;
207 while ((line = packet_read_line(fd, NULL))) {
59556548 208 if (starts_with(line, "shallow "))
745f7a8c 209 continue;
59556548 210 if (starts_with(line, "unshallow "))
745f7a8c 211 continue;
1dd73e20 212 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
213 }
214 }
215}
216
745f7a8c
NTND
217static enum ack_type get_ack(int fd, unsigned char *result_sha1)
218{
74543a04
JK
219 int len;
220 char *line = packet_read_line(fd, &len);
82e56767 221 const char *arg;
745f7a8c
NTND
222
223 if (!len)
1dd73e20 224 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
225 if (!strcmp(line, "NAK"))
226 return NAK;
82e56767
JK
227 if (skip_prefix(line, "ACK ", &arg)) {
228 if (!get_sha1_hex(arg, result_sha1)) {
229 arg += 40;
230 len -= arg - line;
231 if (len < 1)
030e9dd6 232 return ACK;
82e56767 233 if (strstr(arg, "continue"))
745f7a8c 234 return ACK_continue;
82e56767 235 if (strstr(arg, "common"))
745f7a8c 236 return ACK_common;
82e56767 237 if (strstr(arg, "ready"))
745f7a8c
NTND
238 return ACK_ready;
239 return ACK;
240 }
241 }
1dd73e20 242 die(_("git fetch_pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
243}
244
245static void send_request(struct fetch_pack_args *args,
246 int fd, struct strbuf *buf)
247{
248 if (args->stateless_rpc) {
249 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
250 packet_flush(fd);
251 } else
cdf4fb8e 252 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
253}
254
255static void insert_one_alternate_ref(const struct ref *ref, void *unused)
256{
f4e54d02 257 rev_list_insert_ref(NULL, ref->old_oid.hash);
745f7a8c
NTND
258}
259
260#define INITIAL_FLUSH 16
261#define PIPESAFE_FLUSH 32
262#define LARGE_FLUSH 1024
263
264static int next_flush(struct fetch_pack_args *args, int count)
265{
266 int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
267
268 if (count < flush_limit)
269 count <<= 1;
270 else
271 count += flush_limit;
272 return count;
273}
274
275static int find_common(struct fetch_pack_args *args,
276 int fd[2], unsigned char *result_sha1,
277 struct ref *refs)
278{
279 int fetching;
280 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
281 const unsigned char *sha1;
282 unsigned in_vain = 0;
283 int got_continue = 0;
284 int got_ready = 0;
285 struct strbuf req_buf = STRBUF_INIT;
286 size_t state_len = 0;
287
288 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 289 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
290 if (marked)
291 for_each_ref(clear_marks, NULL);
292 marked = 1;
293
b1b49c6e 294 for_each_ref(rev_list_insert_ref_oid, NULL);
745f7a8c
NTND
295 for_each_alternate_ref(insert_one_alternate_ref, NULL);
296
297 fetching = 0;
298 for ( ; refs ; refs = refs->next) {
f4e54d02 299 unsigned char *remote = refs->old_oid.hash;
745f7a8c
NTND
300 const char *remote_hex;
301 struct object *o;
302
303 /*
304 * If that object is complete (i.e. it is an ancestor of a
305 * local ref), we tell them we have it but do not have to
306 * tell them about its ancestors, which they already know
307 * about.
308 *
309 * We use lookup_object here because we are only
310 * interested in the case we *know* the object is
311 * reachable and we have already scanned it.
312 */
313 if (((o = lookup_object(remote)) != NULL) &&
314 (o->flags & COMPLETE)) {
315 continue;
316 }
317
318 remote_hex = sha1_to_hex(remote);
319 if (!fetching) {
320 struct strbuf c = STRBUF_INIT;
321 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
322 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
323 if (no_done) strbuf_addstr(&c, " no-done");
324 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
325 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
326 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
327 if (args->no_progress) strbuf_addstr(&c, " no-progress");
328 if (args->include_tag) strbuf_addstr(&c, " include-tag");
329 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 330 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
745f7a8c
NTND
331 if (agent_supported) strbuf_addf(&c, " agent=%s",
332 git_user_agent_sanitized());
333 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
334 strbuf_release(&c);
335 } else
336 packet_buf_write(&req_buf, "want %s\n", remote_hex);
337 fetching++;
338 }
339
340 if (!fetching) {
341 strbuf_release(&req_buf);
342 packet_flush(fd[1]);
343 return 1;
344 }
345
346 if (is_repository_shallow())
1a30f5a2 347 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
348 if (args->depth > 0)
349 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882
NTND
350 if (args->deepen_since) {
351 unsigned long max_age = approxidate(args->deepen_since);
352 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
353 }
745f7a8c
NTND
354 packet_buf_flush(&req_buf);
355 state_len = req_buf.len;
356
79891cb9 357 if (args->deepen) {
74543a04 358 char *line;
ae021d87 359 const char *arg;
745f7a8c
NTND
360 unsigned char sha1[20];
361
362 send_request(args, fd[1], &req_buf);
74543a04 363 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87
JK
364 if (skip_prefix(line, "shallow ", &arg)) {
365 if (get_sha1_hex(arg, sha1))
1dd73e20 366 die(_("invalid shallow line: %s"), line);
745f7a8c
NTND
367 register_shallow(sha1);
368 continue;
369 }
ae021d87
JK
370 if (skip_prefix(line, "unshallow ", &arg)) {
371 if (get_sha1_hex(arg, sha1))
1dd73e20 372 die(_("invalid unshallow line: %s"), line);
745f7a8c 373 if (!lookup_object(sha1))
1dd73e20 374 die(_("object not found: %s"), line);
745f7a8c
NTND
375 /* make sure that it is parsed as shallow */
376 if (!parse_object(sha1))
1dd73e20 377 die(_("error in object: %s"), line);
745f7a8c 378 if (unregister_shallow(sha1))
1dd73e20 379 die(_("no shallow found: %s"), line);
745f7a8c
NTND
380 continue;
381 }
1dd73e20 382 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
383 }
384 } else if (!args->stateless_rpc)
385 send_request(args, fd[1], &req_buf);
386
387 if (!args->stateless_rpc) {
388 /* If we aren't using the stateless-rpc interface
389 * we don't need to retain the headers.
390 */
391 strbuf_setlen(&req_buf, 0);
392 state_len = 0;
393 }
394
395 flushes = 0;
396 retval = -1;
397 while ((sha1 = get_rev())) {
398 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
0d789a5b 399 print_verbose(args, "have %s", sha1_to_hex(sha1));
745f7a8c
NTND
400 in_vain++;
401 if (flush_at <= ++count) {
402 int ack;
403
404 packet_buf_flush(&req_buf);
405 send_request(args, fd[1], &req_buf);
406 strbuf_setlen(&req_buf, state_len);
407 flushes++;
408 flush_at = next_flush(args, count);
409
410 /*
411 * We keep one window "ahead" of the other side, and
412 * will wait for an ACK only on the next one
413 */
414 if (!args->stateless_rpc && count == INITIAL_FLUSH)
415 continue;
416
417 consume_shallow_list(args, fd[0]);
418 do {
419 ack = get_ack(fd[0], result_sha1);
0d789a5b 420 if (ack)
1dd73e20
NTND
421 print_verbose(args, _("got %s %d %s"), "ack",
422 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
423 switch (ack) {
424 case ACK:
425 flushes = 0;
426 multi_ack = 0;
427 retval = 0;
428 goto done;
429 case ACK_common:
430 case ACK_ready:
431 case ACK_continue: {
432 struct commit *commit =
433 lookup_commit(result_sha1);
434 if (!commit)
1dd73e20 435 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
745f7a8c
NTND
436 if (args->stateless_rpc
437 && ack == ACK_common
438 && !(commit->object.flags & COMMON)) {
439 /* We need to replay the have for this object
440 * on the next RPC request so the peer knows
441 * it is in common with us.
442 */
443 const char *hex = sha1_to_hex(result_sha1);
444 packet_buf_write(&req_buf, "have %s\n", hex);
445 state_len = req_buf.len;
446 }
447 mark_common(commit, 0, 1);
448 retval = 0;
449 in_vain = 0;
450 got_continue = 1;
451 if (ack == ACK_ready) {
099327b5 452 clear_prio_queue(&rev_list);
745f7a8c
NTND
453 got_ready = 1;
454 }
455 break;
456 }
457 }
458 } while (ack);
459 flushes--;
460 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 461 print_verbose(args, _("giving up"));
745f7a8c
NTND
462 break; /* give up */
463 }
464 }
465 }
466done:
467 if (!got_ready || !no_done) {
468 packet_buf_write(&req_buf, "done\n");
469 send_request(args, fd[1], &req_buf);
470 }
1dd73e20 471 print_verbose(args, _("done"));
745f7a8c
NTND
472 if (retval != 0) {
473 multi_ack = 0;
474 flushes++;
475 }
476 strbuf_release(&req_buf);
477
ff62eca7
NTND
478 if (!got_ready || !no_done)
479 consume_shallow_list(args, fd[0]);
745f7a8c
NTND
480 while (flushes || multi_ack) {
481 int ack = get_ack(fd[0], result_sha1);
482 if (ack) {
1dd73e20
NTND
483 print_verbose(args, _("got %s (%d) %s"), "ack",
484 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
485 if (ack == ACK)
486 return 0;
487 multi_ack = 1;
488 continue;
489 }
490 flushes--;
491 }
492 /* it is no error to fetch into a completely empty repo */
493 return count ? retval : 0;
494}
495
496static struct commit_list *complete;
497
6e20a51a 498static int mark_complete(const unsigned char *sha1)
745f7a8c
NTND
499{
500 struct object *o = parse_object(sha1);
501
502 while (o && o->type == OBJ_TAG) {
503 struct tag *t = (struct tag *) o;
504 if (!t->tagged)
505 break; /* broken repository */
506 o->flags |= COMPLETE;
ed1c9977 507 o = parse_object(t->tagged->oid.hash);
745f7a8c
NTND
508 }
509 if (o && o->type == OBJ_COMMIT) {
510 struct commit *commit = (struct commit *)o;
511 if (!(commit->object.flags & COMPLETE)) {
512 commit->object.flags |= COMPLETE;
16445242 513 commit_list_insert(commit, &complete);
745f7a8c
NTND
514 }
515 }
516 return 0;
517}
518
f8ee4d85
MH
519static int mark_complete_oid(const char *refname, const struct object_id *oid,
520 int flag, void *cb_data)
521{
6e20a51a 522 return mark_complete(oid->hash);
f8ee4d85
MH
523}
524
745f7a8c
NTND
525static void mark_recent_complete_commits(struct fetch_pack_args *args,
526 unsigned long cutoff)
527{
528 while (complete && cutoff <= complete->item->date) {
1dd73e20 529 print_verbose(args, _("Marking %s as complete"),
0d789a5b 530 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
531 pop_most_recent_commit(&complete, COMPLETE);
532 }
533}
534
745f7a8c 535static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
536 struct ref **refs,
537 struct ref **sought, int nr_sought)
745f7a8c
NTND
538{
539 struct ref *newlist = NULL;
540 struct ref **newtail = &newlist;
541 struct ref *ref, *next;
f2db854d 542 int i;
745f7a8c 543
f2db854d 544 i = 0;
745f7a8c
NTND
545 for (ref = *refs; ref; ref = next) {
546 int keep = 0;
547 next = ref->next;
f2db854d 548
50e19a83 549 if (starts_with(ref->name, "refs/") &&
4c224081 550 check_refname_format(ref->name, 0))
745f7a8c
NTND
551 ; /* trash */
552 else {
f2db854d
JH
553 while (i < nr_sought) {
554 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
555 if (cmp < 0)
556 break; /* definitely do not have it */
557 else if (cmp == 0) {
558 keep = 1; /* definitely have it */
f2db854d 559 sought[i]->matched = 1;
745f7a8c 560 }
f2db854d 561 i++;
745f7a8c
NTND
562 }
563 }
564
f2db854d 565 if (!keep && args->fetch_all &&
79891cb9 566 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
567 keep = 1;
568
569 if (keep) {
570 *newtail = ref;
571 ref->next = NULL;
572 newtail = &ref->next;
573 } else {
574 free(ref);
575 }
576 }
577
6e7b66ee 578 /* Append unmatched requests to the list */
68ee6289
FM
579 if ((allow_unadvertised_object_request &
580 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
6e7b66ee 581 for (i = 0; i < nr_sought; i++) {
b7916422
JK
582 unsigned char sha1[20];
583
6e7b66ee
JH
584 ref = sought[i];
585 if (ref->matched)
586 continue;
b7916422
JK
587 if (get_sha1_hex(ref->name, sha1) ||
588 ref->name[40] != '\0' ||
f4e54d02 589 hashcmp(sha1, ref->old_oid.hash))
6e7b66ee
JH
590 continue;
591
592 ref->matched = 1;
c3c17bf1
JK
593 *newtail = copy_ref(ref);
594 newtail = &(*newtail)->next;
6e7b66ee
JH
595 }
596 }
745f7a8c
NTND
597 *refs = newlist;
598}
599
600static void mark_alternate_complete(const struct ref *ref, void *unused)
601{
f4e54d02 602 mark_complete(ref->old_oid.hash);
745f7a8c
NTND
603}
604
605static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
606 struct ref **refs,
607 struct ref **sought, int nr_sought)
745f7a8c
NTND
608{
609 struct ref *ref;
610 int retval;
611 unsigned long cutoff = 0;
612
613 save_commit_buffer = 0;
614
615 for (ref = *refs; ref; ref = ref->next) {
616 struct object *o;
617
f4e54d02 618 if (!has_object_file(&ref->old_oid))
012a1bb5
JH
619 continue;
620
f4e54d02 621 o = parse_object(ref->old_oid.hash);
745f7a8c
NTND
622 if (!o)
623 continue;
624
625 /* We already have it -- which may mean that we were
626 * in sync with the other side at some time after
627 * that (it is OK if we guess wrong here).
628 */
629 if (o->type == OBJ_COMMIT) {
630 struct commit *commit = (struct commit *)o;
631 if (!cutoff || cutoff < commit->date)
632 cutoff = commit->date;
633 }
634 }
635
79891cb9 636 if (!args->deepen) {
f8ee4d85 637 for_each_ref(mark_complete_oid, NULL);
745f7a8c 638 for_each_alternate_ref(mark_alternate_complete, NULL);
16445242 639 commit_list_sort_by_date(&complete);
745f7a8c
NTND
640 if (cutoff)
641 mark_recent_complete_commits(args, cutoff);
642 }
643
644 /*
645 * Mark all complete remote refs as common refs.
646 * Don't mark them common yet; the server has to be told so first.
647 */
648 for (ref = *refs; ref; ref = ref->next) {
f4e54d02 649 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
745f7a8c
NTND
650 NULL, 0);
651
652 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
653 continue;
654
655 if (!(o->flags & SEEN)) {
656 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
657
658 mark_common((struct commit *)o, 1, 1);
659 }
660 }
661
f2db854d 662 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
663
664 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
f4e54d02 665 const unsigned char *remote = ref->old_oid.hash;
745f7a8c
NTND
666 struct object *o;
667
668 o = lookup_object(remote);
669 if (!o || !(o->flags & COMPLETE)) {
670 retval = 0;
0d789a5b
NTND
671 print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
672 ref->name);
745f7a8c
NTND
673 continue;
674 }
1dd73e20 675 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
0d789a5b 676 ref->name);
745f7a8c
NTND
677 }
678 return retval;
679}
680
681static int sideband_demux(int in, int out, void *data)
682{
683 int *xd = data;
684
685 int ret = recv_sideband("fetch-pack", xd[0], out);
686 close(out);
687 return ret;
688}
689
690static int get_pack(struct fetch_pack_args *args,
691 int xd[2], char **pack_lockfile)
692{
693 struct async demux;
745f7a8c 694 int do_keep = args->keep_pack;
984a43b9
JK
695 const char *cmd_name;
696 struct pack_header header;
697 int pass_header = 0;
d3180279 698 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 699 int ret;
745f7a8c
NTND
700
701 memset(&demux, 0, sizeof(demux));
702 if (use_sideband) {
703 /* xd[] is talking with upload-pack; subprocess reads from
704 * xd[0], spits out band#2 to stderr, and feeds us band#1
705 * through demux->out.
706 */
707 demux.proc = sideband_demux;
708 demux.data = xd;
709 demux.out = -1;
710 if (start_async(&demux))
1dd73e20 711 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
712 }
713 else
714 demux.out = xd[0];
715
745f7a8c 716 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
717
718 if (read_pack_header(demux.out, &header))
1dd73e20 719 die(_("protocol error: bad pack header"));
984a43b9 720 pass_header = 1;
745f7a8c
NTND
721 if (ntohl(header.hdr_entries) < unpack_limit)
722 do_keep = 0;
723 else
724 do_keep = 1;
725 }
726
6035d6aa 727 if (alternate_shallow_file) {
984a43b9
JK
728 argv_array_push(&cmd.args, "--shallow-file");
729 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
730 }
731
745f7a8c
NTND
732 if (do_keep) {
733 if (pack_lockfile)
734 cmd.out = -1;
984a43b9
JK
735 cmd_name = "index-pack";
736 argv_array_push(&cmd.args, cmd_name);
737 argv_array_push(&cmd.args, "--stdin");
745f7a8c 738 if (!args->quiet && !args->no_progress)
984a43b9 739 argv_array_push(&cmd.args, "-v");
745f7a8c 740 if (args->use_thin_pack)
984a43b9 741 argv_array_push(&cmd.args, "--fix-thin");
745f7a8c 742 if (args->lock_pack || unpack_limit) {
984a43b9
JK
743 char hostname[256];
744 if (gethostname(hostname, sizeof(hostname)))
745 xsnprintf(hostname, sizeof(hostname), "localhost");
746 argv_array_pushf(&cmd.args,
747 "--keep=fetch-pack %"PRIuMAX " on %s",
748 (uintmax_t)getpid(), hostname);
745f7a8c 749 }
c6807a40 750 if (args->check_self_contained_and_connected)
984a43b9 751 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
745f7a8c
NTND
752 }
753 else {
984a43b9
JK
754 cmd_name = "unpack-objects";
755 argv_array_push(&cmd.args, cmd_name);
745f7a8c 756 if (args->quiet || args->no_progress)
984a43b9 757 argv_array_push(&cmd.args, "-q");
c6807a40 758 args->check_self_contained_and_connected = 0;
745f7a8c 759 }
984a43b9
JK
760
761 if (pass_header)
762 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
763 ntohl(header.hdr_version),
764 ntohl(header.hdr_entries));
745f7a8c
NTND
765 if (fetch_fsck_objects >= 0
766 ? fetch_fsck_objects
767 : transfer_fsck_objects >= 0
768 ? transfer_fsck_objects
769 : 0)
984a43b9 770 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
771
772 cmd.in = demux.out;
773 cmd.git_cmd = 1;
774 if (start_command(&cmd))
1dd73e20 775 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
776 if (do_keep && pack_lockfile) {
777 *pack_lockfile = index_pack_lockfile(cmd.out);
778 close(cmd.out);
779 }
780
37cb1dd6
JL
781 if (!use_sideband)
782 /* Closed by start_command() */
783 xd[0] = -1;
784
c6807a40
NTND
785 ret = finish_command(&cmd);
786 if (!ret || (args->check_self_contained_and_connected && ret == 1))
787 args->self_contained_and_connected =
788 args->check_self_contained_and_connected &&
789 ret == 0;
790 else
1dd73e20 791 die(_("%s failed"), cmd_name);
745f7a8c 792 if (use_sideband && finish_async(&demux))
1dd73e20 793 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
794 return 0;
795}
796
f2db854d
JH
797static int cmp_ref_by_name(const void *a_, const void *b_)
798{
799 const struct ref *a = *((const struct ref **)a_);
800 const struct ref *b = *((const struct ref **)b_);
801 return strcmp(a->name, b->name);
802}
803
745f7a8c
NTND
804static struct ref *do_fetch_pack(struct fetch_pack_args *args,
805 int fd[2],
806 const struct ref *orig_ref,
f2db854d 807 struct ref **sought, int nr_sought,
beea4152 808 struct shallow_info *si,
745f7a8c
NTND
809 char **pack_lockfile)
810{
811 struct ref *ref = copy_ref_list(orig_ref);
812 unsigned char sha1[20];
813 const char *agent_feature;
814 int agent_len;
815
816 sort_ref_list(&ref, ref_compare_name);
f2db854d 817 qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
745f7a8c 818
eb86a507 819 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 820 die(_("Server does not support shallow clients"));
508ea882 821 if (args->depth > 0 || args->deepen_since)
79891cb9 822 args->deepen = 1;
745f7a8c 823 if (server_supports("multi_ack_detailed")) {
1dd73e20 824 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
825 multi_ack = 2;
826 if (server_supports("no-done")) {
1dd73e20 827 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
828 if (args->stateless_rpc)
829 no_done = 1;
830 }
831 }
832 else if (server_supports("multi_ack")) {
1dd73e20 833 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
834 multi_ack = 1;
835 }
836 if (server_supports("side-band-64k")) {
1dd73e20 837 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
838 use_sideband = 2;
839 }
840 else if (server_supports("side-band")) {
1dd73e20 841 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
842 use_sideband = 1;
843 }
6e7b66ee 844 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 845 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 846 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 847 }
68ee6289 848 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 849 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
850 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
851 }
745f7a8c
NTND
852 if (!server_supports("thin-pack"))
853 args->use_thin_pack = 0;
854 if (!server_supports("no-progress"))
855 args->no_progress = 0;
856 if (!server_supports("include-tag"))
857 args->include_tag = 0;
0d789a5b 858 if (server_supports("ofs-delta"))
1dd73e20 859 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 860 else
745f7a8c
NTND
861 prefer_ofs_delta = 0;
862
863 if ((agent_feature = server_feature_value("agent", &agent_len))) {
864 agent_supported = 1;
0d789a5b 865 if (agent_len)
1dd73e20 866 print_verbose(args, _("Server version is %.*s"),
0d789a5b 867 agent_len, agent_feature);
745f7a8c 868 }
508ea882
NTND
869 if (server_supports("deepen-since"))
870 deepen_since_ok = 1;
871 else if (args->deepen_since)
872 die(_("Server does not support --shallow-since"));
745f7a8c 873
f2db854d 874 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
875 packet_flush(fd[1]);
876 goto all_done;
877 }
878 if (find_common(args, fd, sha1, ref) < 0)
879 if (!args->keep_pack)
880 /* When cloning, it is not unusual to have
881 * no common commit.
882 */
1dd73e20 883 warning(_("no common commits"));
745f7a8c
NTND
884
885 if (args->stateless_rpc)
886 packet_flush(fd[1]);
79891cb9 887 if (args->deepen)
1a30f5a2
NTND
888 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
889 NULL);
4820a33b 890 else if (si->nr_ours || si->nr_theirs)
beea4152 891 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
892 else
893 alternate_shallow_file = NULL;
745f7a8c 894 if (get_pack(args, fd, pack_lockfile))
1dd73e20 895 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
896
897 all_done:
898 return ref;
899}
900
f44af51d 901static void fetch_pack_config(void)
745f7a8c 902{
f44af51d
TA
903 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
904 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
905 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
906 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
907 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 908
f44af51d 909 git_config(git_default_config, NULL);
745f7a8c
NTND
910}
911
745f7a8c
NTND
912static void fetch_pack_setup(void)
913{
914 static int did_setup;
915 if (did_setup)
916 return;
f44af51d 917 fetch_pack_config();
745f7a8c
NTND
918 if (0 <= transfer_unpack_limit)
919 unpack_limit = transfer_unpack_limit;
920 else if (0 <= fetch_unpack_limit)
921 unpack_limit = fetch_unpack_limit;
922 did_setup = 1;
923}
924
f2db854d
JH
925static int remove_duplicates_in_refs(struct ref **ref, int nr)
926{
927 struct string_list names = STRING_LIST_INIT_NODUP;
928 int src, dst;
929
930 for (src = dst = 0; src < nr; src++) {
931 struct string_list_item *item;
932 item = string_list_insert(&names, ref[src]->name);
933 if (item->util)
934 continue; /* already have it */
935 item->util = ref[src];
936 if (src != dst)
937 ref[dst] = ref[src];
938 dst++;
939 }
940 for (src = dst; src < nr; src++)
941 ref[src] = NULL;
942 string_list_clear(&names, 0);
943 return dst;
944}
945
beea4152 946static void update_shallow(struct fetch_pack_args *args,
4820a33b 947 struct ref **sought, int nr_sought,
beea4152 948 struct shallow_info *si)
a796ccee 949{
4820a33b
NTND
950 struct sha1_array ref = SHA1_ARRAY_INIT;
951 int *status;
beea4152
NTND
952 int i;
953
79891cb9 954 if (args->deepen && alternate_shallow_file) {
a796ccee 955 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 956 unlink_or_warn(git_path_shallow());
a796ccee
NTND
957 rollback_lock_file(&shallow_lock);
958 } else
959 commit_lock_file(&shallow_lock);
960 return;
961 }
beea4152
NTND
962
963 if (!si->shallow || !si->shallow->nr)
964 return;
965
beea4152
NTND
966 if (args->cloning) {
967 /*
968 * remote is shallow, but this is a clone, there are
969 * no objects in repo to worry about. Accept any
970 * shallow points that exist in the pack (iow in repo
971 * after get_pack() and reprepare_packed_git())
972 */
973 struct sha1_array extra = SHA1_ARRAY_INIT;
974 unsigned char (*sha1)[20] = si->shallow->sha1;
975 for (i = 0; i < si->shallow->nr; i++)
976 if (has_sha1_file(sha1[i]))
977 sha1_array_append(&extra, sha1[i]);
978 if (extra.nr) {
979 setup_alternate_shallow(&shallow_lock,
980 &alternate_shallow_file,
981 &extra);
982 commit_lock_file(&shallow_lock);
983 }
984 sha1_array_clear(&extra);
985 return;
986 }
4820a33b
NTND
987
988 if (!si->nr_ours && !si->nr_theirs)
989 return;
990
991 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
992 if (!si->nr_ours && !si->nr_theirs)
993 return;
994 for (i = 0; i < nr_sought; i++)
f4e54d02 995 sha1_array_append(&ref, sought[i]->old_oid.hash);
4820a33b
NTND
996 si->ref = &ref;
997
48d25cae
NTND
998 if (args->update_shallow) {
999 /*
1000 * remote is also shallow, .git/shallow may be updated
1001 * so all refs can be accepted. Make sure we only add
1002 * shallow roots that are actually reachable from new
1003 * refs.
1004 */
1005 struct sha1_array extra = SHA1_ARRAY_INIT;
1006 unsigned char (*sha1)[20] = si->shallow->sha1;
1007 assign_shallow_commits_to_refs(si, NULL, NULL);
1008 if (!si->nr_ours && !si->nr_theirs) {
1009 sha1_array_clear(&ref);
1010 return;
1011 }
1012 for (i = 0; i < si->nr_ours; i++)
1013 sha1_array_append(&extra, sha1[si->ours[i]]);
1014 for (i = 0; i < si->nr_theirs; i++)
1015 sha1_array_append(&extra, sha1[si->theirs[i]]);
1016 setup_alternate_shallow(&shallow_lock,
1017 &alternate_shallow_file,
1018 &extra);
1019 commit_lock_file(&shallow_lock);
1020 sha1_array_clear(&extra);
1021 sha1_array_clear(&ref);
1022 return;
1023 }
1024
4820a33b
NTND
1025 /*
1026 * remote is also shallow, check what ref is safe to update
1027 * without updating .git/shallow
1028 */
1029 status = xcalloc(nr_sought, sizeof(*status));
1030 assign_shallow_commits_to_refs(si, NULL, status);
1031 if (si->nr_ours || si->nr_theirs) {
1032 for (i = 0; i < nr_sought; i++)
1033 if (status[i])
1034 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1035 }
1036 free(status);
1037 sha1_array_clear(&ref);
a796ccee
NTND
1038}
1039
745f7a8c
NTND
1040struct ref *fetch_pack(struct fetch_pack_args *args,
1041 int fd[], struct child_process *conn,
1042 const struct ref *ref,
1043 const char *dest,
f2db854d 1044 struct ref **sought, int nr_sought,
beea4152 1045 struct sha1_array *shallow,
745f7a8c
NTND
1046 char **pack_lockfile)
1047{
745f7a8c 1048 struct ref *ref_cpy;
beea4152 1049 struct shallow_info si;
745f7a8c
NTND
1050
1051 fetch_pack_setup();
f2db854d
JH
1052 if (nr_sought)
1053 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1054
1055 if (!ref) {
1056 packet_flush(fd[1]);
1dd73e20 1057 die(_("no matching remote head"));
745f7a8c 1058 }
beea4152
NTND
1059 prepare_shallow_info(&si, shallow);
1060 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1061 &si, pack_lockfile);
745f7a8c 1062 reprepare_packed_git();
4820a33b 1063 update_shallow(args, sought, nr_sought, &si);
beea4152 1064 clear_shallow_info(&si);
745f7a8c
NTND
1065 return ref_cpy;
1066}