fetch_refs_via_pack: call report_unmatched_refs
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
697cc8ef 2#include "lockfile.h"
745f7a8c
NTND
3#include "refs.h"
4#include "pkt-line.h"
5#include "commit.h"
6#include "tag.h"
7#include "exec_cmd.h"
8#include "pack.h"
9#include "sideband.h"
10#include "fetch-pack.h"
11#include "remote.h"
12#include "run-command.h"
47a59185 13#include "connect.h"
745f7a8c
NTND
14#include "transport.h"
15#include "version.h"
099327b5 16#include "prio-queue.h"
beea4152 17#include "sha1-array.h"
745f7a8c
NTND
18
19static int transfer_unpack_limit = -1;
20static int fetch_unpack_limit = -1;
21static int unpack_limit = 100;
22static int prefer_ofs_delta = 1;
23static int no_done;
508ea882 24static int deepen_since_ok;
a45a2600 25static int deepen_not_ok;
745f7a8c
NTND
26static int fetch_fsck_objects = -1;
27static int transfer_fsck_objects = -1;
28static int agent_supported;
6035d6aa
NTND
29static struct lock_file shallow_lock;
30static const char *alternate_shallow_file;
745f7a8c 31
208acbfb 32/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
33#define COMPLETE (1U << 0)
34#define COMMON (1U << 1)
35#define COMMON_REF (1U << 2)
36#define SEEN (1U << 3)
37#define POPPED (1U << 4)
38
39static int marked;
40
41/*
42 * After sending this many "have"s if we do not get any new ACK , we
43 * give up traversing our history.
44 */
45#define MAX_IN_VAIN 256
46
099327b5 47static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
48static int non_common_revs, multi_ack, use_sideband;
49/* Allow specifying sha1 if it is a ref tip. */
50#define ALLOW_TIP_SHA1 01
68ee6289
FM
51/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
52#define ALLOW_REACHABLE_SHA1 02
7199c093 53static unsigned int allow_unadvertised_object_request;
745f7a8c 54
0d789a5b
NTND
55__attribute__((format (printf, 2, 3)))
56static inline void print_verbose(const struct fetch_pack_args *args,
57 const char *fmt, ...)
58{
59 va_list params;
60
61 if (!args->verbose)
62 return;
63
64 va_start(params, fmt);
65 vfprintf(stderr, fmt, params);
66 va_end(params);
67 fputc('\n', stderr);
68}
69
745f7a8c
NTND
70static void rev_list_push(struct commit *commit, int mark)
71{
72 if (!(commit->object.flags & mark)) {
73 commit->object.flags |= mark;
74
0064053b
JK
75 if (parse_commit(commit))
76 return;
745f7a8c 77
099327b5 78 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
79
80 if (!(commit->object.flags & COMMON))
81 non_common_revs++;
82 }
83}
84
c38cd1c8 85static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
745f7a8c
NTND
86{
87 struct object *o = deref_tag(parse_object(sha1), refname, 0);
88
89 if (o && o->type == OBJ_COMMIT)
90 rev_list_push((struct commit *)o, SEEN);
91
92 return 0;
93}
94
b1b49c6e
MH
95static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
96 int flag, void *cb_data)
745f7a8c 97{
c38cd1c8 98 return rev_list_insert_ref(refname, oid->hash);
b1b49c6e
MH
99}
100
c50fb6ce
MH
101static int clear_marks(const char *refname, const struct object_id *oid,
102 int flag, void *cb_data)
745f7a8c 103{
c50fb6ce 104 struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
745f7a8c
NTND
105
106 if (o && o->type == OBJ_COMMIT)
107 clear_commit_marks((struct commit *)o,
108 COMMON | COMMON_REF | SEEN | POPPED);
109 return 0;
110}
111
112/*
113 This function marks a rev and its ancestors as common.
114 In some cases, it is desirable to mark only the ancestors (for example
115 when only the server does not yet know that they are common).
116*/
117
118static void mark_common(struct commit *commit,
119 int ancestors_only, int dont_parse)
120{
121 if (commit != NULL && !(commit->object.flags & COMMON)) {
122 struct object *o = (struct object *)commit;
123
124 if (!ancestors_only)
125 o->flags |= COMMON;
126
127 if (!(o->flags & SEEN))
128 rev_list_push(commit, SEEN);
129 else {
130 struct commit_list *parents;
131
132 if (!ancestors_only && !(o->flags & POPPED))
133 non_common_revs--;
134 if (!o->parsed && !dont_parse)
135 if (parse_commit(commit))
136 return;
137
138 for (parents = commit->parents;
139 parents;
140 parents = parents->next)
141 mark_common(parents->item, 0, dont_parse);
142 }
143 }
144}
145
146/*
147 Get the next rev to send, ignoring the common.
148*/
149
150static const unsigned char *get_rev(void)
151{
152 struct commit *commit = NULL;
153
154 while (commit == NULL) {
155 unsigned int mark;
156 struct commit_list *parents;
157
099327b5 158 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
159 return NULL;
160
099327b5 161 commit = prio_queue_get(&rev_list);
0064053b 162 parse_commit(commit);
745f7a8c
NTND
163 parents = commit->parents;
164
165 commit->object.flags |= POPPED;
166 if (!(commit->object.flags & COMMON))
167 non_common_revs--;
168
169 if (commit->object.flags & COMMON) {
170 /* do not send "have", and ignore ancestors */
171 commit = NULL;
172 mark = COMMON | SEEN;
173 } else if (commit->object.flags & COMMON_REF)
174 /* send "have", and ignore ancestors */
175 mark = COMMON | SEEN;
176 else
177 /* send "have", also for its ancestors */
178 mark = SEEN;
179
180 while (parents) {
181 if (!(parents->item->object.flags & SEEN))
182 rev_list_push(parents->item, mark);
183 if (mark & COMMON)
184 mark_common(parents->item, 1, 0);
185 parents = parents->next;
186 }
745f7a8c
NTND
187 }
188
ed1c9977 189 return commit->object.oid.hash;
745f7a8c
NTND
190}
191
192enum ack_type {
193 NAK = 0,
194 ACK,
195 ACK_continue,
196 ACK_common,
197 ACK_ready
198};
199
200static void consume_shallow_list(struct fetch_pack_args *args, int fd)
201{
79891cb9 202 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
203 /* If we sent a depth we will get back "duplicate"
204 * shallow and unshallow commands every time there
205 * is a block of have lines exchanged.
206 */
74543a04
JK
207 char *line;
208 while ((line = packet_read_line(fd, NULL))) {
59556548 209 if (starts_with(line, "shallow "))
745f7a8c 210 continue;
59556548 211 if (starts_with(line, "unshallow "))
745f7a8c 212 continue;
1dd73e20 213 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
214 }
215 }
216}
217
745f7a8c
NTND
218static enum ack_type get_ack(int fd, unsigned char *result_sha1)
219{
74543a04
JK
220 int len;
221 char *line = packet_read_line(fd, &len);
82e56767 222 const char *arg;
745f7a8c
NTND
223
224 if (!len)
1dd73e20 225 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
226 if (!strcmp(line, "NAK"))
227 return NAK;
82e56767
JK
228 if (skip_prefix(line, "ACK ", &arg)) {
229 if (!get_sha1_hex(arg, result_sha1)) {
230 arg += 40;
231 len -= arg - line;
232 if (len < 1)
030e9dd6 233 return ACK;
82e56767 234 if (strstr(arg, "continue"))
745f7a8c 235 return ACK_continue;
82e56767 236 if (strstr(arg, "common"))
745f7a8c 237 return ACK_common;
82e56767 238 if (strstr(arg, "ready"))
745f7a8c
NTND
239 return ACK_ready;
240 return ACK;
241 }
242 }
dfbfb9f3 243 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
244}
245
246static void send_request(struct fetch_pack_args *args,
247 int fd, struct strbuf *buf)
248{
249 if (args->stateless_rpc) {
250 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
251 packet_flush(fd);
252 } else
cdf4fb8e 253 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
254}
255
256static void insert_one_alternate_ref(const struct ref *ref, void *unused)
257{
f4e54d02 258 rev_list_insert_ref(NULL, ref->old_oid.hash);
745f7a8c
NTND
259}
260
261#define INITIAL_FLUSH 16
262#define PIPESAFE_FLUSH 32
da470981 263#define LARGE_FLUSH 16384
745f7a8c
NTND
264
265static int next_flush(struct fetch_pack_args *args, int count)
266{
da470981
JT
267 if (args->stateless_rpc) {
268 if (count < LARGE_FLUSH)
269 count <<= 1;
270 else
271 count = count * 11 / 10;
272 } else {
273 if (count < PIPESAFE_FLUSH)
274 count <<= 1;
275 else
276 count += PIPESAFE_FLUSH;
277 }
745f7a8c
NTND
278 return count;
279}
280
281static int find_common(struct fetch_pack_args *args,
282 int fd[2], unsigned char *result_sha1,
283 struct ref *refs)
284{
285 int fetching;
286 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
287 const unsigned char *sha1;
288 unsigned in_vain = 0;
289 int got_continue = 0;
290 int got_ready = 0;
291 struct strbuf req_buf = STRBUF_INIT;
292 size_t state_len = 0;
293
294 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 295 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
296 if (marked)
297 for_each_ref(clear_marks, NULL);
298 marked = 1;
299
b1b49c6e 300 for_each_ref(rev_list_insert_ref_oid, NULL);
745f7a8c
NTND
301 for_each_alternate_ref(insert_one_alternate_ref, NULL);
302
303 fetching = 0;
304 for ( ; refs ; refs = refs->next) {
f4e54d02 305 unsigned char *remote = refs->old_oid.hash;
745f7a8c
NTND
306 const char *remote_hex;
307 struct object *o;
308
309 /*
310 * If that object is complete (i.e. it is an ancestor of a
311 * local ref), we tell them we have it but do not have to
312 * tell them about its ancestors, which they already know
313 * about.
314 *
315 * We use lookup_object here because we are only
316 * interested in the case we *know* the object is
317 * reachable and we have already scanned it.
318 */
319 if (((o = lookup_object(remote)) != NULL) &&
320 (o->flags & COMPLETE)) {
321 continue;
322 }
323
324 remote_hex = sha1_to_hex(remote);
325 if (!fetching) {
326 struct strbuf c = STRBUF_INIT;
327 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
328 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
329 if (no_done) strbuf_addstr(&c, " no-done");
330 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
331 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 332 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
333 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
334 if (args->no_progress) strbuf_addstr(&c, " no-progress");
335 if (args->include_tag) strbuf_addstr(&c, " include-tag");
336 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 337 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 338 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
339 if (agent_supported) strbuf_addf(&c, " agent=%s",
340 git_user_agent_sanitized());
341 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
342 strbuf_release(&c);
343 } else
344 packet_buf_write(&req_buf, "want %s\n", remote_hex);
345 fetching++;
346 }
347
348 if (!fetching) {
349 strbuf_release(&req_buf);
350 packet_flush(fd[1]);
351 return 1;
352 }
353
354 if (is_repository_shallow())
1a30f5a2 355 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
356 if (args->depth > 0)
357 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882
NTND
358 if (args->deepen_since) {
359 unsigned long max_age = approxidate(args->deepen_since);
360 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
361 }
a45a2600
NTND
362 if (args->deepen_not) {
363 int i;
364 for (i = 0; i < args->deepen_not->nr; i++) {
365 struct string_list_item *s = args->deepen_not->items + i;
366 packet_buf_write(&req_buf, "deepen-not %s", s->string);
367 }
368 }
745f7a8c
NTND
369 packet_buf_flush(&req_buf);
370 state_len = req_buf.len;
371
79891cb9 372 if (args->deepen) {
74543a04 373 char *line;
ae021d87 374 const char *arg;
745f7a8c
NTND
375 unsigned char sha1[20];
376
377 send_request(args, fd[1], &req_buf);
74543a04 378 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87
JK
379 if (skip_prefix(line, "shallow ", &arg)) {
380 if (get_sha1_hex(arg, sha1))
1dd73e20 381 die(_("invalid shallow line: %s"), line);
745f7a8c
NTND
382 register_shallow(sha1);
383 continue;
384 }
ae021d87
JK
385 if (skip_prefix(line, "unshallow ", &arg)) {
386 if (get_sha1_hex(arg, sha1))
1dd73e20 387 die(_("invalid unshallow line: %s"), line);
745f7a8c 388 if (!lookup_object(sha1))
1dd73e20 389 die(_("object not found: %s"), line);
745f7a8c
NTND
390 /* make sure that it is parsed as shallow */
391 if (!parse_object(sha1))
1dd73e20 392 die(_("error in object: %s"), line);
745f7a8c 393 if (unregister_shallow(sha1))
1dd73e20 394 die(_("no shallow found: %s"), line);
745f7a8c
NTND
395 continue;
396 }
1dd73e20 397 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
398 }
399 } else if (!args->stateless_rpc)
400 send_request(args, fd[1], &req_buf);
401
402 if (!args->stateless_rpc) {
403 /* If we aren't using the stateless-rpc interface
404 * we don't need to retain the headers.
405 */
406 strbuf_setlen(&req_buf, 0);
407 state_len = 0;
408 }
409
410 flushes = 0;
411 retval = -1;
412 while ((sha1 = get_rev())) {
413 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
0d789a5b 414 print_verbose(args, "have %s", sha1_to_hex(sha1));
745f7a8c
NTND
415 in_vain++;
416 if (flush_at <= ++count) {
417 int ack;
418
419 packet_buf_flush(&req_buf);
420 send_request(args, fd[1], &req_buf);
421 strbuf_setlen(&req_buf, state_len);
422 flushes++;
423 flush_at = next_flush(args, count);
424
425 /*
426 * We keep one window "ahead" of the other side, and
427 * will wait for an ACK only on the next one
428 */
429 if (!args->stateless_rpc && count == INITIAL_FLUSH)
430 continue;
431
432 consume_shallow_list(args, fd[0]);
433 do {
434 ack = get_ack(fd[0], result_sha1);
0d789a5b 435 if (ack)
1dd73e20
NTND
436 print_verbose(args, _("got %s %d %s"), "ack",
437 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
438 switch (ack) {
439 case ACK:
440 flushes = 0;
441 multi_ack = 0;
442 retval = 0;
443 goto done;
444 case ACK_common:
445 case ACK_ready:
446 case ACK_continue: {
447 struct commit *commit =
448 lookup_commit(result_sha1);
449 if (!commit)
1dd73e20 450 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
745f7a8c
NTND
451 if (args->stateless_rpc
452 && ack == ACK_common
453 && !(commit->object.flags & COMMON)) {
454 /* We need to replay the have for this object
455 * on the next RPC request so the peer knows
456 * it is in common with us.
457 */
458 const char *hex = sha1_to_hex(result_sha1);
459 packet_buf_write(&req_buf, "have %s\n", hex);
460 state_len = req_buf.len;
06b3d386
JT
461 /*
462 * Reset in_vain because an ack
463 * for this commit has not been
464 * seen.
465 */
466 in_vain = 0;
467 } else if (!args->stateless_rpc
468 || ack != ACK_common)
469 in_vain = 0;
745f7a8c
NTND
470 mark_common(commit, 0, 1);
471 retval = 0;
745f7a8c
NTND
472 got_continue = 1;
473 if (ack == ACK_ready) {
099327b5 474 clear_prio_queue(&rev_list);
745f7a8c
NTND
475 got_ready = 1;
476 }
477 break;
478 }
479 }
480 } while (ack);
481 flushes--;
482 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 483 print_verbose(args, _("giving up"));
745f7a8c
NTND
484 break; /* give up */
485 }
486 }
487 }
488done:
489 if (!got_ready || !no_done) {
490 packet_buf_write(&req_buf, "done\n");
491 send_request(args, fd[1], &req_buf);
492 }
1dd73e20 493 print_verbose(args, _("done"));
745f7a8c
NTND
494 if (retval != 0) {
495 multi_ack = 0;
496 flushes++;
497 }
498 strbuf_release(&req_buf);
499
ff62eca7
NTND
500 if (!got_ready || !no_done)
501 consume_shallow_list(args, fd[0]);
745f7a8c
NTND
502 while (flushes || multi_ack) {
503 int ack = get_ack(fd[0], result_sha1);
504 if (ack) {
1dd73e20
NTND
505 print_verbose(args, _("got %s (%d) %s"), "ack",
506 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
507 if (ack == ACK)
508 return 0;
509 multi_ack = 1;
510 continue;
511 }
512 flushes--;
513 }
514 /* it is no error to fetch into a completely empty repo */
515 return count ? retval : 0;
516}
517
518static struct commit_list *complete;
519
6e20a51a 520static int mark_complete(const unsigned char *sha1)
745f7a8c
NTND
521{
522 struct object *o = parse_object(sha1);
523
524 while (o && o->type == OBJ_TAG) {
525 struct tag *t = (struct tag *) o;
526 if (!t->tagged)
527 break; /* broken repository */
528 o->flags |= COMPLETE;
ed1c9977 529 o = parse_object(t->tagged->oid.hash);
745f7a8c
NTND
530 }
531 if (o && o->type == OBJ_COMMIT) {
532 struct commit *commit = (struct commit *)o;
533 if (!(commit->object.flags & COMPLETE)) {
534 commit->object.flags |= COMPLETE;
16445242 535 commit_list_insert(commit, &complete);
745f7a8c
NTND
536 }
537 }
538 return 0;
539}
540
f8ee4d85
MH
541static int mark_complete_oid(const char *refname, const struct object_id *oid,
542 int flag, void *cb_data)
543{
6e20a51a 544 return mark_complete(oid->hash);
f8ee4d85
MH
545}
546
745f7a8c
NTND
547static void mark_recent_complete_commits(struct fetch_pack_args *args,
548 unsigned long cutoff)
549{
550 while (complete && cutoff <= complete->item->date) {
1dd73e20 551 print_verbose(args, _("Marking %s as complete"),
0d789a5b 552 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
553 pop_most_recent_commit(&complete, COMPLETE);
554 }
555}
556
745f7a8c 557static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
558 struct ref **refs,
559 struct ref **sought, int nr_sought)
745f7a8c
NTND
560{
561 struct ref *newlist = NULL;
562 struct ref **newtail = &newlist;
563 struct ref *ref, *next;
f2db854d 564 int i;
745f7a8c 565
f2db854d 566 i = 0;
745f7a8c
NTND
567 for (ref = *refs; ref; ref = next) {
568 int keep = 0;
569 next = ref->next;
f2db854d 570
50e19a83 571 if (starts_with(ref->name, "refs/") &&
4c224081 572 check_refname_format(ref->name, 0))
745f7a8c
NTND
573 ; /* trash */
574 else {
f2db854d
JH
575 while (i < nr_sought) {
576 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
577 if (cmp < 0)
578 break; /* definitely do not have it */
579 else if (cmp == 0) {
580 keep = 1; /* definitely have it */
f2db854d 581 sought[i]->matched = 1;
745f7a8c 582 }
f2db854d 583 i++;
745f7a8c
NTND
584 }
585 }
586
f2db854d 587 if (!keep && args->fetch_all &&
79891cb9 588 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
589 keep = 1;
590
591 if (keep) {
592 *newtail = ref;
593 ref->next = NULL;
594 newtail = &ref->next;
595 } else {
596 free(ref);
597 }
598 }
599
6e7b66ee 600 /* Append unmatched requests to the list */
68ee6289
FM
601 if ((allow_unadvertised_object_request &
602 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
6e7b66ee 603 for (i = 0; i < nr_sought; i++) {
b7916422
JK
604 unsigned char sha1[20];
605
6e7b66ee
JH
606 ref = sought[i];
607 if (ref->matched)
608 continue;
b7916422
JK
609 if (get_sha1_hex(ref->name, sha1) ||
610 ref->name[40] != '\0' ||
f4e54d02 611 hashcmp(sha1, ref->old_oid.hash))
6e7b66ee
JH
612 continue;
613
614 ref->matched = 1;
c3c17bf1
JK
615 *newtail = copy_ref(ref);
616 newtail = &(*newtail)->next;
6e7b66ee
JH
617 }
618 }
745f7a8c
NTND
619 *refs = newlist;
620}
621
622static void mark_alternate_complete(const struct ref *ref, void *unused)
623{
f4e54d02 624 mark_complete(ref->old_oid.hash);
745f7a8c
NTND
625}
626
627static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
628 struct ref **refs,
629 struct ref **sought, int nr_sought)
745f7a8c
NTND
630{
631 struct ref *ref;
632 int retval;
633 unsigned long cutoff = 0;
634
635 save_commit_buffer = 0;
636
637 for (ref = *refs; ref; ref = ref->next) {
638 struct object *o;
639
f4e54d02 640 if (!has_object_file(&ref->old_oid))
012a1bb5
JH
641 continue;
642
f4e54d02 643 o = parse_object(ref->old_oid.hash);
745f7a8c
NTND
644 if (!o)
645 continue;
646
647 /* We already have it -- which may mean that we were
648 * in sync with the other side at some time after
649 * that (it is OK if we guess wrong here).
650 */
651 if (o->type == OBJ_COMMIT) {
652 struct commit *commit = (struct commit *)o;
653 if (!cutoff || cutoff < commit->date)
654 cutoff = commit->date;
655 }
656 }
657
79891cb9 658 if (!args->deepen) {
f8ee4d85 659 for_each_ref(mark_complete_oid, NULL);
745f7a8c 660 for_each_alternate_ref(mark_alternate_complete, NULL);
16445242 661 commit_list_sort_by_date(&complete);
745f7a8c
NTND
662 if (cutoff)
663 mark_recent_complete_commits(args, cutoff);
664 }
665
666 /*
667 * Mark all complete remote refs as common refs.
668 * Don't mark them common yet; the server has to be told so first.
669 */
670 for (ref = *refs; ref; ref = ref->next) {
f4e54d02 671 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
745f7a8c
NTND
672 NULL, 0);
673
674 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
675 continue;
676
677 if (!(o->flags & SEEN)) {
678 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
679
680 mark_common((struct commit *)o, 1, 1);
681 }
682 }
683
f2db854d 684 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
685
686 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
f4e54d02 687 const unsigned char *remote = ref->old_oid.hash;
745f7a8c
NTND
688 struct object *o;
689
690 o = lookup_object(remote);
691 if (!o || !(o->flags & COMPLETE)) {
692 retval = 0;
0d789a5b
NTND
693 print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
694 ref->name);
745f7a8c
NTND
695 continue;
696 }
1dd73e20 697 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
0d789a5b 698 ref->name);
745f7a8c
NTND
699 }
700 return retval;
701}
702
703static int sideband_demux(int in, int out, void *data)
704{
705 int *xd = data;
9ff18faf 706 int ret;
745f7a8c 707
9ff18faf 708 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
709 close(out);
710 return ret;
711}
712
713static int get_pack(struct fetch_pack_args *args,
714 int xd[2], char **pack_lockfile)
715{
716 struct async demux;
745f7a8c 717 int do_keep = args->keep_pack;
984a43b9
JK
718 const char *cmd_name;
719 struct pack_header header;
720 int pass_header = 0;
d3180279 721 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 722 int ret;
745f7a8c
NTND
723
724 memset(&demux, 0, sizeof(demux));
725 if (use_sideband) {
726 /* xd[] is talking with upload-pack; subprocess reads from
727 * xd[0], spits out band#2 to stderr, and feeds us band#1
728 * through demux->out.
729 */
730 demux.proc = sideband_demux;
731 demux.data = xd;
732 demux.out = -1;
df857572 733 demux.isolate_sigpipe = 1;
745f7a8c 734 if (start_async(&demux))
1dd73e20 735 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
736 }
737 else
738 demux.out = xd[0];
739
745f7a8c 740 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
741
742 if (read_pack_header(demux.out, &header))
1dd73e20 743 die(_("protocol error: bad pack header"));
984a43b9 744 pass_header = 1;
745f7a8c
NTND
745 if (ntohl(header.hdr_entries) < unpack_limit)
746 do_keep = 0;
747 else
748 do_keep = 1;
749 }
750
6035d6aa 751 if (alternate_shallow_file) {
984a43b9
JK
752 argv_array_push(&cmd.args, "--shallow-file");
753 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
754 }
755
745f7a8c
NTND
756 if (do_keep) {
757 if (pack_lockfile)
758 cmd.out = -1;
984a43b9
JK
759 cmd_name = "index-pack";
760 argv_array_push(&cmd.args, cmd_name);
761 argv_array_push(&cmd.args, "--stdin");
745f7a8c 762 if (!args->quiet && !args->no_progress)
984a43b9 763 argv_array_push(&cmd.args, "-v");
745f7a8c 764 if (args->use_thin_pack)
984a43b9 765 argv_array_push(&cmd.args, "--fix-thin");
745f7a8c 766 if (args->lock_pack || unpack_limit) {
984a43b9
JK
767 char hostname[256];
768 if (gethostname(hostname, sizeof(hostname)))
769 xsnprintf(hostname, sizeof(hostname), "localhost");
770 argv_array_pushf(&cmd.args,
771 "--keep=fetch-pack %"PRIuMAX " on %s",
772 (uintmax_t)getpid(), hostname);
745f7a8c 773 }
c6807a40 774 if (args->check_self_contained_and_connected)
984a43b9 775 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
745f7a8c
NTND
776 }
777 else {
984a43b9
JK
778 cmd_name = "unpack-objects";
779 argv_array_push(&cmd.args, cmd_name);
745f7a8c 780 if (args->quiet || args->no_progress)
984a43b9 781 argv_array_push(&cmd.args, "-q");
c6807a40 782 args->check_self_contained_and_connected = 0;
745f7a8c 783 }
984a43b9
JK
784
785 if (pass_header)
786 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
787 ntohl(header.hdr_version),
788 ntohl(header.hdr_entries));
745f7a8c
NTND
789 if (fetch_fsck_objects >= 0
790 ? fetch_fsck_objects
791 : transfer_fsck_objects >= 0
792 ? transfer_fsck_objects
793 : 0)
984a43b9 794 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
795
796 cmd.in = demux.out;
797 cmd.git_cmd = 1;
798 if (start_command(&cmd))
1dd73e20 799 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
800 if (do_keep && pack_lockfile) {
801 *pack_lockfile = index_pack_lockfile(cmd.out);
802 close(cmd.out);
803 }
804
37cb1dd6
JL
805 if (!use_sideband)
806 /* Closed by start_command() */
807 xd[0] = -1;
808
c6807a40
NTND
809 ret = finish_command(&cmd);
810 if (!ret || (args->check_self_contained_and_connected && ret == 1))
811 args->self_contained_and_connected =
812 args->check_self_contained_and_connected &&
813 ret == 0;
814 else
1dd73e20 815 die(_("%s failed"), cmd_name);
745f7a8c 816 if (use_sideband && finish_async(&demux))
1dd73e20 817 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
818 return 0;
819}
820
f2db854d
JH
821static int cmp_ref_by_name(const void *a_, const void *b_)
822{
823 const struct ref *a = *((const struct ref **)a_);
824 const struct ref *b = *((const struct ref **)b_);
825 return strcmp(a->name, b->name);
826}
827
745f7a8c
NTND
828static struct ref *do_fetch_pack(struct fetch_pack_args *args,
829 int fd[2],
830 const struct ref *orig_ref,
f2db854d 831 struct ref **sought, int nr_sought,
beea4152 832 struct shallow_info *si,
745f7a8c
NTND
833 char **pack_lockfile)
834{
835 struct ref *ref = copy_ref_list(orig_ref);
836 unsigned char sha1[20];
837 const char *agent_feature;
838 int agent_len;
839
840 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 841 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 842
eb86a507 843 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 844 die(_("Server does not support shallow clients"));
a45a2600 845 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 846 args->deepen = 1;
745f7a8c 847 if (server_supports("multi_ack_detailed")) {
1dd73e20 848 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
849 multi_ack = 2;
850 if (server_supports("no-done")) {
1dd73e20 851 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
852 if (args->stateless_rpc)
853 no_done = 1;
854 }
855 }
856 else if (server_supports("multi_ack")) {
1dd73e20 857 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
858 multi_ack = 1;
859 }
860 if (server_supports("side-band-64k")) {
1dd73e20 861 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
862 use_sideband = 2;
863 }
864 else if (server_supports("side-band")) {
1dd73e20 865 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
866 use_sideband = 1;
867 }
6e7b66ee 868 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 869 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 870 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 871 }
68ee6289 872 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 873 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
874 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
875 }
745f7a8c
NTND
876 if (!server_supports("thin-pack"))
877 args->use_thin_pack = 0;
878 if (!server_supports("no-progress"))
879 args->no_progress = 0;
880 if (!server_supports("include-tag"))
881 args->include_tag = 0;
0d789a5b 882 if (server_supports("ofs-delta"))
1dd73e20 883 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 884 else
745f7a8c
NTND
885 prefer_ofs_delta = 0;
886
887 if ((agent_feature = server_feature_value("agent", &agent_len))) {
888 agent_supported = 1;
0d789a5b 889 if (agent_len)
1dd73e20 890 print_verbose(args, _("Server version is %.*s"),
0d789a5b 891 agent_len, agent_feature);
745f7a8c 892 }
508ea882
NTND
893 if (server_supports("deepen-since"))
894 deepen_since_ok = 1;
895 else if (args->deepen_since)
896 die(_("Server does not support --shallow-since"));
a45a2600
NTND
897 if (server_supports("deepen-not"))
898 deepen_not_ok = 1;
899 else if (args->deepen_not)
900 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
901 if (!server_supports("deepen-relative") && args->deepen_relative)
902 die(_("Server does not support --deepen"));
745f7a8c 903
f2db854d 904 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
905 packet_flush(fd[1]);
906 goto all_done;
907 }
908 if (find_common(args, fd, sha1, ref) < 0)
909 if (!args->keep_pack)
910 /* When cloning, it is not unusual to have
911 * no common commit.
912 */
1dd73e20 913 warning(_("no common commits"));
745f7a8c
NTND
914
915 if (args->stateless_rpc)
916 packet_flush(fd[1]);
79891cb9 917 if (args->deepen)
1a30f5a2
NTND
918 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
919 NULL);
4820a33b 920 else if (si->nr_ours || si->nr_theirs)
beea4152 921 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
922 else
923 alternate_shallow_file = NULL;
745f7a8c 924 if (get_pack(args, fd, pack_lockfile))
1dd73e20 925 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
926
927 all_done:
928 return ref;
929}
930
f44af51d 931static void fetch_pack_config(void)
745f7a8c 932{
f44af51d
TA
933 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
934 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
935 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
936 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
937 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 938
f44af51d 939 git_config(git_default_config, NULL);
745f7a8c
NTND
940}
941
745f7a8c
NTND
942static void fetch_pack_setup(void)
943{
944 static int did_setup;
945 if (did_setup)
946 return;
f44af51d 947 fetch_pack_config();
745f7a8c
NTND
948 if (0 <= transfer_unpack_limit)
949 unpack_limit = transfer_unpack_limit;
950 else if (0 <= fetch_unpack_limit)
951 unpack_limit = fetch_unpack_limit;
952 did_setup = 1;
953}
954
f2db854d
JH
955static int remove_duplicates_in_refs(struct ref **ref, int nr)
956{
957 struct string_list names = STRING_LIST_INIT_NODUP;
958 int src, dst;
959
960 for (src = dst = 0; src < nr; src++) {
961 struct string_list_item *item;
962 item = string_list_insert(&names, ref[src]->name);
963 if (item->util)
964 continue; /* already have it */
965 item->util = ref[src];
966 if (src != dst)
967 ref[dst] = ref[src];
968 dst++;
969 }
970 for (src = dst; src < nr; src++)
971 ref[src] = NULL;
972 string_list_clear(&names, 0);
973 return dst;
974}
975
beea4152 976static void update_shallow(struct fetch_pack_args *args,
4820a33b 977 struct ref **sought, int nr_sought,
beea4152 978 struct shallow_info *si)
a796ccee 979{
4820a33b
NTND
980 struct sha1_array ref = SHA1_ARRAY_INIT;
981 int *status;
beea4152
NTND
982 int i;
983
79891cb9 984 if (args->deepen && alternate_shallow_file) {
a796ccee 985 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 986 unlink_or_warn(git_path_shallow());
a796ccee
NTND
987 rollback_lock_file(&shallow_lock);
988 } else
989 commit_lock_file(&shallow_lock);
990 return;
991 }
beea4152
NTND
992
993 if (!si->shallow || !si->shallow->nr)
994 return;
995
beea4152
NTND
996 if (args->cloning) {
997 /*
998 * remote is shallow, but this is a clone, there are
999 * no objects in repo to worry about. Accept any
1000 * shallow points that exist in the pack (iow in repo
1001 * after get_pack() and reprepare_packed_git())
1002 */
1003 struct sha1_array extra = SHA1_ARRAY_INIT;
1004 unsigned char (*sha1)[20] = si->shallow->sha1;
1005 for (i = 0; i < si->shallow->nr; i++)
1006 if (has_sha1_file(sha1[i]))
1007 sha1_array_append(&extra, sha1[i]);
1008 if (extra.nr) {
1009 setup_alternate_shallow(&shallow_lock,
1010 &alternate_shallow_file,
1011 &extra);
1012 commit_lock_file(&shallow_lock);
1013 }
1014 sha1_array_clear(&extra);
1015 return;
1016 }
4820a33b
NTND
1017
1018 if (!si->nr_ours && !si->nr_theirs)
1019 return;
1020
1021 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1022 if (!si->nr_ours && !si->nr_theirs)
1023 return;
1024 for (i = 0; i < nr_sought; i++)
f4e54d02 1025 sha1_array_append(&ref, sought[i]->old_oid.hash);
4820a33b
NTND
1026 si->ref = &ref;
1027
48d25cae
NTND
1028 if (args->update_shallow) {
1029 /*
1030 * remote is also shallow, .git/shallow may be updated
1031 * so all refs can be accepted. Make sure we only add
1032 * shallow roots that are actually reachable from new
1033 * refs.
1034 */
1035 struct sha1_array extra = SHA1_ARRAY_INIT;
1036 unsigned char (*sha1)[20] = si->shallow->sha1;
1037 assign_shallow_commits_to_refs(si, NULL, NULL);
1038 if (!si->nr_ours && !si->nr_theirs) {
1039 sha1_array_clear(&ref);
1040 return;
1041 }
1042 for (i = 0; i < si->nr_ours; i++)
1043 sha1_array_append(&extra, sha1[si->ours[i]]);
1044 for (i = 0; i < si->nr_theirs; i++)
1045 sha1_array_append(&extra, sha1[si->theirs[i]]);
1046 setup_alternate_shallow(&shallow_lock,
1047 &alternate_shallow_file,
1048 &extra);
1049 commit_lock_file(&shallow_lock);
1050 sha1_array_clear(&extra);
1051 sha1_array_clear(&ref);
1052 return;
1053 }
1054
4820a33b
NTND
1055 /*
1056 * remote is also shallow, check what ref is safe to update
1057 * without updating .git/shallow
1058 */
1059 status = xcalloc(nr_sought, sizeof(*status));
1060 assign_shallow_commits_to_refs(si, NULL, status);
1061 if (si->nr_ours || si->nr_theirs) {
1062 for (i = 0; i < nr_sought; i++)
1063 if (status[i])
1064 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1065 }
1066 free(status);
1067 sha1_array_clear(&ref);
a796ccee
NTND
1068}
1069
745f7a8c
NTND
1070struct ref *fetch_pack(struct fetch_pack_args *args,
1071 int fd[], struct child_process *conn,
1072 const struct ref *ref,
1073 const char *dest,
f2db854d 1074 struct ref **sought, int nr_sought,
beea4152 1075 struct sha1_array *shallow,
745f7a8c
NTND
1076 char **pack_lockfile)
1077{
745f7a8c 1078 struct ref *ref_cpy;
beea4152 1079 struct shallow_info si;
745f7a8c
NTND
1080
1081 fetch_pack_setup();
f2db854d
JH
1082 if (nr_sought)
1083 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1084
1085 if (!ref) {
1086 packet_flush(fd[1]);
1dd73e20 1087 die(_("no matching remote head"));
745f7a8c 1088 }
beea4152
NTND
1089 prepare_shallow_info(&si, shallow);
1090 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1091 &si, pack_lockfile);
745f7a8c 1092 reprepare_packed_git();
4820a33b 1093 update_shallow(args, sought, nr_sought, &si);
beea4152 1094 clear_shallow_info(&si);
745f7a8c
NTND
1095 return ref_cpy;
1096}
e860d96b
MM
1097
1098int report_unmatched_refs(struct ref **sought, int nr_sought)
1099{
1100 int i, ret = 0;
1101
1102 for (i = 0; i < nr_sought; i++) {
1103 if (!sought[i] || sought[i]->matched)
1104 continue;
1105 error(_("no such remote ref %s"), sought[i]->name);
1106 ret = 1;
1107 }
1108 return ret;
1109}