for_each_alternate_ref: replace transport code with for-each-ref
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
697cc8ef 2#include "lockfile.h"
745f7a8c
NTND
3#include "refs.h"
4#include "pkt-line.h"
5#include "commit.h"
6#include "tag.h"
7#include "exec_cmd.h"
8#include "pack.h"
9#include "sideband.h"
10#include "fetch-pack.h"
11#include "remote.h"
12#include "run-command.h"
47a59185 13#include "connect.h"
745f7a8c
NTND
14#include "transport.h"
15#include "version.h"
099327b5 16#include "prio-queue.h"
beea4152 17#include "sha1-array.h"
745f7a8c
NTND
18
19static int transfer_unpack_limit = -1;
20static int fetch_unpack_limit = -1;
21static int unpack_limit = 100;
22static int prefer_ofs_delta = 1;
23static int no_done;
508ea882 24static int deepen_since_ok;
a45a2600 25static int deepen_not_ok;
745f7a8c
NTND
26static int fetch_fsck_objects = -1;
27static int transfer_fsck_objects = -1;
28static int agent_supported;
6035d6aa
NTND
29static struct lock_file shallow_lock;
30static const char *alternate_shallow_file;
745f7a8c 31
208acbfb 32/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
33#define COMPLETE (1U << 0)
34#define COMMON (1U << 1)
35#define COMMON_REF (1U << 2)
36#define SEEN (1U << 3)
37#define POPPED (1U << 4)
38
39static int marked;
40
41/*
42 * After sending this many "have"s if we do not get any new ACK , we
43 * give up traversing our history.
44 */
45#define MAX_IN_VAIN 256
46
099327b5 47static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
48static int non_common_revs, multi_ack, use_sideband;
49/* Allow specifying sha1 if it is a ref tip. */
50#define ALLOW_TIP_SHA1 01
68ee6289
FM
51/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
52#define ALLOW_REACHABLE_SHA1 02
7199c093 53static unsigned int allow_unadvertised_object_request;
745f7a8c 54
0d789a5b
NTND
55__attribute__((format (printf, 2, 3)))
56static inline void print_verbose(const struct fetch_pack_args *args,
57 const char *fmt, ...)
58{
59 va_list params;
60
61 if (!args->verbose)
62 return;
63
64 va_start(params, fmt);
65 vfprintf(stderr, fmt, params);
66 va_end(params);
67 fputc('\n', stderr);
68}
69
745f7a8c
NTND
70static void rev_list_push(struct commit *commit, int mark)
71{
72 if (!(commit->object.flags & mark)) {
73 commit->object.flags |= mark;
74
0064053b
JK
75 if (parse_commit(commit))
76 return;
745f7a8c 77
099327b5 78 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
79
80 if (!(commit->object.flags & COMMON))
81 non_common_revs++;
82 }
83}
84
c38cd1c8 85static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
745f7a8c
NTND
86{
87 struct object *o = deref_tag(parse_object(sha1), refname, 0);
88
89 if (o && o->type == OBJ_COMMIT)
90 rev_list_push((struct commit *)o, SEEN);
91
92 return 0;
93}
94
b1b49c6e
MH
95static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
96 int flag, void *cb_data)
745f7a8c 97{
c38cd1c8 98 return rev_list_insert_ref(refname, oid->hash);
b1b49c6e
MH
99}
100
c50fb6ce
MH
101static int clear_marks(const char *refname, const struct object_id *oid,
102 int flag, void *cb_data)
745f7a8c 103{
c50fb6ce 104 struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
745f7a8c
NTND
105
106 if (o && o->type == OBJ_COMMIT)
107 clear_commit_marks((struct commit *)o,
108 COMMON | COMMON_REF | SEEN | POPPED);
109 return 0;
110}
111
112/*
113 This function marks a rev and its ancestors as common.
114 In some cases, it is desirable to mark only the ancestors (for example
115 when only the server does not yet know that they are common).
116*/
117
118static void mark_common(struct commit *commit,
119 int ancestors_only, int dont_parse)
120{
121 if (commit != NULL && !(commit->object.flags & COMMON)) {
122 struct object *o = (struct object *)commit;
123
124 if (!ancestors_only)
125 o->flags |= COMMON;
126
127 if (!(o->flags & SEEN))
128 rev_list_push(commit, SEEN);
129 else {
130 struct commit_list *parents;
131
132 if (!ancestors_only && !(o->flags & POPPED))
133 non_common_revs--;
134 if (!o->parsed && !dont_parse)
135 if (parse_commit(commit))
136 return;
137
138 for (parents = commit->parents;
139 parents;
140 parents = parents->next)
141 mark_common(parents->item, 0, dont_parse);
142 }
143 }
144}
145
146/*
147 Get the next rev to send, ignoring the common.
148*/
149
150static const unsigned char *get_rev(void)
151{
152 struct commit *commit = NULL;
153
154 while (commit == NULL) {
155 unsigned int mark;
156 struct commit_list *parents;
157
099327b5 158 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
159 return NULL;
160
099327b5 161 commit = prio_queue_get(&rev_list);
0064053b 162 parse_commit(commit);
745f7a8c
NTND
163 parents = commit->parents;
164
165 commit->object.flags |= POPPED;
166 if (!(commit->object.flags & COMMON))
167 non_common_revs--;
168
169 if (commit->object.flags & COMMON) {
170 /* do not send "have", and ignore ancestors */
171 commit = NULL;
172 mark = COMMON | SEEN;
173 } else if (commit->object.flags & COMMON_REF)
174 /* send "have", and ignore ancestors */
175 mark = COMMON | SEEN;
176 else
177 /* send "have", also for its ancestors */
178 mark = SEEN;
179
180 while (parents) {
181 if (!(parents->item->object.flags & SEEN))
182 rev_list_push(parents->item, mark);
183 if (mark & COMMON)
184 mark_common(parents->item, 1, 0);
185 parents = parents->next;
186 }
745f7a8c
NTND
187 }
188
ed1c9977 189 return commit->object.oid.hash;
745f7a8c
NTND
190}
191
192enum ack_type {
193 NAK = 0,
194 ACK,
195 ACK_continue,
196 ACK_common,
197 ACK_ready
198};
199
200static void consume_shallow_list(struct fetch_pack_args *args, int fd)
201{
79891cb9 202 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
203 /* If we sent a depth we will get back "duplicate"
204 * shallow and unshallow commands every time there
205 * is a block of have lines exchanged.
206 */
74543a04
JK
207 char *line;
208 while ((line = packet_read_line(fd, NULL))) {
59556548 209 if (starts_with(line, "shallow "))
745f7a8c 210 continue;
59556548 211 if (starts_with(line, "unshallow "))
745f7a8c 212 continue;
1dd73e20 213 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
214 }
215 }
216}
217
745f7a8c
NTND
218static enum ack_type get_ack(int fd, unsigned char *result_sha1)
219{
74543a04
JK
220 int len;
221 char *line = packet_read_line(fd, &len);
82e56767 222 const char *arg;
745f7a8c
NTND
223
224 if (!len)
1dd73e20 225 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
226 if (!strcmp(line, "NAK"))
227 return NAK;
82e56767
JK
228 if (skip_prefix(line, "ACK ", &arg)) {
229 if (!get_sha1_hex(arg, result_sha1)) {
230 arg += 40;
231 len -= arg - line;
232 if (len < 1)
030e9dd6 233 return ACK;
82e56767 234 if (strstr(arg, "continue"))
745f7a8c 235 return ACK_continue;
82e56767 236 if (strstr(arg, "common"))
745f7a8c 237 return ACK_common;
82e56767 238 if (strstr(arg, "ready"))
745f7a8c
NTND
239 return ACK_ready;
240 return ACK;
241 }
242 }
dfbfb9f3 243 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
244}
245
246static void send_request(struct fetch_pack_args *args,
247 int fd, struct strbuf *buf)
248{
249 if (args->stateless_rpc) {
250 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
251 packet_flush(fd);
252 } else
cdf4fb8e 253 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
254}
255
2429d63a
JK
256static void insert_one_alternate_ref(const char *refname,
257 const struct object_id *oid,
258 void *unused)
745f7a8c 259{
2429d63a 260 rev_list_insert_ref(NULL, oid->hash);
745f7a8c
NTND
261}
262
263#define INITIAL_FLUSH 16
264#define PIPESAFE_FLUSH 32
da470981 265#define LARGE_FLUSH 16384
745f7a8c
NTND
266
267static int next_flush(struct fetch_pack_args *args, int count)
268{
da470981
JT
269 if (args->stateless_rpc) {
270 if (count < LARGE_FLUSH)
271 count <<= 1;
272 else
273 count = count * 11 / 10;
274 } else {
275 if (count < PIPESAFE_FLUSH)
276 count <<= 1;
277 else
278 count += PIPESAFE_FLUSH;
279 }
745f7a8c
NTND
280 return count;
281}
282
283static int find_common(struct fetch_pack_args *args,
284 int fd[2], unsigned char *result_sha1,
285 struct ref *refs)
286{
287 int fetching;
288 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
289 const unsigned char *sha1;
290 unsigned in_vain = 0;
291 int got_continue = 0;
292 int got_ready = 0;
293 struct strbuf req_buf = STRBUF_INIT;
294 size_t state_len = 0;
295
296 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 297 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
298 if (marked)
299 for_each_ref(clear_marks, NULL);
300 marked = 1;
301
b1b49c6e 302 for_each_ref(rev_list_insert_ref_oid, NULL);
745f7a8c
NTND
303 for_each_alternate_ref(insert_one_alternate_ref, NULL);
304
305 fetching = 0;
306 for ( ; refs ; refs = refs->next) {
f4e54d02 307 unsigned char *remote = refs->old_oid.hash;
745f7a8c
NTND
308 const char *remote_hex;
309 struct object *o;
310
311 /*
312 * If that object is complete (i.e. it is an ancestor of a
313 * local ref), we tell them we have it but do not have to
314 * tell them about its ancestors, which they already know
315 * about.
316 *
317 * We use lookup_object here because we are only
318 * interested in the case we *know* the object is
319 * reachable and we have already scanned it.
320 */
321 if (((o = lookup_object(remote)) != NULL) &&
322 (o->flags & COMPLETE)) {
323 continue;
324 }
325
326 remote_hex = sha1_to_hex(remote);
327 if (!fetching) {
328 struct strbuf c = STRBUF_INIT;
329 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
330 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
331 if (no_done) strbuf_addstr(&c, " no-done");
332 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
333 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 334 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
335 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
336 if (args->no_progress) strbuf_addstr(&c, " no-progress");
337 if (args->include_tag) strbuf_addstr(&c, " include-tag");
338 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 339 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 340 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
341 if (agent_supported) strbuf_addf(&c, " agent=%s",
342 git_user_agent_sanitized());
343 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
344 strbuf_release(&c);
345 } else
346 packet_buf_write(&req_buf, "want %s\n", remote_hex);
347 fetching++;
348 }
349
350 if (!fetching) {
351 strbuf_release(&req_buf);
352 packet_flush(fd[1]);
353 return 1;
354 }
355
356 if (is_repository_shallow())
1a30f5a2 357 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
358 if (args->depth > 0)
359 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882
NTND
360 if (args->deepen_since) {
361 unsigned long max_age = approxidate(args->deepen_since);
362 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
363 }
a45a2600
NTND
364 if (args->deepen_not) {
365 int i;
366 for (i = 0; i < args->deepen_not->nr; i++) {
367 struct string_list_item *s = args->deepen_not->items + i;
368 packet_buf_write(&req_buf, "deepen-not %s", s->string);
369 }
370 }
745f7a8c
NTND
371 packet_buf_flush(&req_buf);
372 state_len = req_buf.len;
373
79891cb9 374 if (args->deepen) {
74543a04 375 char *line;
ae021d87 376 const char *arg;
745f7a8c
NTND
377 unsigned char sha1[20];
378
379 send_request(args, fd[1], &req_buf);
74543a04 380 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87
JK
381 if (skip_prefix(line, "shallow ", &arg)) {
382 if (get_sha1_hex(arg, sha1))
1dd73e20 383 die(_("invalid shallow line: %s"), line);
745f7a8c
NTND
384 register_shallow(sha1);
385 continue;
386 }
ae021d87
JK
387 if (skip_prefix(line, "unshallow ", &arg)) {
388 if (get_sha1_hex(arg, sha1))
1dd73e20 389 die(_("invalid unshallow line: %s"), line);
745f7a8c 390 if (!lookup_object(sha1))
1dd73e20 391 die(_("object not found: %s"), line);
745f7a8c
NTND
392 /* make sure that it is parsed as shallow */
393 if (!parse_object(sha1))
1dd73e20 394 die(_("error in object: %s"), line);
745f7a8c 395 if (unregister_shallow(sha1))
1dd73e20 396 die(_("no shallow found: %s"), line);
745f7a8c
NTND
397 continue;
398 }
1dd73e20 399 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
400 }
401 } else if (!args->stateless_rpc)
402 send_request(args, fd[1], &req_buf);
403
404 if (!args->stateless_rpc) {
405 /* If we aren't using the stateless-rpc interface
406 * we don't need to retain the headers.
407 */
408 strbuf_setlen(&req_buf, 0);
409 state_len = 0;
410 }
411
412 flushes = 0;
413 retval = -1;
414 while ((sha1 = get_rev())) {
415 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
0d789a5b 416 print_verbose(args, "have %s", sha1_to_hex(sha1));
745f7a8c
NTND
417 in_vain++;
418 if (flush_at <= ++count) {
419 int ack;
420
421 packet_buf_flush(&req_buf);
422 send_request(args, fd[1], &req_buf);
423 strbuf_setlen(&req_buf, state_len);
424 flushes++;
425 flush_at = next_flush(args, count);
426
427 /*
428 * We keep one window "ahead" of the other side, and
429 * will wait for an ACK only on the next one
430 */
431 if (!args->stateless_rpc && count == INITIAL_FLUSH)
432 continue;
433
434 consume_shallow_list(args, fd[0]);
435 do {
436 ack = get_ack(fd[0], result_sha1);
0d789a5b 437 if (ack)
1dd73e20
NTND
438 print_verbose(args, _("got %s %d %s"), "ack",
439 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
440 switch (ack) {
441 case ACK:
442 flushes = 0;
443 multi_ack = 0;
444 retval = 0;
445 goto done;
446 case ACK_common:
447 case ACK_ready:
448 case ACK_continue: {
449 struct commit *commit =
450 lookup_commit(result_sha1);
451 if (!commit)
1dd73e20 452 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
745f7a8c
NTND
453 if (args->stateless_rpc
454 && ack == ACK_common
455 && !(commit->object.flags & COMMON)) {
456 /* We need to replay the have for this object
457 * on the next RPC request so the peer knows
458 * it is in common with us.
459 */
460 const char *hex = sha1_to_hex(result_sha1);
461 packet_buf_write(&req_buf, "have %s\n", hex);
462 state_len = req_buf.len;
06b3d386
JT
463 /*
464 * Reset in_vain because an ack
465 * for this commit has not been
466 * seen.
467 */
468 in_vain = 0;
469 } else if (!args->stateless_rpc
470 || ack != ACK_common)
471 in_vain = 0;
745f7a8c
NTND
472 mark_common(commit, 0, 1);
473 retval = 0;
745f7a8c
NTND
474 got_continue = 1;
475 if (ack == ACK_ready) {
099327b5 476 clear_prio_queue(&rev_list);
745f7a8c
NTND
477 got_ready = 1;
478 }
479 break;
480 }
481 }
482 } while (ack);
483 flushes--;
484 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 485 print_verbose(args, _("giving up"));
745f7a8c
NTND
486 break; /* give up */
487 }
488 }
489 }
490done:
491 if (!got_ready || !no_done) {
492 packet_buf_write(&req_buf, "done\n");
493 send_request(args, fd[1], &req_buf);
494 }
1dd73e20 495 print_verbose(args, _("done"));
745f7a8c
NTND
496 if (retval != 0) {
497 multi_ack = 0;
498 flushes++;
499 }
500 strbuf_release(&req_buf);
501
ff62eca7
NTND
502 if (!got_ready || !no_done)
503 consume_shallow_list(args, fd[0]);
745f7a8c
NTND
504 while (flushes || multi_ack) {
505 int ack = get_ack(fd[0], result_sha1);
506 if (ack) {
1dd73e20
NTND
507 print_verbose(args, _("got %s (%d) %s"), "ack",
508 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
509 if (ack == ACK)
510 return 0;
511 multi_ack = 1;
512 continue;
513 }
514 flushes--;
515 }
516 /* it is no error to fetch into a completely empty repo */
517 return count ? retval : 0;
518}
519
520static struct commit_list *complete;
521
6e20a51a 522static int mark_complete(const unsigned char *sha1)
745f7a8c
NTND
523{
524 struct object *o = parse_object(sha1);
525
526 while (o && o->type == OBJ_TAG) {
527 struct tag *t = (struct tag *) o;
528 if (!t->tagged)
529 break; /* broken repository */
530 o->flags |= COMPLETE;
ed1c9977 531 o = parse_object(t->tagged->oid.hash);
745f7a8c
NTND
532 }
533 if (o && o->type == OBJ_COMMIT) {
534 struct commit *commit = (struct commit *)o;
535 if (!(commit->object.flags & COMPLETE)) {
536 commit->object.flags |= COMPLETE;
16445242 537 commit_list_insert(commit, &complete);
745f7a8c
NTND
538 }
539 }
540 return 0;
541}
542
f8ee4d85
MH
543static int mark_complete_oid(const char *refname, const struct object_id *oid,
544 int flag, void *cb_data)
545{
6e20a51a 546 return mark_complete(oid->hash);
f8ee4d85
MH
547}
548
745f7a8c
NTND
549static void mark_recent_complete_commits(struct fetch_pack_args *args,
550 unsigned long cutoff)
551{
552 while (complete && cutoff <= complete->item->date) {
1dd73e20 553 print_verbose(args, _("Marking %s as complete"),
0d789a5b 554 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
555 pop_most_recent_commit(&complete, COMPLETE);
556 }
557}
558
745f7a8c 559static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
560 struct ref **refs,
561 struct ref **sought, int nr_sought)
745f7a8c
NTND
562{
563 struct ref *newlist = NULL;
564 struct ref **newtail = &newlist;
565 struct ref *ref, *next;
f2db854d 566 int i;
745f7a8c 567
f2db854d 568 i = 0;
745f7a8c
NTND
569 for (ref = *refs; ref; ref = next) {
570 int keep = 0;
571 next = ref->next;
f2db854d 572
50e19a83 573 if (starts_with(ref->name, "refs/") &&
4c224081 574 check_refname_format(ref->name, 0))
745f7a8c
NTND
575 ; /* trash */
576 else {
f2db854d
JH
577 while (i < nr_sought) {
578 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
579 if (cmp < 0)
580 break; /* definitely do not have it */
581 else if (cmp == 0) {
582 keep = 1; /* definitely have it */
f2db854d 583 sought[i]->matched = 1;
745f7a8c 584 }
f2db854d 585 i++;
745f7a8c
NTND
586 }
587 }
588
f2db854d 589 if (!keep && args->fetch_all &&
79891cb9 590 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
591 keep = 1;
592
593 if (keep) {
594 *newtail = ref;
595 ref->next = NULL;
596 newtail = &ref->next;
597 } else {
598 free(ref);
599 }
600 }
601
6e7b66ee 602 /* Append unmatched requests to the list */
68ee6289
FM
603 if ((allow_unadvertised_object_request &
604 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
6e7b66ee 605 for (i = 0; i < nr_sought; i++) {
b7916422
JK
606 unsigned char sha1[20];
607
6e7b66ee
JH
608 ref = sought[i];
609 if (ref->matched)
610 continue;
b7916422
JK
611 if (get_sha1_hex(ref->name, sha1) ||
612 ref->name[40] != '\0' ||
f4e54d02 613 hashcmp(sha1, ref->old_oid.hash))
6e7b66ee
JH
614 continue;
615
616 ref->matched = 1;
c3c17bf1
JK
617 *newtail = copy_ref(ref);
618 newtail = &(*newtail)->next;
6e7b66ee
JH
619 }
620 }
745f7a8c
NTND
621 *refs = newlist;
622}
623
2429d63a
JK
624static void mark_alternate_complete(const char *refname,
625 const struct object_id *oid,
626 void *unused)
745f7a8c 627{
2429d63a 628 mark_complete(oid->hash);
745f7a8c
NTND
629}
630
631static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
632 struct ref **refs,
633 struct ref **sought, int nr_sought)
745f7a8c
NTND
634{
635 struct ref *ref;
636 int retval;
637 unsigned long cutoff = 0;
638
639 save_commit_buffer = 0;
640
641 for (ref = *refs; ref; ref = ref->next) {
642 struct object *o;
643
f4e54d02 644 if (!has_object_file(&ref->old_oid))
012a1bb5
JH
645 continue;
646
f4e54d02 647 o = parse_object(ref->old_oid.hash);
745f7a8c
NTND
648 if (!o)
649 continue;
650
651 /* We already have it -- which may mean that we were
652 * in sync with the other side at some time after
653 * that (it is OK if we guess wrong here).
654 */
655 if (o->type == OBJ_COMMIT) {
656 struct commit *commit = (struct commit *)o;
657 if (!cutoff || cutoff < commit->date)
658 cutoff = commit->date;
659 }
660 }
661
79891cb9 662 if (!args->deepen) {
f8ee4d85 663 for_each_ref(mark_complete_oid, NULL);
745f7a8c 664 for_each_alternate_ref(mark_alternate_complete, NULL);
16445242 665 commit_list_sort_by_date(&complete);
745f7a8c
NTND
666 if (cutoff)
667 mark_recent_complete_commits(args, cutoff);
668 }
669
670 /*
671 * Mark all complete remote refs as common refs.
672 * Don't mark them common yet; the server has to be told so first.
673 */
674 for (ref = *refs; ref; ref = ref->next) {
f4e54d02 675 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
745f7a8c
NTND
676 NULL, 0);
677
678 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
679 continue;
680
681 if (!(o->flags & SEEN)) {
682 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
683
684 mark_common((struct commit *)o, 1, 1);
685 }
686 }
687
f2db854d 688 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
689
690 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
f4e54d02 691 const unsigned char *remote = ref->old_oid.hash;
745f7a8c
NTND
692 struct object *o;
693
694 o = lookup_object(remote);
695 if (!o || !(o->flags & COMPLETE)) {
696 retval = 0;
0d789a5b
NTND
697 print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
698 ref->name);
745f7a8c
NTND
699 continue;
700 }
1dd73e20 701 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
0d789a5b 702 ref->name);
745f7a8c
NTND
703 }
704 return retval;
705}
706
707static int sideband_demux(int in, int out, void *data)
708{
709 int *xd = data;
9ff18faf 710 int ret;
745f7a8c 711
9ff18faf 712 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
713 close(out);
714 return ret;
715}
716
717static int get_pack(struct fetch_pack_args *args,
718 int xd[2], char **pack_lockfile)
719{
720 struct async demux;
745f7a8c 721 int do_keep = args->keep_pack;
984a43b9
JK
722 const char *cmd_name;
723 struct pack_header header;
724 int pass_header = 0;
d3180279 725 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 726 int ret;
745f7a8c
NTND
727
728 memset(&demux, 0, sizeof(demux));
729 if (use_sideband) {
730 /* xd[] is talking with upload-pack; subprocess reads from
731 * xd[0], spits out band#2 to stderr, and feeds us band#1
732 * through demux->out.
733 */
734 demux.proc = sideband_demux;
735 demux.data = xd;
736 demux.out = -1;
df857572 737 demux.isolate_sigpipe = 1;
745f7a8c 738 if (start_async(&demux))
1dd73e20 739 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
740 }
741 else
742 demux.out = xd[0];
743
745f7a8c 744 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
745
746 if (read_pack_header(demux.out, &header))
1dd73e20 747 die(_("protocol error: bad pack header"));
984a43b9 748 pass_header = 1;
745f7a8c
NTND
749 if (ntohl(header.hdr_entries) < unpack_limit)
750 do_keep = 0;
751 else
752 do_keep = 1;
753 }
754
6035d6aa 755 if (alternate_shallow_file) {
984a43b9
JK
756 argv_array_push(&cmd.args, "--shallow-file");
757 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
758 }
759
745f7a8c
NTND
760 if (do_keep) {
761 if (pack_lockfile)
762 cmd.out = -1;
984a43b9
JK
763 cmd_name = "index-pack";
764 argv_array_push(&cmd.args, cmd_name);
765 argv_array_push(&cmd.args, "--stdin");
745f7a8c 766 if (!args->quiet && !args->no_progress)
984a43b9 767 argv_array_push(&cmd.args, "-v");
745f7a8c 768 if (args->use_thin_pack)
984a43b9 769 argv_array_push(&cmd.args, "--fix-thin");
745f7a8c 770 if (args->lock_pack || unpack_limit) {
984a43b9
JK
771 char hostname[256];
772 if (gethostname(hostname, sizeof(hostname)))
773 xsnprintf(hostname, sizeof(hostname), "localhost");
774 argv_array_pushf(&cmd.args,
775 "--keep=fetch-pack %"PRIuMAX " on %s",
776 (uintmax_t)getpid(), hostname);
745f7a8c 777 }
c6807a40 778 if (args->check_self_contained_and_connected)
984a43b9 779 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
745f7a8c
NTND
780 }
781 else {
984a43b9
JK
782 cmd_name = "unpack-objects";
783 argv_array_push(&cmd.args, cmd_name);
745f7a8c 784 if (args->quiet || args->no_progress)
984a43b9 785 argv_array_push(&cmd.args, "-q");
c6807a40 786 args->check_self_contained_and_connected = 0;
745f7a8c 787 }
984a43b9
JK
788
789 if (pass_header)
790 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
791 ntohl(header.hdr_version),
792 ntohl(header.hdr_entries));
745f7a8c
NTND
793 if (fetch_fsck_objects >= 0
794 ? fetch_fsck_objects
795 : transfer_fsck_objects >= 0
796 ? transfer_fsck_objects
797 : 0)
984a43b9 798 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
799
800 cmd.in = demux.out;
801 cmd.git_cmd = 1;
802 if (start_command(&cmd))
1dd73e20 803 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
804 if (do_keep && pack_lockfile) {
805 *pack_lockfile = index_pack_lockfile(cmd.out);
806 close(cmd.out);
807 }
808
37cb1dd6
JL
809 if (!use_sideband)
810 /* Closed by start_command() */
811 xd[0] = -1;
812
c6807a40
NTND
813 ret = finish_command(&cmd);
814 if (!ret || (args->check_self_contained_and_connected && ret == 1))
815 args->self_contained_and_connected =
816 args->check_self_contained_and_connected &&
817 ret == 0;
818 else
1dd73e20 819 die(_("%s failed"), cmd_name);
745f7a8c 820 if (use_sideband && finish_async(&demux))
1dd73e20 821 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
822 return 0;
823}
824
f2db854d
JH
825static int cmp_ref_by_name(const void *a_, const void *b_)
826{
827 const struct ref *a = *((const struct ref **)a_);
828 const struct ref *b = *((const struct ref **)b_);
829 return strcmp(a->name, b->name);
830}
831
745f7a8c
NTND
832static struct ref *do_fetch_pack(struct fetch_pack_args *args,
833 int fd[2],
834 const struct ref *orig_ref,
f2db854d 835 struct ref **sought, int nr_sought,
beea4152 836 struct shallow_info *si,
745f7a8c
NTND
837 char **pack_lockfile)
838{
839 struct ref *ref = copy_ref_list(orig_ref);
840 unsigned char sha1[20];
841 const char *agent_feature;
842 int agent_len;
843
844 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 845 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 846
eb86a507 847 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 848 die(_("Server does not support shallow clients"));
a45a2600 849 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 850 args->deepen = 1;
745f7a8c 851 if (server_supports("multi_ack_detailed")) {
1dd73e20 852 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
853 multi_ack = 2;
854 if (server_supports("no-done")) {
1dd73e20 855 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
856 if (args->stateless_rpc)
857 no_done = 1;
858 }
859 }
860 else if (server_supports("multi_ack")) {
1dd73e20 861 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
862 multi_ack = 1;
863 }
864 if (server_supports("side-band-64k")) {
1dd73e20 865 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
866 use_sideband = 2;
867 }
868 else if (server_supports("side-band")) {
1dd73e20 869 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
870 use_sideband = 1;
871 }
6e7b66ee 872 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 873 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 874 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 875 }
68ee6289 876 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 877 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
878 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
879 }
745f7a8c
NTND
880 if (!server_supports("thin-pack"))
881 args->use_thin_pack = 0;
882 if (!server_supports("no-progress"))
883 args->no_progress = 0;
884 if (!server_supports("include-tag"))
885 args->include_tag = 0;
0d789a5b 886 if (server_supports("ofs-delta"))
1dd73e20 887 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 888 else
745f7a8c
NTND
889 prefer_ofs_delta = 0;
890
891 if ((agent_feature = server_feature_value("agent", &agent_len))) {
892 agent_supported = 1;
0d789a5b 893 if (agent_len)
1dd73e20 894 print_verbose(args, _("Server version is %.*s"),
0d789a5b 895 agent_len, agent_feature);
745f7a8c 896 }
508ea882
NTND
897 if (server_supports("deepen-since"))
898 deepen_since_ok = 1;
899 else if (args->deepen_since)
900 die(_("Server does not support --shallow-since"));
a45a2600
NTND
901 if (server_supports("deepen-not"))
902 deepen_not_ok = 1;
903 else if (args->deepen_not)
904 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
905 if (!server_supports("deepen-relative") && args->deepen_relative)
906 die(_("Server does not support --deepen"));
745f7a8c 907
f2db854d 908 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
909 packet_flush(fd[1]);
910 goto all_done;
911 }
912 if (find_common(args, fd, sha1, ref) < 0)
913 if (!args->keep_pack)
914 /* When cloning, it is not unusual to have
915 * no common commit.
916 */
1dd73e20 917 warning(_("no common commits"));
745f7a8c
NTND
918
919 if (args->stateless_rpc)
920 packet_flush(fd[1]);
79891cb9 921 if (args->deepen)
1a30f5a2
NTND
922 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
923 NULL);
4820a33b 924 else if (si->nr_ours || si->nr_theirs)
beea4152 925 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
926 else
927 alternate_shallow_file = NULL;
745f7a8c 928 if (get_pack(args, fd, pack_lockfile))
1dd73e20 929 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
930
931 all_done:
932 return ref;
933}
934
f44af51d 935static void fetch_pack_config(void)
745f7a8c 936{
f44af51d
TA
937 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
938 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
939 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
940 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
941 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 942
f44af51d 943 git_config(git_default_config, NULL);
745f7a8c
NTND
944}
945
745f7a8c
NTND
946static void fetch_pack_setup(void)
947{
948 static int did_setup;
949 if (did_setup)
950 return;
f44af51d 951 fetch_pack_config();
745f7a8c
NTND
952 if (0 <= transfer_unpack_limit)
953 unpack_limit = transfer_unpack_limit;
954 else if (0 <= fetch_unpack_limit)
955 unpack_limit = fetch_unpack_limit;
956 did_setup = 1;
957}
958
f2db854d
JH
959static int remove_duplicates_in_refs(struct ref **ref, int nr)
960{
961 struct string_list names = STRING_LIST_INIT_NODUP;
962 int src, dst;
963
964 for (src = dst = 0; src < nr; src++) {
965 struct string_list_item *item;
966 item = string_list_insert(&names, ref[src]->name);
967 if (item->util)
968 continue; /* already have it */
969 item->util = ref[src];
970 if (src != dst)
971 ref[dst] = ref[src];
972 dst++;
973 }
974 for (src = dst; src < nr; src++)
975 ref[src] = NULL;
976 string_list_clear(&names, 0);
977 return dst;
978}
979
beea4152 980static void update_shallow(struct fetch_pack_args *args,
4820a33b 981 struct ref **sought, int nr_sought,
beea4152 982 struct shallow_info *si)
a796ccee 983{
4820a33b
NTND
984 struct sha1_array ref = SHA1_ARRAY_INIT;
985 int *status;
beea4152
NTND
986 int i;
987
79891cb9 988 if (args->deepen && alternate_shallow_file) {
a796ccee 989 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 990 unlink_or_warn(git_path_shallow());
a796ccee
NTND
991 rollback_lock_file(&shallow_lock);
992 } else
993 commit_lock_file(&shallow_lock);
994 return;
995 }
beea4152
NTND
996
997 if (!si->shallow || !si->shallow->nr)
998 return;
999
beea4152
NTND
1000 if (args->cloning) {
1001 /*
1002 * remote is shallow, but this is a clone, there are
1003 * no objects in repo to worry about. Accept any
1004 * shallow points that exist in the pack (iow in repo
1005 * after get_pack() and reprepare_packed_git())
1006 */
1007 struct sha1_array extra = SHA1_ARRAY_INIT;
1008 unsigned char (*sha1)[20] = si->shallow->sha1;
1009 for (i = 0; i < si->shallow->nr; i++)
1010 if (has_sha1_file(sha1[i]))
1011 sha1_array_append(&extra, sha1[i]);
1012 if (extra.nr) {
1013 setup_alternate_shallow(&shallow_lock,
1014 &alternate_shallow_file,
1015 &extra);
1016 commit_lock_file(&shallow_lock);
1017 }
1018 sha1_array_clear(&extra);
1019 return;
1020 }
4820a33b
NTND
1021
1022 if (!si->nr_ours && !si->nr_theirs)
1023 return;
1024
1025 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1026 if (!si->nr_ours && !si->nr_theirs)
1027 return;
1028 for (i = 0; i < nr_sought; i++)
f4e54d02 1029 sha1_array_append(&ref, sought[i]->old_oid.hash);
4820a33b
NTND
1030 si->ref = &ref;
1031
48d25cae
NTND
1032 if (args->update_shallow) {
1033 /*
1034 * remote is also shallow, .git/shallow may be updated
1035 * so all refs can be accepted. Make sure we only add
1036 * shallow roots that are actually reachable from new
1037 * refs.
1038 */
1039 struct sha1_array extra = SHA1_ARRAY_INIT;
1040 unsigned char (*sha1)[20] = si->shallow->sha1;
1041 assign_shallow_commits_to_refs(si, NULL, NULL);
1042 if (!si->nr_ours && !si->nr_theirs) {
1043 sha1_array_clear(&ref);
1044 return;
1045 }
1046 for (i = 0; i < si->nr_ours; i++)
1047 sha1_array_append(&extra, sha1[si->ours[i]]);
1048 for (i = 0; i < si->nr_theirs; i++)
1049 sha1_array_append(&extra, sha1[si->theirs[i]]);
1050 setup_alternate_shallow(&shallow_lock,
1051 &alternate_shallow_file,
1052 &extra);
1053 commit_lock_file(&shallow_lock);
1054 sha1_array_clear(&extra);
1055 sha1_array_clear(&ref);
1056 return;
1057 }
1058
4820a33b
NTND
1059 /*
1060 * remote is also shallow, check what ref is safe to update
1061 * without updating .git/shallow
1062 */
1063 status = xcalloc(nr_sought, sizeof(*status));
1064 assign_shallow_commits_to_refs(si, NULL, status);
1065 if (si->nr_ours || si->nr_theirs) {
1066 for (i = 0; i < nr_sought; i++)
1067 if (status[i])
1068 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1069 }
1070 free(status);
1071 sha1_array_clear(&ref);
a796ccee
NTND
1072}
1073
745f7a8c
NTND
1074struct ref *fetch_pack(struct fetch_pack_args *args,
1075 int fd[], struct child_process *conn,
1076 const struct ref *ref,
1077 const char *dest,
f2db854d 1078 struct ref **sought, int nr_sought,
beea4152 1079 struct sha1_array *shallow,
745f7a8c
NTND
1080 char **pack_lockfile)
1081{
745f7a8c 1082 struct ref *ref_cpy;
beea4152 1083 struct shallow_info si;
745f7a8c
NTND
1084
1085 fetch_pack_setup();
f2db854d
JH
1086 if (nr_sought)
1087 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1088
1089 if (!ref) {
1090 packet_flush(fd[1]);
1dd73e20 1091 die(_("no matching remote head"));
745f7a8c 1092 }
beea4152
NTND
1093 prepare_shallow_info(&si, shallow);
1094 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1095 &si, pack_lockfile);
745f7a8c 1096 reprepare_packed_git();
4820a33b 1097 update_shallow(args, sought, nr_sought, &si);
beea4152 1098 clear_shallow_info(&si);
745f7a8c
NTND
1099 return ref_cpy;
1100}