upload-pack: add get_reachable_list()
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
697cc8ef 2#include "lockfile.h"
745f7a8c
NTND
3#include "refs.h"
4#include "pkt-line.h"
5#include "commit.h"
6#include "tag.h"
7#include "exec_cmd.h"
8#include "pack.h"
9#include "sideband.h"
10#include "fetch-pack.h"
11#include "remote.h"
12#include "run-command.h"
47a59185 13#include "connect.h"
745f7a8c
NTND
14#include "transport.h"
15#include "version.h"
099327b5 16#include "prio-queue.h"
beea4152 17#include "sha1-array.h"
745f7a8c
NTND
18
19static int transfer_unpack_limit = -1;
20static int fetch_unpack_limit = -1;
21static int unpack_limit = 100;
22static int prefer_ofs_delta = 1;
23static int no_done;
508ea882 24static int deepen_since_ok;
a45a2600 25static int deepen_not_ok;
745f7a8c
NTND
26static int fetch_fsck_objects = -1;
27static int transfer_fsck_objects = -1;
28static int agent_supported;
6035d6aa
NTND
29static struct lock_file shallow_lock;
30static const char *alternate_shallow_file;
745f7a8c 31
208acbfb 32/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
33#define COMPLETE (1U << 0)
34#define COMMON (1U << 1)
35#define COMMON_REF (1U << 2)
36#define SEEN (1U << 3)
37#define POPPED (1U << 4)
38
39static int marked;
40
41/*
42 * After sending this many "have"s if we do not get any new ACK , we
43 * give up traversing our history.
44 */
45#define MAX_IN_VAIN 256
46
099327b5 47static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
48static int non_common_revs, multi_ack, use_sideband;
49/* Allow specifying sha1 if it is a ref tip. */
50#define ALLOW_TIP_SHA1 01
68ee6289
FM
51/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
52#define ALLOW_REACHABLE_SHA1 02
7199c093 53static unsigned int allow_unadvertised_object_request;
745f7a8c 54
0d789a5b
NTND
55__attribute__((format (printf, 2, 3)))
56static inline void print_verbose(const struct fetch_pack_args *args,
57 const char *fmt, ...)
58{
59 va_list params;
60
61 if (!args->verbose)
62 return;
63
64 va_start(params, fmt);
65 vfprintf(stderr, fmt, params);
66 va_end(params);
67 fputc('\n', stderr);
68}
69
745f7a8c
NTND
70static void rev_list_push(struct commit *commit, int mark)
71{
72 if (!(commit->object.flags & mark)) {
73 commit->object.flags |= mark;
74
0064053b
JK
75 if (parse_commit(commit))
76 return;
745f7a8c 77
099327b5 78 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
79
80 if (!(commit->object.flags & COMMON))
81 non_common_revs++;
82 }
83}
84
c38cd1c8 85static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
745f7a8c
NTND
86{
87 struct object *o = deref_tag(parse_object(sha1), refname, 0);
88
89 if (o && o->type == OBJ_COMMIT)
90 rev_list_push((struct commit *)o, SEEN);
91
92 return 0;
93}
94
b1b49c6e
MH
95static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
96 int flag, void *cb_data)
745f7a8c 97{
c38cd1c8 98 return rev_list_insert_ref(refname, oid->hash);
b1b49c6e
MH
99}
100
c50fb6ce
MH
101static int clear_marks(const char *refname, const struct object_id *oid,
102 int flag, void *cb_data)
745f7a8c 103{
c50fb6ce 104 struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
745f7a8c
NTND
105
106 if (o && o->type == OBJ_COMMIT)
107 clear_commit_marks((struct commit *)o,
108 COMMON | COMMON_REF | SEEN | POPPED);
109 return 0;
110}
111
112/*
113 This function marks a rev and its ancestors as common.
114 In some cases, it is desirable to mark only the ancestors (for example
115 when only the server does not yet know that they are common).
116*/
117
118static void mark_common(struct commit *commit,
119 int ancestors_only, int dont_parse)
120{
121 if (commit != NULL && !(commit->object.flags & COMMON)) {
122 struct object *o = (struct object *)commit;
123
124 if (!ancestors_only)
125 o->flags |= COMMON;
126
127 if (!(o->flags & SEEN))
128 rev_list_push(commit, SEEN);
129 else {
130 struct commit_list *parents;
131
132 if (!ancestors_only && !(o->flags & POPPED))
133 non_common_revs--;
134 if (!o->parsed && !dont_parse)
135 if (parse_commit(commit))
136 return;
137
138 for (parents = commit->parents;
139 parents;
140 parents = parents->next)
141 mark_common(parents->item, 0, dont_parse);
142 }
143 }
144}
145
146/*
147 Get the next rev to send, ignoring the common.
148*/
149
150static const unsigned char *get_rev(void)
151{
152 struct commit *commit = NULL;
153
154 while (commit == NULL) {
155 unsigned int mark;
156 struct commit_list *parents;
157
099327b5 158 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
159 return NULL;
160
099327b5 161 commit = prio_queue_get(&rev_list);
0064053b 162 parse_commit(commit);
745f7a8c
NTND
163 parents = commit->parents;
164
165 commit->object.flags |= POPPED;
166 if (!(commit->object.flags & COMMON))
167 non_common_revs--;
168
169 if (commit->object.flags & COMMON) {
170 /* do not send "have", and ignore ancestors */
171 commit = NULL;
172 mark = COMMON | SEEN;
173 } else if (commit->object.flags & COMMON_REF)
174 /* send "have", and ignore ancestors */
175 mark = COMMON | SEEN;
176 else
177 /* send "have", also for its ancestors */
178 mark = SEEN;
179
180 while (parents) {
181 if (!(parents->item->object.flags & SEEN))
182 rev_list_push(parents->item, mark);
183 if (mark & COMMON)
184 mark_common(parents->item, 1, 0);
185 parents = parents->next;
186 }
745f7a8c
NTND
187 }
188
ed1c9977 189 return commit->object.oid.hash;
745f7a8c
NTND
190}
191
192enum ack_type {
193 NAK = 0,
194 ACK,
195 ACK_continue,
196 ACK_common,
197 ACK_ready
198};
199
200static void consume_shallow_list(struct fetch_pack_args *args, int fd)
201{
79891cb9 202 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
203 /* If we sent a depth we will get back "duplicate"
204 * shallow and unshallow commands every time there
205 * is a block of have lines exchanged.
206 */
74543a04
JK
207 char *line;
208 while ((line = packet_read_line(fd, NULL))) {
59556548 209 if (starts_with(line, "shallow "))
745f7a8c 210 continue;
59556548 211 if (starts_with(line, "unshallow "))
745f7a8c 212 continue;
1dd73e20 213 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
214 }
215 }
216}
217
745f7a8c
NTND
218static enum ack_type get_ack(int fd, unsigned char *result_sha1)
219{
74543a04
JK
220 int len;
221 char *line = packet_read_line(fd, &len);
82e56767 222 const char *arg;
745f7a8c
NTND
223
224 if (!len)
1dd73e20 225 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
226 if (!strcmp(line, "NAK"))
227 return NAK;
82e56767
JK
228 if (skip_prefix(line, "ACK ", &arg)) {
229 if (!get_sha1_hex(arg, result_sha1)) {
230 arg += 40;
231 len -= arg - line;
232 if (len < 1)
030e9dd6 233 return ACK;
82e56767 234 if (strstr(arg, "continue"))
745f7a8c 235 return ACK_continue;
82e56767 236 if (strstr(arg, "common"))
745f7a8c 237 return ACK_common;
82e56767 238 if (strstr(arg, "ready"))
745f7a8c
NTND
239 return ACK_ready;
240 return ACK;
241 }
242 }
1dd73e20 243 die(_("git fetch_pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
244}
245
246static void send_request(struct fetch_pack_args *args,
247 int fd, struct strbuf *buf)
248{
249 if (args->stateless_rpc) {
250 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
251 packet_flush(fd);
252 } else
cdf4fb8e 253 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
254}
255
256static void insert_one_alternate_ref(const struct ref *ref, void *unused)
257{
f4e54d02 258 rev_list_insert_ref(NULL, ref->old_oid.hash);
745f7a8c
NTND
259}
260
261#define INITIAL_FLUSH 16
262#define PIPESAFE_FLUSH 32
263#define LARGE_FLUSH 1024
264
265static int next_flush(struct fetch_pack_args *args, int count)
266{
267 int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
268
269 if (count < flush_limit)
270 count <<= 1;
271 else
272 count += flush_limit;
273 return count;
274}
275
276static int find_common(struct fetch_pack_args *args,
277 int fd[2], unsigned char *result_sha1,
278 struct ref *refs)
279{
280 int fetching;
281 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
282 const unsigned char *sha1;
283 unsigned in_vain = 0;
284 int got_continue = 0;
285 int got_ready = 0;
286 struct strbuf req_buf = STRBUF_INIT;
287 size_t state_len = 0;
288
289 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 290 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
291 if (marked)
292 for_each_ref(clear_marks, NULL);
293 marked = 1;
294
b1b49c6e 295 for_each_ref(rev_list_insert_ref_oid, NULL);
745f7a8c
NTND
296 for_each_alternate_ref(insert_one_alternate_ref, NULL);
297
298 fetching = 0;
299 for ( ; refs ; refs = refs->next) {
f4e54d02 300 unsigned char *remote = refs->old_oid.hash;
745f7a8c
NTND
301 const char *remote_hex;
302 struct object *o;
303
304 /*
305 * If that object is complete (i.e. it is an ancestor of a
306 * local ref), we tell them we have it but do not have to
307 * tell them about its ancestors, which they already know
308 * about.
309 *
310 * We use lookup_object here because we are only
311 * interested in the case we *know* the object is
312 * reachable and we have already scanned it.
313 */
314 if (((o = lookup_object(remote)) != NULL) &&
315 (o->flags & COMPLETE)) {
316 continue;
317 }
318
319 remote_hex = sha1_to_hex(remote);
320 if (!fetching) {
321 struct strbuf c = STRBUF_INIT;
322 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
323 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
324 if (no_done) strbuf_addstr(&c, " no-done");
325 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
326 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
327 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
328 if (args->no_progress) strbuf_addstr(&c, " no-progress");
329 if (args->include_tag) strbuf_addstr(&c, " include-tag");
330 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 331 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 332 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
333 if (agent_supported) strbuf_addf(&c, " agent=%s",
334 git_user_agent_sanitized());
335 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
336 strbuf_release(&c);
337 } else
338 packet_buf_write(&req_buf, "want %s\n", remote_hex);
339 fetching++;
340 }
341
342 if (!fetching) {
343 strbuf_release(&req_buf);
344 packet_flush(fd[1]);
345 return 1;
346 }
347
348 if (is_repository_shallow())
1a30f5a2 349 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
350 if (args->depth > 0)
351 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882
NTND
352 if (args->deepen_since) {
353 unsigned long max_age = approxidate(args->deepen_since);
354 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
355 }
a45a2600
NTND
356 if (args->deepen_not) {
357 int i;
358 for (i = 0; i < args->deepen_not->nr; i++) {
359 struct string_list_item *s = args->deepen_not->items + i;
360 packet_buf_write(&req_buf, "deepen-not %s", s->string);
361 }
362 }
745f7a8c
NTND
363 packet_buf_flush(&req_buf);
364 state_len = req_buf.len;
365
79891cb9 366 if (args->deepen) {
74543a04 367 char *line;
ae021d87 368 const char *arg;
745f7a8c
NTND
369 unsigned char sha1[20];
370
371 send_request(args, fd[1], &req_buf);
74543a04 372 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87
JK
373 if (skip_prefix(line, "shallow ", &arg)) {
374 if (get_sha1_hex(arg, sha1))
1dd73e20 375 die(_("invalid shallow line: %s"), line);
745f7a8c
NTND
376 register_shallow(sha1);
377 continue;
378 }
ae021d87
JK
379 if (skip_prefix(line, "unshallow ", &arg)) {
380 if (get_sha1_hex(arg, sha1))
1dd73e20 381 die(_("invalid unshallow line: %s"), line);
745f7a8c 382 if (!lookup_object(sha1))
1dd73e20 383 die(_("object not found: %s"), line);
745f7a8c
NTND
384 /* make sure that it is parsed as shallow */
385 if (!parse_object(sha1))
1dd73e20 386 die(_("error in object: %s"), line);
745f7a8c 387 if (unregister_shallow(sha1))
1dd73e20 388 die(_("no shallow found: %s"), line);
745f7a8c
NTND
389 continue;
390 }
1dd73e20 391 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
392 }
393 } else if (!args->stateless_rpc)
394 send_request(args, fd[1], &req_buf);
395
396 if (!args->stateless_rpc) {
397 /* If we aren't using the stateless-rpc interface
398 * we don't need to retain the headers.
399 */
400 strbuf_setlen(&req_buf, 0);
401 state_len = 0;
402 }
403
404 flushes = 0;
405 retval = -1;
406 while ((sha1 = get_rev())) {
407 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
0d789a5b 408 print_verbose(args, "have %s", sha1_to_hex(sha1));
745f7a8c
NTND
409 in_vain++;
410 if (flush_at <= ++count) {
411 int ack;
412
413 packet_buf_flush(&req_buf);
414 send_request(args, fd[1], &req_buf);
415 strbuf_setlen(&req_buf, state_len);
416 flushes++;
417 flush_at = next_flush(args, count);
418
419 /*
420 * We keep one window "ahead" of the other side, and
421 * will wait for an ACK only on the next one
422 */
423 if (!args->stateless_rpc && count == INITIAL_FLUSH)
424 continue;
425
426 consume_shallow_list(args, fd[0]);
427 do {
428 ack = get_ack(fd[0], result_sha1);
0d789a5b 429 if (ack)
1dd73e20
NTND
430 print_verbose(args, _("got %s %d %s"), "ack",
431 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
432 switch (ack) {
433 case ACK:
434 flushes = 0;
435 multi_ack = 0;
436 retval = 0;
437 goto done;
438 case ACK_common:
439 case ACK_ready:
440 case ACK_continue: {
441 struct commit *commit =
442 lookup_commit(result_sha1);
443 if (!commit)
1dd73e20 444 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
745f7a8c
NTND
445 if (args->stateless_rpc
446 && ack == ACK_common
447 && !(commit->object.flags & COMMON)) {
448 /* We need to replay the have for this object
449 * on the next RPC request so the peer knows
450 * it is in common with us.
451 */
452 const char *hex = sha1_to_hex(result_sha1);
453 packet_buf_write(&req_buf, "have %s\n", hex);
454 state_len = req_buf.len;
455 }
456 mark_common(commit, 0, 1);
457 retval = 0;
458 in_vain = 0;
459 got_continue = 1;
460 if (ack == ACK_ready) {
099327b5 461 clear_prio_queue(&rev_list);
745f7a8c
NTND
462 got_ready = 1;
463 }
464 break;
465 }
466 }
467 } while (ack);
468 flushes--;
469 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 470 print_verbose(args, _("giving up"));
745f7a8c
NTND
471 break; /* give up */
472 }
473 }
474 }
475done:
476 if (!got_ready || !no_done) {
477 packet_buf_write(&req_buf, "done\n");
478 send_request(args, fd[1], &req_buf);
479 }
1dd73e20 480 print_verbose(args, _("done"));
745f7a8c
NTND
481 if (retval != 0) {
482 multi_ack = 0;
483 flushes++;
484 }
485 strbuf_release(&req_buf);
486
ff62eca7
NTND
487 if (!got_ready || !no_done)
488 consume_shallow_list(args, fd[0]);
745f7a8c
NTND
489 while (flushes || multi_ack) {
490 int ack = get_ack(fd[0], result_sha1);
491 if (ack) {
1dd73e20
NTND
492 print_verbose(args, _("got %s (%d) %s"), "ack",
493 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
494 if (ack == ACK)
495 return 0;
496 multi_ack = 1;
497 continue;
498 }
499 flushes--;
500 }
501 /* it is no error to fetch into a completely empty repo */
502 return count ? retval : 0;
503}
504
505static struct commit_list *complete;
506
6e20a51a 507static int mark_complete(const unsigned char *sha1)
745f7a8c
NTND
508{
509 struct object *o = parse_object(sha1);
510
511 while (o && o->type == OBJ_TAG) {
512 struct tag *t = (struct tag *) o;
513 if (!t->tagged)
514 break; /* broken repository */
515 o->flags |= COMPLETE;
ed1c9977 516 o = parse_object(t->tagged->oid.hash);
745f7a8c
NTND
517 }
518 if (o && o->type == OBJ_COMMIT) {
519 struct commit *commit = (struct commit *)o;
520 if (!(commit->object.flags & COMPLETE)) {
521 commit->object.flags |= COMPLETE;
16445242 522 commit_list_insert(commit, &complete);
745f7a8c
NTND
523 }
524 }
525 return 0;
526}
527
f8ee4d85
MH
528static int mark_complete_oid(const char *refname, const struct object_id *oid,
529 int flag, void *cb_data)
530{
6e20a51a 531 return mark_complete(oid->hash);
f8ee4d85
MH
532}
533
745f7a8c
NTND
534static void mark_recent_complete_commits(struct fetch_pack_args *args,
535 unsigned long cutoff)
536{
537 while (complete && cutoff <= complete->item->date) {
1dd73e20 538 print_verbose(args, _("Marking %s as complete"),
0d789a5b 539 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
540 pop_most_recent_commit(&complete, COMPLETE);
541 }
542}
543
745f7a8c 544static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
545 struct ref **refs,
546 struct ref **sought, int nr_sought)
745f7a8c
NTND
547{
548 struct ref *newlist = NULL;
549 struct ref **newtail = &newlist;
550 struct ref *ref, *next;
f2db854d 551 int i;
745f7a8c 552
f2db854d 553 i = 0;
745f7a8c
NTND
554 for (ref = *refs; ref; ref = next) {
555 int keep = 0;
556 next = ref->next;
f2db854d 557
50e19a83 558 if (starts_with(ref->name, "refs/") &&
4c224081 559 check_refname_format(ref->name, 0))
745f7a8c
NTND
560 ; /* trash */
561 else {
f2db854d
JH
562 while (i < nr_sought) {
563 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
564 if (cmp < 0)
565 break; /* definitely do not have it */
566 else if (cmp == 0) {
567 keep = 1; /* definitely have it */
f2db854d 568 sought[i]->matched = 1;
745f7a8c 569 }
f2db854d 570 i++;
745f7a8c
NTND
571 }
572 }
573
f2db854d 574 if (!keep && args->fetch_all &&
79891cb9 575 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
576 keep = 1;
577
578 if (keep) {
579 *newtail = ref;
580 ref->next = NULL;
581 newtail = &ref->next;
582 } else {
583 free(ref);
584 }
585 }
586
6e7b66ee 587 /* Append unmatched requests to the list */
68ee6289
FM
588 if ((allow_unadvertised_object_request &
589 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
6e7b66ee 590 for (i = 0; i < nr_sought; i++) {
b7916422
JK
591 unsigned char sha1[20];
592
6e7b66ee
JH
593 ref = sought[i];
594 if (ref->matched)
595 continue;
b7916422
JK
596 if (get_sha1_hex(ref->name, sha1) ||
597 ref->name[40] != '\0' ||
f4e54d02 598 hashcmp(sha1, ref->old_oid.hash))
6e7b66ee
JH
599 continue;
600
601 ref->matched = 1;
c3c17bf1
JK
602 *newtail = copy_ref(ref);
603 newtail = &(*newtail)->next;
6e7b66ee
JH
604 }
605 }
745f7a8c
NTND
606 *refs = newlist;
607}
608
609static void mark_alternate_complete(const struct ref *ref, void *unused)
610{
f4e54d02 611 mark_complete(ref->old_oid.hash);
745f7a8c
NTND
612}
613
614static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
615 struct ref **refs,
616 struct ref **sought, int nr_sought)
745f7a8c
NTND
617{
618 struct ref *ref;
619 int retval;
620 unsigned long cutoff = 0;
621
622 save_commit_buffer = 0;
623
624 for (ref = *refs; ref; ref = ref->next) {
625 struct object *o;
626
f4e54d02 627 if (!has_object_file(&ref->old_oid))
012a1bb5
JH
628 continue;
629
f4e54d02 630 o = parse_object(ref->old_oid.hash);
745f7a8c
NTND
631 if (!o)
632 continue;
633
634 /* We already have it -- which may mean that we were
635 * in sync with the other side at some time after
636 * that (it is OK if we guess wrong here).
637 */
638 if (o->type == OBJ_COMMIT) {
639 struct commit *commit = (struct commit *)o;
640 if (!cutoff || cutoff < commit->date)
641 cutoff = commit->date;
642 }
643 }
644
79891cb9 645 if (!args->deepen) {
f8ee4d85 646 for_each_ref(mark_complete_oid, NULL);
745f7a8c 647 for_each_alternate_ref(mark_alternate_complete, NULL);
16445242 648 commit_list_sort_by_date(&complete);
745f7a8c
NTND
649 if (cutoff)
650 mark_recent_complete_commits(args, cutoff);
651 }
652
653 /*
654 * Mark all complete remote refs as common refs.
655 * Don't mark them common yet; the server has to be told so first.
656 */
657 for (ref = *refs; ref; ref = ref->next) {
f4e54d02 658 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
745f7a8c
NTND
659 NULL, 0);
660
661 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
662 continue;
663
664 if (!(o->flags & SEEN)) {
665 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
666
667 mark_common((struct commit *)o, 1, 1);
668 }
669 }
670
f2db854d 671 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
672
673 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
f4e54d02 674 const unsigned char *remote = ref->old_oid.hash;
745f7a8c
NTND
675 struct object *o;
676
677 o = lookup_object(remote);
678 if (!o || !(o->flags & COMPLETE)) {
679 retval = 0;
0d789a5b
NTND
680 print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
681 ref->name);
745f7a8c
NTND
682 continue;
683 }
1dd73e20 684 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
0d789a5b 685 ref->name);
745f7a8c
NTND
686 }
687 return retval;
688}
689
690static int sideband_demux(int in, int out, void *data)
691{
692 int *xd = data;
693
694 int ret = recv_sideband("fetch-pack", xd[0], out);
695 close(out);
696 return ret;
697}
698
699static int get_pack(struct fetch_pack_args *args,
700 int xd[2], char **pack_lockfile)
701{
702 struct async demux;
745f7a8c 703 int do_keep = args->keep_pack;
984a43b9
JK
704 const char *cmd_name;
705 struct pack_header header;
706 int pass_header = 0;
d3180279 707 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 708 int ret;
745f7a8c
NTND
709
710 memset(&demux, 0, sizeof(demux));
711 if (use_sideband) {
712 /* xd[] is talking with upload-pack; subprocess reads from
713 * xd[0], spits out band#2 to stderr, and feeds us band#1
714 * through demux->out.
715 */
716 demux.proc = sideband_demux;
717 demux.data = xd;
718 demux.out = -1;
719 if (start_async(&demux))
1dd73e20 720 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
721 }
722 else
723 demux.out = xd[0];
724
745f7a8c 725 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
726
727 if (read_pack_header(demux.out, &header))
1dd73e20 728 die(_("protocol error: bad pack header"));
984a43b9 729 pass_header = 1;
745f7a8c
NTND
730 if (ntohl(header.hdr_entries) < unpack_limit)
731 do_keep = 0;
732 else
733 do_keep = 1;
734 }
735
6035d6aa 736 if (alternate_shallow_file) {
984a43b9
JK
737 argv_array_push(&cmd.args, "--shallow-file");
738 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
739 }
740
745f7a8c
NTND
741 if (do_keep) {
742 if (pack_lockfile)
743 cmd.out = -1;
984a43b9
JK
744 cmd_name = "index-pack";
745 argv_array_push(&cmd.args, cmd_name);
746 argv_array_push(&cmd.args, "--stdin");
745f7a8c 747 if (!args->quiet && !args->no_progress)
984a43b9 748 argv_array_push(&cmd.args, "-v");
745f7a8c 749 if (args->use_thin_pack)
984a43b9 750 argv_array_push(&cmd.args, "--fix-thin");
745f7a8c 751 if (args->lock_pack || unpack_limit) {
984a43b9
JK
752 char hostname[256];
753 if (gethostname(hostname, sizeof(hostname)))
754 xsnprintf(hostname, sizeof(hostname), "localhost");
755 argv_array_pushf(&cmd.args,
756 "--keep=fetch-pack %"PRIuMAX " on %s",
757 (uintmax_t)getpid(), hostname);
745f7a8c 758 }
c6807a40 759 if (args->check_self_contained_and_connected)
984a43b9 760 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
745f7a8c
NTND
761 }
762 else {
984a43b9
JK
763 cmd_name = "unpack-objects";
764 argv_array_push(&cmd.args, cmd_name);
745f7a8c 765 if (args->quiet || args->no_progress)
984a43b9 766 argv_array_push(&cmd.args, "-q");
c6807a40 767 args->check_self_contained_and_connected = 0;
745f7a8c 768 }
984a43b9
JK
769
770 if (pass_header)
771 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
772 ntohl(header.hdr_version),
773 ntohl(header.hdr_entries));
745f7a8c
NTND
774 if (fetch_fsck_objects >= 0
775 ? fetch_fsck_objects
776 : transfer_fsck_objects >= 0
777 ? transfer_fsck_objects
778 : 0)
984a43b9 779 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
780
781 cmd.in = demux.out;
782 cmd.git_cmd = 1;
783 if (start_command(&cmd))
1dd73e20 784 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
785 if (do_keep && pack_lockfile) {
786 *pack_lockfile = index_pack_lockfile(cmd.out);
787 close(cmd.out);
788 }
789
37cb1dd6
JL
790 if (!use_sideband)
791 /* Closed by start_command() */
792 xd[0] = -1;
793
c6807a40
NTND
794 ret = finish_command(&cmd);
795 if (!ret || (args->check_self_contained_and_connected && ret == 1))
796 args->self_contained_and_connected =
797 args->check_self_contained_and_connected &&
798 ret == 0;
799 else
1dd73e20 800 die(_("%s failed"), cmd_name);
745f7a8c 801 if (use_sideband && finish_async(&demux))
1dd73e20 802 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
803 return 0;
804}
805
f2db854d
JH
806static int cmp_ref_by_name(const void *a_, const void *b_)
807{
808 const struct ref *a = *((const struct ref **)a_);
809 const struct ref *b = *((const struct ref **)b_);
810 return strcmp(a->name, b->name);
811}
812
745f7a8c
NTND
813static struct ref *do_fetch_pack(struct fetch_pack_args *args,
814 int fd[2],
815 const struct ref *orig_ref,
f2db854d 816 struct ref **sought, int nr_sought,
beea4152 817 struct shallow_info *si,
745f7a8c
NTND
818 char **pack_lockfile)
819{
820 struct ref *ref = copy_ref_list(orig_ref);
821 unsigned char sha1[20];
822 const char *agent_feature;
823 int agent_len;
824
825 sort_ref_list(&ref, ref_compare_name);
f2db854d 826 qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
745f7a8c 827
eb86a507 828 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 829 die(_("Server does not support shallow clients"));
a45a2600 830 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 831 args->deepen = 1;
745f7a8c 832 if (server_supports("multi_ack_detailed")) {
1dd73e20 833 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
834 multi_ack = 2;
835 if (server_supports("no-done")) {
1dd73e20 836 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
837 if (args->stateless_rpc)
838 no_done = 1;
839 }
840 }
841 else if (server_supports("multi_ack")) {
1dd73e20 842 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
843 multi_ack = 1;
844 }
845 if (server_supports("side-band-64k")) {
1dd73e20 846 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
847 use_sideband = 2;
848 }
849 else if (server_supports("side-band")) {
1dd73e20 850 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
851 use_sideband = 1;
852 }
6e7b66ee 853 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 854 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 855 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 856 }
68ee6289 857 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 858 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
859 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
860 }
745f7a8c
NTND
861 if (!server_supports("thin-pack"))
862 args->use_thin_pack = 0;
863 if (!server_supports("no-progress"))
864 args->no_progress = 0;
865 if (!server_supports("include-tag"))
866 args->include_tag = 0;
0d789a5b 867 if (server_supports("ofs-delta"))
1dd73e20 868 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 869 else
745f7a8c
NTND
870 prefer_ofs_delta = 0;
871
872 if ((agent_feature = server_feature_value("agent", &agent_len))) {
873 agent_supported = 1;
0d789a5b 874 if (agent_len)
1dd73e20 875 print_verbose(args, _("Server version is %.*s"),
0d789a5b 876 agent_len, agent_feature);
745f7a8c 877 }
508ea882
NTND
878 if (server_supports("deepen-since"))
879 deepen_since_ok = 1;
880 else if (args->deepen_since)
881 die(_("Server does not support --shallow-since"));
a45a2600
NTND
882 if (server_supports("deepen-not"))
883 deepen_not_ok = 1;
884 else if (args->deepen_not)
885 die(_("Server does not support --shallow-exclude"));
745f7a8c 886
f2db854d 887 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
888 packet_flush(fd[1]);
889 goto all_done;
890 }
891 if (find_common(args, fd, sha1, ref) < 0)
892 if (!args->keep_pack)
893 /* When cloning, it is not unusual to have
894 * no common commit.
895 */
1dd73e20 896 warning(_("no common commits"));
745f7a8c
NTND
897
898 if (args->stateless_rpc)
899 packet_flush(fd[1]);
79891cb9 900 if (args->deepen)
1a30f5a2
NTND
901 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
902 NULL);
4820a33b 903 else if (si->nr_ours || si->nr_theirs)
beea4152 904 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
905 else
906 alternate_shallow_file = NULL;
745f7a8c 907 if (get_pack(args, fd, pack_lockfile))
1dd73e20 908 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
909
910 all_done:
911 return ref;
912}
913
f44af51d 914static void fetch_pack_config(void)
745f7a8c 915{
f44af51d
TA
916 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
917 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
918 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
919 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
920 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 921
f44af51d 922 git_config(git_default_config, NULL);
745f7a8c
NTND
923}
924
745f7a8c
NTND
925static void fetch_pack_setup(void)
926{
927 static int did_setup;
928 if (did_setup)
929 return;
f44af51d 930 fetch_pack_config();
745f7a8c
NTND
931 if (0 <= transfer_unpack_limit)
932 unpack_limit = transfer_unpack_limit;
933 else if (0 <= fetch_unpack_limit)
934 unpack_limit = fetch_unpack_limit;
935 did_setup = 1;
936}
937
f2db854d
JH
938static int remove_duplicates_in_refs(struct ref **ref, int nr)
939{
940 struct string_list names = STRING_LIST_INIT_NODUP;
941 int src, dst;
942
943 for (src = dst = 0; src < nr; src++) {
944 struct string_list_item *item;
945 item = string_list_insert(&names, ref[src]->name);
946 if (item->util)
947 continue; /* already have it */
948 item->util = ref[src];
949 if (src != dst)
950 ref[dst] = ref[src];
951 dst++;
952 }
953 for (src = dst; src < nr; src++)
954 ref[src] = NULL;
955 string_list_clear(&names, 0);
956 return dst;
957}
958
beea4152 959static void update_shallow(struct fetch_pack_args *args,
4820a33b 960 struct ref **sought, int nr_sought,
beea4152 961 struct shallow_info *si)
a796ccee 962{
4820a33b
NTND
963 struct sha1_array ref = SHA1_ARRAY_INIT;
964 int *status;
beea4152
NTND
965 int i;
966
79891cb9 967 if (args->deepen && alternate_shallow_file) {
a796ccee 968 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 969 unlink_or_warn(git_path_shallow());
a796ccee
NTND
970 rollback_lock_file(&shallow_lock);
971 } else
972 commit_lock_file(&shallow_lock);
973 return;
974 }
beea4152
NTND
975
976 if (!si->shallow || !si->shallow->nr)
977 return;
978
beea4152
NTND
979 if (args->cloning) {
980 /*
981 * remote is shallow, but this is a clone, there are
982 * no objects in repo to worry about. Accept any
983 * shallow points that exist in the pack (iow in repo
984 * after get_pack() and reprepare_packed_git())
985 */
986 struct sha1_array extra = SHA1_ARRAY_INIT;
987 unsigned char (*sha1)[20] = si->shallow->sha1;
988 for (i = 0; i < si->shallow->nr; i++)
989 if (has_sha1_file(sha1[i]))
990 sha1_array_append(&extra, sha1[i]);
991 if (extra.nr) {
992 setup_alternate_shallow(&shallow_lock,
993 &alternate_shallow_file,
994 &extra);
995 commit_lock_file(&shallow_lock);
996 }
997 sha1_array_clear(&extra);
998 return;
999 }
4820a33b
NTND
1000
1001 if (!si->nr_ours && !si->nr_theirs)
1002 return;
1003
1004 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1005 if (!si->nr_ours && !si->nr_theirs)
1006 return;
1007 for (i = 0; i < nr_sought; i++)
f4e54d02 1008 sha1_array_append(&ref, sought[i]->old_oid.hash);
4820a33b
NTND
1009 si->ref = &ref;
1010
48d25cae
NTND
1011 if (args->update_shallow) {
1012 /*
1013 * remote is also shallow, .git/shallow may be updated
1014 * so all refs can be accepted. Make sure we only add
1015 * shallow roots that are actually reachable from new
1016 * refs.
1017 */
1018 struct sha1_array extra = SHA1_ARRAY_INIT;
1019 unsigned char (*sha1)[20] = si->shallow->sha1;
1020 assign_shallow_commits_to_refs(si, NULL, NULL);
1021 if (!si->nr_ours && !si->nr_theirs) {
1022 sha1_array_clear(&ref);
1023 return;
1024 }
1025 for (i = 0; i < si->nr_ours; i++)
1026 sha1_array_append(&extra, sha1[si->ours[i]]);
1027 for (i = 0; i < si->nr_theirs; i++)
1028 sha1_array_append(&extra, sha1[si->theirs[i]]);
1029 setup_alternate_shallow(&shallow_lock,
1030 &alternate_shallow_file,
1031 &extra);
1032 commit_lock_file(&shallow_lock);
1033 sha1_array_clear(&extra);
1034 sha1_array_clear(&ref);
1035 return;
1036 }
1037
4820a33b
NTND
1038 /*
1039 * remote is also shallow, check what ref is safe to update
1040 * without updating .git/shallow
1041 */
1042 status = xcalloc(nr_sought, sizeof(*status));
1043 assign_shallow_commits_to_refs(si, NULL, status);
1044 if (si->nr_ours || si->nr_theirs) {
1045 for (i = 0; i < nr_sought; i++)
1046 if (status[i])
1047 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1048 }
1049 free(status);
1050 sha1_array_clear(&ref);
a796ccee
NTND
1051}
1052
745f7a8c
NTND
1053struct ref *fetch_pack(struct fetch_pack_args *args,
1054 int fd[], struct child_process *conn,
1055 const struct ref *ref,
1056 const char *dest,
f2db854d 1057 struct ref **sought, int nr_sought,
beea4152 1058 struct sha1_array *shallow,
745f7a8c
NTND
1059 char **pack_lockfile)
1060{
745f7a8c 1061 struct ref *ref_cpy;
beea4152 1062 struct shallow_info si;
745f7a8c
NTND
1063
1064 fetch_pack_setup();
f2db854d
JH
1065 if (nr_sought)
1066 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1067
1068 if (!ref) {
1069 packet_flush(fd[1]);
1dd73e20 1070 die(_("no matching remote head"));
745f7a8c 1071 }
beea4152
NTND
1072 prepare_shallow_info(&si, shallow);
1073 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1074 &si, pack_lockfile);
745f7a8c 1075 reprepare_packed_git();
4820a33b 1076 update_shallow(args, sought, nr_sought, &si);
beea4152 1077 clear_shallow_info(&si);
745f7a8c
NTND
1078 return ref_cpy;
1079}