unpack-trees: batch fetching of missing blobs
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
b2141fc1 2#include "config.h"
697cc8ef 3#include "lockfile.h"
745f7a8c
NTND
4#include "refs.h"
5#include "pkt-line.h"
6#include "commit.h"
7#include "tag.h"
8#include "exec_cmd.h"
9#include "pack.h"
10#include "sideband.h"
11#include "fetch-pack.h"
12#include "remote.h"
13#include "run-command.h"
47a59185 14#include "connect.h"
745f7a8c
NTND
15#include "transport.h"
16#include "version.h"
099327b5 17#include "prio-queue.h"
beea4152 18#include "sha1-array.h"
fdb69d33 19#include "oidset.h"
0abe14f6 20#include "packfile.h"
745f7a8c
NTND
21
22static int transfer_unpack_limit = -1;
23static int fetch_unpack_limit = -1;
24static int unpack_limit = 100;
25static int prefer_ofs_delta = 1;
26static int no_done;
508ea882 27static int deepen_since_ok;
a45a2600 28static int deepen_not_ok;
745f7a8c
NTND
29static int fetch_fsck_objects = -1;
30static int transfer_fsck_objects = -1;
31static int agent_supported;
640d8b72 32static int server_supports_filtering;
6035d6aa
NTND
33static struct lock_file shallow_lock;
34static const char *alternate_shallow_file;
745f7a8c 35
208acbfb 36/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
37#define COMPLETE (1U << 0)
38#define COMMON (1U << 1)
39#define COMMON_REF (1U << 2)
40#define SEEN (1U << 3)
41#define POPPED (1U << 4)
41a078c6 42#define ALTERNATE (1U << 5)
745f7a8c
NTND
43
44static int marked;
45
46/*
47 * After sending this many "have"s if we do not get any new ACK , we
48 * give up traversing our history.
49 */
50#define MAX_IN_VAIN 256
51
099327b5 52static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
53static int non_common_revs, multi_ack, use_sideband;
54/* Allow specifying sha1 if it is a ref tip. */
55#define ALLOW_TIP_SHA1 01
68ee6289
FM
56/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
57#define ALLOW_REACHABLE_SHA1 02
7199c093 58static unsigned int allow_unadvertised_object_request;
745f7a8c 59
0d789a5b
NTND
60__attribute__((format (printf, 2, 3)))
61static inline void print_verbose(const struct fetch_pack_args *args,
62 const char *fmt, ...)
63{
64 va_list params;
65
66 if (!args->verbose)
67 return;
68
69 va_start(params, fmt);
70 vfprintf(stderr, fmt, params);
71 va_end(params);
72 fputc('\n', stderr);
73}
74
41a078c6
JK
75struct alternate_object_cache {
76 struct object **items;
77 size_t nr, alloc;
78};
79
80static void cache_one_alternate(const char *refname,
81 const struct object_id *oid,
82 void *vcache)
83{
84 struct alternate_object_cache *cache = vcache;
c251c83d 85 struct object *obj = parse_object(oid);
41a078c6
JK
86
87 if (!obj || (obj->flags & ALTERNATE))
88 return;
89
90 obj->flags |= ALTERNATE;
91 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
92 cache->items[cache->nr++] = obj;
93}
94
95static void for_each_cached_alternate(void (*cb)(struct object *))
96{
97 static int initialized;
98 static struct alternate_object_cache cache;
99 size_t i;
100
101 if (!initialized) {
102 for_each_alternate_ref(cache_one_alternate, &cache);
103 initialized = 1;
104 }
105
106 for (i = 0; i < cache.nr; i++)
107 cb(cache.items[i]);
108}
109
745f7a8c
NTND
110static void rev_list_push(struct commit *commit, int mark)
111{
112 if (!(commit->object.flags & mark)) {
113 commit->object.flags |= mark;
114
0064053b
JK
115 if (parse_commit(commit))
116 return;
745f7a8c 117
099327b5 118 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
119
120 if (!(commit->object.flags & COMMON))
121 non_common_revs++;
122 }
123}
124
1b283377 125static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
745f7a8c 126{
c251c83d 127 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
128
129 if (o && o->type == OBJ_COMMIT)
130 rev_list_push((struct commit *)o, SEEN);
131
132 return 0;
133}
134
b1b49c6e
MH
135static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
136 int flag, void *cb_data)
745f7a8c 137{
1b283377 138 return rev_list_insert_ref(refname, oid);
b1b49c6e
MH
139}
140
c50fb6ce
MH
141static int clear_marks(const char *refname, const struct object_id *oid,
142 int flag, void *cb_data)
745f7a8c 143{
c251c83d 144 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
145
146 if (o && o->type == OBJ_COMMIT)
147 clear_commit_marks((struct commit *)o,
148 COMMON | COMMON_REF | SEEN | POPPED);
149 return 0;
150}
151
152/*
153 This function marks a rev and its ancestors as common.
154 In some cases, it is desirable to mark only the ancestors (for example
155 when only the server does not yet know that they are common).
156*/
157
158static void mark_common(struct commit *commit,
159 int ancestors_only, int dont_parse)
160{
161 if (commit != NULL && !(commit->object.flags & COMMON)) {
162 struct object *o = (struct object *)commit;
163
164 if (!ancestors_only)
165 o->flags |= COMMON;
166
167 if (!(o->flags & SEEN))
168 rev_list_push(commit, SEEN);
169 else {
170 struct commit_list *parents;
171
172 if (!ancestors_only && !(o->flags & POPPED))
173 non_common_revs--;
174 if (!o->parsed && !dont_parse)
175 if (parse_commit(commit))
176 return;
177
178 for (parents = commit->parents;
179 parents;
180 parents = parents->next)
181 mark_common(parents->item, 0, dont_parse);
182 }
183 }
184}
185
186/*
187 Get the next rev to send, ignoring the common.
188*/
189
1b283377 190static const struct object_id *get_rev(void)
745f7a8c
NTND
191{
192 struct commit *commit = NULL;
193
194 while (commit == NULL) {
195 unsigned int mark;
196 struct commit_list *parents;
197
099327b5 198 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
199 return NULL;
200
099327b5 201 commit = prio_queue_get(&rev_list);
0064053b 202 parse_commit(commit);
745f7a8c
NTND
203 parents = commit->parents;
204
205 commit->object.flags |= POPPED;
206 if (!(commit->object.flags & COMMON))
207 non_common_revs--;
208
209 if (commit->object.flags & COMMON) {
210 /* do not send "have", and ignore ancestors */
211 commit = NULL;
212 mark = COMMON | SEEN;
213 } else if (commit->object.flags & COMMON_REF)
214 /* send "have", and ignore ancestors */
215 mark = COMMON | SEEN;
216 else
217 /* send "have", also for its ancestors */
218 mark = SEEN;
219
220 while (parents) {
221 if (!(parents->item->object.flags & SEEN))
222 rev_list_push(parents->item, mark);
223 if (mark & COMMON)
224 mark_common(parents->item, 1, 0);
225 parents = parents->next;
226 }
745f7a8c
NTND
227 }
228
1b283377 229 return &commit->object.oid;
745f7a8c
NTND
230}
231
232enum ack_type {
233 NAK = 0,
234 ACK,
235 ACK_continue,
236 ACK_common,
237 ACK_ready
238};
239
240static void consume_shallow_list(struct fetch_pack_args *args, int fd)
241{
79891cb9 242 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
243 /* If we sent a depth we will get back "duplicate"
244 * shallow and unshallow commands every time there
245 * is a block of have lines exchanged.
246 */
74543a04
JK
247 char *line;
248 while ((line = packet_read_line(fd, NULL))) {
59556548 249 if (starts_with(line, "shallow "))
745f7a8c 250 continue;
59556548 251 if (starts_with(line, "unshallow "))
745f7a8c 252 continue;
1dd73e20 253 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
254 }
255 }
256}
257
1b283377 258static enum ack_type get_ack(int fd, struct object_id *result_oid)
745f7a8c 259{
74543a04
JK
260 int len;
261 char *line = packet_read_line(fd, &len);
82e56767 262 const char *arg;
745f7a8c
NTND
263
264 if (!len)
1dd73e20 265 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
266 if (!strcmp(line, "NAK"))
267 return NAK;
82e56767 268 if (skip_prefix(line, "ACK ", &arg)) {
1b283377 269 if (!get_oid_hex(arg, result_oid)) {
82e56767
JK
270 arg += 40;
271 len -= arg - line;
272 if (len < 1)
030e9dd6 273 return ACK;
82e56767 274 if (strstr(arg, "continue"))
745f7a8c 275 return ACK_continue;
82e56767 276 if (strstr(arg, "common"))
745f7a8c 277 return ACK_common;
82e56767 278 if (strstr(arg, "ready"))
745f7a8c
NTND
279 return ACK_ready;
280 return ACK;
281 }
282 }
8e2c7bef
JT
283 if (skip_prefix(line, "ERR ", &arg))
284 die(_("remote error: %s"), arg);
dfbfb9f3 285 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
286}
287
288static void send_request(struct fetch_pack_args *args,
289 int fd, struct strbuf *buf)
290{
291 if (args->stateless_rpc) {
292 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
293 packet_flush(fd);
294 } else
cdf4fb8e 295 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
296}
297
41a078c6 298static void insert_one_alternate_object(struct object *obj)
745f7a8c 299{
1b283377 300 rev_list_insert_ref(NULL, &obj->oid);
745f7a8c
NTND
301}
302
303#define INITIAL_FLUSH 16
304#define PIPESAFE_FLUSH 32
da470981 305#define LARGE_FLUSH 16384
745f7a8c
NTND
306
307static int next_flush(struct fetch_pack_args *args, int count)
308{
da470981
JT
309 if (args->stateless_rpc) {
310 if (count < LARGE_FLUSH)
311 count <<= 1;
312 else
313 count = count * 11 / 10;
314 } else {
315 if (count < PIPESAFE_FLUSH)
316 count <<= 1;
317 else
318 count += PIPESAFE_FLUSH;
319 }
745f7a8c
NTND
320 return count;
321}
322
323static int find_common(struct fetch_pack_args *args,
1b283377 324 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
325 struct ref *refs)
326{
327 int fetching;
328 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 329 const struct object_id *oid;
745f7a8c
NTND
330 unsigned in_vain = 0;
331 int got_continue = 0;
332 int got_ready = 0;
333 struct strbuf req_buf = STRBUF_INIT;
334 size_t state_len = 0;
335
336 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 337 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
338 if (marked)
339 for_each_ref(clear_marks, NULL);
340 marked = 1;
341
b1b49c6e 342 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 343 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
344
345 fetching = 0;
346 for ( ; refs ; refs = refs->next) {
1b283377 347 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
348 const char *remote_hex;
349 struct object *o;
350
351 /*
352 * If that object is complete (i.e. it is an ancestor of a
353 * local ref), we tell them we have it but do not have to
354 * tell them about its ancestors, which they already know
355 * about.
356 *
357 * We use lookup_object here because we are only
358 * interested in the case we *know* the object is
359 * reachable and we have already scanned it.
360 */
1b283377 361 if (((o = lookup_object(remote->hash)) != NULL) &&
745f7a8c
NTND
362 (o->flags & COMPLETE)) {
363 continue;
364 }
365
1b283377 366 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
367 if (!fetching) {
368 struct strbuf c = STRBUF_INIT;
369 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
370 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
371 if (no_done) strbuf_addstr(&c, " no-done");
372 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
373 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 374 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
375 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
376 if (args->no_progress) strbuf_addstr(&c, " no-progress");
377 if (args->include_tag) strbuf_addstr(&c, " include-tag");
378 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 379 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 380 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
381 if (agent_supported) strbuf_addf(&c, " agent=%s",
382 git_user_agent_sanitized());
640d8b72
JH
383 if (args->filter_options.choice)
384 strbuf_addstr(&c, " filter");
745f7a8c
NTND
385 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
386 strbuf_release(&c);
387 } else
388 packet_buf_write(&req_buf, "want %s\n", remote_hex);
389 fetching++;
390 }
391
392 if (!fetching) {
393 strbuf_release(&req_buf);
394 packet_flush(fd[1]);
395 return 1;
396 }
397
398 if (is_repository_shallow())
1a30f5a2 399 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
400 if (args->depth > 0)
401 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882 402 if (args->deepen_since) {
dddbad72 403 timestamp_t max_age = approxidate(args->deepen_since);
cb71f8bd 404 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
508ea882 405 }
a45a2600
NTND
406 if (args->deepen_not) {
407 int i;
408 for (i = 0; i < args->deepen_not->nr; i++) {
409 struct string_list_item *s = args->deepen_not->items + i;
410 packet_buf_write(&req_buf, "deepen-not %s", s->string);
411 }
412 }
640d8b72
JH
413 if (server_supports_filtering && args->filter_options.choice)
414 packet_buf_write(&req_buf, "filter %s",
415 args->filter_options.filter_spec);
745f7a8c
NTND
416 packet_buf_flush(&req_buf);
417 state_len = req_buf.len;
418
79891cb9 419 if (args->deepen) {
74543a04 420 char *line;
ae021d87 421 const char *arg;
1b283377 422 struct object_id oid;
745f7a8c
NTND
423
424 send_request(args, fd[1], &req_buf);
74543a04 425 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87 426 if (skip_prefix(line, "shallow ", &arg)) {
1b283377 427 if (get_oid_hex(arg, &oid))
1dd73e20 428 die(_("invalid shallow line: %s"), line);
e92b848c 429 register_shallow(&oid);
745f7a8c
NTND
430 continue;
431 }
ae021d87 432 if (skip_prefix(line, "unshallow ", &arg)) {
1b283377 433 if (get_oid_hex(arg, &oid))
1dd73e20 434 die(_("invalid unshallow line: %s"), line);
1b283377 435 if (!lookup_object(oid.hash))
1dd73e20 436 die(_("object not found: %s"), line);
745f7a8c 437 /* make sure that it is parsed as shallow */
c251c83d 438 if (!parse_object(&oid))
1dd73e20 439 die(_("error in object: %s"), line);
e92b848c 440 if (unregister_shallow(&oid))
1dd73e20 441 die(_("no shallow found: %s"), line);
745f7a8c
NTND
442 continue;
443 }
1dd73e20 444 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
445 }
446 } else if (!args->stateless_rpc)
447 send_request(args, fd[1], &req_buf);
448
449 if (!args->stateless_rpc) {
450 /* If we aren't using the stateless-rpc interface
451 * we don't need to retain the headers.
452 */
453 strbuf_setlen(&req_buf, 0);
454 state_len = 0;
455 }
456
457 flushes = 0;
458 retval = -1;
88e2f9ed
JT
459 if (args->no_dependents)
460 goto done;
1b283377 461 while ((oid = get_rev())) {
462 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
463 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
464 in_vain++;
465 if (flush_at <= ++count) {
466 int ack;
467
468 packet_buf_flush(&req_buf);
469 send_request(args, fd[1], &req_buf);
470 strbuf_setlen(&req_buf, state_len);
471 flushes++;
472 flush_at = next_flush(args, count);
473
474 /*
475 * We keep one window "ahead" of the other side, and
476 * will wait for an ACK only on the next one
477 */
478 if (!args->stateless_rpc && count == INITIAL_FLUSH)
479 continue;
480
481 consume_shallow_list(args, fd[0]);
482 do {
1b283377 483 ack = get_ack(fd[0], result_oid);
0d789a5b 484 if (ack)
1dd73e20 485 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 486 ack, oid_to_hex(result_oid));
745f7a8c
NTND
487 switch (ack) {
488 case ACK:
489 flushes = 0;
490 multi_ack = 0;
491 retval = 0;
492 goto done;
493 case ACK_common:
494 case ACK_ready:
495 case ACK_continue: {
496 struct commit *commit =
bc83266a 497 lookup_commit(result_oid);
745f7a8c 498 if (!commit)
1b283377 499 die(_("invalid commit %s"), oid_to_hex(result_oid));
745f7a8c
NTND
500 if (args->stateless_rpc
501 && ack == ACK_common
502 && !(commit->object.flags & COMMON)) {
503 /* We need to replay the have for this object
504 * on the next RPC request so the peer knows
505 * it is in common with us.
506 */
1b283377 507 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
508 packet_buf_write(&req_buf, "have %s\n", hex);
509 state_len = req_buf.len;
06b3d386
JT
510 /*
511 * Reset in_vain because an ack
512 * for this commit has not been
513 * seen.
514 */
515 in_vain = 0;
516 } else if (!args->stateless_rpc
517 || ack != ACK_common)
518 in_vain = 0;
745f7a8c
NTND
519 mark_common(commit, 0, 1);
520 retval = 0;
745f7a8c
NTND
521 got_continue = 1;
522 if (ack == ACK_ready) {
099327b5 523 clear_prio_queue(&rev_list);
745f7a8c
NTND
524 got_ready = 1;
525 }
526 break;
527 }
528 }
529 } while (ack);
530 flushes--;
531 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 532 print_verbose(args, _("giving up"));
745f7a8c
NTND
533 break; /* give up */
534 }
535 }
536 }
537done:
538 if (!got_ready || !no_done) {
539 packet_buf_write(&req_buf, "done\n");
540 send_request(args, fd[1], &req_buf);
541 }
1dd73e20 542 print_verbose(args, _("done"));
745f7a8c
NTND
543 if (retval != 0) {
544 multi_ack = 0;
545 flushes++;
546 }
547 strbuf_release(&req_buf);
548
ff62eca7
NTND
549 if (!got_ready || !no_done)
550 consume_shallow_list(args, fd[0]);
745f7a8c 551 while (flushes || multi_ack) {
1b283377 552 int ack = get_ack(fd[0], result_oid);
745f7a8c 553 if (ack) {
1dd73e20 554 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 555 ack, oid_to_hex(result_oid));
745f7a8c
NTND
556 if (ack == ACK)
557 return 0;
558 multi_ack = 1;
559 continue;
560 }
561 flushes--;
562 }
563 /* it is no error to fetch into a completely empty repo */
564 return count ? retval : 0;
565}
566
567static struct commit_list *complete;
568
1b283377 569static int mark_complete(const struct object_id *oid)
745f7a8c 570{
c251c83d 571 struct object *o = parse_object(oid);
745f7a8c
NTND
572
573 while (o && o->type == OBJ_TAG) {
574 struct tag *t = (struct tag *) o;
575 if (!t->tagged)
576 break; /* broken repository */
577 o->flags |= COMPLETE;
c251c83d 578 o = parse_object(&t->tagged->oid);
745f7a8c
NTND
579 }
580 if (o && o->type == OBJ_COMMIT) {
581 struct commit *commit = (struct commit *)o;
582 if (!(commit->object.flags & COMPLETE)) {
583 commit->object.flags |= COMPLETE;
16445242 584 commit_list_insert(commit, &complete);
745f7a8c
NTND
585 }
586 }
587 return 0;
588}
589
f8ee4d85
MH
590static int mark_complete_oid(const char *refname, const struct object_id *oid,
591 int flag, void *cb_data)
592{
1b283377 593 return mark_complete(oid);
f8ee4d85
MH
594}
595
745f7a8c 596static void mark_recent_complete_commits(struct fetch_pack_args *args,
dddbad72 597 timestamp_t cutoff)
745f7a8c
NTND
598{
599 while (complete && cutoff <= complete->item->date) {
1dd73e20 600 print_verbose(args, _("Marking %s as complete"),
0d789a5b 601 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
602 pop_most_recent_commit(&complete, COMPLETE);
603 }
604}
605
fdb69d33
JT
606static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
607{
608 for (; refs; refs = refs->next)
609 oidset_insert(oids, &refs->old_oid);
610}
611
612static int tip_oids_contain(struct oidset *tip_oids,
613 struct ref *unmatched, struct ref *newlist,
614 const struct object_id *id)
615{
616 /*
617 * Note that this only looks at the ref lists the first time it's
618 * called. This works out in filter_refs() because even though it may
619 * add to "newlist" between calls, the additions will always be for
620 * oids that are already in the set.
621 */
9e6fabde 622 if (!tip_oids->map.map.tablesize) {
fdb69d33
JT
623 add_refs_to_oidset(tip_oids, unmatched);
624 add_refs_to_oidset(tip_oids, newlist);
625 }
626 return oidset_contains(tip_oids, id);
627}
628
745f7a8c 629static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
630 struct ref **refs,
631 struct ref **sought, int nr_sought)
745f7a8c
NTND
632{
633 struct ref *newlist = NULL;
634 struct ref **newtail = &newlist;
fdb69d33 635 struct ref *unmatched = NULL;
745f7a8c 636 struct ref *ref, *next;
fdb69d33 637 struct oidset tip_oids = OIDSET_INIT;
f2db854d 638 int i;
745f7a8c 639
f2db854d 640 i = 0;
745f7a8c
NTND
641 for (ref = *refs; ref; ref = next) {
642 int keep = 0;
643 next = ref->next;
f2db854d 644
50e19a83 645 if (starts_with(ref->name, "refs/") &&
4c224081 646 check_refname_format(ref->name, 0))
745f7a8c
NTND
647 ; /* trash */
648 else {
f2db854d
JH
649 while (i < nr_sought) {
650 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
651 if (cmp < 0)
652 break; /* definitely do not have it */
653 else if (cmp == 0) {
654 keep = 1; /* definitely have it */
d56583de 655 sought[i]->match_status = REF_MATCHED;
745f7a8c 656 }
f2db854d 657 i++;
745f7a8c
NTND
658 }
659 }
660
f2db854d 661 if (!keep && args->fetch_all &&
79891cb9 662 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
663 keep = 1;
664
665 if (keep) {
666 *newtail = ref;
667 ref->next = NULL;
668 newtail = &ref->next;
669 } else {
fdb69d33
JT
670 ref->next = unmatched;
671 unmatched = ref;
745f7a8c
NTND
672 }
673 }
674
6e7b66ee 675 /* Append unmatched requests to the list */
d56583de 676 for (i = 0; i < nr_sought; i++) {
1b283377 677 struct object_id oid;
678 const char *p;
b7916422 679
d56583de
MM
680 ref = sought[i];
681 if (ref->match_status != REF_NOT_MATCHED)
682 continue;
1b283377 683 if (parse_oid_hex(ref->name, &oid, &p) ||
684 *p != '\0' ||
685 oidcmp(&oid, &ref->old_oid))
d56583de 686 continue;
6e7b66ee 687
d56583de 688 if ((allow_unadvertised_object_request &
fdb69d33
JT
689 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
690 tip_oids_contain(&tip_oids, unmatched, newlist,
691 &ref->old_oid)) {
d56583de 692 ref->match_status = REF_MATCHED;
c3c17bf1
JK
693 *newtail = copy_ref(ref);
694 newtail = &(*newtail)->next;
d56583de
MM
695 } else {
696 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
697 }
698 }
fdb69d33
JT
699
700 oidset_clear(&tip_oids);
701 for (ref = unmatched; ref; ref = next) {
702 next = ref->next;
703 free(ref);
704 }
705
745f7a8c
NTND
706 *refs = newlist;
707}
708
41a078c6 709static void mark_alternate_complete(struct object *obj)
745f7a8c 710{
1b283377 711 mark_complete(&obj->oid);
745f7a8c
NTND
712}
713
714static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
715 struct ref **refs,
716 struct ref **sought, int nr_sought)
745f7a8c
NTND
717{
718 struct ref *ref;
719 int retval;
dddbad72 720 timestamp_t cutoff = 0;
745f7a8c
NTND
721
722 save_commit_buffer = 0;
723
724 for (ref = *refs; ref; ref = ref->next) {
725 struct object *o;
726
f4e54d02 727 if (!has_object_file(&ref->old_oid))
012a1bb5
JH
728 continue;
729
c251c83d 730 o = parse_object(&ref->old_oid);
745f7a8c
NTND
731 if (!o)
732 continue;
733
734 /* We already have it -- which may mean that we were
735 * in sync with the other side at some time after
736 * that (it is OK if we guess wrong here).
737 */
738 if (o->type == OBJ_COMMIT) {
739 struct commit *commit = (struct commit *)o;
740 if (!cutoff || cutoff < commit->date)
741 cutoff = commit->date;
742 }
743 }
744
88e2f9ed
JT
745 if (!args->no_dependents) {
746 if (!args->deepen) {
747 for_each_ref(mark_complete_oid, NULL);
748 for_each_cached_alternate(mark_alternate_complete);
749 commit_list_sort_by_date(&complete);
750 if (cutoff)
751 mark_recent_complete_commits(args, cutoff);
752 }
745f7a8c 753
88e2f9ed
JT
754 /*
755 * Mark all complete remote refs as common refs.
756 * Don't mark them common yet; the server has to be told so first.
757 */
758 for (ref = *refs; ref; ref = ref->next) {
759 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
760 NULL, 0);
745f7a8c 761
88e2f9ed
JT
762 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
763 continue;
745f7a8c 764
88e2f9ed
JT
765 if (!(o->flags & SEEN)) {
766 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
745f7a8c 767
88e2f9ed
JT
768 mark_common((struct commit *)o, 1, 1);
769 }
745f7a8c
NTND
770 }
771 }
772
f2db854d 773 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
774
775 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 776 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
777 struct object *o;
778
1b283377 779 o = lookup_object(remote->hash);
745f7a8c
NTND
780 if (!o || !(o->flags & COMPLETE)) {
781 retval = 0;
1b283377 782 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 783 ref->name);
745f7a8c
NTND
784 continue;
785 }
1b283377 786 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 787 ref->name);
745f7a8c
NTND
788 }
789 return retval;
790}
791
792static int sideband_demux(int in, int out, void *data)
793{
794 int *xd = data;
9ff18faf 795 int ret;
745f7a8c 796
9ff18faf 797 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
798 close(out);
799 return ret;
800}
801
802static int get_pack(struct fetch_pack_args *args,
803 int xd[2], char **pack_lockfile)
804{
805 struct async demux;
745f7a8c 806 int do_keep = args->keep_pack;
984a43b9
JK
807 const char *cmd_name;
808 struct pack_header header;
809 int pass_header = 0;
d3180279 810 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 811 int ret;
745f7a8c
NTND
812
813 memset(&demux, 0, sizeof(demux));
814 if (use_sideband) {
815 /* xd[] is talking with upload-pack; subprocess reads from
816 * xd[0], spits out band#2 to stderr, and feeds us band#1
817 * through demux->out.
818 */
819 demux.proc = sideband_demux;
820 demux.data = xd;
821 demux.out = -1;
df857572 822 demux.isolate_sigpipe = 1;
745f7a8c 823 if (start_async(&demux))
1dd73e20 824 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
825 }
826 else
827 demux.out = xd[0];
828
745f7a8c 829 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
830
831 if (read_pack_header(demux.out, &header))
1dd73e20 832 die(_("protocol error: bad pack header"));
984a43b9 833 pass_header = 1;
745f7a8c
NTND
834 if (ntohl(header.hdr_entries) < unpack_limit)
835 do_keep = 0;
836 else
837 do_keep = 1;
838 }
839
6035d6aa 840 if (alternate_shallow_file) {
984a43b9
JK
841 argv_array_push(&cmd.args, "--shallow-file");
842 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
843 }
844
88e2f9ed 845 if (do_keep || args->from_promisor) {
745f7a8c
NTND
846 if (pack_lockfile)
847 cmd.out = -1;
984a43b9
JK
848 cmd_name = "index-pack";
849 argv_array_push(&cmd.args, cmd_name);
850 argv_array_push(&cmd.args, "--stdin");
745f7a8c 851 if (!args->quiet && !args->no_progress)
984a43b9 852 argv_array_push(&cmd.args, "-v");
745f7a8c 853 if (args->use_thin_pack)
984a43b9 854 argv_array_push(&cmd.args, "--fix-thin");
88e2f9ed 855 if (do_keep && (args->lock_pack || unpack_limit)) {
da25bdb7 856 char hostname[HOST_NAME_MAX + 1];
5781a9a2 857 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
858 xsnprintf(hostname, sizeof(hostname), "localhost");
859 argv_array_pushf(&cmd.args,
860 "--keep=fetch-pack %"PRIuMAX " on %s",
861 (uintmax_t)getpid(), hostname);
745f7a8c 862 }
c6807a40 863 if (args->check_self_contained_and_connected)
984a43b9 864 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
88e2f9ed
JT
865 if (args->from_promisor)
866 argv_array_push(&cmd.args, "--promisor");
745f7a8c
NTND
867 }
868 else {
984a43b9
JK
869 cmd_name = "unpack-objects";
870 argv_array_push(&cmd.args, cmd_name);
745f7a8c 871 if (args->quiet || args->no_progress)
984a43b9 872 argv_array_push(&cmd.args, "-q");
c6807a40 873 args->check_self_contained_and_connected = 0;
745f7a8c 874 }
984a43b9
JK
875
876 if (pass_header)
877 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
878 ntohl(header.hdr_version),
879 ntohl(header.hdr_entries));
745f7a8c
NTND
880 if (fetch_fsck_objects >= 0
881 ? fetch_fsck_objects
882 : transfer_fsck_objects >= 0
883 ? transfer_fsck_objects
884 : 0)
984a43b9 885 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
886
887 cmd.in = demux.out;
888 cmd.git_cmd = 1;
889 if (start_command(&cmd))
1dd73e20 890 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
891 if (do_keep && pack_lockfile) {
892 *pack_lockfile = index_pack_lockfile(cmd.out);
893 close(cmd.out);
894 }
895
37cb1dd6
JL
896 if (!use_sideband)
897 /* Closed by start_command() */
898 xd[0] = -1;
899
c6807a40
NTND
900 ret = finish_command(&cmd);
901 if (!ret || (args->check_self_contained_and_connected && ret == 1))
902 args->self_contained_and_connected =
903 args->check_self_contained_and_connected &&
904 ret == 0;
905 else
1dd73e20 906 die(_("%s failed"), cmd_name);
745f7a8c 907 if (use_sideband && finish_async(&demux))
1dd73e20 908 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
909 return 0;
910}
911
f2db854d
JH
912static int cmp_ref_by_name(const void *a_, const void *b_)
913{
914 const struct ref *a = *((const struct ref **)a_);
915 const struct ref *b = *((const struct ref **)b_);
916 return strcmp(a->name, b->name);
917}
918
745f7a8c
NTND
919static struct ref *do_fetch_pack(struct fetch_pack_args *args,
920 int fd[2],
921 const struct ref *orig_ref,
f2db854d 922 struct ref **sought, int nr_sought,
beea4152 923 struct shallow_info *si,
745f7a8c
NTND
924 char **pack_lockfile)
925{
926 struct ref *ref = copy_ref_list(orig_ref);
1b283377 927 struct object_id oid;
745f7a8c
NTND
928 const char *agent_feature;
929 int agent_len;
930
931 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 932 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 933
eb86a507 934 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 935 die(_("Server does not support shallow clients"));
a45a2600 936 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 937 args->deepen = 1;
745f7a8c 938 if (server_supports("multi_ack_detailed")) {
1dd73e20 939 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
940 multi_ack = 2;
941 if (server_supports("no-done")) {
1dd73e20 942 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
943 if (args->stateless_rpc)
944 no_done = 1;
945 }
946 }
947 else if (server_supports("multi_ack")) {
1dd73e20 948 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
949 multi_ack = 1;
950 }
951 if (server_supports("side-band-64k")) {
1dd73e20 952 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
953 use_sideband = 2;
954 }
955 else if (server_supports("side-band")) {
1dd73e20 956 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
957 use_sideband = 1;
958 }
6e7b66ee 959 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 960 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 961 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 962 }
68ee6289 963 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 964 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
965 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
966 }
745f7a8c
NTND
967 if (!server_supports("thin-pack"))
968 args->use_thin_pack = 0;
969 if (!server_supports("no-progress"))
970 args->no_progress = 0;
971 if (!server_supports("include-tag"))
972 args->include_tag = 0;
0d789a5b 973 if (server_supports("ofs-delta"))
1dd73e20 974 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 975 else
745f7a8c
NTND
976 prefer_ofs_delta = 0;
977
640d8b72
JH
978 if (server_supports("filter")) {
979 server_supports_filtering = 1;
980 print_verbose(args, _("Server supports filter"));
981 } else if (args->filter_options.choice) {
982 warning("filtering not recognized by server, ignoring");
983 }
984
745f7a8c
NTND
985 if ((agent_feature = server_feature_value("agent", &agent_len))) {
986 agent_supported = 1;
0d789a5b 987 if (agent_len)
1dd73e20 988 print_verbose(args, _("Server version is %.*s"),
0d789a5b 989 agent_len, agent_feature);
745f7a8c 990 }
508ea882
NTND
991 if (server_supports("deepen-since"))
992 deepen_since_ok = 1;
993 else if (args->deepen_since)
994 die(_("Server does not support --shallow-since"));
a45a2600
NTND
995 if (server_supports("deepen-not"))
996 deepen_not_ok = 1;
997 else if (args->deepen_not)
998 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
999 if (!server_supports("deepen-relative") && args->deepen_relative)
1000 die(_("Server does not support --deepen"));
745f7a8c 1001
f2db854d 1002 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
1003 packet_flush(fd[1]);
1004 goto all_done;
1005 }
1b283377 1006 if (find_common(args, fd, &oid, ref) < 0)
745f7a8c
NTND
1007 if (!args->keep_pack)
1008 /* When cloning, it is not unusual to have
1009 * no common commit.
1010 */
1dd73e20 1011 warning(_("no common commits"));
745f7a8c
NTND
1012
1013 if (args->stateless_rpc)
1014 packet_flush(fd[1]);
79891cb9 1015 if (args->deepen)
1a30f5a2
NTND
1016 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1017 NULL);
4820a33b 1018 else if (si->nr_ours || si->nr_theirs)
beea4152 1019 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
1020 else
1021 alternate_shallow_file = NULL;
745f7a8c 1022 if (get_pack(args, fd, pack_lockfile))
1dd73e20 1023 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
1024
1025 all_done:
1026 return ref;
1027}
1028
f44af51d 1029static void fetch_pack_config(void)
745f7a8c 1030{
f44af51d
TA
1031 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1032 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1033 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1034 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1035 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 1036
f44af51d 1037 git_config(git_default_config, NULL);
745f7a8c
NTND
1038}
1039
745f7a8c
NTND
1040static void fetch_pack_setup(void)
1041{
1042 static int did_setup;
1043 if (did_setup)
1044 return;
f44af51d 1045 fetch_pack_config();
745f7a8c
NTND
1046 if (0 <= transfer_unpack_limit)
1047 unpack_limit = transfer_unpack_limit;
1048 else if (0 <= fetch_unpack_limit)
1049 unpack_limit = fetch_unpack_limit;
1050 did_setup = 1;
1051}
1052
f2db854d
JH
1053static int remove_duplicates_in_refs(struct ref **ref, int nr)
1054{
1055 struct string_list names = STRING_LIST_INIT_NODUP;
1056 int src, dst;
1057
1058 for (src = dst = 0; src < nr; src++) {
1059 struct string_list_item *item;
1060 item = string_list_insert(&names, ref[src]->name);
1061 if (item->util)
1062 continue; /* already have it */
1063 item->util = ref[src];
1064 if (src != dst)
1065 ref[dst] = ref[src];
1066 dst++;
1067 }
1068 for (src = dst; src < nr; src++)
1069 ref[src] = NULL;
1070 string_list_clear(&names, 0);
1071 return dst;
1072}
1073
beea4152 1074static void update_shallow(struct fetch_pack_args *args,
4820a33b 1075 struct ref **sought, int nr_sought,
beea4152 1076 struct shallow_info *si)
a796ccee 1077{
910650d2 1078 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1079 int *status;
beea4152
NTND
1080 int i;
1081
79891cb9 1082 if (args->deepen && alternate_shallow_file) {
a796ccee 1083 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 1084 unlink_or_warn(git_path_shallow());
a796ccee
NTND
1085 rollback_lock_file(&shallow_lock);
1086 } else
1087 commit_lock_file(&shallow_lock);
1088 return;
1089 }
beea4152
NTND
1090
1091 if (!si->shallow || !si->shallow->nr)
1092 return;
1093
beea4152
NTND
1094 if (args->cloning) {
1095 /*
1096 * remote is shallow, but this is a clone, there are
1097 * no objects in repo to worry about. Accept any
1098 * shallow points that exist in the pack (iow in repo
1099 * after get_pack() and reprepare_packed_git())
1100 */
910650d2 1101 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1102 struct object_id *oid = si->shallow->oid;
beea4152 1103 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1104 if (has_object_file(&oid[i]))
910650d2 1105 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1106 if (extra.nr) {
1107 setup_alternate_shallow(&shallow_lock,
1108 &alternate_shallow_file,
1109 &extra);
1110 commit_lock_file(&shallow_lock);
1111 }
910650d2 1112 oid_array_clear(&extra);
beea4152
NTND
1113 return;
1114 }
4820a33b
NTND
1115
1116 if (!si->nr_ours && !si->nr_theirs)
1117 return;
1118
1119 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1120 if (!si->nr_ours && !si->nr_theirs)
1121 return;
1122 for (i = 0; i < nr_sought; i++)
910650d2 1123 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1124 si->ref = &ref;
1125
48d25cae
NTND
1126 if (args->update_shallow) {
1127 /*
1128 * remote is also shallow, .git/shallow may be updated
1129 * so all refs can be accepted. Make sure we only add
1130 * shallow roots that are actually reachable from new
1131 * refs.
1132 */
910650d2 1133 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1134 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1135 assign_shallow_commits_to_refs(si, NULL, NULL);
1136 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1137 oid_array_clear(&ref);
48d25cae
NTND
1138 return;
1139 }
1140 for (i = 0; i < si->nr_ours; i++)
910650d2 1141 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1142 for (i = 0; i < si->nr_theirs; i++)
910650d2 1143 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1144 setup_alternate_shallow(&shallow_lock,
1145 &alternate_shallow_file,
1146 &extra);
1147 commit_lock_file(&shallow_lock);
910650d2 1148 oid_array_clear(&extra);
1149 oid_array_clear(&ref);
48d25cae
NTND
1150 return;
1151 }
1152
4820a33b
NTND
1153 /*
1154 * remote is also shallow, check what ref is safe to update
1155 * without updating .git/shallow
1156 */
1157 status = xcalloc(nr_sought, sizeof(*status));
1158 assign_shallow_commits_to_refs(si, NULL, status);
1159 if (si->nr_ours || si->nr_theirs) {
1160 for (i = 0; i < nr_sought; i++)
1161 if (status[i])
1162 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1163 }
1164 free(status);
910650d2 1165 oid_array_clear(&ref);
a796ccee
NTND
1166}
1167
745f7a8c
NTND
1168struct ref *fetch_pack(struct fetch_pack_args *args,
1169 int fd[], struct child_process *conn,
1170 const struct ref *ref,
1171 const char *dest,
f2db854d 1172 struct ref **sought, int nr_sought,
910650d2 1173 struct oid_array *shallow,
745f7a8c
NTND
1174 char **pack_lockfile)
1175{
745f7a8c 1176 struct ref *ref_cpy;
beea4152 1177 struct shallow_info si;
745f7a8c
NTND
1178
1179 fetch_pack_setup();
f2db854d
JH
1180 if (nr_sought)
1181 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1182
1183 if (!ref) {
1184 packet_flush(fd[1]);
1dd73e20 1185 die(_("no matching remote head"));
745f7a8c 1186 }
beea4152
NTND
1187 prepare_shallow_info(&si, shallow);
1188 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1189 &si, pack_lockfile);
745f7a8c 1190 reprepare_packed_git();
4820a33b 1191 update_shallow(args, sought, nr_sought, &si);
beea4152 1192 clear_shallow_info(&si);
745f7a8c
NTND
1193 return ref_cpy;
1194}
e860d96b
MM
1195
1196int report_unmatched_refs(struct ref **sought, int nr_sought)
1197{
1198 int i, ret = 0;
1199
1200 for (i = 0; i < nr_sought; i++) {
d56583de 1201 if (!sought[i])
e860d96b 1202 continue;
d56583de
MM
1203 switch (sought[i]->match_status) {
1204 case REF_MATCHED:
1205 continue;
1206 case REF_NOT_MATCHED:
1207 error(_("no such remote ref %s"), sought[i]->name);
1208 break;
1209 case REF_UNADVERTISED_NOT_ALLOWED:
1210 error(_("Server does not allow request for unadvertised object %s"),
1211 sought[i]->name);
1212 break;
1213 }
e860d96b
MM
1214 ret = 1;
1215 }
1216 return ret;
1217}