object: add repository argument to lookup_object
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
a49d2834 2#include "repository.h"
b2141fc1 3#include "config.h"
697cc8ef 4#include "lockfile.h"
745f7a8c
NTND
5#include "refs.h"
6#include "pkt-line.h"
7#include "commit.h"
8#include "tag.h"
d807c4a0 9#include "exec-cmd.h"
745f7a8c
NTND
10#include "pack.h"
11#include "sideband.h"
12#include "fetch-pack.h"
13#include "remote.h"
14#include "run-command.h"
47a59185 15#include "connect.h"
745f7a8c
NTND
16#include "transport.h"
17#include "version.h"
099327b5 18#include "prio-queue.h"
beea4152 19#include "sha1-array.h"
fdb69d33 20#include "oidset.h"
0abe14f6 21#include "packfile.h"
cbd53a21 22#include "object-store.h"
745f7a8c
NTND
23
24static int transfer_unpack_limit = -1;
25static int fetch_unpack_limit = -1;
26static int unpack_limit = 100;
27static int prefer_ofs_delta = 1;
28static int no_done;
508ea882 29static int deepen_since_ok;
a45a2600 30static int deepen_not_ok;
745f7a8c
NTND
31static int fetch_fsck_objects = -1;
32static int transfer_fsck_objects = -1;
33static int agent_supported;
640d8b72 34static int server_supports_filtering;
6035d6aa
NTND
35static struct lock_file shallow_lock;
36static const char *alternate_shallow_file;
745f7a8c 37
208acbfb 38/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
39#define COMPLETE (1U << 0)
40#define COMMON (1U << 1)
41#define COMMON_REF (1U << 2)
42#define SEEN (1U << 3)
43#define POPPED (1U << 4)
41a078c6 44#define ALTERNATE (1U << 5)
745f7a8c
NTND
45
46static int marked;
47
48/*
49 * After sending this many "have"s if we do not get any new ACK , we
50 * give up traversing our history.
51 */
52#define MAX_IN_VAIN 256
53
099327b5 54static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
55static int non_common_revs, multi_ack, use_sideband;
56/* Allow specifying sha1 if it is a ref tip. */
57#define ALLOW_TIP_SHA1 01
68ee6289
FM
58/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
59#define ALLOW_REACHABLE_SHA1 02
7199c093 60static unsigned int allow_unadvertised_object_request;
745f7a8c 61
0d789a5b
NTND
62__attribute__((format (printf, 2, 3)))
63static inline void print_verbose(const struct fetch_pack_args *args,
64 const char *fmt, ...)
65{
66 va_list params;
67
68 if (!args->verbose)
69 return;
70
71 va_start(params, fmt);
72 vfprintf(stderr, fmt, params);
73 va_end(params);
74 fputc('\n', stderr);
75}
76
41a078c6
JK
77struct alternate_object_cache {
78 struct object **items;
79 size_t nr, alloc;
80};
81
82static void cache_one_alternate(const char *refname,
83 const struct object_id *oid,
84 void *vcache)
85{
86 struct alternate_object_cache *cache = vcache;
109cd76d 87 struct object *obj = parse_object(the_repository, oid);
41a078c6
JK
88
89 if (!obj || (obj->flags & ALTERNATE))
90 return;
91
92 obj->flags |= ALTERNATE;
93 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
94 cache->items[cache->nr++] = obj;
95}
96
97static void for_each_cached_alternate(void (*cb)(struct object *))
98{
99 static int initialized;
100 static struct alternate_object_cache cache;
101 size_t i;
102
103 if (!initialized) {
104 for_each_alternate_ref(cache_one_alternate, &cache);
105 initialized = 1;
106 }
107
108 for (i = 0; i < cache.nr; i++)
109 cb(cache.items[i]);
110}
111
745f7a8c
NTND
112static void rev_list_push(struct commit *commit, int mark)
113{
114 if (!(commit->object.flags & mark)) {
115 commit->object.flags |= mark;
116
0064053b
JK
117 if (parse_commit(commit))
118 return;
745f7a8c 119
099327b5 120 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
121
122 if (!(commit->object.flags & COMMON))
123 non_common_revs++;
124 }
125}
126
1b283377 127static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
745f7a8c 128{
109cd76d
SB
129 struct object *o = deref_tag(parse_object(the_repository, oid),
130 refname, 0);
745f7a8c
NTND
131
132 if (o && o->type == OBJ_COMMIT)
133 rev_list_push((struct commit *)o, SEEN);
134
135 return 0;
136}
137
b1b49c6e
MH
138static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
139 int flag, void *cb_data)
745f7a8c 140{
1b283377 141 return rev_list_insert_ref(refname, oid);
b1b49c6e
MH
142}
143
c50fb6ce
MH
144static int clear_marks(const char *refname, const struct object_id *oid,
145 int flag, void *cb_data)
745f7a8c 146{
109cd76d
SB
147 struct object *o = deref_tag(parse_object(the_repository, oid),
148 refname, 0);
745f7a8c
NTND
149
150 if (o && o->type == OBJ_COMMIT)
151 clear_commit_marks((struct commit *)o,
152 COMMON | COMMON_REF | SEEN | POPPED);
153 return 0;
154}
155
156/*
157 This function marks a rev and its ancestors as common.
158 In some cases, it is desirable to mark only the ancestors (for example
159 when only the server does not yet know that they are common).
160*/
161
162static void mark_common(struct commit *commit,
163 int ancestors_only, int dont_parse)
164{
165 if (commit != NULL && !(commit->object.flags & COMMON)) {
166 struct object *o = (struct object *)commit;
167
168 if (!ancestors_only)
169 o->flags |= COMMON;
170
171 if (!(o->flags & SEEN))
172 rev_list_push(commit, SEEN);
173 else {
174 struct commit_list *parents;
175
176 if (!ancestors_only && !(o->flags & POPPED))
177 non_common_revs--;
178 if (!o->parsed && !dont_parse)
179 if (parse_commit(commit))
180 return;
181
182 for (parents = commit->parents;
183 parents;
184 parents = parents->next)
185 mark_common(parents->item, 0, dont_parse);
186 }
187 }
188}
189
190/*
191 Get the next rev to send, ignoring the common.
192*/
193
1b283377 194static const struct object_id *get_rev(void)
745f7a8c
NTND
195{
196 struct commit *commit = NULL;
197
198 while (commit == NULL) {
199 unsigned int mark;
200 struct commit_list *parents;
201
099327b5 202 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
203 return NULL;
204
099327b5 205 commit = prio_queue_get(&rev_list);
0064053b 206 parse_commit(commit);
745f7a8c
NTND
207 parents = commit->parents;
208
209 commit->object.flags |= POPPED;
210 if (!(commit->object.flags & COMMON))
211 non_common_revs--;
212
213 if (commit->object.flags & COMMON) {
214 /* do not send "have", and ignore ancestors */
215 commit = NULL;
216 mark = COMMON | SEEN;
217 } else if (commit->object.flags & COMMON_REF)
218 /* send "have", and ignore ancestors */
219 mark = COMMON | SEEN;
220 else
221 /* send "have", also for its ancestors */
222 mark = SEEN;
223
224 while (parents) {
225 if (!(parents->item->object.flags & SEEN))
226 rev_list_push(parents->item, mark);
227 if (mark & COMMON)
228 mark_common(parents->item, 1, 0);
229 parents = parents->next;
230 }
745f7a8c
NTND
231 }
232
1b283377 233 return &commit->object.oid;
745f7a8c
NTND
234}
235
236enum ack_type {
237 NAK = 0,
238 ACK,
239 ACK_continue,
240 ACK_common,
241 ACK_ready
242};
243
244static void consume_shallow_list(struct fetch_pack_args *args, int fd)
245{
79891cb9 246 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
247 /* If we sent a depth we will get back "duplicate"
248 * shallow and unshallow commands every time there
249 * is a block of have lines exchanged.
250 */
74543a04
JK
251 char *line;
252 while ((line = packet_read_line(fd, NULL))) {
59556548 253 if (starts_with(line, "shallow "))
745f7a8c 254 continue;
59556548 255 if (starts_with(line, "unshallow "))
745f7a8c 256 continue;
1dd73e20 257 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
258 }
259 }
260}
261
1b283377 262static enum ack_type get_ack(int fd, struct object_id *result_oid)
745f7a8c 263{
74543a04
JK
264 int len;
265 char *line = packet_read_line(fd, &len);
82e56767 266 const char *arg;
745f7a8c 267
bc9d4dc5
JK
268 if (!line)
269 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
745f7a8c
NTND
270 if (!strcmp(line, "NAK"))
271 return NAK;
82e56767 272 if (skip_prefix(line, "ACK ", &arg)) {
1b283377 273 if (!get_oid_hex(arg, result_oid)) {
82e56767
JK
274 arg += 40;
275 len -= arg - line;
276 if (len < 1)
030e9dd6 277 return ACK;
82e56767 278 if (strstr(arg, "continue"))
745f7a8c 279 return ACK_continue;
82e56767 280 if (strstr(arg, "common"))
745f7a8c 281 return ACK_common;
82e56767 282 if (strstr(arg, "ready"))
745f7a8c
NTND
283 return ACK_ready;
284 return ACK;
285 }
286 }
8e2c7bef
JT
287 if (skip_prefix(line, "ERR ", &arg))
288 die(_("remote error: %s"), arg);
dfbfb9f3 289 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
290}
291
292static void send_request(struct fetch_pack_args *args,
293 int fd, struct strbuf *buf)
294{
295 if (args->stateless_rpc) {
296 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
297 packet_flush(fd);
298 } else
cdf4fb8e 299 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
300}
301
41a078c6 302static void insert_one_alternate_object(struct object *obj)
745f7a8c 303{
1b283377 304 rev_list_insert_ref(NULL, &obj->oid);
745f7a8c
NTND
305}
306
307#define INITIAL_FLUSH 16
308#define PIPESAFE_FLUSH 32
da470981 309#define LARGE_FLUSH 16384
745f7a8c 310
685fbd32 311static int next_flush(int stateless_rpc, int count)
745f7a8c 312{
685fbd32 313 if (stateless_rpc) {
da470981
JT
314 if (count < LARGE_FLUSH)
315 count <<= 1;
316 else
317 count = count * 11 / 10;
318 } else {
319 if (count < PIPESAFE_FLUSH)
320 count <<= 1;
321 else
322 count += PIPESAFE_FLUSH;
323 }
745f7a8c
NTND
324 return count;
325}
326
327static int find_common(struct fetch_pack_args *args,
1b283377 328 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
329 struct ref *refs)
330{
331 int fetching;
332 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 333 const struct object_id *oid;
745f7a8c
NTND
334 unsigned in_vain = 0;
335 int got_continue = 0;
336 int got_ready = 0;
337 struct strbuf req_buf = STRBUF_INIT;
338 size_t state_len = 0;
339
340 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 341 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
342 if (marked)
343 for_each_ref(clear_marks, NULL);
344 marked = 1;
345
b1b49c6e 346 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 347 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
348
349 fetching = 0;
350 for ( ; refs ; refs = refs->next) {
1b283377 351 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
352 const char *remote_hex;
353 struct object *o;
354
355 /*
356 * If that object is complete (i.e. it is an ancestor of a
357 * local ref), we tell them we have it but do not have to
358 * tell them about its ancestors, which they already know
359 * about.
360 *
361 * We use lookup_object here because we are only
362 * interested in the case we *know* the object is
363 * reachable and we have already scanned it.
364 */
5abddd1e 365 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
745f7a8c
NTND
366 (o->flags & COMPLETE)) {
367 continue;
368 }
369
1b283377 370 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
371 if (!fetching) {
372 struct strbuf c = STRBUF_INIT;
373 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
374 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
375 if (no_done) strbuf_addstr(&c, " no-done");
376 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
377 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 378 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
379 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
380 if (args->no_progress) strbuf_addstr(&c, " no-progress");
381 if (args->include_tag) strbuf_addstr(&c, " include-tag");
382 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 383 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 384 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
385 if (agent_supported) strbuf_addf(&c, " agent=%s",
386 git_user_agent_sanitized());
640d8b72
JH
387 if (args->filter_options.choice)
388 strbuf_addstr(&c, " filter");
745f7a8c
NTND
389 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
390 strbuf_release(&c);
391 } else
392 packet_buf_write(&req_buf, "want %s\n", remote_hex);
393 fetching++;
394 }
395
396 if (!fetching) {
397 strbuf_release(&req_buf);
398 packet_flush(fd[1]);
399 return 1;
400 }
401
c8813487 402 if (is_repository_shallow(the_repository))
1a30f5a2 403 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
404 if (args->depth > 0)
405 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882 406 if (args->deepen_since) {
dddbad72 407 timestamp_t max_age = approxidate(args->deepen_since);
cb71f8bd 408 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
508ea882 409 }
a45a2600
NTND
410 if (args->deepen_not) {
411 int i;
412 for (i = 0; i < args->deepen_not->nr; i++) {
413 struct string_list_item *s = args->deepen_not->items + i;
414 packet_buf_write(&req_buf, "deepen-not %s", s->string);
415 }
416 }
640d8b72
JH
417 if (server_supports_filtering && args->filter_options.choice)
418 packet_buf_write(&req_buf, "filter %s",
419 args->filter_options.filter_spec);
745f7a8c
NTND
420 packet_buf_flush(&req_buf);
421 state_len = req_buf.len;
422
79891cb9 423 if (args->deepen) {
74543a04 424 char *line;
ae021d87 425 const char *arg;
1b283377 426 struct object_id oid;
745f7a8c
NTND
427
428 send_request(args, fd[1], &req_buf);
74543a04 429 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87 430 if (skip_prefix(line, "shallow ", &arg)) {
1b283377 431 if (get_oid_hex(arg, &oid))
1dd73e20 432 die(_("invalid shallow line: %s"), line);
19143f13 433 register_shallow(the_repository, &oid);
745f7a8c
NTND
434 continue;
435 }
ae021d87 436 if (skip_prefix(line, "unshallow ", &arg)) {
1b283377 437 if (get_oid_hex(arg, &oid))
1dd73e20 438 die(_("invalid unshallow line: %s"), line);
5abddd1e 439 if (!lookup_object(the_repository, oid.hash))
1dd73e20 440 die(_("object not found: %s"), line);
745f7a8c 441 /* make sure that it is parsed as shallow */
109cd76d 442 if (!parse_object(the_repository, &oid))
1dd73e20 443 die(_("error in object: %s"), line);
e92b848c 444 if (unregister_shallow(&oid))
1dd73e20 445 die(_("no shallow found: %s"), line);
745f7a8c
NTND
446 continue;
447 }
1dd73e20 448 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
449 }
450 } else if (!args->stateless_rpc)
451 send_request(args, fd[1], &req_buf);
452
453 if (!args->stateless_rpc) {
454 /* If we aren't using the stateless-rpc interface
455 * we don't need to retain the headers.
456 */
457 strbuf_setlen(&req_buf, 0);
458 state_len = 0;
459 }
460
461 flushes = 0;
462 retval = -1;
88e2f9ed
JT
463 if (args->no_dependents)
464 goto done;
1b283377 465 while ((oid = get_rev())) {
466 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
467 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
468 in_vain++;
469 if (flush_at <= ++count) {
470 int ack;
471
472 packet_buf_flush(&req_buf);
473 send_request(args, fd[1], &req_buf);
474 strbuf_setlen(&req_buf, state_len);
475 flushes++;
685fbd32 476 flush_at = next_flush(args->stateless_rpc, count);
745f7a8c
NTND
477
478 /*
479 * We keep one window "ahead" of the other side, and
480 * will wait for an ACK only on the next one
481 */
482 if (!args->stateless_rpc && count == INITIAL_FLUSH)
483 continue;
484
485 consume_shallow_list(args, fd[0]);
486 do {
1b283377 487 ack = get_ack(fd[0], result_oid);
0d789a5b 488 if (ack)
1dd73e20 489 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 490 ack, oid_to_hex(result_oid));
745f7a8c
NTND
491 switch (ack) {
492 case ACK:
493 flushes = 0;
494 multi_ack = 0;
495 retval = 0;
496 goto done;
497 case ACK_common:
498 case ACK_ready:
499 case ACK_continue: {
500 struct commit *commit =
bc83266a 501 lookup_commit(result_oid);
745f7a8c 502 if (!commit)
1b283377 503 die(_("invalid commit %s"), oid_to_hex(result_oid));
745f7a8c
NTND
504 if (args->stateless_rpc
505 && ack == ACK_common
506 && !(commit->object.flags & COMMON)) {
507 /* We need to replay the have for this object
508 * on the next RPC request so the peer knows
509 * it is in common with us.
510 */
1b283377 511 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
512 packet_buf_write(&req_buf, "have %s\n", hex);
513 state_len = req_buf.len;
06b3d386
JT
514 /*
515 * Reset in_vain because an ack
516 * for this commit has not been
517 * seen.
518 */
519 in_vain = 0;
520 } else if (!args->stateless_rpc
521 || ack != ACK_common)
522 in_vain = 0;
745f7a8c
NTND
523 mark_common(commit, 0, 1);
524 retval = 0;
745f7a8c
NTND
525 got_continue = 1;
526 if (ack == ACK_ready) {
099327b5 527 clear_prio_queue(&rev_list);
745f7a8c
NTND
528 got_ready = 1;
529 }
530 break;
531 }
532 }
533 } while (ack);
534 flushes--;
535 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 536 print_verbose(args, _("giving up"));
745f7a8c
NTND
537 break; /* give up */
538 }
539 }
540 }
541done:
542 if (!got_ready || !no_done) {
543 packet_buf_write(&req_buf, "done\n");
544 send_request(args, fd[1], &req_buf);
545 }
1dd73e20 546 print_verbose(args, _("done"));
745f7a8c
NTND
547 if (retval != 0) {
548 multi_ack = 0;
549 flushes++;
550 }
551 strbuf_release(&req_buf);
552
ff62eca7
NTND
553 if (!got_ready || !no_done)
554 consume_shallow_list(args, fd[0]);
745f7a8c 555 while (flushes || multi_ack) {
1b283377 556 int ack = get_ack(fd[0], result_oid);
745f7a8c 557 if (ack) {
1dd73e20 558 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 559 ack, oid_to_hex(result_oid));
745f7a8c
NTND
560 if (ack == ACK)
561 return 0;
562 multi_ack = 1;
563 continue;
564 }
565 flushes--;
566 }
567 /* it is no error to fetch into a completely empty repo */
568 return count ? retval : 0;
569}
570
571static struct commit_list *complete;
572
1b283377 573static int mark_complete(const struct object_id *oid)
745f7a8c 574{
109cd76d 575 struct object *o = parse_object(the_repository, oid);
745f7a8c
NTND
576
577 while (o && o->type == OBJ_TAG) {
578 struct tag *t = (struct tag *) o;
579 if (!t->tagged)
580 break; /* broken repository */
581 o->flags |= COMPLETE;
109cd76d 582 o = parse_object(the_repository, &t->tagged->oid);
745f7a8c
NTND
583 }
584 if (o && o->type == OBJ_COMMIT) {
585 struct commit *commit = (struct commit *)o;
586 if (!(commit->object.flags & COMPLETE)) {
587 commit->object.flags |= COMPLETE;
16445242 588 commit_list_insert(commit, &complete);
745f7a8c
NTND
589 }
590 }
591 return 0;
592}
593
f8ee4d85
MH
594static int mark_complete_oid(const char *refname, const struct object_id *oid,
595 int flag, void *cb_data)
596{
1b283377 597 return mark_complete(oid);
f8ee4d85
MH
598}
599
745f7a8c 600static void mark_recent_complete_commits(struct fetch_pack_args *args,
dddbad72 601 timestamp_t cutoff)
745f7a8c
NTND
602{
603 while (complete && cutoff <= complete->item->date) {
1dd73e20 604 print_verbose(args, _("Marking %s as complete"),
0d789a5b 605 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
606 pop_most_recent_commit(&complete, COMPLETE);
607 }
608}
609
fdb69d33
JT
610static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
611{
612 for (; refs; refs = refs->next)
613 oidset_insert(oids, &refs->old_oid);
614}
615
616static int tip_oids_contain(struct oidset *tip_oids,
617 struct ref *unmatched, struct ref *newlist,
618 const struct object_id *id)
619{
620 /*
621 * Note that this only looks at the ref lists the first time it's
622 * called. This works out in filter_refs() because even though it may
623 * add to "newlist" between calls, the additions will always be for
624 * oids that are already in the set.
625 */
9e6fabde 626 if (!tip_oids->map.map.tablesize) {
fdb69d33
JT
627 add_refs_to_oidset(tip_oids, unmatched);
628 add_refs_to_oidset(tip_oids, newlist);
629 }
630 return oidset_contains(tip_oids, id);
631}
632
745f7a8c 633static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
634 struct ref **refs,
635 struct ref **sought, int nr_sought)
745f7a8c
NTND
636{
637 struct ref *newlist = NULL;
638 struct ref **newtail = &newlist;
fdb69d33 639 struct ref *unmatched = NULL;
745f7a8c 640 struct ref *ref, *next;
fdb69d33 641 struct oidset tip_oids = OIDSET_INIT;
f2db854d 642 int i;
745f7a8c 643
f2db854d 644 i = 0;
745f7a8c
NTND
645 for (ref = *refs; ref; ref = next) {
646 int keep = 0;
647 next = ref->next;
f2db854d 648
50e19a83 649 if (starts_with(ref->name, "refs/") &&
4c224081 650 check_refname_format(ref->name, 0))
745f7a8c
NTND
651 ; /* trash */
652 else {
f2db854d
JH
653 while (i < nr_sought) {
654 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
655 if (cmp < 0)
656 break; /* definitely do not have it */
657 else if (cmp == 0) {
658 keep = 1; /* definitely have it */
d56583de 659 sought[i]->match_status = REF_MATCHED;
745f7a8c 660 }
f2db854d 661 i++;
745f7a8c 662 }
745f7a8c 663
e9502c0a
JK
664 if (!keep && args->fetch_all &&
665 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
666 keep = 1;
667 }
745f7a8c
NTND
668
669 if (keep) {
670 *newtail = ref;
671 ref->next = NULL;
672 newtail = &ref->next;
673 } else {
fdb69d33
JT
674 ref->next = unmatched;
675 unmatched = ref;
745f7a8c
NTND
676 }
677 }
678
6e7b66ee 679 /* Append unmatched requests to the list */
d56583de 680 for (i = 0; i < nr_sought; i++) {
1b283377 681 struct object_id oid;
682 const char *p;
b7916422 683
d56583de
MM
684 ref = sought[i];
685 if (ref->match_status != REF_NOT_MATCHED)
686 continue;
1b283377 687 if (parse_oid_hex(ref->name, &oid, &p) ||
688 *p != '\0' ||
689 oidcmp(&oid, &ref->old_oid))
d56583de 690 continue;
6e7b66ee 691
d56583de 692 if ((allow_unadvertised_object_request &
fdb69d33
JT
693 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
694 tip_oids_contain(&tip_oids, unmatched, newlist,
695 &ref->old_oid)) {
d56583de 696 ref->match_status = REF_MATCHED;
c3c17bf1
JK
697 *newtail = copy_ref(ref);
698 newtail = &(*newtail)->next;
d56583de
MM
699 } else {
700 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
701 }
702 }
fdb69d33
JT
703
704 oidset_clear(&tip_oids);
705 for (ref = unmatched; ref; ref = next) {
706 next = ref->next;
707 free(ref);
708 }
709
745f7a8c
NTND
710 *refs = newlist;
711}
712
41a078c6 713static void mark_alternate_complete(struct object *obj)
745f7a8c 714{
1b283377 715 mark_complete(&obj->oid);
745f7a8c
NTND
716}
717
024aa469
TI
718struct loose_object_iter {
719 struct oidset *loose_object_set;
720 struct ref *refs;
721};
722
723/*
724 * If the number of refs is not larger than the number of loose objects,
725 * this function stops inserting.
726 */
727static int add_loose_objects_to_set(const struct object_id *oid,
728 const char *path,
729 void *data)
730{
731 struct loose_object_iter *iter = data;
732 oidset_insert(iter->loose_object_set, oid);
733 if (iter->refs == NULL)
734 return 1;
735
736 iter->refs = iter->refs->next;
737 return 0;
738}
739
745f7a8c 740static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
741 struct ref **refs,
742 struct ref **sought, int nr_sought)
745f7a8c
NTND
743{
744 struct ref *ref;
745 int retval;
a1c6d7c1 746 int old_save_commit_buffer = save_commit_buffer;
dddbad72 747 timestamp_t cutoff = 0;
024aa469
TI
748 struct oidset loose_oid_set = OIDSET_INIT;
749 int use_oidset = 0;
750 struct loose_object_iter iter = {&loose_oid_set, *refs};
751
752 /* Enumerate all loose objects or know refs are not so many. */
753 use_oidset = !for_each_loose_object(add_loose_objects_to_set,
754 &iter, 0);
745f7a8c
NTND
755
756 save_commit_buffer = 0;
757
758 for (ref = *refs; ref; ref = ref->next) {
759 struct object *o;
024aa469 760 unsigned int flags = OBJECT_INFO_QUICK;
745f7a8c 761
024aa469
TI
762 if (use_oidset &&
763 !oidset_contains(&loose_oid_set, &ref->old_oid)) {
764 /*
765 * I know this does not exist in the loose form,
766 * so check if it exists in a non-loose form.
767 */
768 flags |= OBJECT_INFO_IGNORE_LOOSE;
769 }
012a1bb5 770
024aa469
TI
771 if (!has_object_file_with_flags(&ref->old_oid, flags))
772 continue;
109cd76d 773 o = parse_object(the_repository, &ref->old_oid);
745f7a8c
NTND
774 if (!o)
775 continue;
776
777 /* We already have it -- which may mean that we were
778 * in sync with the other side at some time after
779 * that (it is OK if we guess wrong here).
780 */
781 if (o->type == OBJ_COMMIT) {
782 struct commit *commit = (struct commit *)o;
783 if (!cutoff || cutoff < commit->date)
784 cutoff = commit->date;
785 }
786 }
787
024aa469
TI
788 oidset_clear(&loose_oid_set);
789
88e2f9ed
JT
790 if (!args->no_dependents) {
791 if (!args->deepen) {
792 for_each_ref(mark_complete_oid, NULL);
793 for_each_cached_alternate(mark_alternate_complete);
794 commit_list_sort_by_date(&complete);
795 if (cutoff)
796 mark_recent_complete_commits(args, cutoff);
797 }
745f7a8c 798
88e2f9ed
JT
799 /*
800 * Mark all complete remote refs as common refs.
801 * Don't mark them common yet; the server has to be told so first.
802 */
803 for (ref = *refs; ref; ref = ref->next) {
5abddd1e
SB
804 struct object *o = deref_tag(lookup_object(the_repository,
805 ref->old_oid.hash),
88e2f9ed 806 NULL, 0);
745f7a8c 807
88e2f9ed
JT
808 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
809 continue;
745f7a8c 810
88e2f9ed
JT
811 if (!(o->flags & SEEN)) {
812 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
745f7a8c 813
88e2f9ed
JT
814 mark_common((struct commit *)o, 1, 1);
815 }
745f7a8c
NTND
816 }
817 }
818
f2db854d 819 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
820
821 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 822 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
823 struct object *o;
824
5abddd1e 825 o = lookup_object(the_repository, remote->hash);
745f7a8c
NTND
826 if (!o || !(o->flags & COMPLETE)) {
827 retval = 0;
1b283377 828 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 829 ref->name);
745f7a8c
NTND
830 continue;
831 }
1b283377 832 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 833 ref->name);
745f7a8c 834 }
a1c6d7c1
JT
835
836 save_commit_buffer = old_save_commit_buffer;
837
745f7a8c
NTND
838 return retval;
839}
840
841static int sideband_demux(int in, int out, void *data)
842{
843 int *xd = data;
9ff18faf 844 int ret;
745f7a8c 845
9ff18faf 846 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
847 close(out);
848 return ret;
849}
850
851static int get_pack(struct fetch_pack_args *args,
852 int xd[2], char **pack_lockfile)
853{
854 struct async demux;
745f7a8c 855 int do_keep = args->keep_pack;
984a43b9
JK
856 const char *cmd_name;
857 struct pack_header header;
858 int pass_header = 0;
d3180279 859 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 860 int ret;
745f7a8c
NTND
861
862 memset(&demux, 0, sizeof(demux));
863 if (use_sideband) {
864 /* xd[] is talking with upload-pack; subprocess reads from
865 * xd[0], spits out band#2 to stderr, and feeds us band#1
866 * through demux->out.
867 */
868 demux.proc = sideband_demux;
869 demux.data = xd;
870 demux.out = -1;
df857572 871 demux.isolate_sigpipe = 1;
745f7a8c 872 if (start_async(&demux))
1dd73e20 873 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
874 }
875 else
876 demux.out = xd[0];
877
745f7a8c 878 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
879
880 if (read_pack_header(demux.out, &header))
1dd73e20 881 die(_("protocol error: bad pack header"));
984a43b9 882 pass_header = 1;
745f7a8c
NTND
883 if (ntohl(header.hdr_entries) < unpack_limit)
884 do_keep = 0;
885 else
886 do_keep = 1;
887 }
888
6035d6aa 889 if (alternate_shallow_file) {
984a43b9
JK
890 argv_array_push(&cmd.args, "--shallow-file");
891 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
892 }
893
88e2f9ed 894 if (do_keep || args->from_promisor) {
745f7a8c
NTND
895 if (pack_lockfile)
896 cmd.out = -1;
984a43b9
JK
897 cmd_name = "index-pack";
898 argv_array_push(&cmd.args, cmd_name);
899 argv_array_push(&cmd.args, "--stdin");
745f7a8c 900 if (!args->quiet && !args->no_progress)
984a43b9 901 argv_array_push(&cmd.args, "-v");
745f7a8c 902 if (args->use_thin_pack)
984a43b9 903 argv_array_push(&cmd.args, "--fix-thin");
88e2f9ed 904 if (do_keep && (args->lock_pack || unpack_limit)) {
da25bdb7 905 char hostname[HOST_NAME_MAX + 1];
5781a9a2 906 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
907 xsnprintf(hostname, sizeof(hostname), "localhost");
908 argv_array_pushf(&cmd.args,
909 "--keep=fetch-pack %"PRIuMAX " on %s",
910 (uintmax_t)getpid(), hostname);
745f7a8c 911 }
c6807a40 912 if (args->check_self_contained_and_connected)
984a43b9 913 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
88e2f9ed
JT
914 if (args->from_promisor)
915 argv_array_push(&cmd.args, "--promisor");
745f7a8c
NTND
916 }
917 else {
984a43b9
JK
918 cmd_name = "unpack-objects";
919 argv_array_push(&cmd.args, cmd_name);
745f7a8c 920 if (args->quiet || args->no_progress)
984a43b9 921 argv_array_push(&cmd.args, "-q");
c6807a40 922 args->check_self_contained_and_connected = 0;
745f7a8c 923 }
984a43b9
JK
924
925 if (pass_header)
926 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
927 ntohl(header.hdr_version),
928 ntohl(header.hdr_entries));
745f7a8c
NTND
929 if (fetch_fsck_objects >= 0
930 ? fetch_fsck_objects
931 : transfer_fsck_objects >= 0
932 ? transfer_fsck_objects
98a2ea46
JT
933 : 0) {
934 if (args->from_promisor)
935 /*
936 * We cannot use --strict in index-pack because it
937 * checks both broken objects and links, but we only
938 * want to check for broken objects.
939 */
940 argv_array_push(&cmd.args, "--fsck-objects");
941 else
942 argv_array_push(&cmd.args, "--strict");
943 }
745f7a8c
NTND
944
945 cmd.in = demux.out;
946 cmd.git_cmd = 1;
947 if (start_command(&cmd))
1dd73e20 948 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
949 if (do_keep && pack_lockfile) {
950 *pack_lockfile = index_pack_lockfile(cmd.out);
951 close(cmd.out);
952 }
953
37cb1dd6
JL
954 if (!use_sideband)
955 /* Closed by start_command() */
956 xd[0] = -1;
957
c6807a40
NTND
958 ret = finish_command(&cmd);
959 if (!ret || (args->check_self_contained_and_connected && ret == 1))
960 args->self_contained_and_connected =
961 args->check_self_contained_and_connected &&
962 ret == 0;
963 else
1dd73e20 964 die(_("%s failed"), cmd_name);
745f7a8c 965 if (use_sideband && finish_async(&demux))
1dd73e20 966 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
967 return 0;
968}
969
f2db854d
JH
970static int cmp_ref_by_name(const void *a_, const void *b_)
971{
972 const struct ref *a = *((const struct ref **)a_);
973 const struct ref *b = *((const struct ref **)b_);
974 return strcmp(a->name, b->name);
975}
976
745f7a8c
NTND
977static struct ref *do_fetch_pack(struct fetch_pack_args *args,
978 int fd[2],
979 const struct ref *orig_ref,
f2db854d 980 struct ref **sought, int nr_sought,
beea4152 981 struct shallow_info *si,
745f7a8c
NTND
982 char **pack_lockfile)
983{
984 struct ref *ref = copy_ref_list(orig_ref);
1b283377 985 struct object_id oid;
745f7a8c
NTND
986 const char *agent_feature;
987 int agent_len;
988
989 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 990 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 991
c8813487 992 if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
1dd73e20 993 die(_("Server does not support shallow clients"));
a45a2600 994 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 995 args->deepen = 1;
745f7a8c 996 if (server_supports("multi_ack_detailed")) {
1dd73e20 997 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
998 multi_ack = 2;
999 if (server_supports("no-done")) {
1dd73e20 1000 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
1001 if (args->stateless_rpc)
1002 no_done = 1;
1003 }
1004 }
1005 else if (server_supports("multi_ack")) {
1dd73e20 1006 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
1007 multi_ack = 1;
1008 }
1009 if (server_supports("side-band-64k")) {
1dd73e20 1010 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
1011 use_sideband = 2;
1012 }
1013 else if (server_supports("side-band")) {
1dd73e20 1014 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
1015 use_sideband = 1;
1016 }
6e7b66ee 1017 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 1018 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 1019 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 1020 }
68ee6289 1021 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 1022 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
1023 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1024 }
745f7a8c
NTND
1025 if (!server_supports("thin-pack"))
1026 args->use_thin_pack = 0;
1027 if (!server_supports("no-progress"))
1028 args->no_progress = 0;
1029 if (!server_supports("include-tag"))
1030 args->include_tag = 0;
0d789a5b 1031 if (server_supports("ofs-delta"))
1dd73e20 1032 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 1033 else
745f7a8c
NTND
1034 prefer_ofs_delta = 0;
1035
640d8b72
JH
1036 if (server_supports("filter")) {
1037 server_supports_filtering = 1;
1038 print_verbose(args, _("Server supports filter"));
1039 } else if (args->filter_options.choice) {
1040 warning("filtering not recognized by server, ignoring");
1041 }
1042
745f7a8c
NTND
1043 if ((agent_feature = server_feature_value("agent", &agent_len))) {
1044 agent_supported = 1;
0d789a5b 1045 if (agent_len)
1dd73e20 1046 print_verbose(args, _("Server version is %.*s"),
0d789a5b 1047 agent_len, agent_feature);
745f7a8c 1048 }
508ea882
NTND
1049 if (server_supports("deepen-since"))
1050 deepen_since_ok = 1;
1051 else if (args->deepen_since)
1052 die(_("Server does not support --shallow-since"));
a45a2600
NTND
1053 if (server_supports("deepen-not"))
1054 deepen_not_ok = 1;
1055 else if (args->deepen_not)
1056 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
1057 if (!server_supports("deepen-relative") && args->deepen_relative)
1058 die(_("Server does not support --deepen"));
745f7a8c 1059
f2db854d 1060 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
1061 packet_flush(fd[1]);
1062 goto all_done;
1063 }
1b283377 1064 if (find_common(args, fd, &oid, ref) < 0)
745f7a8c
NTND
1065 if (!args->keep_pack)
1066 /* When cloning, it is not unusual to have
1067 * no common commit.
1068 */
1dd73e20 1069 warning(_("no common commits"));
745f7a8c
NTND
1070
1071 if (args->stateless_rpc)
1072 packet_flush(fd[1]);
79891cb9 1073 if (args->deepen)
1a30f5a2
NTND
1074 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1075 NULL);
4820a33b 1076 else if (si->nr_ours || si->nr_theirs)
beea4152 1077 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
1078 else
1079 alternate_shallow_file = NULL;
745f7a8c 1080 if (get_pack(args, fd, pack_lockfile))
1dd73e20 1081 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
1082
1083 all_done:
1084 return ref;
1085}
1086
f7e20501
BW
1087static void add_shallow_requests(struct strbuf *req_buf,
1088 const struct fetch_pack_args *args)
1089{
b16b60f7 1090 if (is_repository_shallow(the_repository))
f7e20501
BW
1091 write_shallow_commits(req_buf, 1, NULL);
1092 if (args->depth > 0)
1093 packet_buf_write(req_buf, "deepen %d", args->depth);
1094 if (args->deepen_since) {
1095 timestamp_t max_age = approxidate(args->deepen_since);
1096 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1097 }
1098 if (args->deepen_not) {
1099 int i;
1100 for (i = 0; i < args->deepen_not->nr; i++) {
1101 struct string_list_item *s = args->deepen_not->items + i;
1102 packet_buf_write(req_buf, "deepen-not %s", s->string);
1103 }
1104 }
1105}
1106
685fbd32
BW
1107static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1108{
1109 for ( ; wants ; wants = wants->next) {
1110 const struct object_id *remote = &wants->old_oid;
1111 const char *remote_hex;
1112 struct object *o;
1113
1114 /*
1115 * If that object is complete (i.e. it is an ancestor of a
1116 * local ref), we tell them we have it but do not have to
1117 * tell them about its ancestors, which they already know
1118 * about.
1119 *
1120 * We use lookup_object here because we are only
1121 * interested in the case we *know* the object is
1122 * reachable and we have already scanned it.
1123 */
5abddd1e 1124 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
685fbd32
BW
1125 (o->flags & COMPLETE)) {
1126 continue;
1127 }
1128
1129 remote_hex = oid_to_hex(remote);
1130 packet_buf_write(req_buf, "want %s\n", remote_hex);
1131 }
1132}
1133
1134static void add_common(struct strbuf *req_buf, struct oidset *common)
1135{
1136 struct oidset_iter iter;
1137 const struct object_id *oid;
1138 oidset_iter_init(common, &iter);
1139
1140 while ((oid = oidset_iter_next(&iter))) {
1141 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1142 }
1143}
1144
1145static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1146{
1147 int ret = 0;
1148 int haves_added = 0;
1149 const struct object_id *oid;
1150
1151 while ((oid = get_rev())) {
1152 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1153 if (++haves_added >= *haves_to_send)
1154 break;
1155 }
1156
1157 *in_vain += haves_added;
1158 if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1159 /* Send Done */
1160 packet_buf_write(req_buf, "done\n");
1161 ret = 1;
1162 }
1163
1164 /* Increase haves to send on next round */
1165 *haves_to_send = next_flush(1, *haves_to_send);
1166
1167 return ret;
1168}
1169
1170static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1171 const struct ref *wants, struct oidset *common,
1172 int *haves_to_send, int *in_vain)
1173{
1174 int ret = 0;
1175 struct strbuf req_buf = STRBUF_INIT;
1176
1177 if (server_supports_v2("fetch", 1))
1178 packet_buf_write(&req_buf, "command=fetch");
1179 if (server_supports_v2("agent", 0))
1180 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
5e3548ef
BW
1181 if (args->server_options && args->server_options->nr &&
1182 server_supports_v2("server-option", 1)) {
1183 int i;
1184 for (i = 0; i < args->server_options->nr; i++)
1185 packet_write_fmt(fd_out, "server-option=%s",
1186 args->server_options->items[i].string);
1187 }
685fbd32
BW
1188
1189 packet_buf_delim(&req_buf);
1190 if (args->use_thin_pack)
1191 packet_buf_write(&req_buf, "thin-pack");
1192 if (args->no_progress)
1193 packet_buf_write(&req_buf, "no-progress");
1194 if (args->include_tag)
1195 packet_buf_write(&req_buf, "include-tag");
1196 if (prefer_ofs_delta)
1197 packet_buf_write(&req_buf, "ofs-delta");
1198
f7e20501
BW
1199 /* Add shallow-info and deepen request */
1200 if (server_supports_feature("fetch", "shallow", 0))
1201 add_shallow_requests(&req_buf, args);
b16b60f7 1202 else if (is_repository_shallow(the_repository) || args->deepen)
f7e20501
BW
1203 die(_("Server does not support shallow requests"));
1204
ba95710a
JT
1205 /* Add filter */
1206 if (server_supports_feature("fetch", "filter", 0) &&
1207 args->filter_options.choice) {
1208 print_verbose(args, _("Server supports filter"));
1209 packet_buf_write(&req_buf, "filter %s",
1210 args->filter_options.filter_spec);
1211 } else if (args->filter_options.choice) {
1212 warning("filtering not recognized by server, ignoring");
1213 }
1214
685fbd32
BW
1215 /* add wants */
1216 add_wants(wants, &req_buf);
1217
ba95710a
JT
1218 if (args->no_dependents) {
1219 packet_buf_write(&req_buf, "done");
1220 ret = 1;
1221 } else {
1222 /* Add all of the common commits we've found in previous rounds */
1223 add_common(&req_buf, common);
685fbd32 1224
ba95710a
JT
1225 /* Add initial haves */
1226 ret = add_haves(&req_buf, haves_to_send, in_vain);
1227 }
685fbd32
BW
1228
1229 /* Send request */
1230 packet_buf_flush(&req_buf);
1231 write_or_die(fd_out, req_buf.buf, req_buf.len);
1232
1233 strbuf_release(&req_buf);
1234 return ret;
1235}
1236
1237/*
1238 * Processes a section header in a server's response and checks if it matches
1239 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1240 * not consumed); if 0, the line will be consumed and the function will die if
1241 * the section header doesn't match what was expected.
1242 */
1243static int process_section_header(struct packet_reader *reader,
1244 const char *section, int peek)
1245{
1246 int ret;
1247
1248 if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
f7e20501 1249 die("error reading section header '%s'", section);
685fbd32
BW
1250
1251 ret = !strcmp(reader->line, section);
1252
1253 if (!peek) {
1254 if (!ret)
1255 die("expected '%s', received '%s'",
1256 section, reader->line);
1257 packet_reader_read(reader);
1258 }
1259
1260 return ret;
1261}
1262
1263static int process_acks(struct packet_reader *reader, struct oidset *common)
1264{
1265 /* received */
1266 int received_ready = 0;
1267 int received_ack = 0;
1268
1269 process_section_header(reader, "acknowledgments", 0);
1270 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1271 const char *arg;
1272
1273 if (!strcmp(reader->line, "NAK"))
1274 continue;
1275
1276 if (skip_prefix(reader->line, "ACK ", &arg)) {
1277 struct object_id oid;
1278 if (!get_oid_hex(arg, &oid)) {
1279 struct commit *commit;
1280 oidset_insert(common, &oid);
1281 commit = lookup_commit(&oid);
1282 mark_common(commit, 0, 1);
1283 }
1284 continue;
1285 }
1286
1287 if (!strcmp(reader->line, "ready")) {
1288 clear_prio_queue(&rev_list);
1289 received_ready = 1;
1290 continue;
1291 }
1292
1293 die("unexpected acknowledgment line: '%s'", reader->line);
1294 }
1295
1296 if (reader->status != PACKET_READ_FLUSH &&
1297 reader->status != PACKET_READ_DELIM)
1298 die("error processing acks: %d", reader->status);
1299
1300 /* return 0 if no common, 1 if there are common, or 2 if ready */
1301 return received_ready ? 2 : (received_ack ? 1 : 0);
1302}
1303
f7e20501
BW
1304static void receive_shallow_info(struct fetch_pack_args *args,
1305 struct packet_reader *reader)
1306{
1307 process_section_header(reader, "shallow-info", 0);
1308 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1309 const char *arg;
1310 struct object_id oid;
1311
1312 if (skip_prefix(reader->line, "shallow ", &arg)) {
1313 if (get_oid_hex(arg, &oid))
1314 die(_("invalid shallow line: %s"), reader->line);
b16b60f7 1315 register_shallow(the_repository, &oid);
f7e20501
BW
1316 continue;
1317 }
1318 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1319 if (get_oid_hex(arg, &oid))
1320 die(_("invalid unshallow line: %s"), reader->line);
5abddd1e 1321 if (!lookup_object(the_repository, oid.hash))
f7e20501
BW
1322 die(_("object not found: %s"), reader->line);
1323 /* make sure that it is parsed as shallow */
109cd76d 1324 if (!parse_object(the_repository, &oid))
f7e20501
BW
1325 die(_("error in object: %s"), reader->line);
1326 if (unregister_shallow(&oid))
1327 die(_("no shallow found: %s"), reader->line);
1328 continue;
1329 }
1330 die(_("expected shallow/unshallow, got %s"), reader->line);
1331 }
1332
1333 if (reader->status != PACKET_READ_FLUSH &&
1334 reader->status != PACKET_READ_DELIM)
1335 die("error processing shallow info: %d", reader->status);
1336
1337 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1338 args->deepen = 1;
1339}
1340
685fbd32
BW
1341enum fetch_state {
1342 FETCH_CHECK_LOCAL = 0,
1343 FETCH_SEND_REQUEST,
1344 FETCH_PROCESS_ACKS,
1345 FETCH_GET_PACK,
1346 FETCH_DONE,
1347};
1348
1349static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1350 int fd[2],
1351 const struct ref *orig_ref,
1352 struct ref **sought, int nr_sought,
1353 char **pack_lockfile)
1354{
1355 struct ref *ref = copy_ref_list(orig_ref);
1356 enum fetch_state state = FETCH_CHECK_LOCAL;
1357 struct oidset common = OIDSET_INIT;
1358 struct packet_reader reader;
1359 int in_vain = 0;
1360 int haves_to_send = INITIAL_FLUSH;
1361 packet_reader_init(&reader, fd[0], NULL, 0,
1362 PACKET_READ_CHOMP_NEWLINE);
1363
1364 while (state != FETCH_DONE) {
1365 switch (state) {
1366 case FETCH_CHECK_LOCAL:
1367 sort_ref_list(&ref, ref_compare_name);
1368 QSORT(sought, nr_sought, cmp_ref_by_name);
1369
1370 /* v2 supports these by default */
1371 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1372 use_sideband = 2;
f7e20501
BW
1373 if (args->depth > 0 || args->deepen_since || args->deepen_not)
1374 args->deepen = 1;
685fbd32
BW
1375
1376 if (marked)
1377 for_each_ref(clear_marks, NULL);
1378 marked = 1;
1379
1380 for_each_ref(rev_list_insert_ref_oid, NULL);
1381 for_each_cached_alternate(insert_one_alternate_object);
1382
1383 /* Filter 'ref' by 'sought' and those that aren't local */
1384 if (everything_local(args, &ref, sought, nr_sought))
1385 state = FETCH_DONE;
1386 else
1387 state = FETCH_SEND_REQUEST;
1388 break;
1389 case FETCH_SEND_REQUEST:
1390 if (send_fetch_request(fd[1], args, ref, &common,
1391 &haves_to_send, &in_vain))
1392 state = FETCH_GET_PACK;
1393 else
1394 state = FETCH_PROCESS_ACKS;
1395 break;
1396 case FETCH_PROCESS_ACKS:
1397 /* Process ACKs/NAKs */
1398 switch (process_acks(&reader, &common)) {
1399 case 2:
1400 state = FETCH_GET_PACK;
1401 break;
1402 case 1:
1403 in_vain = 0;
1404 /* fallthrough */
1405 default:
1406 state = FETCH_SEND_REQUEST;
1407 break;
1408 }
1409 break;
1410 case FETCH_GET_PACK:
f7e20501
BW
1411 /* Check for shallow-info section */
1412 if (process_section_header(&reader, "shallow-info", 1))
1413 receive_shallow_info(args, &reader);
1414
685fbd32
BW
1415 /* get the pack */
1416 process_section_header(&reader, "packfile", 0);
1417 if (get_pack(args, fd, pack_lockfile))
1418 die(_("git fetch-pack: fetch failed."));
1419
1420 state = FETCH_DONE;
1421 break;
1422 case FETCH_DONE:
1423 continue;
1424 }
1425 }
1426
1427 oidset_clear(&common);
1428 return ref;
1429}
1430
f44af51d 1431static void fetch_pack_config(void)
745f7a8c 1432{
f44af51d
TA
1433 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1434 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1435 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1436 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1437 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 1438
f44af51d 1439 git_config(git_default_config, NULL);
745f7a8c
NTND
1440}
1441
745f7a8c
NTND
1442static void fetch_pack_setup(void)
1443{
1444 static int did_setup;
1445 if (did_setup)
1446 return;
f44af51d 1447 fetch_pack_config();
745f7a8c
NTND
1448 if (0 <= transfer_unpack_limit)
1449 unpack_limit = transfer_unpack_limit;
1450 else if (0 <= fetch_unpack_limit)
1451 unpack_limit = fetch_unpack_limit;
1452 did_setup = 1;
1453}
1454
f2db854d
JH
1455static int remove_duplicates_in_refs(struct ref **ref, int nr)
1456{
1457 struct string_list names = STRING_LIST_INIT_NODUP;
1458 int src, dst;
1459
1460 for (src = dst = 0; src < nr; src++) {
1461 struct string_list_item *item;
1462 item = string_list_insert(&names, ref[src]->name);
1463 if (item->util)
1464 continue; /* already have it */
1465 item->util = ref[src];
1466 if (src != dst)
1467 ref[dst] = ref[src];
1468 dst++;
1469 }
1470 for (src = dst; src < nr; src++)
1471 ref[src] = NULL;
1472 string_list_clear(&names, 0);
1473 return dst;
1474}
1475
beea4152 1476static void update_shallow(struct fetch_pack_args *args,
4820a33b 1477 struct ref **sought, int nr_sought,
beea4152 1478 struct shallow_info *si)
a796ccee 1479{
910650d2 1480 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1481 int *status;
beea4152
NTND
1482 int i;
1483
79891cb9 1484 if (args->deepen && alternate_shallow_file) {
a796ccee 1485 if (*alternate_shallow_file == '\0') { /* --unshallow */
102de880 1486 unlink_or_warn(git_path_shallow(the_repository));
a796ccee
NTND
1487 rollback_lock_file(&shallow_lock);
1488 } else
1489 commit_lock_file(&shallow_lock);
1490 return;
1491 }
beea4152
NTND
1492
1493 if (!si->shallow || !si->shallow->nr)
1494 return;
1495
beea4152
NTND
1496 if (args->cloning) {
1497 /*
1498 * remote is shallow, but this is a clone, there are
1499 * no objects in repo to worry about. Accept any
1500 * shallow points that exist in the pack (iow in repo
1501 * after get_pack() and reprepare_packed_git())
1502 */
910650d2 1503 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1504 struct object_id *oid = si->shallow->oid;
beea4152 1505 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1506 if (has_object_file(&oid[i]))
910650d2 1507 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1508 if (extra.nr) {
1509 setup_alternate_shallow(&shallow_lock,
1510 &alternate_shallow_file,
1511 &extra);
1512 commit_lock_file(&shallow_lock);
1513 }
910650d2 1514 oid_array_clear(&extra);
beea4152
NTND
1515 return;
1516 }
4820a33b
NTND
1517
1518 if (!si->nr_ours && !si->nr_theirs)
1519 return;
1520
1521 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1522 if (!si->nr_ours && !si->nr_theirs)
1523 return;
1524 for (i = 0; i < nr_sought; i++)
910650d2 1525 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1526 si->ref = &ref;
1527
48d25cae
NTND
1528 if (args->update_shallow) {
1529 /*
1530 * remote is also shallow, .git/shallow may be updated
1531 * so all refs can be accepted. Make sure we only add
1532 * shallow roots that are actually reachable from new
1533 * refs.
1534 */
910650d2 1535 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1536 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1537 assign_shallow_commits_to_refs(si, NULL, NULL);
1538 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1539 oid_array_clear(&ref);
48d25cae
NTND
1540 return;
1541 }
1542 for (i = 0; i < si->nr_ours; i++)
910650d2 1543 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1544 for (i = 0; i < si->nr_theirs; i++)
910650d2 1545 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1546 setup_alternate_shallow(&shallow_lock,
1547 &alternate_shallow_file,
1548 &extra);
1549 commit_lock_file(&shallow_lock);
910650d2 1550 oid_array_clear(&extra);
1551 oid_array_clear(&ref);
48d25cae
NTND
1552 return;
1553 }
1554
4820a33b
NTND
1555 /*
1556 * remote is also shallow, check what ref is safe to update
1557 * without updating .git/shallow
1558 */
1559 status = xcalloc(nr_sought, sizeof(*status));
1560 assign_shallow_commits_to_refs(si, NULL, status);
1561 if (si->nr_ours || si->nr_theirs) {
1562 for (i = 0; i < nr_sought; i++)
1563 if (status[i])
1564 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1565 }
1566 free(status);
910650d2 1567 oid_array_clear(&ref);
a796ccee
NTND
1568}
1569
745f7a8c
NTND
1570struct ref *fetch_pack(struct fetch_pack_args *args,
1571 int fd[], struct child_process *conn,
1572 const struct ref *ref,
1573 const char *dest,
f2db854d 1574 struct ref **sought, int nr_sought,
910650d2 1575 struct oid_array *shallow,
685fbd32
BW
1576 char **pack_lockfile,
1577 enum protocol_version version)
745f7a8c 1578{
745f7a8c 1579 struct ref *ref_cpy;
beea4152 1580 struct shallow_info si;
745f7a8c
NTND
1581
1582 fetch_pack_setup();
f2db854d
JH
1583 if (nr_sought)
1584 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1585
1586 if (!ref) {
1587 packet_flush(fd[1]);
1dd73e20 1588 die(_("no matching remote head"));
745f7a8c 1589 }
beea4152 1590 prepare_shallow_info(&si, shallow);
685fbd32
BW
1591 if (version == protocol_v2)
1592 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1593 pack_lockfile);
1594 else
1595 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1596 &si, pack_lockfile);
a49d2834 1597 reprepare_packed_git(the_repository);
4820a33b 1598 update_shallow(args, sought, nr_sought, &si);
beea4152 1599 clear_shallow_info(&si);
745f7a8c
NTND
1600 return ref_cpy;
1601}
e860d96b
MM
1602
1603int report_unmatched_refs(struct ref **sought, int nr_sought)
1604{
1605 int i, ret = 0;
1606
1607 for (i = 0; i < nr_sought; i++) {
d56583de 1608 if (!sought[i])
e860d96b 1609 continue;
d56583de
MM
1610 switch (sought[i]->match_status) {
1611 case REF_MATCHED:
1612 continue;
1613 case REF_NOT_MATCHED:
1614 error(_("no such remote ref %s"), sought[i]->name);
1615 break;
1616 case REF_UNADVERTISED_NOT_ALLOWED:
1617 error(_("Server does not allow request for unadvertised object %s"),
1618 sought[i]->name);
1619 break;
1620 }
e860d96b
MM
1621 ret = 1;
1622 }
1623 return ret;
1624}