commit: add repository argument to lookup_commit
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
a49d2834 2#include "repository.h"
b2141fc1 3#include "config.h"
697cc8ef 4#include "lockfile.h"
745f7a8c
NTND
5#include "refs.h"
6#include "pkt-line.h"
7#include "commit.h"
8#include "tag.h"
d807c4a0 9#include "exec-cmd.h"
745f7a8c
NTND
10#include "pack.h"
11#include "sideband.h"
12#include "fetch-pack.h"
13#include "remote.h"
14#include "run-command.h"
47a59185 15#include "connect.h"
745f7a8c
NTND
16#include "transport.h"
17#include "version.h"
099327b5 18#include "prio-queue.h"
beea4152 19#include "sha1-array.h"
fdb69d33 20#include "oidset.h"
0abe14f6 21#include "packfile.h"
cbd53a21 22#include "object-store.h"
745f7a8c
NTND
23
24static int transfer_unpack_limit = -1;
25static int fetch_unpack_limit = -1;
26static int unpack_limit = 100;
27static int prefer_ofs_delta = 1;
28static int no_done;
508ea882 29static int deepen_since_ok;
a45a2600 30static int deepen_not_ok;
745f7a8c
NTND
31static int fetch_fsck_objects = -1;
32static int transfer_fsck_objects = -1;
33static int agent_supported;
640d8b72 34static int server_supports_filtering;
6035d6aa
NTND
35static struct lock_file shallow_lock;
36static const char *alternate_shallow_file;
745f7a8c 37
208acbfb 38/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
39#define COMPLETE (1U << 0)
40#define COMMON (1U << 1)
41#define COMMON_REF (1U << 2)
42#define SEEN (1U << 3)
43#define POPPED (1U << 4)
41a078c6 44#define ALTERNATE (1U << 5)
745f7a8c
NTND
45
46static int marked;
47
48/*
49 * After sending this many "have"s if we do not get any new ACK , we
50 * give up traversing our history.
51 */
52#define MAX_IN_VAIN 256
53
099327b5 54static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
55static int non_common_revs, multi_ack, use_sideband;
56/* Allow specifying sha1 if it is a ref tip. */
57#define ALLOW_TIP_SHA1 01
68ee6289
FM
58/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
59#define ALLOW_REACHABLE_SHA1 02
7199c093 60static unsigned int allow_unadvertised_object_request;
745f7a8c 61
0d789a5b
NTND
62__attribute__((format (printf, 2, 3)))
63static inline void print_verbose(const struct fetch_pack_args *args,
64 const char *fmt, ...)
65{
66 va_list params;
67
68 if (!args->verbose)
69 return;
70
71 va_start(params, fmt);
72 vfprintf(stderr, fmt, params);
73 va_end(params);
74 fputc('\n', stderr);
75}
76
41a078c6
JK
77struct alternate_object_cache {
78 struct object **items;
79 size_t nr, alloc;
80};
81
82static void cache_one_alternate(const char *refname,
83 const struct object_id *oid,
84 void *vcache)
85{
86 struct alternate_object_cache *cache = vcache;
109cd76d 87 struct object *obj = parse_object(the_repository, oid);
41a078c6
JK
88
89 if (!obj || (obj->flags & ALTERNATE))
90 return;
91
92 obj->flags |= ALTERNATE;
93 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
94 cache->items[cache->nr++] = obj;
95}
96
97static void for_each_cached_alternate(void (*cb)(struct object *))
98{
99 static int initialized;
100 static struct alternate_object_cache cache;
101 size_t i;
102
103 if (!initialized) {
104 for_each_alternate_ref(cache_one_alternate, &cache);
105 initialized = 1;
106 }
107
108 for (i = 0; i < cache.nr; i++)
109 cb(cache.items[i]);
110}
111
745f7a8c
NTND
112static void rev_list_push(struct commit *commit, int mark)
113{
114 if (!(commit->object.flags & mark)) {
115 commit->object.flags |= mark;
116
0064053b
JK
117 if (parse_commit(commit))
118 return;
745f7a8c 119
099327b5 120 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
121
122 if (!(commit->object.flags & COMMON))
123 non_common_revs++;
124 }
125}
126
1b283377 127static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
745f7a8c 128{
109cd76d
SB
129 struct object *o = deref_tag(parse_object(the_repository, oid),
130 refname, 0);
745f7a8c
NTND
131
132 if (o && o->type == OBJ_COMMIT)
133 rev_list_push((struct commit *)o, SEEN);
134
135 return 0;
136}
137
b1b49c6e
MH
138static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
139 int flag, void *cb_data)
745f7a8c 140{
1b283377 141 return rev_list_insert_ref(refname, oid);
b1b49c6e
MH
142}
143
c50fb6ce
MH
144static int clear_marks(const char *refname, const struct object_id *oid,
145 int flag, void *cb_data)
745f7a8c 146{
109cd76d
SB
147 struct object *o = deref_tag(parse_object(the_repository, oid),
148 refname, 0);
745f7a8c
NTND
149
150 if (o && o->type == OBJ_COMMIT)
151 clear_commit_marks((struct commit *)o,
152 COMMON | COMMON_REF | SEEN | POPPED);
153 return 0;
154}
155
156/*
157 This function marks a rev and its ancestors as common.
158 In some cases, it is desirable to mark only the ancestors (for example
159 when only the server does not yet know that they are common).
160*/
161
162static void mark_common(struct commit *commit,
163 int ancestors_only, int dont_parse)
164{
165 if (commit != NULL && !(commit->object.flags & COMMON)) {
166 struct object *o = (struct object *)commit;
167
168 if (!ancestors_only)
169 o->flags |= COMMON;
170
171 if (!(o->flags & SEEN))
172 rev_list_push(commit, SEEN);
173 else {
174 struct commit_list *parents;
175
176 if (!ancestors_only && !(o->flags & POPPED))
177 non_common_revs--;
178 if (!o->parsed && !dont_parse)
179 if (parse_commit(commit))
180 return;
181
182 for (parents = commit->parents;
183 parents;
184 parents = parents->next)
185 mark_common(parents->item, 0, dont_parse);
186 }
187 }
188}
189
190/*
191 Get the next rev to send, ignoring the common.
192*/
193
1b283377 194static const struct object_id *get_rev(void)
745f7a8c
NTND
195{
196 struct commit *commit = NULL;
197
198 while (commit == NULL) {
199 unsigned int mark;
200 struct commit_list *parents;
201
099327b5 202 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
203 return NULL;
204
099327b5 205 commit = prio_queue_get(&rev_list);
0064053b 206 parse_commit(commit);
745f7a8c
NTND
207 parents = commit->parents;
208
209 commit->object.flags |= POPPED;
210 if (!(commit->object.flags & COMMON))
211 non_common_revs--;
212
213 if (commit->object.flags & COMMON) {
214 /* do not send "have", and ignore ancestors */
215 commit = NULL;
216 mark = COMMON | SEEN;
217 } else if (commit->object.flags & COMMON_REF)
218 /* send "have", and ignore ancestors */
219 mark = COMMON | SEEN;
220 else
221 /* send "have", also for its ancestors */
222 mark = SEEN;
223
224 while (parents) {
225 if (!(parents->item->object.flags & SEEN))
226 rev_list_push(parents->item, mark);
227 if (mark & COMMON)
228 mark_common(parents->item, 1, 0);
229 parents = parents->next;
230 }
745f7a8c
NTND
231 }
232
1b283377 233 return &commit->object.oid;
745f7a8c
NTND
234}
235
236enum ack_type {
237 NAK = 0,
238 ACK,
239 ACK_continue,
240 ACK_common,
241 ACK_ready
242};
243
244static void consume_shallow_list(struct fetch_pack_args *args, int fd)
245{
79891cb9 246 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
247 /* If we sent a depth we will get back "duplicate"
248 * shallow and unshallow commands every time there
249 * is a block of have lines exchanged.
250 */
74543a04
JK
251 char *line;
252 while ((line = packet_read_line(fd, NULL))) {
59556548 253 if (starts_with(line, "shallow "))
745f7a8c 254 continue;
59556548 255 if (starts_with(line, "unshallow "))
745f7a8c 256 continue;
1dd73e20 257 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
258 }
259 }
260}
261
1b283377 262static enum ack_type get_ack(int fd, struct object_id *result_oid)
745f7a8c 263{
74543a04
JK
264 int len;
265 char *line = packet_read_line(fd, &len);
82e56767 266 const char *arg;
745f7a8c 267
bc9d4dc5
JK
268 if (!line)
269 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
745f7a8c
NTND
270 if (!strcmp(line, "NAK"))
271 return NAK;
82e56767 272 if (skip_prefix(line, "ACK ", &arg)) {
1b283377 273 if (!get_oid_hex(arg, result_oid)) {
82e56767
JK
274 arg += 40;
275 len -= arg - line;
276 if (len < 1)
030e9dd6 277 return ACK;
82e56767 278 if (strstr(arg, "continue"))
745f7a8c 279 return ACK_continue;
82e56767 280 if (strstr(arg, "common"))
745f7a8c 281 return ACK_common;
82e56767 282 if (strstr(arg, "ready"))
745f7a8c
NTND
283 return ACK_ready;
284 return ACK;
285 }
286 }
8e2c7bef
JT
287 if (skip_prefix(line, "ERR ", &arg))
288 die(_("remote error: %s"), arg);
dfbfb9f3 289 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
290}
291
292static void send_request(struct fetch_pack_args *args,
293 int fd, struct strbuf *buf)
294{
295 if (args->stateless_rpc) {
296 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
297 packet_flush(fd);
298 } else
cdf4fb8e 299 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
300}
301
41a078c6 302static void insert_one_alternate_object(struct object *obj)
745f7a8c 303{
1b283377 304 rev_list_insert_ref(NULL, &obj->oid);
745f7a8c
NTND
305}
306
307#define INITIAL_FLUSH 16
308#define PIPESAFE_FLUSH 32
da470981 309#define LARGE_FLUSH 16384
745f7a8c 310
685fbd32 311static int next_flush(int stateless_rpc, int count)
745f7a8c 312{
685fbd32 313 if (stateless_rpc) {
da470981
JT
314 if (count < LARGE_FLUSH)
315 count <<= 1;
316 else
317 count = count * 11 / 10;
318 } else {
319 if (count < PIPESAFE_FLUSH)
320 count <<= 1;
321 else
322 count += PIPESAFE_FLUSH;
323 }
745f7a8c
NTND
324 return count;
325}
326
327static int find_common(struct fetch_pack_args *args,
1b283377 328 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
329 struct ref *refs)
330{
331 int fetching;
332 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 333 const struct object_id *oid;
745f7a8c
NTND
334 unsigned in_vain = 0;
335 int got_continue = 0;
336 int got_ready = 0;
337 struct strbuf req_buf = STRBUF_INIT;
338 size_t state_len = 0;
339
340 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 341 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
342 if (marked)
343 for_each_ref(clear_marks, NULL);
344 marked = 1;
345
b1b49c6e 346 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 347 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
348
349 fetching = 0;
350 for ( ; refs ; refs = refs->next) {
1b283377 351 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
352 const char *remote_hex;
353 struct object *o;
354
355 /*
356 * If that object is complete (i.e. it is an ancestor of a
357 * local ref), we tell them we have it but do not have to
358 * tell them about its ancestors, which they already know
359 * about.
360 *
361 * We use lookup_object here because we are only
362 * interested in the case we *know* the object is
363 * reachable and we have already scanned it.
364 */
5abddd1e 365 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
745f7a8c
NTND
366 (o->flags & COMPLETE)) {
367 continue;
368 }
369
1b283377 370 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
371 if (!fetching) {
372 struct strbuf c = STRBUF_INIT;
373 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
374 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
375 if (no_done) strbuf_addstr(&c, " no-done");
376 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
377 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 378 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
379 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
380 if (args->no_progress) strbuf_addstr(&c, " no-progress");
381 if (args->include_tag) strbuf_addstr(&c, " include-tag");
382 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 383 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 384 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
385 if (agent_supported) strbuf_addf(&c, " agent=%s",
386 git_user_agent_sanitized());
640d8b72
JH
387 if (args->filter_options.choice)
388 strbuf_addstr(&c, " filter");
745f7a8c
NTND
389 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
390 strbuf_release(&c);
391 } else
392 packet_buf_write(&req_buf, "want %s\n", remote_hex);
393 fetching++;
394 }
395
396 if (!fetching) {
397 strbuf_release(&req_buf);
398 packet_flush(fd[1]);
399 return 1;
400 }
401
c8813487 402 if (is_repository_shallow(the_repository))
1a30f5a2 403 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
404 if (args->depth > 0)
405 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882 406 if (args->deepen_since) {
dddbad72 407 timestamp_t max_age = approxidate(args->deepen_since);
cb71f8bd 408 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
508ea882 409 }
a45a2600
NTND
410 if (args->deepen_not) {
411 int i;
412 for (i = 0; i < args->deepen_not->nr; i++) {
413 struct string_list_item *s = args->deepen_not->items + i;
414 packet_buf_write(&req_buf, "deepen-not %s", s->string);
415 }
416 }
640d8b72
JH
417 if (server_supports_filtering && args->filter_options.choice)
418 packet_buf_write(&req_buf, "filter %s",
419 args->filter_options.filter_spec);
745f7a8c
NTND
420 packet_buf_flush(&req_buf);
421 state_len = req_buf.len;
422
79891cb9 423 if (args->deepen) {
74543a04 424 char *line;
ae021d87 425 const char *arg;
1b283377 426 struct object_id oid;
745f7a8c
NTND
427
428 send_request(args, fd[1], &req_buf);
74543a04 429 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87 430 if (skip_prefix(line, "shallow ", &arg)) {
1b283377 431 if (get_oid_hex(arg, &oid))
1dd73e20 432 die(_("invalid shallow line: %s"), line);
19143f13 433 register_shallow(the_repository, &oid);
745f7a8c
NTND
434 continue;
435 }
ae021d87 436 if (skip_prefix(line, "unshallow ", &arg)) {
1b283377 437 if (get_oid_hex(arg, &oid))
1dd73e20 438 die(_("invalid unshallow line: %s"), line);
5abddd1e 439 if (!lookup_object(the_repository, oid.hash))
1dd73e20 440 die(_("object not found: %s"), line);
745f7a8c 441 /* make sure that it is parsed as shallow */
109cd76d 442 if (!parse_object(the_repository, &oid))
1dd73e20 443 die(_("error in object: %s"), line);
e92b848c 444 if (unregister_shallow(&oid))
1dd73e20 445 die(_("no shallow found: %s"), line);
745f7a8c
NTND
446 continue;
447 }
1dd73e20 448 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
449 }
450 } else if (!args->stateless_rpc)
451 send_request(args, fd[1], &req_buf);
452
453 if (!args->stateless_rpc) {
454 /* If we aren't using the stateless-rpc interface
455 * we don't need to retain the headers.
456 */
457 strbuf_setlen(&req_buf, 0);
458 state_len = 0;
459 }
460
461 flushes = 0;
462 retval = -1;
88e2f9ed
JT
463 if (args->no_dependents)
464 goto done;
1b283377 465 while ((oid = get_rev())) {
466 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
467 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
468 in_vain++;
469 if (flush_at <= ++count) {
470 int ack;
471
472 packet_buf_flush(&req_buf);
473 send_request(args, fd[1], &req_buf);
474 strbuf_setlen(&req_buf, state_len);
475 flushes++;
685fbd32 476 flush_at = next_flush(args->stateless_rpc, count);
745f7a8c
NTND
477
478 /*
479 * We keep one window "ahead" of the other side, and
480 * will wait for an ACK only on the next one
481 */
482 if (!args->stateless_rpc && count == INITIAL_FLUSH)
483 continue;
484
485 consume_shallow_list(args, fd[0]);
486 do {
1b283377 487 ack = get_ack(fd[0], result_oid);
0d789a5b 488 if (ack)
1dd73e20 489 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 490 ack, oid_to_hex(result_oid));
745f7a8c
NTND
491 switch (ack) {
492 case ACK:
493 flushes = 0;
494 multi_ack = 0;
495 retval = 0;
496 goto done;
497 case ACK_common:
498 case ACK_ready:
499 case ACK_continue: {
500 struct commit *commit =
c1f5eb49
SB
501 lookup_commit(the_repository,
502 result_oid);
745f7a8c 503 if (!commit)
1b283377 504 die(_("invalid commit %s"), oid_to_hex(result_oid));
745f7a8c
NTND
505 if (args->stateless_rpc
506 && ack == ACK_common
507 && !(commit->object.flags & COMMON)) {
508 /* We need to replay the have for this object
509 * on the next RPC request so the peer knows
510 * it is in common with us.
511 */
1b283377 512 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
513 packet_buf_write(&req_buf, "have %s\n", hex);
514 state_len = req_buf.len;
06b3d386
JT
515 /*
516 * Reset in_vain because an ack
517 * for this commit has not been
518 * seen.
519 */
520 in_vain = 0;
521 } else if (!args->stateless_rpc
522 || ack != ACK_common)
523 in_vain = 0;
745f7a8c
NTND
524 mark_common(commit, 0, 1);
525 retval = 0;
745f7a8c
NTND
526 got_continue = 1;
527 if (ack == ACK_ready) {
099327b5 528 clear_prio_queue(&rev_list);
745f7a8c
NTND
529 got_ready = 1;
530 }
531 break;
532 }
533 }
534 } while (ack);
535 flushes--;
536 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 537 print_verbose(args, _("giving up"));
745f7a8c
NTND
538 break; /* give up */
539 }
540 }
541 }
542done:
543 if (!got_ready || !no_done) {
544 packet_buf_write(&req_buf, "done\n");
545 send_request(args, fd[1], &req_buf);
546 }
1dd73e20 547 print_verbose(args, _("done"));
745f7a8c
NTND
548 if (retval != 0) {
549 multi_ack = 0;
550 flushes++;
551 }
552 strbuf_release(&req_buf);
553
ff62eca7
NTND
554 if (!got_ready || !no_done)
555 consume_shallow_list(args, fd[0]);
745f7a8c 556 while (flushes || multi_ack) {
1b283377 557 int ack = get_ack(fd[0], result_oid);
745f7a8c 558 if (ack) {
1dd73e20 559 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 560 ack, oid_to_hex(result_oid));
745f7a8c
NTND
561 if (ack == ACK)
562 return 0;
563 multi_ack = 1;
564 continue;
565 }
566 flushes--;
567 }
568 /* it is no error to fetch into a completely empty repo */
569 return count ? retval : 0;
570}
571
572static struct commit_list *complete;
573
1b283377 574static int mark_complete(const struct object_id *oid)
745f7a8c 575{
109cd76d 576 struct object *o = parse_object(the_repository, oid);
745f7a8c
NTND
577
578 while (o && o->type == OBJ_TAG) {
579 struct tag *t = (struct tag *) o;
580 if (!t->tagged)
581 break; /* broken repository */
582 o->flags |= COMPLETE;
109cd76d 583 o = parse_object(the_repository, &t->tagged->oid);
745f7a8c
NTND
584 }
585 if (o && o->type == OBJ_COMMIT) {
586 struct commit *commit = (struct commit *)o;
587 if (!(commit->object.flags & COMPLETE)) {
588 commit->object.flags |= COMPLETE;
16445242 589 commit_list_insert(commit, &complete);
745f7a8c
NTND
590 }
591 }
592 return 0;
593}
594
f8ee4d85
MH
595static int mark_complete_oid(const char *refname, const struct object_id *oid,
596 int flag, void *cb_data)
597{
1b283377 598 return mark_complete(oid);
f8ee4d85
MH
599}
600
745f7a8c 601static void mark_recent_complete_commits(struct fetch_pack_args *args,
dddbad72 602 timestamp_t cutoff)
745f7a8c
NTND
603{
604 while (complete && cutoff <= complete->item->date) {
1dd73e20 605 print_verbose(args, _("Marking %s as complete"),
0d789a5b 606 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
607 pop_most_recent_commit(&complete, COMPLETE);
608 }
609}
610
fdb69d33
JT
611static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
612{
613 for (; refs; refs = refs->next)
614 oidset_insert(oids, &refs->old_oid);
615}
616
617static int tip_oids_contain(struct oidset *tip_oids,
618 struct ref *unmatched, struct ref *newlist,
619 const struct object_id *id)
620{
621 /*
622 * Note that this only looks at the ref lists the first time it's
623 * called. This works out in filter_refs() because even though it may
624 * add to "newlist" between calls, the additions will always be for
625 * oids that are already in the set.
626 */
9e6fabde 627 if (!tip_oids->map.map.tablesize) {
fdb69d33
JT
628 add_refs_to_oidset(tip_oids, unmatched);
629 add_refs_to_oidset(tip_oids, newlist);
630 }
631 return oidset_contains(tip_oids, id);
632}
633
745f7a8c 634static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
635 struct ref **refs,
636 struct ref **sought, int nr_sought)
745f7a8c
NTND
637{
638 struct ref *newlist = NULL;
639 struct ref **newtail = &newlist;
fdb69d33 640 struct ref *unmatched = NULL;
745f7a8c 641 struct ref *ref, *next;
fdb69d33 642 struct oidset tip_oids = OIDSET_INIT;
f2db854d 643 int i;
745f7a8c 644
f2db854d 645 i = 0;
745f7a8c
NTND
646 for (ref = *refs; ref; ref = next) {
647 int keep = 0;
648 next = ref->next;
f2db854d 649
50e19a83 650 if (starts_with(ref->name, "refs/") &&
4c224081 651 check_refname_format(ref->name, 0))
745f7a8c
NTND
652 ; /* trash */
653 else {
f2db854d
JH
654 while (i < nr_sought) {
655 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
656 if (cmp < 0)
657 break; /* definitely do not have it */
658 else if (cmp == 0) {
659 keep = 1; /* definitely have it */
d56583de 660 sought[i]->match_status = REF_MATCHED;
745f7a8c 661 }
f2db854d 662 i++;
745f7a8c 663 }
745f7a8c 664
e9502c0a
JK
665 if (!keep && args->fetch_all &&
666 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
667 keep = 1;
668 }
745f7a8c
NTND
669
670 if (keep) {
671 *newtail = ref;
672 ref->next = NULL;
673 newtail = &ref->next;
674 } else {
fdb69d33
JT
675 ref->next = unmatched;
676 unmatched = ref;
745f7a8c
NTND
677 }
678 }
679
6e7b66ee 680 /* Append unmatched requests to the list */
d56583de 681 for (i = 0; i < nr_sought; i++) {
1b283377 682 struct object_id oid;
683 const char *p;
b7916422 684
d56583de
MM
685 ref = sought[i];
686 if (ref->match_status != REF_NOT_MATCHED)
687 continue;
1b283377 688 if (parse_oid_hex(ref->name, &oid, &p) ||
689 *p != '\0' ||
690 oidcmp(&oid, &ref->old_oid))
d56583de 691 continue;
6e7b66ee 692
d56583de 693 if ((allow_unadvertised_object_request &
fdb69d33
JT
694 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
695 tip_oids_contain(&tip_oids, unmatched, newlist,
696 &ref->old_oid)) {
d56583de 697 ref->match_status = REF_MATCHED;
c3c17bf1
JK
698 *newtail = copy_ref(ref);
699 newtail = &(*newtail)->next;
d56583de
MM
700 } else {
701 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
702 }
703 }
fdb69d33
JT
704
705 oidset_clear(&tip_oids);
706 for (ref = unmatched; ref; ref = next) {
707 next = ref->next;
708 free(ref);
709 }
710
745f7a8c
NTND
711 *refs = newlist;
712}
713
41a078c6 714static void mark_alternate_complete(struct object *obj)
745f7a8c 715{
1b283377 716 mark_complete(&obj->oid);
745f7a8c
NTND
717}
718
024aa469
TI
719struct loose_object_iter {
720 struct oidset *loose_object_set;
721 struct ref *refs;
722};
723
724/*
725 * If the number of refs is not larger than the number of loose objects,
726 * this function stops inserting.
727 */
728static int add_loose_objects_to_set(const struct object_id *oid,
729 const char *path,
730 void *data)
731{
732 struct loose_object_iter *iter = data;
733 oidset_insert(iter->loose_object_set, oid);
734 if (iter->refs == NULL)
735 return 1;
736
737 iter->refs = iter->refs->next;
738 return 0;
739}
740
745f7a8c 741static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
742 struct ref **refs,
743 struct ref **sought, int nr_sought)
745f7a8c
NTND
744{
745 struct ref *ref;
746 int retval;
a1c6d7c1 747 int old_save_commit_buffer = save_commit_buffer;
dddbad72 748 timestamp_t cutoff = 0;
024aa469
TI
749 struct oidset loose_oid_set = OIDSET_INIT;
750 int use_oidset = 0;
751 struct loose_object_iter iter = {&loose_oid_set, *refs};
752
753 /* Enumerate all loose objects or know refs are not so many. */
754 use_oidset = !for_each_loose_object(add_loose_objects_to_set,
755 &iter, 0);
745f7a8c
NTND
756
757 save_commit_buffer = 0;
758
759 for (ref = *refs; ref; ref = ref->next) {
760 struct object *o;
024aa469 761 unsigned int flags = OBJECT_INFO_QUICK;
745f7a8c 762
024aa469
TI
763 if (use_oidset &&
764 !oidset_contains(&loose_oid_set, &ref->old_oid)) {
765 /*
766 * I know this does not exist in the loose form,
767 * so check if it exists in a non-loose form.
768 */
769 flags |= OBJECT_INFO_IGNORE_LOOSE;
770 }
012a1bb5 771
024aa469
TI
772 if (!has_object_file_with_flags(&ref->old_oid, flags))
773 continue;
109cd76d 774 o = parse_object(the_repository, &ref->old_oid);
745f7a8c
NTND
775 if (!o)
776 continue;
777
778 /* We already have it -- which may mean that we were
779 * in sync with the other side at some time after
780 * that (it is OK if we guess wrong here).
781 */
782 if (o->type == OBJ_COMMIT) {
783 struct commit *commit = (struct commit *)o;
784 if (!cutoff || cutoff < commit->date)
785 cutoff = commit->date;
786 }
787 }
788
024aa469
TI
789 oidset_clear(&loose_oid_set);
790
88e2f9ed
JT
791 if (!args->no_dependents) {
792 if (!args->deepen) {
793 for_each_ref(mark_complete_oid, NULL);
794 for_each_cached_alternate(mark_alternate_complete);
795 commit_list_sort_by_date(&complete);
796 if (cutoff)
797 mark_recent_complete_commits(args, cutoff);
798 }
745f7a8c 799
88e2f9ed
JT
800 /*
801 * Mark all complete remote refs as common refs.
802 * Don't mark them common yet; the server has to be told so first.
803 */
804 for (ref = *refs; ref; ref = ref->next) {
5abddd1e
SB
805 struct object *o = deref_tag(lookup_object(the_repository,
806 ref->old_oid.hash),
88e2f9ed 807 NULL, 0);
745f7a8c 808
88e2f9ed
JT
809 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
810 continue;
745f7a8c 811
88e2f9ed
JT
812 if (!(o->flags & SEEN)) {
813 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
745f7a8c 814
88e2f9ed
JT
815 mark_common((struct commit *)o, 1, 1);
816 }
745f7a8c
NTND
817 }
818 }
819
f2db854d 820 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
821
822 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 823 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
824 struct object *o;
825
5abddd1e 826 o = lookup_object(the_repository, remote->hash);
745f7a8c
NTND
827 if (!o || !(o->flags & COMPLETE)) {
828 retval = 0;
1b283377 829 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 830 ref->name);
745f7a8c
NTND
831 continue;
832 }
1b283377 833 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 834 ref->name);
745f7a8c 835 }
a1c6d7c1
JT
836
837 save_commit_buffer = old_save_commit_buffer;
838
745f7a8c
NTND
839 return retval;
840}
841
842static int sideband_demux(int in, int out, void *data)
843{
844 int *xd = data;
9ff18faf 845 int ret;
745f7a8c 846
9ff18faf 847 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
848 close(out);
849 return ret;
850}
851
852static int get_pack(struct fetch_pack_args *args,
853 int xd[2], char **pack_lockfile)
854{
855 struct async demux;
745f7a8c 856 int do_keep = args->keep_pack;
984a43b9
JK
857 const char *cmd_name;
858 struct pack_header header;
859 int pass_header = 0;
d3180279 860 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 861 int ret;
745f7a8c
NTND
862
863 memset(&demux, 0, sizeof(demux));
864 if (use_sideband) {
865 /* xd[] is talking with upload-pack; subprocess reads from
866 * xd[0], spits out band#2 to stderr, and feeds us band#1
867 * through demux->out.
868 */
869 demux.proc = sideband_demux;
870 demux.data = xd;
871 demux.out = -1;
df857572 872 demux.isolate_sigpipe = 1;
745f7a8c 873 if (start_async(&demux))
1dd73e20 874 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
875 }
876 else
877 demux.out = xd[0];
878
745f7a8c 879 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
880
881 if (read_pack_header(demux.out, &header))
1dd73e20 882 die(_("protocol error: bad pack header"));
984a43b9 883 pass_header = 1;
745f7a8c
NTND
884 if (ntohl(header.hdr_entries) < unpack_limit)
885 do_keep = 0;
886 else
887 do_keep = 1;
888 }
889
6035d6aa 890 if (alternate_shallow_file) {
984a43b9
JK
891 argv_array_push(&cmd.args, "--shallow-file");
892 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
893 }
894
88e2f9ed 895 if (do_keep || args->from_promisor) {
745f7a8c
NTND
896 if (pack_lockfile)
897 cmd.out = -1;
984a43b9
JK
898 cmd_name = "index-pack";
899 argv_array_push(&cmd.args, cmd_name);
900 argv_array_push(&cmd.args, "--stdin");
745f7a8c 901 if (!args->quiet && !args->no_progress)
984a43b9 902 argv_array_push(&cmd.args, "-v");
745f7a8c 903 if (args->use_thin_pack)
984a43b9 904 argv_array_push(&cmd.args, "--fix-thin");
88e2f9ed 905 if (do_keep && (args->lock_pack || unpack_limit)) {
da25bdb7 906 char hostname[HOST_NAME_MAX + 1];
5781a9a2 907 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
908 xsnprintf(hostname, sizeof(hostname), "localhost");
909 argv_array_pushf(&cmd.args,
910 "--keep=fetch-pack %"PRIuMAX " on %s",
911 (uintmax_t)getpid(), hostname);
745f7a8c 912 }
c6807a40 913 if (args->check_self_contained_and_connected)
984a43b9 914 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
88e2f9ed
JT
915 if (args->from_promisor)
916 argv_array_push(&cmd.args, "--promisor");
745f7a8c
NTND
917 }
918 else {
984a43b9
JK
919 cmd_name = "unpack-objects";
920 argv_array_push(&cmd.args, cmd_name);
745f7a8c 921 if (args->quiet || args->no_progress)
984a43b9 922 argv_array_push(&cmd.args, "-q");
c6807a40 923 args->check_self_contained_and_connected = 0;
745f7a8c 924 }
984a43b9
JK
925
926 if (pass_header)
927 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
928 ntohl(header.hdr_version),
929 ntohl(header.hdr_entries));
745f7a8c
NTND
930 if (fetch_fsck_objects >= 0
931 ? fetch_fsck_objects
932 : transfer_fsck_objects >= 0
933 ? transfer_fsck_objects
98a2ea46
JT
934 : 0) {
935 if (args->from_promisor)
936 /*
937 * We cannot use --strict in index-pack because it
938 * checks both broken objects and links, but we only
939 * want to check for broken objects.
940 */
941 argv_array_push(&cmd.args, "--fsck-objects");
942 else
943 argv_array_push(&cmd.args, "--strict");
944 }
745f7a8c
NTND
945
946 cmd.in = demux.out;
947 cmd.git_cmd = 1;
948 if (start_command(&cmd))
1dd73e20 949 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
950 if (do_keep && pack_lockfile) {
951 *pack_lockfile = index_pack_lockfile(cmd.out);
952 close(cmd.out);
953 }
954
37cb1dd6
JL
955 if (!use_sideband)
956 /* Closed by start_command() */
957 xd[0] = -1;
958
c6807a40
NTND
959 ret = finish_command(&cmd);
960 if (!ret || (args->check_self_contained_and_connected && ret == 1))
961 args->self_contained_and_connected =
962 args->check_self_contained_and_connected &&
963 ret == 0;
964 else
1dd73e20 965 die(_("%s failed"), cmd_name);
745f7a8c 966 if (use_sideband && finish_async(&demux))
1dd73e20 967 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
968 return 0;
969}
970
f2db854d
JH
971static int cmp_ref_by_name(const void *a_, const void *b_)
972{
973 const struct ref *a = *((const struct ref **)a_);
974 const struct ref *b = *((const struct ref **)b_);
975 return strcmp(a->name, b->name);
976}
977
745f7a8c
NTND
978static struct ref *do_fetch_pack(struct fetch_pack_args *args,
979 int fd[2],
980 const struct ref *orig_ref,
f2db854d 981 struct ref **sought, int nr_sought,
beea4152 982 struct shallow_info *si,
745f7a8c
NTND
983 char **pack_lockfile)
984{
985 struct ref *ref = copy_ref_list(orig_ref);
1b283377 986 struct object_id oid;
745f7a8c
NTND
987 const char *agent_feature;
988 int agent_len;
989
990 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 991 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 992
c8813487 993 if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
1dd73e20 994 die(_("Server does not support shallow clients"));
a45a2600 995 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 996 args->deepen = 1;
745f7a8c 997 if (server_supports("multi_ack_detailed")) {
1dd73e20 998 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
999 multi_ack = 2;
1000 if (server_supports("no-done")) {
1dd73e20 1001 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
1002 if (args->stateless_rpc)
1003 no_done = 1;
1004 }
1005 }
1006 else if (server_supports("multi_ack")) {
1dd73e20 1007 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
1008 multi_ack = 1;
1009 }
1010 if (server_supports("side-band-64k")) {
1dd73e20 1011 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
1012 use_sideband = 2;
1013 }
1014 else if (server_supports("side-band")) {
1dd73e20 1015 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
1016 use_sideband = 1;
1017 }
6e7b66ee 1018 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 1019 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 1020 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 1021 }
68ee6289 1022 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 1023 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
1024 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1025 }
745f7a8c
NTND
1026 if (!server_supports("thin-pack"))
1027 args->use_thin_pack = 0;
1028 if (!server_supports("no-progress"))
1029 args->no_progress = 0;
1030 if (!server_supports("include-tag"))
1031 args->include_tag = 0;
0d789a5b 1032 if (server_supports("ofs-delta"))
1dd73e20 1033 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 1034 else
745f7a8c
NTND
1035 prefer_ofs_delta = 0;
1036
640d8b72
JH
1037 if (server_supports("filter")) {
1038 server_supports_filtering = 1;
1039 print_verbose(args, _("Server supports filter"));
1040 } else if (args->filter_options.choice) {
1041 warning("filtering not recognized by server, ignoring");
1042 }
1043
745f7a8c
NTND
1044 if ((agent_feature = server_feature_value("agent", &agent_len))) {
1045 agent_supported = 1;
0d789a5b 1046 if (agent_len)
1dd73e20 1047 print_verbose(args, _("Server version is %.*s"),
0d789a5b 1048 agent_len, agent_feature);
745f7a8c 1049 }
508ea882
NTND
1050 if (server_supports("deepen-since"))
1051 deepen_since_ok = 1;
1052 else if (args->deepen_since)
1053 die(_("Server does not support --shallow-since"));
a45a2600
NTND
1054 if (server_supports("deepen-not"))
1055 deepen_not_ok = 1;
1056 else if (args->deepen_not)
1057 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
1058 if (!server_supports("deepen-relative") && args->deepen_relative)
1059 die(_("Server does not support --deepen"));
745f7a8c 1060
f2db854d 1061 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
1062 packet_flush(fd[1]);
1063 goto all_done;
1064 }
1b283377 1065 if (find_common(args, fd, &oid, ref) < 0)
745f7a8c
NTND
1066 if (!args->keep_pack)
1067 /* When cloning, it is not unusual to have
1068 * no common commit.
1069 */
1dd73e20 1070 warning(_("no common commits"));
745f7a8c
NTND
1071
1072 if (args->stateless_rpc)
1073 packet_flush(fd[1]);
79891cb9 1074 if (args->deepen)
1a30f5a2
NTND
1075 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1076 NULL);
4820a33b 1077 else if (si->nr_ours || si->nr_theirs)
beea4152 1078 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
1079 else
1080 alternate_shallow_file = NULL;
745f7a8c 1081 if (get_pack(args, fd, pack_lockfile))
1dd73e20 1082 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
1083
1084 all_done:
1085 return ref;
1086}
1087
f7e20501
BW
1088static void add_shallow_requests(struct strbuf *req_buf,
1089 const struct fetch_pack_args *args)
1090{
b16b60f7 1091 if (is_repository_shallow(the_repository))
f7e20501
BW
1092 write_shallow_commits(req_buf, 1, NULL);
1093 if (args->depth > 0)
1094 packet_buf_write(req_buf, "deepen %d", args->depth);
1095 if (args->deepen_since) {
1096 timestamp_t max_age = approxidate(args->deepen_since);
1097 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1098 }
1099 if (args->deepen_not) {
1100 int i;
1101 for (i = 0; i < args->deepen_not->nr; i++) {
1102 struct string_list_item *s = args->deepen_not->items + i;
1103 packet_buf_write(req_buf, "deepen-not %s", s->string);
1104 }
1105 }
1106}
1107
685fbd32
BW
1108static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1109{
1110 for ( ; wants ; wants = wants->next) {
1111 const struct object_id *remote = &wants->old_oid;
1112 const char *remote_hex;
1113 struct object *o;
1114
1115 /*
1116 * If that object is complete (i.e. it is an ancestor of a
1117 * local ref), we tell them we have it but do not have to
1118 * tell them about its ancestors, which they already know
1119 * about.
1120 *
1121 * We use lookup_object here because we are only
1122 * interested in the case we *know* the object is
1123 * reachable and we have already scanned it.
1124 */
5abddd1e 1125 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
685fbd32
BW
1126 (o->flags & COMPLETE)) {
1127 continue;
1128 }
1129
1130 remote_hex = oid_to_hex(remote);
1131 packet_buf_write(req_buf, "want %s\n", remote_hex);
1132 }
1133}
1134
1135static void add_common(struct strbuf *req_buf, struct oidset *common)
1136{
1137 struct oidset_iter iter;
1138 const struct object_id *oid;
1139 oidset_iter_init(common, &iter);
1140
1141 while ((oid = oidset_iter_next(&iter))) {
1142 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1143 }
1144}
1145
1146static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1147{
1148 int ret = 0;
1149 int haves_added = 0;
1150 const struct object_id *oid;
1151
1152 while ((oid = get_rev())) {
1153 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1154 if (++haves_added >= *haves_to_send)
1155 break;
1156 }
1157
1158 *in_vain += haves_added;
1159 if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1160 /* Send Done */
1161 packet_buf_write(req_buf, "done\n");
1162 ret = 1;
1163 }
1164
1165 /* Increase haves to send on next round */
1166 *haves_to_send = next_flush(1, *haves_to_send);
1167
1168 return ret;
1169}
1170
1171static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1172 const struct ref *wants, struct oidset *common,
1173 int *haves_to_send, int *in_vain)
1174{
1175 int ret = 0;
1176 struct strbuf req_buf = STRBUF_INIT;
1177
1178 if (server_supports_v2("fetch", 1))
1179 packet_buf_write(&req_buf, "command=fetch");
1180 if (server_supports_v2("agent", 0))
1181 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
5e3548ef
BW
1182 if (args->server_options && args->server_options->nr &&
1183 server_supports_v2("server-option", 1)) {
1184 int i;
1185 for (i = 0; i < args->server_options->nr; i++)
1186 packet_write_fmt(fd_out, "server-option=%s",
1187 args->server_options->items[i].string);
1188 }
685fbd32
BW
1189
1190 packet_buf_delim(&req_buf);
1191 if (args->use_thin_pack)
1192 packet_buf_write(&req_buf, "thin-pack");
1193 if (args->no_progress)
1194 packet_buf_write(&req_buf, "no-progress");
1195 if (args->include_tag)
1196 packet_buf_write(&req_buf, "include-tag");
1197 if (prefer_ofs_delta)
1198 packet_buf_write(&req_buf, "ofs-delta");
1199
f7e20501
BW
1200 /* Add shallow-info and deepen request */
1201 if (server_supports_feature("fetch", "shallow", 0))
1202 add_shallow_requests(&req_buf, args);
b16b60f7 1203 else if (is_repository_shallow(the_repository) || args->deepen)
f7e20501
BW
1204 die(_("Server does not support shallow requests"));
1205
ba95710a
JT
1206 /* Add filter */
1207 if (server_supports_feature("fetch", "filter", 0) &&
1208 args->filter_options.choice) {
1209 print_verbose(args, _("Server supports filter"));
1210 packet_buf_write(&req_buf, "filter %s",
1211 args->filter_options.filter_spec);
1212 } else if (args->filter_options.choice) {
1213 warning("filtering not recognized by server, ignoring");
1214 }
1215
685fbd32
BW
1216 /* add wants */
1217 add_wants(wants, &req_buf);
1218
ba95710a
JT
1219 if (args->no_dependents) {
1220 packet_buf_write(&req_buf, "done");
1221 ret = 1;
1222 } else {
1223 /* Add all of the common commits we've found in previous rounds */
1224 add_common(&req_buf, common);
685fbd32 1225
ba95710a
JT
1226 /* Add initial haves */
1227 ret = add_haves(&req_buf, haves_to_send, in_vain);
1228 }
685fbd32
BW
1229
1230 /* Send request */
1231 packet_buf_flush(&req_buf);
1232 write_or_die(fd_out, req_buf.buf, req_buf.len);
1233
1234 strbuf_release(&req_buf);
1235 return ret;
1236}
1237
1238/*
1239 * Processes a section header in a server's response and checks if it matches
1240 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1241 * not consumed); if 0, the line will be consumed and the function will die if
1242 * the section header doesn't match what was expected.
1243 */
1244static int process_section_header(struct packet_reader *reader,
1245 const char *section, int peek)
1246{
1247 int ret;
1248
1249 if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
f7e20501 1250 die("error reading section header '%s'", section);
685fbd32
BW
1251
1252 ret = !strcmp(reader->line, section);
1253
1254 if (!peek) {
1255 if (!ret)
1256 die("expected '%s', received '%s'",
1257 section, reader->line);
1258 packet_reader_read(reader);
1259 }
1260
1261 return ret;
1262}
1263
1264static int process_acks(struct packet_reader *reader, struct oidset *common)
1265{
1266 /* received */
1267 int received_ready = 0;
1268 int received_ack = 0;
1269
1270 process_section_header(reader, "acknowledgments", 0);
1271 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1272 const char *arg;
1273
1274 if (!strcmp(reader->line, "NAK"))
1275 continue;
1276
1277 if (skip_prefix(reader->line, "ACK ", &arg)) {
1278 struct object_id oid;
1279 if (!get_oid_hex(arg, &oid)) {
1280 struct commit *commit;
1281 oidset_insert(common, &oid);
c1f5eb49 1282 commit = lookup_commit(the_repository, &oid);
685fbd32
BW
1283 mark_common(commit, 0, 1);
1284 }
1285 continue;
1286 }
1287
1288 if (!strcmp(reader->line, "ready")) {
1289 clear_prio_queue(&rev_list);
1290 received_ready = 1;
1291 continue;
1292 }
1293
1294 die("unexpected acknowledgment line: '%s'", reader->line);
1295 }
1296
1297 if (reader->status != PACKET_READ_FLUSH &&
1298 reader->status != PACKET_READ_DELIM)
1299 die("error processing acks: %d", reader->status);
1300
1301 /* return 0 if no common, 1 if there are common, or 2 if ready */
1302 return received_ready ? 2 : (received_ack ? 1 : 0);
1303}
1304
f7e20501
BW
1305static void receive_shallow_info(struct fetch_pack_args *args,
1306 struct packet_reader *reader)
1307{
1308 process_section_header(reader, "shallow-info", 0);
1309 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1310 const char *arg;
1311 struct object_id oid;
1312
1313 if (skip_prefix(reader->line, "shallow ", &arg)) {
1314 if (get_oid_hex(arg, &oid))
1315 die(_("invalid shallow line: %s"), reader->line);
b16b60f7 1316 register_shallow(the_repository, &oid);
f7e20501
BW
1317 continue;
1318 }
1319 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1320 if (get_oid_hex(arg, &oid))
1321 die(_("invalid unshallow line: %s"), reader->line);
5abddd1e 1322 if (!lookup_object(the_repository, oid.hash))
f7e20501
BW
1323 die(_("object not found: %s"), reader->line);
1324 /* make sure that it is parsed as shallow */
109cd76d 1325 if (!parse_object(the_repository, &oid))
f7e20501
BW
1326 die(_("error in object: %s"), reader->line);
1327 if (unregister_shallow(&oid))
1328 die(_("no shallow found: %s"), reader->line);
1329 continue;
1330 }
1331 die(_("expected shallow/unshallow, got %s"), reader->line);
1332 }
1333
1334 if (reader->status != PACKET_READ_FLUSH &&
1335 reader->status != PACKET_READ_DELIM)
1336 die("error processing shallow info: %d", reader->status);
1337
1338 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1339 args->deepen = 1;
1340}
1341
685fbd32
BW
1342enum fetch_state {
1343 FETCH_CHECK_LOCAL = 0,
1344 FETCH_SEND_REQUEST,
1345 FETCH_PROCESS_ACKS,
1346 FETCH_GET_PACK,
1347 FETCH_DONE,
1348};
1349
1350static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1351 int fd[2],
1352 const struct ref *orig_ref,
1353 struct ref **sought, int nr_sought,
1354 char **pack_lockfile)
1355{
1356 struct ref *ref = copy_ref_list(orig_ref);
1357 enum fetch_state state = FETCH_CHECK_LOCAL;
1358 struct oidset common = OIDSET_INIT;
1359 struct packet_reader reader;
1360 int in_vain = 0;
1361 int haves_to_send = INITIAL_FLUSH;
1362 packet_reader_init(&reader, fd[0], NULL, 0,
1363 PACKET_READ_CHOMP_NEWLINE);
1364
1365 while (state != FETCH_DONE) {
1366 switch (state) {
1367 case FETCH_CHECK_LOCAL:
1368 sort_ref_list(&ref, ref_compare_name);
1369 QSORT(sought, nr_sought, cmp_ref_by_name);
1370
1371 /* v2 supports these by default */
1372 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1373 use_sideband = 2;
f7e20501
BW
1374 if (args->depth > 0 || args->deepen_since || args->deepen_not)
1375 args->deepen = 1;
685fbd32
BW
1376
1377 if (marked)
1378 for_each_ref(clear_marks, NULL);
1379 marked = 1;
1380
1381 for_each_ref(rev_list_insert_ref_oid, NULL);
1382 for_each_cached_alternate(insert_one_alternate_object);
1383
1384 /* Filter 'ref' by 'sought' and those that aren't local */
1385 if (everything_local(args, &ref, sought, nr_sought))
1386 state = FETCH_DONE;
1387 else
1388 state = FETCH_SEND_REQUEST;
1389 break;
1390 case FETCH_SEND_REQUEST:
1391 if (send_fetch_request(fd[1], args, ref, &common,
1392 &haves_to_send, &in_vain))
1393 state = FETCH_GET_PACK;
1394 else
1395 state = FETCH_PROCESS_ACKS;
1396 break;
1397 case FETCH_PROCESS_ACKS:
1398 /* Process ACKs/NAKs */
1399 switch (process_acks(&reader, &common)) {
1400 case 2:
1401 state = FETCH_GET_PACK;
1402 break;
1403 case 1:
1404 in_vain = 0;
1405 /* fallthrough */
1406 default:
1407 state = FETCH_SEND_REQUEST;
1408 break;
1409 }
1410 break;
1411 case FETCH_GET_PACK:
f7e20501
BW
1412 /* Check for shallow-info section */
1413 if (process_section_header(&reader, "shallow-info", 1))
1414 receive_shallow_info(args, &reader);
1415
685fbd32
BW
1416 /* get the pack */
1417 process_section_header(&reader, "packfile", 0);
1418 if (get_pack(args, fd, pack_lockfile))
1419 die(_("git fetch-pack: fetch failed."));
1420
1421 state = FETCH_DONE;
1422 break;
1423 case FETCH_DONE:
1424 continue;
1425 }
1426 }
1427
1428 oidset_clear(&common);
1429 return ref;
1430}
1431
f44af51d 1432static void fetch_pack_config(void)
745f7a8c 1433{
f44af51d
TA
1434 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1435 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1436 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1437 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1438 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 1439
f44af51d 1440 git_config(git_default_config, NULL);
745f7a8c
NTND
1441}
1442
745f7a8c
NTND
1443static void fetch_pack_setup(void)
1444{
1445 static int did_setup;
1446 if (did_setup)
1447 return;
f44af51d 1448 fetch_pack_config();
745f7a8c
NTND
1449 if (0 <= transfer_unpack_limit)
1450 unpack_limit = transfer_unpack_limit;
1451 else if (0 <= fetch_unpack_limit)
1452 unpack_limit = fetch_unpack_limit;
1453 did_setup = 1;
1454}
1455
f2db854d
JH
1456static int remove_duplicates_in_refs(struct ref **ref, int nr)
1457{
1458 struct string_list names = STRING_LIST_INIT_NODUP;
1459 int src, dst;
1460
1461 for (src = dst = 0; src < nr; src++) {
1462 struct string_list_item *item;
1463 item = string_list_insert(&names, ref[src]->name);
1464 if (item->util)
1465 continue; /* already have it */
1466 item->util = ref[src];
1467 if (src != dst)
1468 ref[dst] = ref[src];
1469 dst++;
1470 }
1471 for (src = dst; src < nr; src++)
1472 ref[src] = NULL;
1473 string_list_clear(&names, 0);
1474 return dst;
1475}
1476
beea4152 1477static void update_shallow(struct fetch_pack_args *args,
4820a33b 1478 struct ref **sought, int nr_sought,
beea4152 1479 struct shallow_info *si)
a796ccee 1480{
910650d2 1481 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1482 int *status;
beea4152
NTND
1483 int i;
1484
79891cb9 1485 if (args->deepen && alternate_shallow_file) {
a796ccee 1486 if (*alternate_shallow_file == '\0') { /* --unshallow */
102de880 1487 unlink_or_warn(git_path_shallow(the_repository));
a796ccee
NTND
1488 rollback_lock_file(&shallow_lock);
1489 } else
1490 commit_lock_file(&shallow_lock);
1491 return;
1492 }
beea4152
NTND
1493
1494 if (!si->shallow || !si->shallow->nr)
1495 return;
1496
beea4152
NTND
1497 if (args->cloning) {
1498 /*
1499 * remote is shallow, but this is a clone, there are
1500 * no objects in repo to worry about. Accept any
1501 * shallow points that exist in the pack (iow in repo
1502 * after get_pack() and reprepare_packed_git())
1503 */
910650d2 1504 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1505 struct object_id *oid = si->shallow->oid;
beea4152 1506 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1507 if (has_object_file(&oid[i]))
910650d2 1508 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1509 if (extra.nr) {
1510 setup_alternate_shallow(&shallow_lock,
1511 &alternate_shallow_file,
1512 &extra);
1513 commit_lock_file(&shallow_lock);
1514 }
910650d2 1515 oid_array_clear(&extra);
beea4152
NTND
1516 return;
1517 }
4820a33b
NTND
1518
1519 if (!si->nr_ours && !si->nr_theirs)
1520 return;
1521
1522 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1523 if (!si->nr_ours && !si->nr_theirs)
1524 return;
1525 for (i = 0; i < nr_sought; i++)
910650d2 1526 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1527 si->ref = &ref;
1528
48d25cae
NTND
1529 if (args->update_shallow) {
1530 /*
1531 * remote is also shallow, .git/shallow may be updated
1532 * so all refs can be accepted. Make sure we only add
1533 * shallow roots that are actually reachable from new
1534 * refs.
1535 */
910650d2 1536 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1537 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1538 assign_shallow_commits_to_refs(si, NULL, NULL);
1539 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1540 oid_array_clear(&ref);
48d25cae
NTND
1541 return;
1542 }
1543 for (i = 0; i < si->nr_ours; i++)
910650d2 1544 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1545 for (i = 0; i < si->nr_theirs; i++)
910650d2 1546 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1547 setup_alternate_shallow(&shallow_lock,
1548 &alternate_shallow_file,
1549 &extra);
1550 commit_lock_file(&shallow_lock);
910650d2 1551 oid_array_clear(&extra);
1552 oid_array_clear(&ref);
48d25cae
NTND
1553 return;
1554 }
1555
4820a33b
NTND
1556 /*
1557 * remote is also shallow, check what ref is safe to update
1558 * without updating .git/shallow
1559 */
1560 status = xcalloc(nr_sought, sizeof(*status));
1561 assign_shallow_commits_to_refs(si, NULL, status);
1562 if (si->nr_ours || si->nr_theirs) {
1563 for (i = 0; i < nr_sought; i++)
1564 if (status[i])
1565 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1566 }
1567 free(status);
910650d2 1568 oid_array_clear(&ref);
a796ccee
NTND
1569}
1570
745f7a8c
NTND
1571struct ref *fetch_pack(struct fetch_pack_args *args,
1572 int fd[], struct child_process *conn,
1573 const struct ref *ref,
1574 const char *dest,
f2db854d 1575 struct ref **sought, int nr_sought,
910650d2 1576 struct oid_array *shallow,
685fbd32
BW
1577 char **pack_lockfile,
1578 enum protocol_version version)
745f7a8c 1579{
745f7a8c 1580 struct ref *ref_cpy;
beea4152 1581 struct shallow_info si;
745f7a8c
NTND
1582
1583 fetch_pack_setup();
f2db854d
JH
1584 if (nr_sought)
1585 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1586
1587 if (!ref) {
1588 packet_flush(fd[1]);
1dd73e20 1589 die(_("no matching remote head"));
745f7a8c 1590 }
beea4152 1591 prepare_shallow_info(&si, shallow);
685fbd32
BW
1592 if (version == protocol_v2)
1593 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1594 pack_lockfile);
1595 else
1596 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1597 &si, pack_lockfile);
a49d2834 1598 reprepare_packed_git(the_repository);
4820a33b 1599 update_shallow(args, sought, nr_sought, &si);
beea4152 1600 clear_shallow_info(&si);
745f7a8c
NTND
1601 return ref_cpy;
1602}
e860d96b
MM
1603
1604int report_unmatched_refs(struct ref **sought, int nr_sought)
1605{
1606 int i, ret = 0;
1607
1608 for (i = 0; i < nr_sought; i++) {
d56583de 1609 if (!sought[i])
e860d96b 1610 continue;
d56583de
MM
1611 switch (sought[i]->match_status) {
1612 case REF_MATCHED:
1613 continue;
1614 case REF_NOT_MATCHED:
1615 error(_("no such remote ref %s"), sought[i]->name);
1616 break;
1617 case REF_UNADVERTISED_NOT_ALLOWED:
1618 error(_("Server does not allow request for unadvertised object %s"),
1619 sought[i]->name);
1620 break;
1621 }
e860d96b
MM
1622 ret = 1;
1623 }
1624 return ret;
1625}