fetch-pack: implement ref-in-want
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
a49d2834 2#include "repository.h"
b2141fc1 3#include "config.h"
697cc8ef 4#include "lockfile.h"
745f7a8c
NTND
5#include "refs.h"
6#include "pkt-line.h"
7#include "commit.h"
8#include "tag.h"
d807c4a0 9#include "exec-cmd.h"
745f7a8c
NTND
10#include "pack.h"
11#include "sideband.h"
12#include "fetch-pack.h"
13#include "remote.h"
14#include "run-command.h"
47a59185 15#include "connect.h"
745f7a8c
NTND
16#include "transport.h"
17#include "version.h"
099327b5 18#include "prio-queue.h"
beea4152 19#include "sha1-array.h"
fdb69d33 20#include "oidset.h"
0abe14f6 21#include "packfile.h"
745f7a8c
NTND
22
23static int transfer_unpack_limit = -1;
24static int fetch_unpack_limit = -1;
25static int unpack_limit = 100;
26static int prefer_ofs_delta = 1;
27static int no_done;
508ea882 28static int deepen_since_ok;
a45a2600 29static int deepen_not_ok;
745f7a8c
NTND
30static int fetch_fsck_objects = -1;
31static int transfer_fsck_objects = -1;
32static int agent_supported;
640d8b72 33static int server_supports_filtering;
6035d6aa
NTND
34static struct lock_file shallow_lock;
35static const char *alternate_shallow_file;
745f7a8c 36
208acbfb 37/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
38#define COMPLETE (1U << 0)
39#define COMMON (1U << 1)
40#define COMMON_REF (1U << 2)
41#define SEEN (1U << 3)
42#define POPPED (1U << 4)
41a078c6 43#define ALTERNATE (1U << 5)
745f7a8c
NTND
44
45static int marked;
46
47/*
48 * After sending this many "have"s if we do not get any new ACK , we
49 * give up traversing our history.
50 */
51#define MAX_IN_VAIN 256
52
099327b5 53static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
54static int non_common_revs, multi_ack, use_sideband;
55/* Allow specifying sha1 if it is a ref tip. */
56#define ALLOW_TIP_SHA1 01
68ee6289
FM
57/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
58#define ALLOW_REACHABLE_SHA1 02
7199c093 59static unsigned int allow_unadvertised_object_request;
745f7a8c 60
0d789a5b
NTND
61__attribute__((format (printf, 2, 3)))
62static inline void print_verbose(const struct fetch_pack_args *args,
63 const char *fmt, ...)
64{
65 va_list params;
66
67 if (!args->verbose)
68 return;
69
70 va_start(params, fmt);
71 vfprintf(stderr, fmt, params);
72 va_end(params);
73 fputc('\n', stderr);
74}
75
41a078c6
JK
76struct alternate_object_cache {
77 struct object **items;
78 size_t nr, alloc;
79};
80
81static void cache_one_alternate(const char *refname,
82 const struct object_id *oid,
83 void *vcache)
84{
85 struct alternate_object_cache *cache = vcache;
c251c83d 86 struct object *obj = parse_object(oid);
41a078c6
JK
87
88 if (!obj || (obj->flags & ALTERNATE))
89 return;
90
91 obj->flags |= ALTERNATE;
92 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
93 cache->items[cache->nr++] = obj;
94}
95
96static void for_each_cached_alternate(void (*cb)(struct object *))
97{
98 static int initialized;
99 static struct alternate_object_cache cache;
100 size_t i;
101
102 if (!initialized) {
103 for_each_alternate_ref(cache_one_alternate, &cache);
104 initialized = 1;
105 }
106
107 for (i = 0; i < cache.nr; i++)
108 cb(cache.items[i]);
109}
110
745f7a8c
NTND
111static void rev_list_push(struct commit *commit, int mark)
112{
113 if (!(commit->object.flags & mark)) {
114 commit->object.flags |= mark;
115
0064053b
JK
116 if (parse_commit(commit))
117 return;
745f7a8c 118
099327b5 119 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
120
121 if (!(commit->object.flags & COMMON))
122 non_common_revs++;
123 }
124}
125
1b283377 126static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
745f7a8c 127{
c251c83d 128 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
129
130 if (o && o->type == OBJ_COMMIT)
131 rev_list_push((struct commit *)o, SEEN);
132
133 return 0;
134}
135
b1b49c6e
MH
136static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
137 int flag, void *cb_data)
745f7a8c 138{
1b283377 139 return rev_list_insert_ref(refname, oid);
b1b49c6e
MH
140}
141
c50fb6ce
MH
142static int clear_marks(const char *refname, const struct object_id *oid,
143 int flag, void *cb_data)
745f7a8c 144{
c251c83d 145 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
146
147 if (o && o->type == OBJ_COMMIT)
148 clear_commit_marks((struct commit *)o,
149 COMMON | COMMON_REF | SEEN | POPPED);
150 return 0;
151}
152
153/*
154 This function marks a rev and its ancestors as common.
155 In some cases, it is desirable to mark only the ancestors (for example
156 when only the server does not yet know that they are common).
157*/
158
159static void mark_common(struct commit *commit,
160 int ancestors_only, int dont_parse)
161{
162 if (commit != NULL && !(commit->object.flags & COMMON)) {
163 struct object *o = (struct object *)commit;
164
165 if (!ancestors_only)
166 o->flags |= COMMON;
167
168 if (!(o->flags & SEEN))
169 rev_list_push(commit, SEEN);
170 else {
171 struct commit_list *parents;
172
173 if (!ancestors_only && !(o->flags & POPPED))
174 non_common_revs--;
175 if (!o->parsed && !dont_parse)
176 if (parse_commit(commit))
177 return;
178
179 for (parents = commit->parents;
180 parents;
181 parents = parents->next)
182 mark_common(parents->item, 0, dont_parse);
183 }
184 }
185}
186
187/*
188 Get the next rev to send, ignoring the common.
189*/
190
1b283377 191static const struct object_id *get_rev(void)
745f7a8c
NTND
192{
193 struct commit *commit = NULL;
194
195 while (commit == NULL) {
196 unsigned int mark;
197 struct commit_list *parents;
198
099327b5 199 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
200 return NULL;
201
099327b5 202 commit = prio_queue_get(&rev_list);
0064053b 203 parse_commit(commit);
745f7a8c
NTND
204 parents = commit->parents;
205
206 commit->object.flags |= POPPED;
207 if (!(commit->object.flags & COMMON))
208 non_common_revs--;
209
210 if (commit->object.flags & COMMON) {
211 /* do not send "have", and ignore ancestors */
212 commit = NULL;
213 mark = COMMON | SEEN;
214 } else if (commit->object.flags & COMMON_REF)
215 /* send "have", and ignore ancestors */
216 mark = COMMON | SEEN;
217 else
218 /* send "have", also for its ancestors */
219 mark = SEEN;
220
221 while (parents) {
222 if (!(parents->item->object.flags & SEEN))
223 rev_list_push(parents->item, mark);
224 if (mark & COMMON)
225 mark_common(parents->item, 1, 0);
226 parents = parents->next;
227 }
745f7a8c
NTND
228 }
229
1b283377 230 return &commit->object.oid;
745f7a8c
NTND
231}
232
233enum ack_type {
234 NAK = 0,
235 ACK,
236 ACK_continue,
237 ACK_common,
238 ACK_ready
239};
240
241static void consume_shallow_list(struct fetch_pack_args *args, int fd)
242{
79891cb9 243 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
244 /* If we sent a depth we will get back "duplicate"
245 * shallow and unshallow commands every time there
246 * is a block of have lines exchanged.
247 */
74543a04
JK
248 char *line;
249 while ((line = packet_read_line(fd, NULL))) {
59556548 250 if (starts_with(line, "shallow "))
745f7a8c 251 continue;
59556548 252 if (starts_with(line, "unshallow "))
745f7a8c 253 continue;
1dd73e20 254 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
255 }
256 }
257}
258
1b283377 259static enum ack_type get_ack(int fd, struct object_id *result_oid)
745f7a8c 260{
74543a04
JK
261 int len;
262 char *line = packet_read_line(fd, &len);
82e56767 263 const char *arg;
745f7a8c 264
bc9d4dc5
JK
265 if (!line)
266 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
745f7a8c
NTND
267 if (!strcmp(line, "NAK"))
268 return NAK;
82e56767 269 if (skip_prefix(line, "ACK ", &arg)) {
1b283377 270 if (!get_oid_hex(arg, result_oid)) {
82e56767
JK
271 arg += 40;
272 len -= arg - line;
273 if (len < 1)
030e9dd6 274 return ACK;
82e56767 275 if (strstr(arg, "continue"))
745f7a8c 276 return ACK_continue;
82e56767 277 if (strstr(arg, "common"))
745f7a8c 278 return ACK_common;
82e56767 279 if (strstr(arg, "ready"))
745f7a8c
NTND
280 return ACK_ready;
281 return ACK;
282 }
283 }
8e2c7bef
JT
284 if (skip_prefix(line, "ERR ", &arg))
285 die(_("remote error: %s"), arg);
dfbfb9f3 286 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
287}
288
289static void send_request(struct fetch_pack_args *args,
290 int fd, struct strbuf *buf)
291{
292 if (args->stateless_rpc) {
293 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
294 packet_flush(fd);
295 } else
cdf4fb8e 296 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
297}
298
41a078c6 299static void insert_one_alternate_object(struct object *obj)
745f7a8c 300{
1b283377 301 rev_list_insert_ref(NULL, &obj->oid);
745f7a8c
NTND
302}
303
304#define INITIAL_FLUSH 16
305#define PIPESAFE_FLUSH 32
da470981 306#define LARGE_FLUSH 16384
745f7a8c 307
685fbd32 308static int next_flush(int stateless_rpc, int count)
745f7a8c 309{
685fbd32 310 if (stateless_rpc) {
da470981
JT
311 if (count < LARGE_FLUSH)
312 count <<= 1;
313 else
314 count = count * 11 / 10;
315 } else {
316 if (count < PIPESAFE_FLUSH)
317 count <<= 1;
318 else
319 count += PIPESAFE_FLUSH;
320 }
745f7a8c
NTND
321 return count;
322}
323
324static int find_common(struct fetch_pack_args *args,
1b283377 325 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
326 struct ref *refs)
327{
328 int fetching;
329 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 330 const struct object_id *oid;
745f7a8c
NTND
331 unsigned in_vain = 0;
332 int got_continue = 0;
333 int got_ready = 0;
334 struct strbuf req_buf = STRBUF_INIT;
335 size_t state_len = 0;
336
337 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 338 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
339 if (marked)
340 for_each_ref(clear_marks, NULL);
341 marked = 1;
342
b1b49c6e 343 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 344 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
345
346 fetching = 0;
347 for ( ; refs ; refs = refs->next) {
1b283377 348 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
349 const char *remote_hex;
350 struct object *o;
351
352 /*
353 * If that object is complete (i.e. it is an ancestor of a
354 * local ref), we tell them we have it but do not have to
355 * tell them about its ancestors, which they already know
356 * about.
357 *
358 * We use lookup_object here because we are only
359 * interested in the case we *know* the object is
360 * reachable and we have already scanned it.
361 */
1b283377 362 if (((o = lookup_object(remote->hash)) != NULL) &&
745f7a8c
NTND
363 (o->flags & COMPLETE)) {
364 continue;
365 }
366
1b283377 367 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
368 if (!fetching) {
369 struct strbuf c = STRBUF_INIT;
370 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
371 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
372 if (no_done) strbuf_addstr(&c, " no-done");
373 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
374 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 375 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
376 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
377 if (args->no_progress) strbuf_addstr(&c, " no-progress");
378 if (args->include_tag) strbuf_addstr(&c, " include-tag");
379 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 380 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 381 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
382 if (agent_supported) strbuf_addf(&c, " agent=%s",
383 git_user_agent_sanitized());
640d8b72
JH
384 if (args->filter_options.choice)
385 strbuf_addstr(&c, " filter");
745f7a8c
NTND
386 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
387 strbuf_release(&c);
388 } else
389 packet_buf_write(&req_buf, "want %s\n", remote_hex);
390 fetching++;
391 }
392
393 if (!fetching) {
394 strbuf_release(&req_buf);
395 packet_flush(fd[1]);
396 return 1;
397 }
398
399 if (is_repository_shallow())
1a30f5a2 400 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
401 if (args->depth > 0)
402 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882 403 if (args->deepen_since) {
dddbad72 404 timestamp_t max_age = approxidate(args->deepen_since);
cb71f8bd 405 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
508ea882 406 }
a45a2600
NTND
407 if (args->deepen_not) {
408 int i;
409 for (i = 0; i < args->deepen_not->nr; i++) {
410 struct string_list_item *s = args->deepen_not->items + i;
411 packet_buf_write(&req_buf, "deepen-not %s", s->string);
412 }
413 }
640d8b72
JH
414 if (server_supports_filtering && args->filter_options.choice)
415 packet_buf_write(&req_buf, "filter %s",
416 args->filter_options.filter_spec);
745f7a8c
NTND
417 packet_buf_flush(&req_buf);
418 state_len = req_buf.len;
419
79891cb9 420 if (args->deepen) {
74543a04 421 char *line;
ae021d87 422 const char *arg;
1b283377 423 struct object_id oid;
745f7a8c
NTND
424
425 send_request(args, fd[1], &req_buf);
74543a04 426 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87 427 if (skip_prefix(line, "shallow ", &arg)) {
1b283377 428 if (get_oid_hex(arg, &oid))
1dd73e20 429 die(_("invalid shallow line: %s"), line);
e92b848c 430 register_shallow(&oid);
745f7a8c
NTND
431 continue;
432 }
ae021d87 433 if (skip_prefix(line, "unshallow ", &arg)) {
1b283377 434 if (get_oid_hex(arg, &oid))
1dd73e20 435 die(_("invalid unshallow line: %s"), line);
1b283377 436 if (!lookup_object(oid.hash))
1dd73e20 437 die(_("object not found: %s"), line);
745f7a8c 438 /* make sure that it is parsed as shallow */
c251c83d 439 if (!parse_object(&oid))
1dd73e20 440 die(_("error in object: %s"), line);
e92b848c 441 if (unregister_shallow(&oid))
1dd73e20 442 die(_("no shallow found: %s"), line);
745f7a8c
NTND
443 continue;
444 }
1dd73e20 445 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
446 }
447 } else if (!args->stateless_rpc)
448 send_request(args, fd[1], &req_buf);
449
450 if (!args->stateless_rpc) {
451 /* If we aren't using the stateless-rpc interface
452 * we don't need to retain the headers.
453 */
454 strbuf_setlen(&req_buf, 0);
455 state_len = 0;
456 }
457
458 flushes = 0;
459 retval = -1;
88e2f9ed
JT
460 if (args->no_dependents)
461 goto done;
1b283377 462 while ((oid = get_rev())) {
463 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
464 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
465 in_vain++;
466 if (flush_at <= ++count) {
467 int ack;
468
469 packet_buf_flush(&req_buf);
470 send_request(args, fd[1], &req_buf);
471 strbuf_setlen(&req_buf, state_len);
472 flushes++;
685fbd32 473 flush_at = next_flush(args->stateless_rpc, count);
745f7a8c
NTND
474
475 /*
476 * We keep one window "ahead" of the other side, and
477 * will wait for an ACK only on the next one
478 */
479 if (!args->stateless_rpc && count == INITIAL_FLUSH)
480 continue;
481
482 consume_shallow_list(args, fd[0]);
483 do {
1b283377 484 ack = get_ack(fd[0], result_oid);
0d789a5b 485 if (ack)
1dd73e20 486 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 487 ack, oid_to_hex(result_oid));
745f7a8c
NTND
488 switch (ack) {
489 case ACK:
490 flushes = 0;
491 multi_ack = 0;
492 retval = 0;
493 goto done;
494 case ACK_common:
495 case ACK_ready:
496 case ACK_continue: {
497 struct commit *commit =
bc83266a 498 lookup_commit(result_oid);
745f7a8c 499 if (!commit)
1b283377 500 die(_("invalid commit %s"), oid_to_hex(result_oid));
745f7a8c
NTND
501 if (args->stateless_rpc
502 && ack == ACK_common
503 && !(commit->object.flags & COMMON)) {
504 /* We need to replay the have for this object
505 * on the next RPC request so the peer knows
506 * it is in common with us.
507 */
1b283377 508 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
509 packet_buf_write(&req_buf, "have %s\n", hex);
510 state_len = req_buf.len;
06b3d386
JT
511 /*
512 * Reset in_vain because an ack
513 * for this commit has not been
514 * seen.
515 */
516 in_vain = 0;
517 } else if (!args->stateless_rpc
518 || ack != ACK_common)
519 in_vain = 0;
745f7a8c
NTND
520 mark_common(commit, 0, 1);
521 retval = 0;
745f7a8c
NTND
522 got_continue = 1;
523 if (ack == ACK_ready) {
099327b5 524 clear_prio_queue(&rev_list);
745f7a8c
NTND
525 got_ready = 1;
526 }
527 break;
528 }
529 }
530 } while (ack);
531 flushes--;
532 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 533 print_verbose(args, _("giving up"));
745f7a8c
NTND
534 break; /* give up */
535 }
536 }
537 }
538done:
539 if (!got_ready || !no_done) {
540 packet_buf_write(&req_buf, "done\n");
541 send_request(args, fd[1], &req_buf);
542 }
1dd73e20 543 print_verbose(args, _("done"));
745f7a8c
NTND
544 if (retval != 0) {
545 multi_ack = 0;
546 flushes++;
547 }
548 strbuf_release(&req_buf);
549
ff62eca7
NTND
550 if (!got_ready || !no_done)
551 consume_shallow_list(args, fd[0]);
745f7a8c 552 while (flushes || multi_ack) {
1b283377 553 int ack = get_ack(fd[0], result_oid);
745f7a8c 554 if (ack) {
1dd73e20 555 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 556 ack, oid_to_hex(result_oid));
745f7a8c
NTND
557 if (ack == ACK)
558 return 0;
559 multi_ack = 1;
560 continue;
561 }
562 flushes--;
563 }
564 /* it is no error to fetch into a completely empty repo */
565 return count ? retval : 0;
566}
567
568static struct commit_list *complete;
569
1b283377 570static int mark_complete(const struct object_id *oid)
745f7a8c 571{
c251c83d 572 struct object *o = parse_object(oid);
745f7a8c
NTND
573
574 while (o && o->type == OBJ_TAG) {
575 struct tag *t = (struct tag *) o;
576 if (!t->tagged)
577 break; /* broken repository */
578 o->flags |= COMPLETE;
c251c83d 579 o = parse_object(&t->tagged->oid);
745f7a8c
NTND
580 }
581 if (o && o->type == OBJ_COMMIT) {
582 struct commit *commit = (struct commit *)o;
583 if (!(commit->object.flags & COMPLETE)) {
584 commit->object.flags |= COMPLETE;
16445242 585 commit_list_insert(commit, &complete);
745f7a8c
NTND
586 }
587 }
588 return 0;
589}
590
f8ee4d85
MH
591static int mark_complete_oid(const char *refname, const struct object_id *oid,
592 int flag, void *cb_data)
593{
1b283377 594 return mark_complete(oid);
f8ee4d85
MH
595}
596
745f7a8c 597static void mark_recent_complete_commits(struct fetch_pack_args *args,
dddbad72 598 timestamp_t cutoff)
745f7a8c
NTND
599{
600 while (complete && cutoff <= complete->item->date) {
1dd73e20 601 print_verbose(args, _("Marking %s as complete"),
0d789a5b 602 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
603 pop_most_recent_commit(&complete, COMPLETE);
604 }
605}
606
fdb69d33
JT
607static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
608{
609 for (; refs; refs = refs->next)
610 oidset_insert(oids, &refs->old_oid);
611}
612
613static int tip_oids_contain(struct oidset *tip_oids,
614 struct ref *unmatched, struct ref *newlist,
615 const struct object_id *id)
616{
617 /*
618 * Note that this only looks at the ref lists the first time it's
619 * called. This works out in filter_refs() because even though it may
620 * add to "newlist" between calls, the additions will always be for
621 * oids that are already in the set.
622 */
9e6fabde 623 if (!tip_oids->map.map.tablesize) {
fdb69d33
JT
624 add_refs_to_oidset(tip_oids, unmatched);
625 add_refs_to_oidset(tip_oids, newlist);
626 }
627 return oidset_contains(tip_oids, id);
628}
629
745f7a8c 630static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
631 struct ref **refs,
632 struct ref **sought, int nr_sought)
745f7a8c
NTND
633{
634 struct ref *newlist = NULL;
635 struct ref **newtail = &newlist;
fdb69d33 636 struct ref *unmatched = NULL;
745f7a8c 637 struct ref *ref, *next;
fdb69d33 638 struct oidset tip_oids = OIDSET_INIT;
f2db854d 639 int i;
745f7a8c 640
f2db854d 641 i = 0;
745f7a8c
NTND
642 for (ref = *refs; ref; ref = next) {
643 int keep = 0;
644 next = ref->next;
f2db854d 645
50e19a83 646 if (starts_with(ref->name, "refs/") &&
4c224081 647 check_refname_format(ref->name, 0))
745f7a8c
NTND
648 ; /* trash */
649 else {
f2db854d
JH
650 while (i < nr_sought) {
651 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
652 if (cmp < 0)
653 break; /* definitely do not have it */
654 else if (cmp == 0) {
655 keep = 1; /* definitely have it */
d56583de 656 sought[i]->match_status = REF_MATCHED;
745f7a8c 657 }
f2db854d 658 i++;
745f7a8c
NTND
659 }
660 }
661
f2db854d 662 if (!keep && args->fetch_all &&
79891cb9 663 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
664 keep = 1;
665
666 if (keep) {
667 *newtail = ref;
668 ref->next = NULL;
669 newtail = &ref->next;
670 } else {
fdb69d33
JT
671 ref->next = unmatched;
672 unmatched = ref;
745f7a8c
NTND
673 }
674 }
675
6e7b66ee 676 /* Append unmatched requests to the list */
d56583de 677 for (i = 0; i < nr_sought; i++) {
1b283377 678 struct object_id oid;
679 const char *p;
b7916422 680
d56583de
MM
681 ref = sought[i];
682 if (ref->match_status != REF_NOT_MATCHED)
683 continue;
1b283377 684 if (parse_oid_hex(ref->name, &oid, &p) ||
685 *p != '\0' ||
686 oidcmp(&oid, &ref->old_oid))
d56583de 687 continue;
6e7b66ee 688
d56583de 689 if ((allow_unadvertised_object_request &
fdb69d33
JT
690 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
691 tip_oids_contain(&tip_oids, unmatched, newlist,
692 &ref->old_oid)) {
d56583de 693 ref->match_status = REF_MATCHED;
c3c17bf1
JK
694 *newtail = copy_ref(ref);
695 newtail = &(*newtail)->next;
d56583de
MM
696 } else {
697 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
698 }
699 }
fdb69d33
JT
700
701 oidset_clear(&tip_oids);
702 for (ref = unmatched; ref; ref = next) {
703 next = ref->next;
704 free(ref);
705 }
706
745f7a8c
NTND
707 *refs = newlist;
708}
709
41a078c6 710static void mark_alternate_complete(struct object *obj)
745f7a8c 711{
1b283377 712 mark_complete(&obj->oid);
745f7a8c
NTND
713}
714
024aa469
TI
715struct loose_object_iter {
716 struct oidset *loose_object_set;
717 struct ref *refs;
718};
719
720/*
721 * If the number of refs is not larger than the number of loose objects,
722 * this function stops inserting.
723 */
724static int add_loose_objects_to_set(const struct object_id *oid,
725 const char *path,
726 void *data)
727{
728 struct loose_object_iter *iter = data;
729 oidset_insert(iter->loose_object_set, oid);
730 if (iter->refs == NULL)
731 return 1;
732
733 iter->refs = iter->refs->next;
734 return 0;
735}
736
745f7a8c 737static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
738 struct ref **refs,
739 struct ref **sought, int nr_sought)
745f7a8c
NTND
740{
741 struct ref *ref;
742 int retval;
a1c6d7c1 743 int old_save_commit_buffer = save_commit_buffer;
dddbad72 744 timestamp_t cutoff = 0;
024aa469
TI
745 struct oidset loose_oid_set = OIDSET_INIT;
746 int use_oidset = 0;
747 struct loose_object_iter iter = {&loose_oid_set, *refs};
748
749 /* Enumerate all loose objects or know refs are not so many. */
750 use_oidset = !for_each_loose_object(add_loose_objects_to_set,
751 &iter, 0);
745f7a8c
NTND
752
753 save_commit_buffer = 0;
754
755 for (ref = *refs; ref; ref = ref->next) {
756 struct object *o;
024aa469 757 unsigned int flags = OBJECT_INFO_QUICK;
745f7a8c 758
024aa469
TI
759 if (use_oidset &&
760 !oidset_contains(&loose_oid_set, &ref->old_oid)) {
761 /*
762 * I know this does not exist in the loose form,
763 * so check if it exists in a non-loose form.
764 */
765 flags |= OBJECT_INFO_IGNORE_LOOSE;
766 }
012a1bb5 767
024aa469
TI
768 if (!has_object_file_with_flags(&ref->old_oid, flags))
769 continue;
c251c83d 770 o = parse_object(&ref->old_oid);
745f7a8c
NTND
771 if (!o)
772 continue;
773
774 /* We already have it -- which may mean that we were
775 * in sync with the other side at some time after
776 * that (it is OK if we guess wrong here).
777 */
778 if (o->type == OBJ_COMMIT) {
779 struct commit *commit = (struct commit *)o;
780 if (!cutoff || cutoff < commit->date)
781 cutoff = commit->date;
782 }
783 }
784
024aa469
TI
785 oidset_clear(&loose_oid_set);
786
88e2f9ed
JT
787 if (!args->no_dependents) {
788 if (!args->deepen) {
789 for_each_ref(mark_complete_oid, NULL);
790 for_each_cached_alternate(mark_alternate_complete);
791 commit_list_sort_by_date(&complete);
792 if (cutoff)
793 mark_recent_complete_commits(args, cutoff);
794 }
745f7a8c 795
88e2f9ed
JT
796 /*
797 * Mark all complete remote refs as common refs.
798 * Don't mark them common yet; the server has to be told so first.
799 */
800 for (ref = *refs; ref; ref = ref->next) {
801 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
802 NULL, 0);
745f7a8c 803
88e2f9ed
JT
804 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
805 continue;
745f7a8c 806
88e2f9ed
JT
807 if (!(o->flags & SEEN)) {
808 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
745f7a8c 809
88e2f9ed
JT
810 mark_common((struct commit *)o, 1, 1);
811 }
745f7a8c
NTND
812 }
813 }
814
f2db854d 815 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
816
817 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 818 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
819 struct object *o;
820
1b283377 821 o = lookup_object(remote->hash);
745f7a8c
NTND
822 if (!o || !(o->flags & COMPLETE)) {
823 retval = 0;
1b283377 824 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 825 ref->name);
745f7a8c
NTND
826 continue;
827 }
1b283377 828 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 829 ref->name);
745f7a8c 830 }
a1c6d7c1
JT
831
832 save_commit_buffer = old_save_commit_buffer;
833
745f7a8c
NTND
834 return retval;
835}
836
837static int sideband_demux(int in, int out, void *data)
838{
839 int *xd = data;
9ff18faf 840 int ret;
745f7a8c 841
9ff18faf 842 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
843 close(out);
844 return ret;
845}
846
847static int get_pack(struct fetch_pack_args *args,
848 int xd[2], char **pack_lockfile)
849{
850 struct async demux;
745f7a8c 851 int do_keep = args->keep_pack;
984a43b9
JK
852 const char *cmd_name;
853 struct pack_header header;
854 int pass_header = 0;
d3180279 855 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 856 int ret;
745f7a8c
NTND
857
858 memset(&demux, 0, sizeof(demux));
859 if (use_sideband) {
860 /* xd[] is talking with upload-pack; subprocess reads from
861 * xd[0], spits out band#2 to stderr, and feeds us band#1
862 * through demux->out.
863 */
864 demux.proc = sideband_demux;
865 demux.data = xd;
866 demux.out = -1;
df857572 867 demux.isolate_sigpipe = 1;
745f7a8c 868 if (start_async(&demux))
1dd73e20 869 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
870 }
871 else
872 demux.out = xd[0];
873
745f7a8c 874 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
875
876 if (read_pack_header(demux.out, &header))
1dd73e20 877 die(_("protocol error: bad pack header"));
984a43b9 878 pass_header = 1;
745f7a8c
NTND
879 if (ntohl(header.hdr_entries) < unpack_limit)
880 do_keep = 0;
881 else
882 do_keep = 1;
883 }
884
6035d6aa 885 if (alternate_shallow_file) {
984a43b9
JK
886 argv_array_push(&cmd.args, "--shallow-file");
887 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
888 }
889
88e2f9ed 890 if (do_keep || args->from_promisor) {
745f7a8c
NTND
891 if (pack_lockfile)
892 cmd.out = -1;
984a43b9
JK
893 cmd_name = "index-pack";
894 argv_array_push(&cmd.args, cmd_name);
895 argv_array_push(&cmd.args, "--stdin");
745f7a8c 896 if (!args->quiet && !args->no_progress)
984a43b9 897 argv_array_push(&cmd.args, "-v");
745f7a8c 898 if (args->use_thin_pack)
984a43b9 899 argv_array_push(&cmd.args, "--fix-thin");
88e2f9ed 900 if (do_keep && (args->lock_pack || unpack_limit)) {
da25bdb7 901 char hostname[HOST_NAME_MAX + 1];
5781a9a2 902 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
903 xsnprintf(hostname, sizeof(hostname), "localhost");
904 argv_array_pushf(&cmd.args,
905 "--keep=fetch-pack %"PRIuMAX " on %s",
906 (uintmax_t)getpid(), hostname);
745f7a8c 907 }
c6807a40 908 if (args->check_self_contained_and_connected)
984a43b9 909 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
88e2f9ed
JT
910 if (args->from_promisor)
911 argv_array_push(&cmd.args, "--promisor");
745f7a8c
NTND
912 }
913 else {
984a43b9
JK
914 cmd_name = "unpack-objects";
915 argv_array_push(&cmd.args, cmd_name);
745f7a8c 916 if (args->quiet || args->no_progress)
984a43b9 917 argv_array_push(&cmd.args, "-q");
c6807a40 918 args->check_self_contained_and_connected = 0;
745f7a8c 919 }
984a43b9
JK
920
921 if (pass_header)
922 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
923 ntohl(header.hdr_version),
924 ntohl(header.hdr_entries));
745f7a8c
NTND
925 if (fetch_fsck_objects >= 0
926 ? fetch_fsck_objects
927 : transfer_fsck_objects >= 0
928 ? transfer_fsck_objects
98a2ea46
JT
929 : 0) {
930 if (args->from_promisor)
931 /*
932 * We cannot use --strict in index-pack because it
933 * checks both broken objects and links, but we only
934 * want to check for broken objects.
935 */
936 argv_array_push(&cmd.args, "--fsck-objects");
937 else
938 argv_array_push(&cmd.args, "--strict");
939 }
745f7a8c
NTND
940
941 cmd.in = demux.out;
942 cmd.git_cmd = 1;
943 if (start_command(&cmd))
1dd73e20 944 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
945 if (do_keep && pack_lockfile) {
946 *pack_lockfile = index_pack_lockfile(cmd.out);
947 close(cmd.out);
948 }
949
37cb1dd6
JL
950 if (!use_sideband)
951 /* Closed by start_command() */
952 xd[0] = -1;
953
c6807a40
NTND
954 ret = finish_command(&cmd);
955 if (!ret || (args->check_self_contained_and_connected && ret == 1))
956 args->self_contained_and_connected =
957 args->check_self_contained_and_connected &&
958 ret == 0;
959 else
1dd73e20 960 die(_("%s failed"), cmd_name);
745f7a8c 961 if (use_sideband && finish_async(&demux))
1dd73e20 962 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
963 return 0;
964}
965
f2db854d
JH
966static int cmp_ref_by_name(const void *a_, const void *b_)
967{
968 const struct ref *a = *((const struct ref **)a_);
969 const struct ref *b = *((const struct ref **)b_);
970 return strcmp(a->name, b->name);
971}
972
745f7a8c
NTND
973static struct ref *do_fetch_pack(struct fetch_pack_args *args,
974 int fd[2],
975 const struct ref *orig_ref,
f2db854d 976 struct ref **sought, int nr_sought,
beea4152 977 struct shallow_info *si,
745f7a8c
NTND
978 char **pack_lockfile)
979{
980 struct ref *ref = copy_ref_list(orig_ref);
1b283377 981 struct object_id oid;
745f7a8c
NTND
982 const char *agent_feature;
983 int agent_len;
984
985 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 986 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 987
eb86a507 988 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 989 die(_("Server does not support shallow clients"));
a45a2600 990 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 991 args->deepen = 1;
745f7a8c 992 if (server_supports("multi_ack_detailed")) {
1dd73e20 993 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
994 multi_ack = 2;
995 if (server_supports("no-done")) {
1dd73e20 996 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
997 if (args->stateless_rpc)
998 no_done = 1;
999 }
1000 }
1001 else if (server_supports("multi_ack")) {
1dd73e20 1002 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
1003 multi_ack = 1;
1004 }
1005 if (server_supports("side-band-64k")) {
1dd73e20 1006 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
1007 use_sideband = 2;
1008 }
1009 else if (server_supports("side-band")) {
1dd73e20 1010 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
1011 use_sideband = 1;
1012 }
6e7b66ee 1013 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 1014 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 1015 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 1016 }
68ee6289 1017 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 1018 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
1019 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1020 }
745f7a8c
NTND
1021 if (!server_supports("thin-pack"))
1022 args->use_thin_pack = 0;
1023 if (!server_supports("no-progress"))
1024 args->no_progress = 0;
1025 if (!server_supports("include-tag"))
1026 args->include_tag = 0;
0d789a5b 1027 if (server_supports("ofs-delta"))
1dd73e20 1028 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 1029 else
745f7a8c
NTND
1030 prefer_ofs_delta = 0;
1031
640d8b72
JH
1032 if (server_supports("filter")) {
1033 server_supports_filtering = 1;
1034 print_verbose(args, _("Server supports filter"));
1035 } else if (args->filter_options.choice) {
1036 warning("filtering not recognized by server, ignoring");
1037 }
1038
745f7a8c
NTND
1039 if ((agent_feature = server_feature_value("agent", &agent_len))) {
1040 agent_supported = 1;
0d789a5b 1041 if (agent_len)
1dd73e20 1042 print_verbose(args, _("Server version is %.*s"),
0d789a5b 1043 agent_len, agent_feature);
745f7a8c 1044 }
508ea882
NTND
1045 if (server_supports("deepen-since"))
1046 deepen_since_ok = 1;
1047 else if (args->deepen_since)
1048 die(_("Server does not support --shallow-since"));
a45a2600
NTND
1049 if (server_supports("deepen-not"))
1050 deepen_not_ok = 1;
1051 else if (args->deepen_not)
1052 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
1053 if (!server_supports("deepen-relative") && args->deepen_relative)
1054 die(_("Server does not support --deepen"));
745f7a8c 1055
f2db854d 1056 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
1057 packet_flush(fd[1]);
1058 goto all_done;
1059 }
1b283377 1060 if (find_common(args, fd, &oid, ref) < 0)
745f7a8c
NTND
1061 if (!args->keep_pack)
1062 /* When cloning, it is not unusual to have
1063 * no common commit.
1064 */
1dd73e20 1065 warning(_("no common commits"));
745f7a8c
NTND
1066
1067 if (args->stateless_rpc)
1068 packet_flush(fd[1]);
79891cb9 1069 if (args->deepen)
1a30f5a2
NTND
1070 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1071 NULL);
4820a33b 1072 else if (si->nr_ours || si->nr_theirs)
beea4152 1073 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
1074 else
1075 alternate_shallow_file = NULL;
745f7a8c 1076 if (get_pack(args, fd, pack_lockfile))
1dd73e20 1077 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
1078
1079 all_done:
1080 return ref;
1081}
1082
f7e20501
BW
1083static void add_shallow_requests(struct strbuf *req_buf,
1084 const struct fetch_pack_args *args)
1085{
1086 if (is_repository_shallow())
1087 write_shallow_commits(req_buf, 1, NULL);
1088 if (args->depth > 0)
1089 packet_buf_write(req_buf, "deepen %d", args->depth);
1090 if (args->deepen_since) {
1091 timestamp_t max_age = approxidate(args->deepen_since);
1092 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1093 }
1094 if (args->deepen_not) {
1095 int i;
1096 for (i = 0; i < args->deepen_not->nr; i++) {
1097 struct string_list_item *s = args->deepen_not->items + i;
1098 packet_buf_write(req_buf, "deepen-not %s", s->string);
1099 }
1100 }
1101}
1102
685fbd32
BW
1103static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1104{
73302051
BW
1105 int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1106
685fbd32
BW
1107 for ( ; wants ; wants = wants->next) {
1108 const struct object_id *remote = &wants->old_oid;
685fbd32
BW
1109 struct object *o;
1110
1111 /*
1112 * If that object is complete (i.e. it is an ancestor of a
1113 * local ref), we tell them we have it but do not have to
1114 * tell them about its ancestors, which they already know
1115 * about.
1116 *
1117 * We use lookup_object here because we are only
1118 * interested in the case we *know* the object is
1119 * reachable and we have already scanned it.
1120 */
1121 if (((o = lookup_object(remote->hash)) != NULL) &&
1122 (o->flags & COMPLETE)) {
1123 continue;
1124 }
1125
73302051
BW
1126 if (!use_ref_in_want || wants->exact_oid)
1127 packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1128 else
1129 packet_buf_write(req_buf, "want-ref %s\n", wants->name);
685fbd32
BW
1130 }
1131}
1132
1133static void add_common(struct strbuf *req_buf, struct oidset *common)
1134{
1135 struct oidset_iter iter;
1136 const struct object_id *oid;
1137 oidset_iter_init(common, &iter);
1138
1139 while ((oid = oidset_iter_next(&iter))) {
1140 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1141 }
1142}
1143
1144static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1145{
1146 int ret = 0;
1147 int haves_added = 0;
1148 const struct object_id *oid;
1149
1150 while ((oid = get_rev())) {
1151 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1152 if (++haves_added >= *haves_to_send)
1153 break;
1154 }
1155
1156 *in_vain += haves_added;
1157 if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1158 /* Send Done */
1159 packet_buf_write(req_buf, "done\n");
1160 ret = 1;
1161 }
1162
1163 /* Increase haves to send on next round */
1164 *haves_to_send = next_flush(1, *haves_to_send);
1165
1166 return ret;
1167}
1168
1169static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1170 const struct ref *wants, struct oidset *common,
1171 int *haves_to_send, int *in_vain)
1172{
1173 int ret = 0;
1174 struct strbuf req_buf = STRBUF_INIT;
1175
1176 if (server_supports_v2("fetch", 1))
1177 packet_buf_write(&req_buf, "command=fetch");
1178 if (server_supports_v2("agent", 0))
1179 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
5e3548ef
BW
1180 if (args->server_options && args->server_options->nr &&
1181 server_supports_v2("server-option", 1)) {
1182 int i;
1183 for (i = 0; i < args->server_options->nr; i++)
1184 packet_write_fmt(fd_out, "server-option=%s",
1185 args->server_options->items[i].string);
1186 }
685fbd32
BW
1187
1188 packet_buf_delim(&req_buf);
1189 if (args->use_thin_pack)
1190 packet_buf_write(&req_buf, "thin-pack");
1191 if (args->no_progress)
1192 packet_buf_write(&req_buf, "no-progress");
1193 if (args->include_tag)
1194 packet_buf_write(&req_buf, "include-tag");
1195 if (prefer_ofs_delta)
1196 packet_buf_write(&req_buf, "ofs-delta");
1197
f7e20501
BW
1198 /* Add shallow-info and deepen request */
1199 if (server_supports_feature("fetch", "shallow", 0))
1200 add_shallow_requests(&req_buf, args);
1201 else if (is_repository_shallow() || args->deepen)
1202 die(_("Server does not support shallow requests"));
1203
ba95710a
JT
1204 /* Add filter */
1205 if (server_supports_feature("fetch", "filter", 0) &&
1206 args->filter_options.choice) {
1207 print_verbose(args, _("Server supports filter"));
1208 packet_buf_write(&req_buf, "filter %s",
1209 args->filter_options.filter_spec);
1210 } else if (args->filter_options.choice) {
1211 warning("filtering not recognized by server, ignoring");
1212 }
1213
685fbd32
BW
1214 /* add wants */
1215 add_wants(wants, &req_buf);
1216
ba95710a
JT
1217 if (args->no_dependents) {
1218 packet_buf_write(&req_buf, "done");
1219 ret = 1;
1220 } else {
1221 /* Add all of the common commits we've found in previous rounds */
1222 add_common(&req_buf, common);
685fbd32 1223
ba95710a
JT
1224 /* Add initial haves */
1225 ret = add_haves(&req_buf, haves_to_send, in_vain);
1226 }
685fbd32
BW
1227
1228 /* Send request */
1229 packet_buf_flush(&req_buf);
1230 write_or_die(fd_out, req_buf.buf, req_buf.len);
1231
1232 strbuf_release(&req_buf);
1233 return ret;
1234}
1235
1236/*
1237 * Processes a section header in a server's response and checks if it matches
1238 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1239 * not consumed); if 0, the line will be consumed and the function will die if
1240 * the section header doesn't match what was expected.
1241 */
1242static int process_section_header(struct packet_reader *reader,
1243 const char *section, int peek)
1244{
1245 int ret;
1246
1247 if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
f7e20501 1248 die("error reading section header '%s'", section);
685fbd32
BW
1249
1250 ret = !strcmp(reader->line, section);
1251
1252 if (!peek) {
1253 if (!ret)
1254 die("expected '%s', received '%s'",
1255 section, reader->line);
1256 packet_reader_read(reader);
1257 }
1258
1259 return ret;
1260}
1261
1262static int process_acks(struct packet_reader *reader, struct oidset *common)
1263{
1264 /* received */
1265 int received_ready = 0;
1266 int received_ack = 0;
1267
1268 process_section_header(reader, "acknowledgments", 0);
1269 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1270 const char *arg;
1271
1272 if (!strcmp(reader->line, "NAK"))
1273 continue;
1274
1275 if (skip_prefix(reader->line, "ACK ", &arg)) {
1276 struct object_id oid;
1277 if (!get_oid_hex(arg, &oid)) {
1278 struct commit *commit;
1279 oidset_insert(common, &oid);
1280 commit = lookup_commit(&oid);
1281 mark_common(commit, 0, 1);
1282 }
1283 continue;
1284 }
1285
1286 if (!strcmp(reader->line, "ready")) {
1287 clear_prio_queue(&rev_list);
1288 received_ready = 1;
1289 continue;
1290 }
1291
1292 die("unexpected acknowledgment line: '%s'", reader->line);
1293 }
1294
1295 if (reader->status != PACKET_READ_FLUSH &&
1296 reader->status != PACKET_READ_DELIM)
1297 die("error processing acks: %d", reader->status);
1298
1299 /* return 0 if no common, 1 if there are common, or 2 if ready */
1300 return received_ready ? 2 : (received_ack ? 1 : 0);
1301}
1302
f7e20501
BW
1303static void receive_shallow_info(struct fetch_pack_args *args,
1304 struct packet_reader *reader)
1305{
1306 process_section_header(reader, "shallow-info", 0);
1307 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1308 const char *arg;
1309 struct object_id oid;
1310
1311 if (skip_prefix(reader->line, "shallow ", &arg)) {
1312 if (get_oid_hex(arg, &oid))
1313 die(_("invalid shallow line: %s"), reader->line);
1314 register_shallow(&oid);
1315 continue;
1316 }
1317 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1318 if (get_oid_hex(arg, &oid))
1319 die(_("invalid unshallow line: %s"), reader->line);
1320 if (!lookup_object(oid.hash))
1321 die(_("object not found: %s"), reader->line);
1322 /* make sure that it is parsed as shallow */
1323 if (!parse_object(&oid))
1324 die(_("error in object: %s"), reader->line);
1325 if (unregister_shallow(&oid))
1326 die(_("no shallow found: %s"), reader->line);
1327 continue;
1328 }
1329 die(_("expected shallow/unshallow, got %s"), reader->line);
1330 }
1331
1332 if (reader->status != PACKET_READ_FLUSH &&
1333 reader->status != PACKET_READ_DELIM)
1334 die("error processing shallow info: %d", reader->status);
1335
1336 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1337 args->deepen = 1;
1338}
1339
73302051
BW
1340static void receive_wanted_refs(struct packet_reader *reader, struct ref *refs)
1341{
1342 process_section_header(reader, "wanted-refs", 0);
1343 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1344 struct object_id oid;
1345 const char *end;
1346 struct ref *r = NULL;
1347
1348 if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1349 die("expected wanted-ref, got '%s'", reader->line);
1350
1351 for (r = refs; r; r = r->next) {
1352 if (!strcmp(end, r->name)) {
1353 oidcpy(&r->old_oid, &oid);
1354 break;
1355 }
1356 }
1357
1358 if (!r)
1359 die("unexpected wanted-ref: '%s'", reader->line);
1360 }
1361
1362 if (reader->status != PACKET_READ_DELIM)
1363 die("error processing wanted refs: %d", reader->status);
1364}
1365
685fbd32
BW
1366enum fetch_state {
1367 FETCH_CHECK_LOCAL = 0,
1368 FETCH_SEND_REQUEST,
1369 FETCH_PROCESS_ACKS,
1370 FETCH_GET_PACK,
1371 FETCH_DONE,
1372};
1373
1374static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1375 int fd[2],
1376 const struct ref *orig_ref,
1377 struct ref **sought, int nr_sought,
1378 char **pack_lockfile)
1379{
1380 struct ref *ref = copy_ref_list(orig_ref);
1381 enum fetch_state state = FETCH_CHECK_LOCAL;
1382 struct oidset common = OIDSET_INIT;
1383 struct packet_reader reader;
1384 int in_vain = 0;
1385 int haves_to_send = INITIAL_FLUSH;
1386 packet_reader_init(&reader, fd[0], NULL, 0,
1387 PACKET_READ_CHOMP_NEWLINE);
1388
1389 while (state != FETCH_DONE) {
1390 switch (state) {
1391 case FETCH_CHECK_LOCAL:
1392 sort_ref_list(&ref, ref_compare_name);
1393 QSORT(sought, nr_sought, cmp_ref_by_name);
1394
1395 /* v2 supports these by default */
1396 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1397 use_sideband = 2;
f7e20501
BW
1398 if (args->depth > 0 || args->deepen_since || args->deepen_not)
1399 args->deepen = 1;
685fbd32
BW
1400
1401 if (marked)
1402 for_each_ref(clear_marks, NULL);
1403 marked = 1;
1404
1405 for_each_ref(rev_list_insert_ref_oid, NULL);
1406 for_each_cached_alternate(insert_one_alternate_object);
1407
1408 /* Filter 'ref' by 'sought' and those that aren't local */
1409 if (everything_local(args, &ref, sought, nr_sought))
1410 state = FETCH_DONE;
1411 else
1412 state = FETCH_SEND_REQUEST;
1413 break;
1414 case FETCH_SEND_REQUEST:
1415 if (send_fetch_request(fd[1], args, ref, &common,
1416 &haves_to_send, &in_vain))
1417 state = FETCH_GET_PACK;
1418 else
1419 state = FETCH_PROCESS_ACKS;
1420 break;
1421 case FETCH_PROCESS_ACKS:
1422 /* Process ACKs/NAKs */
1423 switch (process_acks(&reader, &common)) {
1424 case 2:
1425 state = FETCH_GET_PACK;
1426 break;
1427 case 1:
1428 in_vain = 0;
1429 /* fallthrough */
1430 default:
1431 state = FETCH_SEND_REQUEST;
1432 break;
1433 }
1434 break;
1435 case FETCH_GET_PACK:
f7e20501
BW
1436 /* Check for shallow-info section */
1437 if (process_section_header(&reader, "shallow-info", 1))
1438 receive_shallow_info(args, &reader);
1439
73302051
BW
1440 if (process_section_header(&reader, "wanted-refs", 1))
1441 receive_wanted_refs(&reader, ref);
1442
685fbd32
BW
1443 /* get the pack */
1444 process_section_header(&reader, "packfile", 0);
1445 if (get_pack(args, fd, pack_lockfile))
1446 die(_("git fetch-pack: fetch failed."));
1447
1448 state = FETCH_DONE;
1449 break;
1450 case FETCH_DONE:
1451 continue;
1452 }
1453 }
1454
1455 oidset_clear(&common);
1456 return ref;
1457}
1458
f44af51d 1459static void fetch_pack_config(void)
745f7a8c 1460{
f44af51d
TA
1461 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1462 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1463 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1464 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1465 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 1466
f44af51d 1467 git_config(git_default_config, NULL);
745f7a8c
NTND
1468}
1469
745f7a8c
NTND
1470static void fetch_pack_setup(void)
1471{
1472 static int did_setup;
1473 if (did_setup)
1474 return;
f44af51d 1475 fetch_pack_config();
745f7a8c
NTND
1476 if (0 <= transfer_unpack_limit)
1477 unpack_limit = transfer_unpack_limit;
1478 else if (0 <= fetch_unpack_limit)
1479 unpack_limit = fetch_unpack_limit;
1480 did_setup = 1;
1481}
1482
f2db854d
JH
1483static int remove_duplicates_in_refs(struct ref **ref, int nr)
1484{
1485 struct string_list names = STRING_LIST_INIT_NODUP;
1486 int src, dst;
1487
1488 for (src = dst = 0; src < nr; src++) {
1489 struct string_list_item *item;
1490 item = string_list_insert(&names, ref[src]->name);
1491 if (item->util)
1492 continue; /* already have it */
1493 item->util = ref[src];
1494 if (src != dst)
1495 ref[dst] = ref[src];
1496 dst++;
1497 }
1498 for (src = dst; src < nr; src++)
1499 ref[src] = NULL;
1500 string_list_clear(&names, 0);
1501 return dst;
1502}
1503
beea4152 1504static void update_shallow(struct fetch_pack_args *args,
989b8c44 1505 struct ref *refs,
beea4152 1506 struct shallow_info *si)
a796ccee 1507{
910650d2 1508 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1509 int *status;
beea4152 1510 int i;
989b8c44 1511 struct ref *r;
beea4152 1512
79891cb9 1513 if (args->deepen && alternate_shallow_file) {
a796ccee 1514 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 1515 unlink_or_warn(git_path_shallow());
a796ccee
NTND
1516 rollback_lock_file(&shallow_lock);
1517 } else
1518 commit_lock_file(&shallow_lock);
1519 return;
1520 }
beea4152
NTND
1521
1522 if (!si->shallow || !si->shallow->nr)
1523 return;
1524
beea4152
NTND
1525 if (args->cloning) {
1526 /*
1527 * remote is shallow, but this is a clone, there are
1528 * no objects in repo to worry about. Accept any
1529 * shallow points that exist in the pack (iow in repo
1530 * after get_pack() and reprepare_packed_git())
1531 */
910650d2 1532 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1533 struct object_id *oid = si->shallow->oid;
beea4152 1534 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1535 if (has_object_file(&oid[i]))
910650d2 1536 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1537 if (extra.nr) {
1538 setup_alternate_shallow(&shallow_lock,
1539 &alternate_shallow_file,
1540 &extra);
1541 commit_lock_file(&shallow_lock);
1542 }
910650d2 1543 oid_array_clear(&extra);
beea4152
NTND
1544 return;
1545 }
4820a33b
NTND
1546
1547 if (!si->nr_ours && !si->nr_theirs)
1548 return;
1549
1550 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1551 if (!si->nr_ours && !si->nr_theirs)
1552 return;
989b8c44
BW
1553 for (r = refs; r; r = r->next)
1554 oid_array_append(&ref, &r->old_oid);
4820a33b
NTND
1555 si->ref = &ref;
1556
48d25cae
NTND
1557 if (args->update_shallow) {
1558 /*
1559 * remote is also shallow, .git/shallow may be updated
1560 * so all refs can be accepted. Make sure we only add
1561 * shallow roots that are actually reachable from new
1562 * refs.
1563 */
910650d2 1564 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1565 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1566 assign_shallow_commits_to_refs(si, NULL, NULL);
1567 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1568 oid_array_clear(&ref);
48d25cae
NTND
1569 return;
1570 }
1571 for (i = 0; i < si->nr_ours; i++)
910650d2 1572 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1573 for (i = 0; i < si->nr_theirs; i++)
910650d2 1574 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1575 setup_alternate_shallow(&shallow_lock,
1576 &alternate_shallow_file,
1577 &extra);
1578 commit_lock_file(&shallow_lock);
910650d2 1579 oid_array_clear(&extra);
1580 oid_array_clear(&ref);
48d25cae
NTND
1581 return;
1582 }
1583
4820a33b
NTND
1584 /*
1585 * remote is also shallow, check what ref is safe to update
1586 * without updating .git/shallow
1587 */
989b8c44 1588 status = xcalloc(ref.nr, sizeof(*status));
4820a33b
NTND
1589 assign_shallow_commits_to_refs(si, NULL, status);
1590 if (si->nr_ours || si->nr_theirs) {
989b8c44 1591 for (r = refs, i = 0; r; r = r->next, i++)
4820a33b 1592 if (status[i])
989b8c44 1593 r->status = REF_STATUS_REJECT_SHALLOW;
4820a33b
NTND
1594 }
1595 free(status);
910650d2 1596 oid_array_clear(&ref);
a796ccee
NTND
1597}
1598
745f7a8c
NTND
1599struct ref *fetch_pack(struct fetch_pack_args *args,
1600 int fd[], struct child_process *conn,
1601 const struct ref *ref,
1602 const char *dest,
f2db854d 1603 struct ref **sought, int nr_sought,
910650d2 1604 struct oid_array *shallow,
685fbd32
BW
1605 char **pack_lockfile,
1606 enum protocol_version version)
745f7a8c 1607{
745f7a8c 1608 struct ref *ref_cpy;
beea4152 1609 struct shallow_info si;
745f7a8c
NTND
1610
1611 fetch_pack_setup();
f2db854d
JH
1612 if (nr_sought)
1613 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1614
1615 if (!ref) {
1616 packet_flush(fd[1]);
1dd73e20 1617 die(_("no matching remote head"));
745f7a8c 1618 }
beea4152 1619 prepare_shallow_info(&si, shallow);
685fbd32
BW
1620 if (version == protocol_v2)
1621 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1622 pack_lockfile);
1623 else
1624 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1625 &si, pack_lockfile);
a49d2834 1626 reprepare_packed_git(the_repository);
989b8c44 1627 update_shallow(args, ref_cpy, &si);
beea4152 1628 clear_shallow_info(&si);
745f7a8c
NTND
1629 return ref_cpy;
1630}
e860d96b
MM
1631
1632int report_unmatched_refs(struct ref **sought, int nr_sought)
1633{
1634 int i, ret = 0;
1635
1636 for (i = 0; i < nr_sought; i++) {
d56583de 1637 if (!sought[i])
e860d96b 1638 continue;
d56583de
MM
1639 switch (sought[i]->match_status) {
1640 case REF_MATCHED:
1641 continue;
1642 case REF_NOT_MATCHED:
1643 error(_("no such remote ref %s"), sought[i]->name);
1644 break;
1645 case REF_UNADVERTISED_NOT_ALLOWED:
1646 error(_("Server does not allow request for unadvertised object %s"),
1647 sought[i]->name);
1648 break;
1649 }
e860d96b
MM
1650 ret = 1;
1651 }
1652 return ret;
1653}