fetch-pack: use ref adv. to prune "have" sent
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
a49d2834 2#include "repository.h"
b2141fc1 3#include "config.h"
697cc8ef 4#include "lockfile.h"
745f7a8c
NTND
5#include "refs.h"
6#include "pkt-line.h"
7#include "commit.h"
8#include "tag.h"
d807c4a0 9#include "exec-cmd.h"
745f7a8c
NTND
10#include "pack.h"
11#include "sideband.h"
12#include "fetch-pack.h"
13#include "remote.h"
14#include "run-command.h"
47a59185 15#include "connect.h"
745f7a8c
NTND
16#include "transport.h"
17#include "version.h"
099327b5 18#include "prio-queue.h"
beea4152 19#include "sha1-array.h"
fdb69d33 20#include "oidset.h"
0abe14f6 21#include "packfile.h"
745f7a8c
NTND
22
23static int transfer_unpack_limit = -1;
24static int fetch_unpack_limit = -1;
25static int unpack_limit = 100;
26static int prefer_ofs_delta = 1;
27static int no_done;
508ea882 28static int deepen_since_ok;
a45a2600 29static int deepen_not_ok;
745f7a8c
NTND
30static int fetch_fsck_objects = -1;
31static int transfer_fsck_objects = -1;
32static int agent_supported;
640d8b72 33static int server_supports_filtering;
6035d6aa
NTND
34static struct lock_file shallow_lock;
35static const char *alternate_shallow_file;
745f7a8c 36
208acbfb 37/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
38#define COMPLETE (1U << 0)
39#define COMMON (1U << 1)
40#define COMMON_REF (1U << 2)
41#define SEEN (1U << 3)
42#define POPPED (1U << 4)
41a078c6 43#define ALTERNATE (1U << 5)
745f7a8c
NTND
44
45static int marked;
46
47/*
48 * After sending this many "have"s if we do not get any new ACK , we
49 * give up traversing our history.
50 */
51#define MAX_IN_VAIN 256
52
099327b5 53static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
54static int non_common_revs, multi_ack, use_sideband;
55/* Allow specifying sha1 if it is a ref tip. */
56#define ALLOW_TIP_SHA1 01
68ee6289
FM
57/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
58#define ALLOW_REACHABLE_SHA1 02
7199c093 59static unsigned int allow_unadvertised_object_request;
745f7a8c 60
0d789a5b
NTND
61__attribute__((format (printf, 2, 3)))
62static inline void print_verbose(const struct fetch_pack_args *args,
63 const char *fmt, ...)
64{
65 va_list params;
66
67 if (!args->verbose)
68 return;
69
70 va_start(params, fmt);
71 vfprintf(stderr, fmt, params);
72 va_end(params);
73 fputc('\n', stderr);
74}
75
41a078c6
JK
76struct alternate_object_cache {
77 struct object **items;
78 size_t nr, alloc;
79};
80
81static void cache_one_alternate(const char *refname,
82 const struct object_id *oid,
83 void *vcache)
84{
85 struct alternate_object_cache *cache = vcache;
c251c83d 86 struct object *obj = parse_object(oid);
41a078c6
JK
87
88 if (!obj || (obj->flags & ALTERNATE))
89 return;
90
91 obj->flags |= ALTERNATE;
92 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
93 cache->items[cache->nr++] = obj;
94}
95
96static void for_each_cached_alternate(void (*cb)(struct object *))
97{
98 static int initialized;
99 static struct alternate_object_cache cache;
100 size_t i;
101
102 if (!initialized) {
103 for_each_alternate_ref(cache_one_alternate, &cache);
104 initialized = 1;
105 }
106
107 for (i = 0; i < cache.nr; i++)
108 cb(cache.items[i]);
109}
110
745f7a8c
NTND
111static void rev_list_push(struct commit *commit, int mark)
112{
113 if (!(commit->object.flags & mark)) {
114 commit->object.flags |= mark;
115
0064053b
JK
116 if (parse_commit(commit))
117 return;
745f7a8c 118
099327b5 119 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
120
121 if (!(commit->object.flags & COMMON))
122 non_common_revs++;
123 }
124}
125
1b283377 126static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
745f7a8c 127{
c251c83d 128 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
129
130 if (o && o->type == OBJ_COMMIT)
131 rev_list_push((struct commit *)o, SEEN);
132
133 return 0;
134}
135
b1b49c6e
MH
136static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
137 int flag, void *cb_data)
745f7a8c 138{
1b283377 139 return rev_list_insert_ref(refname, oid);
b1b49c6e
MH
140}
141
c50fb6ce
MH
142static int clear_marks(const char *refname, const struct object_id *oid,
143 int flag, void *cb_data)
745f7a8c 144{
c251c83d 145 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
146
147 if (o && o->type == OBJ_COMMIT)
148 clear_commit_marks((struct commit *)o,
149 COMMON | COMMON_REF | SEEN | POPPED);
150 return 0;
151}
152
153/*
154 This function marks a rev and its ancestors as common.
155 In some cases, it is desirable to mark only the ancestors (for example
156 when only the server does not yet know that they are common).
157*/
158
159static void mark_common(struct commit *commit,
160 int ancestors_only, int dont_parse)
161{
162 if (commit != NULL && !(commit->object.flags & COMMON)) {
163 struct object *o = (struct object *)commit;
164
165 if (!ancestors_only)
166 o->flags |= COMMON;
167
168 if (!(o->flags & SEEN))
169 rev_list_push(commit, SEEN);
170 else {
171 struct commit_list *parents;
172
173 if (!ancestors_only && !(o->flags & POPPED))
174 non_common_revs--;
175 if (!o->parsed && !dont_parse)
176 if (parse_commit(commit))
177 return;
178
179 for (parents = commit->parents;
180 parents;
181 parents = parents->next)
182 mark_common(parents->item, 0, dont_parse);
183 }
184 }
185}
186
187/*
188 Get the next rev to send, ignoring the common.
189*/
190
1b283377 191static const struct object_id *get_rev(void)
745f7a8c
NTND
192{
193 struct commit *commit = NULL;
194
195 while (commit == NULL) {
196 unsigned int mark;
197 struct commit_list *parents;
198
099327b5 199 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
200 return NULL;
201
099327b5 202 commit = prio_queue_get(&rev_list);
0064053b 203 parse_commit(commit);
745f7a8c
NTND
204 parents = commit->parents;
205
206 commit->object.flags |= POPPED;
207 if (!(commit->object.flags & COMMON))
208 non_common_revs--;
209
210 if (commit->object.flags & COMMON) {
211 /* do not send "have", and ignore ancestors */
212 commit = NULL;
213 mark = COMMON | SEEN;
214 } else if (commit->object.flags & COMMON_REF)
215 /* send "have", and ignore ancestors */
216 mark = COMMON | SEEN;
217 else
218 /* send "have", also for its ancestors */
219 mark = SEEN;
220
221 while (parents) {
222 if (!(parents->item->object.flags & SEEN))
223 rev_list_push(parents->item, mark);
224 if (mark & COMMON)
225 mark_common(parents->item, 1, 0);
226 parents = parents->next;
227 }
745f7a8c
NTND
228 }
229
1b283377 230 return &commit->object.oid;
745f7a8c
NTND
231}
232
233enum ack_type {
234 NAK = 0,
235 ACK,
236 ACK_continue,
237 ACK_common,
238 ACK_ready
239};
240
241static void consume_shallow_list(struct fetch_pack_args *args, int fd)
242{
79891cb9 243 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
244 /* If we sent a depth we will get back "duplicate"
245 * shallow and unshallow commands every time there
246 * is a block of have lines exchanged.
247 */
74543a04
JK
248 char *line;
249 while ((line = packet_read_line(fd, NULL))) {
59556548 250 if (starts_with(line, "shallow "))
745f7a8c 251 continue;
59556548 252 if (starts_with(line, "unshallow "))
745f7a8c 253 continue;
1dd73e20 254 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
255 }
256 }
257}
258
1b283377 259static enum ack_type get_ack(int fd, struct object_id *result_oid)
745f7a8c 260{
74543a04
JK
261 int len;
262 char *line = packet_read_line(fd, &len);
82e56767 263 const char *arg;
745f7a8c 264
bc9d4dc5
JK
265 if (!line)
266 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
745f7a8c
NTND
267 if (!strcmp(line, "NAK"))
268 return NAK;
82e56767 269 if (skip_prefix(line, "ACK ", &arg)) {
1b283377 270 if (!get_oid_hex(arg, result_oid)) {
82e56767
JK
271 arg += 40;
272 len -= arg - line;
273 if (len < 1)
030e9dd6 274 return ACK;
82e56767 275 if (strstr(arg, "continue"))
745f7a8c 276 return ACK_continue;
82e56767 277 if (strstr(arg, "common"))
745f7a8c 278 return ACK_common;
82e56767 279 if (strstr(arg, "ready"))
745f7a8c
NTND
280 return ACK_ready;
281 return ACK;
282 }
283 }
8e2c7bef
JT
284 if (skip_prefix(line, "ERR ", &arg))
285 die(_("remote error: %s"), arg);
dfbfb9f3 286 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
287}
288
289static void send_request(struct fetch_pack_args *args,
290 int fd, struct strbuf *buf)
291{
292 if (args->stateless_rpc) {
293 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
294 packet_flush(fd);
295 } else
cdf4fb8e 296 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
297}
298
41a078c6 299static void insert_one_alternate_object(struct object *obj)
745f7a8c 300{
1b283377 301 rev_list_insert_ref(NULL, &obj->oid);
745f7a8c
NTND
302}
303
304#define INITIAL_FLUSH 16
305#define PIPESAFE_FLUSH 32
da470981 306#define LARGE_FLUSH 16384
745f7a8c 307
685fbd32 308static int next_flush(int stateless_rpc, int count)
745f7a8c 309{
685fbd32 310 if (stateless_rpc) {
da470981
JT
311 if (count < LARGE_FLUSH)
312 count <<= 1;
313 else
314 count = count * 11 / 10;
315 } else {
316 if (count < PIPESAFE_FLUSH)
317 count <<= 1;
318 else
319 count += PIPESAFE_FLUSH;
320 }
745f7a8c
NTND
321 return count;
322}
323
324static int find_common(struct fetch_pack_args *args,
1b283377 325 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
326 struct ref *refs)
327{
328 int fetching;
329 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 330 const struct object_id *oid;
745f7a8c
NTND
331 unsigned in_vain = 0;
332 int got_continue = 0;
333 int got_ready = 0;
334 struct strbuf req_buf = STRBUF_INIT;
335 size_t state_len = 0;
336
337 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 338 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c 339
b1b49c6e 340 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 341 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
342
343 fetching = 0;
344 for ( ; refs ; refs = refs->next) {
1b283377 345 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
346 const char *remote_hex;
347 struct object *o;
348
349 /*
350 * If that object is complete (i.e. it is an ancestor of a
351 * local ref), we tell them we have it but do not have to
352 * tell them about its ancestors, which they already know
353 * about.
354 *
355 * We use lookup_object here because we are only
356 * interested in the case we *know* the object is
357 * reachable and we have already scanned it.
358 */
1b283377 359 if (((o = lookup_object(remote->hash)) != NULL) &&
745f7a8c
NTND
360 (o->flags & COMPLETE)) {
361 continue;
362 }
363
1b283377 364 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
365 if (!fetching) {
366 struct strbuf c = STRBUF_INIT;
367 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
368 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
369 if (no_done) strbuf_addstr(&c, " no-done");
370 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
371 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 372 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
373 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
374 if (args->no_progress) strbuf_addstr(&c, " no-progress");
375 if (args->include_tag) strbuf_addstr(&c, " include-tag");
376 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 377 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 378 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
379 if (agent_supported) strbuf_addf(&c, " agent=%s",
380 git_user_agent_sanitized());
640d8b72
JH
381 if (args->filter_options.choice)
382 strbuf_addstr(&c, " filter");
745f7a8c
NTND
383 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
384 strbuf_release(&c);
385 } else
386 packet_buf_write(&req_buf, "want %s\n", remote_hex);
387 fetching++;
388 }
389
390 if (!fetching) {
391 strbuf_release(&req_buf);
392 packet_flush(fd[1]);
393 return 1;
394 }
395
396 if (is_repository_shallow())
1a30f5a2 397 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
398 if (args->depth > 0)
399 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882 400 if (args->deepen_since) {
dddbad72 401 timestamp_t max_age = approxidate(args->deepen_since);
cb71f8bd 402 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
508ea882 403 }
a45a2600
NTND
404 if (args->deepen_not) {
405 int i;
406 for (i = 0; i < args->deepen_not->nr; i++) {
407 struct string_list_item *s = args->deepen_not->items + i;
408 packet_buf_write(&req_buf, "deepen-not %s", s->string);
409 }
410 }
640d8b72
JH
411 if (server_supports_filtering && args->filter_options.choice)
412 packet_buf_write(&req_buf, "filter %s",
413 args->filter_options.filter_spec);
745f7a8c
NTND
414 packet_buf_flush(&req_buf);
415 state_len = req_buf.len;
416
79891cb9 417 if (args->deepen) {
74543a04 418 char *line;
ae021d87 419 const char *arg;
1b283377 420 struct object_id oid;
745f7a8c
NTND
421
422 send_request(args, fd[1], &req_buf);
74543a04 423 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87 424 if (skip_prefix(line, "shallow ", &arg)) {
1b283377 425 if (get_oid_hex(arg, &oid))
1dd73e20 426 die(_("invalid shallow line: %s"), line);
e92b848c 427 register_shallow(&oid);
745f7a8c
NTND
428 continue;
429 }
ae021d87 430 if (skip_prefix(line, "unshallow ", &arg)) {
1b283377 431 if (get_oid_hex(arg, &oid))
1dd73e20 432 die(_("invalid unshallow line: %s"), line);
1b283377 433 if (!lookup_object(oid.hash))
1dd73e20 434 die(_("object not found: %s"), line);
745f7a8c 435 /* make sure that it is parsed as shallow */
c251c83d 436 if (!parse_object(&oid))
1dd73e20 437 die(_("error in object: %s"), line);
e92b848c 438 if (unregister_shallow(&oid))
1dd73e20 439 die(_("no shallow found: %s"), line);
745f7a8c
NTND
440 continue;
441 }
1dd73e20 442 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
443 }
444 } else if (!args->stateless_rpc)
445 send_request(args, fd[1], &req_buf);
446
447 if (!args->stateless_rpc) {
448 /* If we aren't using the stateless-rpc interface
449 * we don't need to retain the headers.
450 */
451 strbuf_setlen(&req_buf, 0);
452 state_len = 0;
453 }
454
455 flushes = 0;
456 retval = -1;
88e2f9ed
JT
457 if (args->no_dependents)
458 goto done;
1b283377 459 while ((oid = get_rev())) {
460 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
461 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
462 in_vain++;
463 if (flush_at <= ++count) {
464 int ack;
465
466 packet_buf_flush(&req_buf);
467 send_request(args, fd[1], &req_buf);
468 strbuf_setlen(&req_buf, state_len);
469 flushes++;
685fbd32 470 flush_at = next_flush(args->stateless_rpc, count);
745f7a8c
NTND
471
472 /*
473 * We keep one window "ahead" of the other side, and
474 * will wait for an ACK only on the next one
475 */
476 if (!args->stateless_rpc && count == INITIAL_FLUSH)
477 continue;
478
479 consume_shallow_list(args, fd[0]);
480 do {
1b283377 481 ack = get_ack(fd[0], result_oid);
0d789a5b 482 if (ack)
1dd73e20 483 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 484 ack, oid_to_hex(result_oid));
745f7a8c
NTND
485 switch (ack) {
486 case ACK:
487 flushes = 0;
488 multi_ack = 0;
489 retval = 0;
490 goto done;
491 case ACK_common:
492 case ACK_ready:
493 case ACK_continue: {
494 struct commit *commit =
bc83266a 495 lookup_commit(result_oid);
745f7a8c 496 if (!commit)
1b283377 497 die(_("invalid commit %s"), oid_to_hex(result_oid));
745f7a8c
NTND
498 if (args->stateless_rpc
499 && ack == ACK_common
500 && !(commit->object.flags & COMMON)) {
501 /* We need to replay the have for this object
502 * on the next RPC request so the peer knows
503 * it is in common with us.
504 */
1b283377 505 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
506 packet_buf_write(&req_buf, "have %s\n", hex);
507 state_len = req_buf.len;
06b3d386
JT
508 /*
509 * Reset in_vain because an ack
510 * for this commit has not been
511 * seen.
512 */
513 in_vain = 0;
514 } else if (!args->stateless_rpc
515 || ack != ACK_common)
516 in_vain = 0;
745f7a8c
NTND
517 mark_common(commit, 0, 1);
518 retval = 0;
745f7a8c 519 got_continue = 1;
21bcf6e4 520 if (ack == ACK_ready)
745f7a8c 521 got_ready = 1;
745f7a8c
NTND
522 break;
523 }
524 }
525 } while (ack);
526 flushes--;
527 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 528 print_verbose(args, _("giving up"));
745f7a8c
NTND
529 break; /* give up */
530 }
21bcf6e4
JT
531 if (got_ready)
532 break;
745f7a8c
NTND
533 }
534 }
535done:
536 if (!got_ready || !no_done) {
537 packet_buf_write(&req_buf, "done\n");
538 send_request(args, fd[1], &req_buf);
539 }
1dd73e20 540 print_verbose(args, _("done"));
745f7a8c
NTND
541 if (retval != 0) {
542 multi_ack = 0;
543 flushes++;
544 }
545 strbuf_release(&req_buf);
546
ff62eca7
NTND
547 if (!got_ready || !no_done)
548 consume_shallow_list(args, fd[0]);
745f7a8c 549 while (flushes || multi_ack) {
1b283377 550 int ack = get_ack(fd[0], result_oid);
745f7a8c 551 if (ack) {
1dd73e20 552 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 553 ack, oid_to_hex(result_oid));
745f7a8c
NTND
554 if (ack == ACK)
555 return 0;
556 multi_ack = 1;
557 continue;
558 }
559 flushes--;
560 }
561 /* it is no error to fetch into a completely empty repo */
562 return count ? retval : 0;
563}
564
565static struct commit_list *complete;
566
1b283377 567static int mark_complete(const struct object_id *oid)
745f7a8c 568{
c251c83d 569 struct object *o = parse_object(oid);
745f7a8c
NTND
570
571 while (o && o->type == OBJ_TAG) {
572 struct tag *t = (struct tag *) o;
573 if (!t->tagged)
574 break; /* broken repository */
575 o->flags |= COMPLETE;
c251c83d 576 o = parse_object(&t->tagged->oid);
745f7a8c
NTND
577 }
578 if (o && o->type == OBJ_COMMIT) {
579 struct commit *commit = (struct commit *)o;
580 if (!(commit->object.flags & COMPLETE)) {
581 commit->object.flags |= COMPLETE;
16445242 582 commit_list_insert(commit, &complete);
745f7a8c
NTND
583 }
584 }
585 return 0;
586}
587
f8ee4d85
MH
588static int mark_complete_oid(const char *refname, const struct object_id *oid,
589 int flag, void *cb_data)
590{
1b283377 591 return mark_complete(oid);
f8ee4d85
MH
592}
593
745f7a8c 594static void mark_recent_complete_commits(struct fetch_pack_args *args,
dddbad72 595 timestamp_t cutoff)
745f7a8c
NTND
596{
597 while (complete && cutoff <= complete->item->date) {
1dd73e20 598 print_verbose(args, _("Marking %s as complete"),
0d789a5b 599 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
600 pop_most_recent_commit(&complete, COMPLETE);
601 }
602}
603
fdb69d33
JT
604static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
605{
606 for (; refs; refs = refs->next)
607 oidset_insert(oids, &refs->old_oid);
608}
609
610static int tip_oids_contain(struct oidset *tip_oids,
611 struct ref *unmatched, struct ref *newlist,
612 const struct object_id *id)
613{
614 /*
615 * Note that this only looks at the ref lists the first time it's
616 * called. This works out in filter_refs() because even though it may
617 * add to "newlist" between calls, the additions will always be for
618 * oids that are already in the set.
619 */
9e6fabde 620 if (!tip_oids->map.map.tablesize) {
fdb69d33
JT
621 add_refs_to_oidset(tip_oids, unmatched);
622 add_refs_to_oidset(tip_oids, newlist);
623 }
624 return oidset_contains(tip_oids, id);
625}
626
745f7a8c 627static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
628 struct ref **refs,
629 struct ref **sought, int nr_sought)
745f7a8c
NTND
630{
631 struct ref *newlist = NULL;
632 struct ref **newtail = &newlist;
fdb69d33 633 struct ref *unmatched = NULL;
745f7a8c 634 struct ref *ref, *next;
fdb69d33 635 struct oidset tip_oids = OIDSET_INIT;
f2db854d 636 int i;
745f7a8c 637
f2db854d 638 i = 0;
745f7a8c
NTND
639 for (ref = *refs; ref; ref = next) {
640 int keep = 0;
641 next = ref->next;
f2db854d 642
50e19a83 643 if (starts_with(ref->name, "refs/") &&
4c224081 644 check_refname_format(ref->name, 0))
745f7a8c
NTND
645 ; /* trash */
646 else {
f2db854d
JH
647 while (i < nr_sought) {
648 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
649 if (cmp < 0)
650 break; /* definitely do not have it */
651 else if (cmp == 0) {
652 keep = 1; /* definitely have it */
d56583de 653 sought[i]->match_status = REF_MATCHED;
745f7a8c 654 }
f2db854d 655 i++;
745f7a8c
NTND
656 }
657 }
658
f2db854d 659 if (!keep && args->fetch_all &&
79891cb9 660 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
661 keep = 1;
662
663 if (keep) {
664 *newtail = ref;
665 ref->next = NULL;
666 newtail = &ref->next;
667 } else {
fdb69d33
JT
668 ref->next = unmatched;
669 unmatched = ref;
745f7a8c
NTND
670 }
671 }
672
6e7b66ee 673 /* Append unmatched requests to the list */
d56583de 674 for (i = 0; i < nr_sought; i++) {
1b283377 675 struct object_id oid;
676 const char *p;
b7916422 677
d56583de
MM
678 ref = sought[i];
679 if (ref->match_status != REF_NOT_MATCHED)
680 continue;
1b283377 681 if (parse_oid_hex(ref->name, &oid, &p) ||
682 *p != '\0' ||
683 oidcmp(&oid, &ref->old_oid))
d56583de 684 continue;
6e7b66ee 685
d56583de 686 if ((allow_unadvertised_object_request &
fdb69d33
JT
687 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
688 tip_oids_contain(&tip_oids, unmatched, newlist,
689 &ref->old_oid)) {
d56583de 690 ref->match_status = REF_MATCHED;
c3c17bf1
JK
691 *newtail = copy_ref(ref);
692 newtail = &(*newtail)->next;
d56583de
MM
693 } else {
694 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
695 }
696 }
fdb69d33
JT
697
698 oidset_clear(&tip_oids);
699 for (ref = unmatched; ref; ref = next) {
700 next = ref->next;
701 free(ref);
702 }
703
745f7a8c
NTND
704 *refs = newlist;
705}
706
41a078c6 707static void mark_alternate_complete(struct object *obj)
745f7a8c 708{
1b283377 709 mark_complete(&obj->oid);
745f7a8c
NTND
710}
711
024aa469
TI
712struct loose_object_iter {
713 struct oidset *loose_object_set;
714 struct ref *refs;
715};
716
717/*
718 * If the number of refs is not larger than the number of loose objects,
719 * this function stops inserting.
720 */
721static int add_loose_objects_to_set(const struct object_id *oid,
722 const char *path,
723 void *data)
724{
725 struct loose_object_iter *iter = data;
726 oidset_insert(iter->loose_object_set, oid);
727 if (iter->refs == NULL)
728 return 1;
729
730 iter->refs = iter->refs->next;
731 return 0;
732}
733
34c29034
JT
734/*
735 * Mark recent commits available locally and reachable from a local ref as
736 * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as
737 * COMMON_REF (otherwise, we are not planning to participate in negotiation, and
738 * thus do not need COMMON_REF marks).
739 *
740 * The cutoff time for recency is determined by this heuristic: it is the
741 * earliest commit time of the objects in refs that are commits and that we know
742 * the commit time of.
743 */
744static void mark_complete_and_common_ref(struct fetch_pack_args *args,
745 struct ref **refs)
745f7a8c
NTND
746{
747 struct ref *ref;
a1c6d7c1 748 int old_save_commit_buffer = save_commit_buffer;
dddbad72 749 timestamp_t cutoff = 0;
024aa469
TI
750 struct oidset loose_oid_set = OIDSET_INIT;
751 int use_oidset = 0;
752 struct loose_object_iter iter = {&loose_oid_set, *refs};
753
754 /* Enumerate all loose objects or know refs are not so many. */
755 use_oidset = !for_each_loose_object(add_loose_objects_to_set,
756 &iter, 0);
745f7a8c
NTND
757
758 save_commit_buffer = 0;
759
760 for (ref = *refs; ref; ref = ref->next) {
761 struct object *o;
024aa469 762 unsigned int flags = OBJECT_INFO_QUICK;
745f7a8c 763
024aa469
TI
764 if (use_oidset &&
765 !oidset_contains(&loose_oid_set, &ref->old_oid)) {
766 /*
767 * I know this does not exist in the loose form,
768 * so check if it exists in a non-loose form.
769 */
770 flags |= OBJECT_INFO_IGNORE_LOOSE;
771 }
012a1bb5 772
024aa469
TI
773 if (!has_object_file_with_flags(&ref->old_oid, flags))
774 continue;
c251c83d 775 o = parse_object(&ref->old_oid);
745f7a8c
NTND
776 if (!o)
777 continue;
778
779 /* We already have it -- which may mean that we were
780 * in sync with the other side at some time after
781 * that (it is OK if we guess wrong here).
782 */
783 if (o->type == OBJ_COMMIT) {
784 struct commit *commit = (struct commit *)o;
785 if (!cutoff || cutoff < commit->date)
786 cutoff = commit->date;
787 }
788 }
789
024aa469
TI
790 oidset_clear(&loose_oid_set);
791
88e2f9ed
JT
792 if (!args->no_dependents) {
793 if (!args->deepen) {
794 for_each_ref(mark_complete_oid, NULL);
795 for_each_cached_alternate(mark_alternate_complete);
796 commit_list_sort_by_date(&complete);
797 if (cutoff)
798 mark_recent_complete_commits(args, cutoff);
799 }
745f7a8c 800
88e2f9ed
JT
801 /*
802 * Mark all complete remote refs as common refs.
803 * Don't mark them common yet; the server has to be told so first.
804 */
805 for (ref = *refs; ref; ref = ref->next) {
806 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
807 NULL, 0);
745f7a8c 808
88e2f9ed
JT
809 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
810 continue;
745f7a8c 811
88e2f9ed
JT
812 if (!(o->flags & SEEN)) {
813 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
745f7a8c 814
88e2f9ed
JT
815 mark_common((struct commit *)o, 1, 1);
816 }
745f7a8c
NTND
817 }
818 }
819
34c29034
JT
820 save_commit_buffer = old_save_commit_buffer;
821}
822
823/*
824 * Returns 1 if every object pointed to by the given remote refs is available
825 * locally and reachable from a local ref, and 0 otherwise.
826 */
827static int everything_local(struct fetch_pack_args *args,
828 struct ref **refs)
829{
830 struct ref *ref;
831 int retval;
745f7a8c
NTND
832
833 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 834 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
835 struct object *o;
836
1b283377 837 o = lookup_object(remote->hash);
745f7a8c
NTND
838 if (!o || !(o->flags & COMPLETE)) {
839 retval = 0;
1b283377 840 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 841 ref->name);
745f7a8c
NTND
842 continue;
843 }
1b283377 844 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 845 ref->name);
745f7a8c 846 }
a1c6d7c1 847
745f7a8c
NTND
848 return retval;
849}
850
851static int sideband_demux(int in, int out, void *data)
852{
853 int *xd = data;
9ff18faf 854 int ret;
745f7a8c 855
9ff18faf 856 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
857 close(out);
858 return ret;
859}
860
861static int get_pack(struct fetch_pack_args *args,
862 int xd[2], char **pack_lockfile)
863{
864 struct async demux;
745f7a8c 865 int do_keep = args->keep_pack;
984a43b9
JK
866 const char *cmd_name;
867 struct pack_header header;
868 int pass_header = 0;
d3180279 869 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 870 int ret;
745f7a8c
NTND
871
872 memset(&demux, 0, sizeof(demux));
873 if (use_sideband) {
874 /* xd[] is talking with upload-pack; subprocess reads from
875 * xd[0], spits out band#2 to stderr, and feeds us band#1
876 * through demux->out.
877 */
878 demux.proc = sideband_demux;
879 demux.data = xd;
880 demux.out = -1;
df857572 881 demux.isolate_sigpipe = 1;
745f7a8c 882 if (start_async(&demux))
1dd73e20 883 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
884 }
885 else
886 demux.out = xd[0];
887
745f7a8c 888 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
889
890 if (read_pack_header(demux.out, &header))
1dd73e20 891 die(_("protocol error: bad pack header"));
984a43b9 892 pass_header = 1;
745f7a8c
NTND
893 if (ntohl(header.hdr_entries) < unpack_limit)
894 do_keep = 0;
895 else
896 do_keep = 1;
897 }
898
6035d6aa 899 if (alternate_shallow_file) {
984a43b9
JK
900 argv_array_push(&cmd.args, "--shallow-file");
901 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
902 }
903
88e2f9ed 904 if (do_keep || args->from_promisor) {
745f7a8c
NTND
905 if (pack_lockfile)
906 cmd.out = -1;
984a43b9
JK
907 cmd_name = "index-pack";
908 argv_array_push(&cmd.args, cmd_name);
909 argv_array_push(&cmd.args, "--stdin");
745f7a8c 910 if (!args->quiet && !args->no_progress)
984a43b9 911 argv_array_push(&cmd.args, "-v");
745f7a8c 912 if (args->use_thin_pack)
984a43b9 913 argv_array_push(&cmd.args, "--fix-thin");
88e2f9ed 914 if (do_keep && (args->lock_pack || unpack_limit)) {
da25bdb7 915 char hostname[HOST_NAME_MAX + 1];
5781a9a2 916 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
917 xsnprintf(hostname, sizeof(hostname), "localhost");
918 argv_array_pushf(&cmd.args,
919 "--keep=fetch-pack %"PRIuMAX " on %s",
920 (uintmax_t)getpid(), hostname);
745f7a8c 921 }
c6807a40 922 if (args->check_self_contained_and_connected)
984a43b9 923 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
88e2f9ed
JT
924 if (args->from_promisor)
925 argv_array_push(&cmd.args, "--promisor");
745f7a8c
NTND
926 }
927 else {
984a43b9
JK
928 cmd_name = "unpack-objects";
929 argv_array_push(&cmd.args, cmd_name);
745f7a8c 930 if (args->quiet || args->no_progress)
984a43b9 931 argv_array_push(&cmd.args, "-q");
c6807a40 932 args->check_self_contained_and_connected = 0;
745f7a8c 933 }
984a43b9
JK
934
935 if (pass_header)
936 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
937 ntohl(header.hdr_version),
938 ntohl(header.hdr_entries));
745f7a8c
NTND
939 if (fetch_fsck_objects >= 0
940 ? fetch_fsck_objects
941 : transfer_fsck_objects >= 0
942 ? transfer_fsck_objects
98a2ea46
JT
943 : 0) {
944 if (args->from_promisor)
945 /*
946 * We cannot use --strict in index-pack because it
947 * checks both broken objects and links, but we only
948 * want to check for broken objects.
949 */
950 argv_array_push(&cmd.args, "--fsck-objects");
951 else
952 argv_array_push(&cmd.args, "--strict");
953 }
745f7a8c
NTND
954
955 cmd.in = demux.out;
956 cmd.git_cmd = 1;
957 if (start_command(&cmd))
1dd73e20 958 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
959 if (do_keep && pack_lockfile) {
960 *pack_lockfile = index_pack_lockfile(cmd.out);
961 close(cmd.out);
962 }
963
37cb1dd6
JL
964 if (!use_sideband)
965 /* Closed by start_command() */
966 xd[0] = -1;
967
c6807a40
NTND
968 ret = finish_command(&cmd);
969 if (!ret || (args->check_self_contained_and_connected && ret == 1))
970 args->self_contained_and_connected =
971 args->check_self_contained_and_connected &&
972 ret == 0;
973 else
1dd73e20 974 die(_("%s failed"), cmd_name);
745f7a8c 975 if (use_sideband && finish_async(&demux))
1dd73e20 976 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
977 return 0;
978}
979
f2db854d
JH
980static int cmp_ref_by_name(const void *a_, const void *b_)
981{
982 const struct ref *a = *((const struct ref **)a_);
983 const struct ref *b = *((const struct ref **)b_);
984 return strcmp(a->name, b->name);
985}
986
745f7a8c
NTND
987static struct ref *do_fetch_pack(struct fetch_pack_args *args,
988 int fd[2],
989 const struct ref *orig_ref,
f2db854d 990 struct ref **sought, int nr_sought,
beea4152 991 struct shallow_info *si,
745f7a8c
NTND
992 char **pack_lockfile)
993{
994 struct ref *ref = copy_ref_list(orig_ref);
1b283377 995 struct object_id oid;
745f7a8c
NTND
996 const char *agent_feature;
997 int agent_len;
998
999 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 1000 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 1001
eb86a507 1002 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 1003 die(_("Server does not support shallow clients"));
a45a2600 1004 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 1005 args->deepen = 1;
745f7a8c 1006 if (server_supports("multi_ack_detailed")) {
1dd73e20 1007 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
1008 multi_ack = 2;
1009 if (server_supports("no-done")) {
1dd73e20 1010 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
1011 if (args->stateless_rpc)
1012 no_done = 1;
1013 }
1014 }
1015 else if (server_supports("multi_ack")) {
1dd73e20 1016 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
1017 multi_ack = 1;
1018 }
1019 if (server_supports("side-band-64k")) {
1dd73e20 1020 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
1021 use_sideband = 2;
1022 }
1023 else if (server_supports("side-band")) {
1dd73e20 1024 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
1025 use_sideband = 1;
1026 }
6e7b66ee 1027 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 1028 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 1029 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 1030 }
68ee6289 1031 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 1032 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
1033 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1034 }
745f7a8c
NTND
1035 if (!server_supports("thin-pack"))
1036 args->use_thin_pack = 0;
1037 if (!server_supports("no-progress"))
1038 args->no_progress = 0;
1039 if (!server_supports("include-tag"))
1040 args->include_tag = 0;
0d789a5b 1041 if (server_supports("ofs-delta"))
1dd73e20 1042 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 1043 else
745f7a8c
NTND
1044 prefer_ofs_delta = 0;
1045
640d8b72
JH
1046 if (server_supports("filter")) {
1047 server_supports_filtering = 1;
1048 print_verbose(args, _("Server supports filter"));
1049 } else if (args->filter_options.choice) {
1050 warning("filtering not recognized by server, ignoring");
1051 }
1052
745f7a8c
NTND
1053 if ((agent_feature = server_feature_value("agent", &agent_len))) {
1054 agent_supported = 1;
0d789a5b 1055 if (agent_len)
1dd73e20 1056 print_verbose(args, _("Server version is %.*s"),
0d789a5b 1057 agent_len, agent_feature);
745f7a8c 1058 }
508ea882
NTND
1059 if (server_supports("deepen-since"))
1060 deepen_since_ok = 1;
1061 else if (args->deepen_since)
1062 die(_("Server does not support --shallow-since"));
a45a2600
NTND
1063 if (server_supports("deepen-not"))
1064 deepen_not_ok = 1;
1065 else if (args->deepen_not)
1066 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
1067 if (!server_supports("deepen-relative") && args->deepen_relative)
1068 die(_("Server does not support --deepen"));
745f7a8c 1069
af008558
JT
1070 if (marked)
1071 for_each_ref(clear_marks, NULL);
1072 marked = 1;
34c29034
JT
1073 mark_complete_and_common_ref(args, &ref);
1074 filter_refs(args, &ref, sought, nr_sought);
1075 if (everything_local(args, &ref)) {
745f7a8c
NTND
1076 packet_flush(fd[1]);
1077 goto all_done;
1078 }
1b283377 1079 if (find_common(args, fd, &oid, ref) < 0)
745f7a8c
NTND
1080 if (!args->keep_pack)
1081 /* When cloning, it is not unusual to have
1082 * no common commit.
1083 */
1dd73e20 1084 warning(_("no common commits"));
745f7a8c
NTND
1085
1086 if (args->stateless_rpc)
1087 packet_flush(fd[1]);
79891cb9 1088 if (args->deepen)
1a30f5a2
NTND
1089 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1090 NULL);
4820a33b 1091 else if (si->nr_ours || si->nr_theirs)
beea4152 1092 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
1093 else
1094 alternate_shallow_file = NULL;
745f7a8c 1095 if (get_pack(args, fd, pack_lockfile))
1dd73e20 1096 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
1097
1098 all_done:
21bcf6e4 1099 clear_prio_queue(&rev_list);
745f7a8c
NTND
1100 return ref;
1101}
1102
f7e20501
BW
1103static void add_shallow_requests(struct strbuf *req_buf,
1104 const struct fetch_pack_args *args)
1105{
1106 if (is_repository_shallow())
1107 write_shallow_commits(req_buf, 1, NULL);
1108 if (args->depth > 0)
1109 packet_buf_write(req_buf, "deepen %d", args->depth);
1110 if (args->deepen_since) {
1111 timestamp_t max_age = approxidate(args->deepen_since);
1112 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1113 }
1114 if (args->deepen_not) {
1115 int i;
1116 for (i = 0; i < args->deepen_not->nr; i++) {
1117 struct string_list_item *s = args->deepen_not->items + i;
1118 packet_buf_write(req_buf, "deepen-not %s", s->string);
1119 }
1120 }
1121}
1122
685fbd32
BW
1123static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1124{
1125 for ( ; wants ; wants = wants->next) {
1126 const struct object_id *remote = &wants->old_oid;
1127 const char *remote_hex;
1128 struct object *o;
1129
1130 /*
1131 * If that object is complete (i.e. it is an ancestor of a
1132 * local ref), we tell them we have it but do not have to
1133 * tell them about its ancestors, which they already know
1134 * about.
1135 *
1136 * We use lookup_object here because we are only
1137 * interested in the case we *know* the object is
1138 * reachable and we have already scanned it.
1139 */
1140 if (((o = lookup_object(remote->hash)) != NULL) &&
1141 (o->flags & COMPLETE)) {
1142 continue;
1143 }
1144
1145 remote_hex = oid_to_hex(remote);
1146 packet_buf_write(req_buf, "want %s\n", remote_hex);
1147 }
1148}
1149
1150static void add_common(struct strbuf *req_buf, struct oidset *common)
1151{
1152 struct oidset_iter iter;
1153 const struct object_id *oid;
1154 oidset_iter_init(common, &iter);
1155
1156 while ((oid = oidset_iter_next(&iter))) {
1157 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1158 }
1159}
1160
1161static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1162{
1163 int ret = 0;
1164 int haves_added = 0;
1165 const struct object_id *oid;
1166
1167 while ((oid = get_rev())) {
1168 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1169 if (++haves_added >= *haves_to_send)
1170 break;
1171 }
1172
1173 *in_vain += haves_added;
1174 if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1175 /* Send Done */
1176 packet_buf_write(req_buf, "done\n");
1177 ret = 1;
1178 }
1179
1180 /* Increase haves to send on next round */
1181 *haves_to_send = next_flush(1, *haves_to_send);
1182
1183 return ret;
1184}
1185
1186static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1187 const struct ref *wants, struct oidset *common,
1188 int *haves_to_send, int *in_vain)
1189{
1190 int ret = 0;
1191 struct strbuf req_buf = STRBUF_INIT;
1192
1193 if (server_supports_v2("fetch", 1))
1194 packet_buf_write(&req_buf, "command=fetch");
1195 if (server_supports_v2("agent", 0))
1196 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
5e3548ef
BW
1197 if (args->server_options && args->server_options->nr &&
1198 server_supports_v2("server-option", 1)) {
1199 int i;
1200 for (i = 0; i < args->server_options->nr; i++)
1201 packet_write_fmt(fd_out, "server-option=%s",
1202 args->server_options->items[i].string);
1203 }
685fbd32
BW
1204
1205 packet_buf_delim(&req_buf);
1206 if (args->use_thin_pack)
1207 packet_buf_write(&req_buf, "thin-pack");
1208 if (args->no_progress)
1209 packet_buf_write(&req_buf, "no-progress");
1210 if (args->include_tag)
1211 packet_buf_write(&req_buf, "include-tag");
1212 if (prefer_ofs_delta)
1213 packet_buf_write(&req_buf, "ofs-delta");
1214
f7e20501
BW
1215 /* Add shallow-info and deepen request */
1216 if (server_supports_feature("fetch", "shallow", 0))
1217 add_shallow_requests(&req_buf, args);
1218 else if (is_repository_shallow() || args->deepen)
1219 die(_("Server does not support shallow requests"));
1220
ba95710a
JT
1221 /* Add filter */
1222 if (server_supports_feature("fetch", "filter", 0) &&
1223 args->filter_options.choice) {
1224 print_verbose(args, _("Server supports filter"));
1225 packet_buf_write(&req_buf, "filter %s",
1226 args->filter_options.filter_spec);
1227 } else if (args->filter_options.choice) {
1228 warning("filtering not recognized by server, ignoring");
1229 }
1230
685fbd32
BW
1231 /* add wants */
1232 add_wants(wants, &req_buf);
1233
ba95710a
JT
1234 if (args->no_dependents) {
1235 packet_buf_write(&req_buf, "done");
1236 ret = 1;
1237 } else {
1238 /* Add all of the common commits we've found in previous rounds */
1239 add_common(&req_buf, common);
685fbd32 1240
ba95710a
JT
1241 /* Add initial haves */
1242 ret = add_haves(&req_buf, haves_to_send, in_vain);
1243 }
685fbd32
BW
1244
1245 /* Send request */
1246 packet_buf_flush(&req_buf);
1247 write_or_die(fd_out, req_buf.buf, req_buf.len);
1248
1249 strbuf_release(&req_buf);
1250 return ret;
1251}
1252
1253/*
1254 * Processes a section header in a server's response and checks if it matches
1255 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1256 * not consumed); if 0, the line will be consumed and the function will die if
1257 * the section header doesn't match what was expected.
1258 */
1259static int process_section_header(struct packet_reader *reader,
1260 const char *section, int peek)
1261{
1262 int ret;
1263
1264 if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
f7e20501 1265 die("error reading section header '%s'", section);
685fbd32
BW
1266
1267 ret = !strcmp(reader->line, section);
1268
1269 if (!peek) {
1270 if (!ret)
1271 die("expected '%s', received '%s'",
1272 section, reader->line);
1273 packet_reader_read(reader);
1274 }
1275
1276 return ret;
1277}
1278
1279static int process_acks(struct packet_reader *reader, struct oidset *common)
1280{
1281 /* received */
1282 int received_ready = 0;
1283 int received_ack = 0;
1284
1285 process_section_header(reader, "acknowledgments", 0);
1286 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1287 const char *arg;
1288
1289 if (!strcmp(reader->line, "NAK"))
1290 continue;
1291
1292 if (skip_prefix(reader->line, "ACK ", &arg)) {
1293 struct object_id oid;
1294 if (!get_oid_hex(arg, &oid)) {
1295 struct commit *commit;
1296 oidset_insert(common, &oid);
1297 commit = lookup_commit(&oid);
1298 mark_common(commit, 0, 1);
1299 }
1300 continue;
1301 }
1302
1303 if (!strcmp(reader->line, "ready")) {
685fbd32
BW
1304 received_ready = 1;
1305 continue;
1306 }
1307
1308 die("unexpected acknowledgment line: '%s'", reader->line);
1309 }
1310
1311 if (reader->status != PACKET_READ_FLUSH &&
1312 reader->status != PACKET_READ_DELIM)
1313 die("error processing acks: %d", reader->status);
1314
1315 /* return 0 if no common, 1 if there are common, or 2 if ready */
1316 return received_ready ? 2 : (received_ack ? 1 : 0);
1317}
1318
f7e20501
BW
1319static void receive_shallow_info(struct fetch_pack_args *args,
1320 struct packet_reader *reader)
1321{
1322 process_section_header(reader, "shallow-info", 0);
1323 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1324 const char *arg;
1325 struct object_id oid;
1326
1327 if (skip_prefix(reader->line, "shallow ", &arg)) {
1328 if (get_oid_hex(arg, &oid))
1329 die(_("invalid shallow line: %s"), reader->line);
1330 register_shallow(&oid);
1331 continue;
1332 }
1333 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1334 if (get_oid_hex(arg, &oid))
1335 die(_("invalid unshallow line: %s"), reader->line);
1336 if (!lookup_object(oid.hash))
1337 die(_("object not found: %s"), reader->line);
1338 /* make sure that it is parsed as shallow */
1339 if (!parse_object(&oid))
1340 die(_("error in object: %s"), reader->line);
1341 if (unregister_shallow(&oid))
1342 die(_("no shallow found: %s"), reader->line);
1343 continue;
1344 }
1345 die(_("expected shallow/unshallow, got %s"), reader->line);
1346 }
1347
1348 if (reader->status != PACKET_READ_FLUSH &&
1349 reader->status != PACKET_READ_DELIM)
1350 die("error processing shallow info: %d", reader->status);
1351
1352 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1353 args->deepen = 1;
1354}
1355
685fbd32
BW
1356enum fetch_state {
1357 FETCH_CHECK_LOCAL = 0,
1358 FETCH_SEND_REQUEST,
1359 FETCH_PROCESS_ACKS,
1360 FETCH_GET_PACK,
1361 FETCH_DONE,
1362};
1363
1364static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1365 int fd[2],
1366 const struct ref *orig_ref,
1367 struct ref **sought, int nr_sought,
1368 char **pack_lockfile)
1369{
1370 struct ref *ref = copy_ref_list(orig_ref);
1371 enum fetch_state state = FETCH_CHECK_LOCAL;
1372 struct oidset common = OIDSET_INIT;
1373 struct packet_reader reader;
1374 int in_vain = 0;
1375 int haves_to_send = INITIAL_FLUSH;
1376 packet_reader_init(&reader, fd[0], NULL, 0,
1377 PACKET_READ_CHOMP_NEWLINE);
1378
1379 while (state != FETCH_DONE) {
1380 switch (state) {
1381 case FETCH_CHECK_LOCAL:
1382 sort_ref_list(&ref, ref_compare_name);
1383 QSORT(sought, nr_sought, cmp_ref_by_name);
1384
1385 /* v2 supports these by default */
1386 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1387 use_sideband = 2;
f7e20501
BW
1388 if (args->depth > 0 || args->deepen_since || args->deepen_not)
1389 args->deepen = 1;
685fbd32
BW
1390
1391 if (marked)
1392 for_each_ref(clear_marks, NULL);
1393 marked = 1;
1394
685fbd32 1395 /* Filter 'ref' by 'sought' and those that aren't local */
34c29034
JT
1396 mark_complete_and_common_ref(args, &ref);
1397 filter_refs(args, &ref, sought, nr_sought);
1398 if (everything_local(args, &ref))
685fbd32
BW
1399 state = FETCH_DONE;
1400 else
1401 state = FETCH_SEND_REQUEST;
af1c90d1
JT
1402
1403 for_each_ref(rev_list_insert_ref_oid, NULL);
1404 for_each_cached_alternate(insert_one_alternate_object);
685fbd32
BW
1405 break;
1406 case FETCH_SEND_REQUEST:
1407 if (send_fetch_request(fd[1], args, ref, &common,
1408 &haves_to_send, &in_vain))
1409 state = FETCH_GET_PACK;
1410 else
1411 state = FETCH_PROCESS_ACKS;
1412 break;
1413 case FETCH_PROCESS_ACKS:
1414 /* Process ACKs/NAKs */
1415 switch (process_acks(&reader, &common)) {
1416 case 2:
1417 state = FETCH_GET_PACK;
1418 break;
1419 case 1:
1420 in_vain = 0;
1421 /* fallthrough */
1422 default:
1423 state = FETCH_SEND_REQUEST;
1424 break;
1425 }
1426 break;
1427 case FETCH_GET_PACK:
f7e20501
BW
1428 /* Check for shallow-info section */
1429 if (process_section_header(&reader, "shallow-info", 1))
1430 receive_shallow_info(args, &reader);
1431
685fbd32
BW
1432 /* get the pack */
1433 process_section_header(&reader, "packfile", 0);
1434 if (get_pack(args, fd, pack_lockfile))
1435 die(_("git fetch-pack: fetch failed."));
1436
1437 state = FETCH_DONE;
1438 break;
1439 case FETCH_DONE:
1440 continue;
1441 }
1442 }
1443
21bcf6e4 1444 clear_prio_queue(&rev_list);
685fbd32
BW
1445 oidset_clear(&common);
1446 return ref;
1447}
1448
f44af51d 1449static void fetch_pack_config(void)
745f7a8c 1450{
f44af51d
TA
1451 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1452 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1453 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1454 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1455 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 1456
f44af51d 1457 git_config(git_default_config, NULL);
745f7a8c
NTND
1458}
1459
745f7a8c
NTND
1460static void fetch_pack_setup(void)
1461{
1462 static int did_setup;
1463 if (did_setup)
1464 return;
f44af51d 1465 fetch_pack_config();
745f7a8c
NTND
1466 if (0 <= transfer_unpack_limit)
1467 unpack_limit = transfer_unpack_limit;
1468 else if (0 <= fetch_unpack_limit)
1469 unpack_limit = fetch_unpack_limit;
1470 did_setup = 1;
1471}
1472
f2db854d
JH
1473static int remove_duplicates_in_refs(struct ref **ref, int nr)
1474{
1475 struct string_list names = STRING_LIST_INIT_NODUP;
1476 int src, dst;
1477
1478 for (src = dst = 0; src < nr; src++) {
1479 struct string_list_item *item;
1480 item = string_list_insert(&names, ref[src]->name);
1481 if (item->util)
1482 continue; /* already have it */
1483 item->util = ref[src];
1484 if (src != dst)
1485 ref[dst] = ref[src];
1486 dst++;
1487 }
1488 for (src = dst; src < nr; src++)
1489 ref[src] = NULL;
1490 string_list_clear(&names, 0);
1491 return dst;
1492}
1493
beea4152 1494static void update_shallow(struct fetch_pack_args *args,
4820a33b 1495 struct ref **sought, int nr_sought,
beea4152 1496 struct shallow_info *si)
a796ccee 1497{
910650d2 1498 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1499 int *status;
beea4152
NTND
1500 int i;
1501
79891cb9 1502 if (args->deepen && alternate_shallow_file) {
a796ccee 1503 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 1504 unlink_or_warn(git_path_shallow());
a796ccee
NTND
1505 rollback_lock_file(&shallow_lock);
1506 } else
1507 commit_lock_file(&shallow_lock);
1508 return;
1509 }
beea4152
NTND
1510
1511 if (!si->shallow || !si->shallow->nr)
1512 return;
1513
beea4152
NTND
1514 if (args->cloning) {
1515 /*
1516 * remote is shallow, but this is a clone, there are
1517 * no objects in repo to worry about. Accept any
1518 * shallow points that exist in the pack (iow in repo
1519 * after get_pack() and reprepare_packed_git())
1520 */
910650d2 1521 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1522 struct object_id *oid = si->shallow->oid;
beea4152 1523 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1524 if (has_object_file(&oid[i]))
910650d2 1525 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1526 if (extra.nr) {
1527 setup_alternate_shallow(&shallow_lock,
1528 &alternate_shallow_file,
1529 &extra);
1530 commit_lock_file(&shallow_lock);
1531 }
910650d2 1532 oid_array_clear(&extra);
beea4152
NTND
1533 return;
1534 }
4820a33b
NTND
1535
1536 if (!si->nr_ours && !si->nr_theirs)
1537 return;
1538
1539 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1540 if (!si->nr_ours && !si->nr_theirs)
1541 return;
1542 for (i = 0; i < nr_sought; i++)
910650d2 1543 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1544 si->ref = &ref;
1545
48d25cae
NTND
1546 if (args->update_shallow) {
1547 /*
1548 * remote is also shallow, .git/shallow may be updated
1549 * so all refs can be accepted. Make sure we only add
1550 * shallow roots that are actually reachable from new
1551 * refs.
1552 */
910650d2 1553 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1554 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1555 assign_shallow_commits_to_refs(si, NULL, NULL);
1556 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1557 oid_array_clear(&ref);
48d25cae
NTND
1558 return;
1559 }
1560 for (i = 0; i < si->nr_ours; i++)
910650d2 1561 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1562 for (i = 0; i < si->nr_theirs; i++)
910650d2 1563 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1564 setup_alternate_shallow(&shallow_lock,
1565 &alternate_shallow_file,
1566 &extra);
1567 commit_lock_file(&shallow_lock);
910650d2 1568 oid_array_clear(&extra);
1569 oid_array_clear(&ref);
48d25cae
NTND
1570 return;
1571 }
1572
4820a33b
NTND
1573 /*
1574 * remote is also shallow, check what ref is safe to update
1575 * without updating .git/shallow
1576 */
1577 status = xcalloc(nr_sought, sizeof(*status));
1578 assign_shallow_commits_to_refs(si, NULL, status);
1579 if (si->nr_ours || si->nr_theirs) {
1580 for (i = 0; i < nr_sought; i++)
1581 if (status[i])
1582 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1583 }
1584 free(status);
910650d2 1585 oid_array_clear(&ref);
a796ccee
NTND
1586}
1587
745f7a8c
NTND
1588struct ref *fetch_pack(struct fetch_pack_args *args,
1589 int fd[], struct child_process *conn,
1590 const struct ref *ref,
1591 const char *dest,
f2db854d 1592 struct ref **sought, int nr_sought,
910650d2 1593 struct oid_array *shallow,
685fbd32
BW
1594 char **pack_lockfile,
1595 enum protocol_version version)
745f7a8c 1596{
745f7a8c 1597 struct ref *ref_cpy;
beea4152 1598 struct shallow_info si;
745f7a8c
NTND
1599
1600 fetch_pack_setup();
f2db854d
JH
1601 if (nr_sought)
1602 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1603
1604 if (!ref) {
1605 packet_flush(fd[1]);
1dd73e20 1606 die(_("no matching remote head"));
745f7a8c 1607 }
beea4152 1608 prepare_shallow_info(&si, shallow);
685fbd32
BW
1609 if (version == protocol_v2)
1610 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1611 pack_lockfile);
1612 else
1613 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1614 &si, pack_lockfile);
a49d2834 1615 reprepare_packed_git(the_repository);
4820a33b 1616 update_shallow(args, sought, nr_sought, &si);
beea4152 1617 clear_shallow_info(&si);
745f7a8c
NTND
1618 return ref_cpy;
1619}
e860d96b
MM
1620
1621int report_unmatched_refs(struct ref **sought, int nr_sought)
1622{
1623 int i, ret = 0;
1624
1625 for (i = 0; i < nr_sought; i++) {
d56583de 1626 if (!sought[i])
e860d96b 1627 continue;
d56583de
MM
1628 switch (sought[i]->match_status) {
1629 case REF_MATCHED:
1630 continue;
1631 case REF_NOT_MATCHED:
1632 error(_("no such remote ref %s"), sought[i]->name);
1633 break;
1634 case REF_UNADVERTISED_NOT_ALLOWED:
1635 error(_("Server does not allow request for unadvertised object %s"),
1636 sought[i]->name);
1637 break;
1638 }
e860d96b
MM
1639 ret = 1;
1640 }
1641 return ret;
1642}