upload-pack: introduce fetch server command
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
b2141fc1 2#include "config.h"
697cc8ef 3#include "lockfile.h"
745f7a8c
NTND
4#include "refs.h"
5#include "pkt-line.h"
6#include "commit.h"
7#include "tag.h"
8#include "exec_cmd.h"
9#include "pack.h"
10#include "sideband.h"
11#include "fetch-pack.h"
12#include "remote.h"
13#include "run-command.h"
47a59185 14#include "connect.h"
745f7a8c
NTND
15#include "transport.h"
16#include "version.h"
099327b5 17#include "prio-queue.h"
beea4152 18#include "sha1-array.h"
fdb69d33 19#include "oidset.h"
0abe14f6 20#include "packfile.h"
745f7a8c
NTND
21
22static int transfer_unpack_limit = -1;
23static int fetch_unpack_limit = -1;
24static int unpack_limit = 100;
25static int prefer_ofs_delta = 1;
26static int no_done;
508ea882 27static int deepen_since_ok;
a45a2600 28static int deepen_not_ok;
745f7a8c
NTND
29static int fetch_fsck_objects = -1;
30static int transfer_fsck_objects = -1;
31static int agent_supported;
6035d6aa
NTND
32static struct lock_file shallow_lock;
33static const char *alternate_shallow_file;
745f7a8c 34
208acbfb 35/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
36#define COMPLETE (1U << 0)
37#define COMMON (1U << 1)
38#define COMMON_REF (1U << 2)
39#define SEEN (1U << 3)
40#define POPPED (1U << 4)
41a078c6 41#define ALTERNATE (1U << 5)
745f7a8c
NTND
42
43static int marked;
44
45/*
46 * After sending this many "have"s if we do not get any new ACK , we
47 * give up traversing our history.
48 */
49#define MAX_IN_VAIN 256
50
099327b5 51static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
52static int non_common_revs, multi_ack, use_sideband;
53/* Allow specifying sha1 if it is a ref tip. */
54#define ALLOW_TIP_SHA1 01
68ee6289
FM
55/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
56#define ALLOW_REACHABLE_SHA1 02
7199c093 57static unsigned int allow_unadvertised_object_request;
745f7a8c 58
0d789a5b
NTND
59__attribute__((format (printf, 2, 3)))
60static inline void print_verbose(const struct fetch_pack_args *args,
61 const char *fmt, ...)
62{
63 va_list params;
64
65 if (!args->verbose)
66 return;
67
68 va_start(params, fmt);
69 vfprintf(stderr, fmt, params);
70 va_end(params);
71 fputc('\n', stderr);
72}
73
41a078c6
JK
74struct alternate_object_cache {
75 struct object **items;
76 size_t nr, alloc;
77};
78
79static void cache_one_alternate(const char *refname,
80 const struct object_id *oid,
81 void *vcache)
82{
83 struct alternate_object_cache *cache = vcache;
c251c83d 84 struct object *obj = parse_object(oid);
41a078c6
JK
85
86 if (!obj || (obj->flags & ALTERNATE))
87 return;
88
89 obj->flags |= ALTERNATE;
90 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
91 cache->items[cache->nr++] = obj;
92}
93
94static void for_each_cached_alternate(void (*cb)(struct object *))
95{
96 static int initialized;
97 static struct alternate_object_cache cache;
98 size_t i;
99
100 if (!initialized) {
101 for_each_alternate_ref(cache_one_alternate, &cache);
102 initialized = 1;
103 }
104
105 for (i = 0; i < cache.nr; i++)
106 cb(cache.items[i]);
107}
108
745f7a8c
NTND
109static void rev_list_push(struct commit *commit, int mark)
110{
111 if (!(commit->object.flags & mark)) {
112 commit->object.flags |= mark;
113
0064053b
JK
114 if (parse_commit(commit))
115 return;
745f7a8c 116
099327b5 117 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
118
119 if (!(commit->object.flags & COMMON))
120 non_common_revs++;
121 }
122}
123
1b283377 124static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
745f7a8c 125{
c251c83d 126 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
127
128 if (o && o->type == OBJ_COMMIT)
129 rev_list_push((struct commit *)o, SEEN);
130
131 return 0;
132}
133
b1b49c6e
MH
134static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
135 int flag, void *cb_data)
745f7a8c 136{
1b283377 137 return rev_list_insert_ref(refname, oid);
b1b49c6e
MH
138}
139
c50fb6ce
MH
140static int clear_marks(const char *refname, const struct object_id *oid,
141 int flag, void *cb_data)
745f7a8c 142{
c251c83d 143 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
144
145 if (o && o->type == OBJ_COMMIT)
146 clear_commit_marks((struct commit *)o,
147 COMMON | COMMON_REF | SEEN | POPPED);
148 return 0;
149}
150
151/*
152 This function marks a rev and its ancestors as common.
153 In some cases, it is desirable to mark only the ancestors (for example
154 when only the server does not yet know that they are common).
155*/
156
157static void mark_common(struct commit *commit,
158 int ancestors_only, int dont_parse)
159{
160 if (commit != NULL && !(commit->object.flags & COMMON)) {
161 struct object *o = (struct object *)commit;
162
163 if (!ancestors_only)
164 o->flags |= COMMON;
165
166 if (!(o->flags & SEEN))
167 rev_list_push(commit, SEEN);
168 else {
169 struct commit_list *parents;
170
171 if (!ancestors_only && !(o->flags & POPPED))
172 non_common_revs--;
173 if (!o->parsed && !dont_parse)
174 if (parse_commit(commit))
175 return;
176
177 for (parents = commit->parents;
178 parents;
179 parents = parents->next)
180 mark_common(parents->item, 0, dont_parse);
181 }
182 }
183}
184
185/*
186 Get the next rev to send, ignoring the common.
187*/
188
1b283377 189static const struct object_id *get_rev(void)
745f7a8c
NTND
190{
191 struct commit *commit = NULL;
192
193 while (commit == NULL) {
194 unsigned int mark;
195 struct commit_list *parents;
196
099327b5 197 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
198 return NULL;
199
099327b5 200 commit = prio_queue_get(&rev_list);
0064053b 201 parse_commit(commit);
745f7a8c
NTND
202 parents = commit->parents;
203
204 commit->object.flags |= POPPED;
205 if (!(commit->object.flags & COMMON))
206 non_common_revs--;
207
208 if (commit->object.flags & COMMON) {
209 /* do not send "have", and ignore ancestors */
210 commit = NULL;
211 mark = COMMON | SEEN;
212 } else if (commit->object.flags & COMMON_REF)
213 /* send "have", and ignore ancestors */
214 mark = COMMON | SEEN;
215 else
216 /* send "have", also for its ancestors */
217 mark = SEEN;
218
219 while (parents) {
220 if (!(parents->item->object.flags & SEEN))
221 rev_list_push(parents->item, mark);
222 if (mark & COMMON)
223 mark_common(parents->item, 1, 0);
224 parents = parents->next;
225 }
745f7a8c
NTND
226 }
227
1b283377 228 return &commit->object.oid;
745f7a8c
NTND
229}
230
231enum ack_type {
232 NAK = 0,
233 ACK,
234 ACK_continue,
235 ACK_common,
236 ACK_ready
237};
238
239static void consume_shallow_list(struct fetch_pack_args *args, int fd)
240{
79891cb9 241 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
242 /* If we sent a depth we will get back "duplicate"
243 * shallow and unshallow commands every time there
244 * is a block of have lines exchanged.
245 */
74543a04
JK
246 char *line;
247 while ((line = packet_read_line(fd, NULL))) {
59556548 248 if (starts_with(line, "shallow "))
745f7a8c 249 continue;
59556548 250 if (starts_with(line, "unshallow "))
745f7a8c 251 continue;
1dd73e20 252 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
253 }
254 }
255}
256
1b283377 257static enum ack_type get_ack(int fd, struct object_id *result_oid)
745f7a8c 258{
74543a04
JK
259 int len;
260 char *line = packet_read_line(fd, &len);
82e56767 261 const char *arg;
745f7a8c
NTND
262
263 if (!len)
1dd73e20 264 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
265 if (!strcmp(line, "NAK"))
266 return NAK;
82e56767 267 if (skip_prefix(line, "ACK ", &arg)) {
1b283377 268 if (!get_oid_hex(arg, result_oid)) {
82e56767
JK
269 arg += 40;
270 len -= arg - line;
271 if (len < 1)
030e9dd6 272 return ACK;
82e56767 273 if (strstr(arg, "continue"))
745f7a8c 274 return ACK_continue;
82e56767 275 if (strstr(arg, "common"))
745f7a8c 276 return ACK_common;
82e56767 277 if (strstr(arg, "ready"))
745f7a8c
NTND
278 return ACK_ready;
279 return ACK;
280 }
281 }
8e2c7bef
JT
282 if (skip_prefix(line, "ERR ", &arg))
283 die(_("remote error: %s"), arg);
dfbfb9f3 284 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
285}
286
287static void send_request(struct fetch_pack_args *args,
288 int fd, struct strbuf *buf)
289{
290 if (args->stateless_rpc) {
291 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
292 packet_flush(fd);
293 } else
cdf4fb8e 294 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
295}
296
41a078c6 297static void insert_one_alternate_object(struct object *obj)
745f7a8c 298{
1b283377 299 rev_list_insert_ref(NULL, &obj->oid);
745f7a8c
NTND
300}
301
302#define INITIAL_FLUSH 16
303#define PIPESAFE_FLUSH 32
da470981 304#define LARGE_FLUSH 16384
745f7a8c
NTND
305
306static int next_flush(struct fetch_pack_args *args, int count)
307{
da470981
JT
308 if (args->stateless_rpc) {
309 if (count < LARGE_FLUSH)
310 count <<= 1;
311 else
312 count = count * 11 / 10;
313 } else {
314 if (count < PIPESAFE_FLUSH)
315 count <<= 1;
316 else
317 count += PIPESAFE_FLUSH;
318 }
745f7a8c
NTND
319 return count;
320}
321
322static int find_common(struct fetch_pack_args *args,
1b283377 323 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
324 struct ref *refs)
325{
326 int fetching;
327 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 328 const struct object_id *oid;
745f7a8c
NTND
329 unsigned in_vain = 0;
330 int got_continue = 0;
331 int got_ready = 0;
332 struct strbuf req_buf = STRBUF_INIT;
333 size_t state_len = 0;
334
335 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 336 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
337 if (marked)
338 for_each_ref(clear_marks, NULL);
339 marked = 1;
340
b1b49c6e 341 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 342 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
343
344 fetching = 0;
345 for ( ; refs ; refs = refs->next) {
1b283377 346 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
347 const char *remote_hex;
348 struct object *o;
349
350 /*
351 * If that object is complete (i.e. it is an ancestor of a
352 * local ref), we tell them we have it but do not have to
353 * tell them about its ancestors, which they already know
354 * about.
355 *
356 * We use lookup_object here because we are only
357 * interested in the case we *know* the object is
358 * reachable and we have already scanned it.
359 */
1b283377 360 if (((o = lookup_object(remote->hash)) != NULL) &&
745f7a8c
NTND
361 (o->flags & COMPLETE)) {
362 continue;
363 }
364
1b283377 365 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
366 if (!fetching) {
367 struct strbuf c = STRBUF_INIT;
368 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
369 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
370 if (no_done) strbuf_addstr(&c, " no-done");
371 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
372 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 373 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
374 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
375 if (args->no_progress) strbuf_addstr(&c, " no-progress");
376 if (args->include_tag) strbuf_addstr(&c, " include-tag");
377 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 378 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 379 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
380 if (agent_supported) strbuf_addf(&c, " agent=%s",
381 git_user_agent_sanitized());
382 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
383 strbuf_release(&c);
384 } else
385 packet_buf_write(&req_buf, "want %s\n", remote_hex);
386 fetching++;
387 }
388
389 if (!fetching) {
390 strbuf_release(&req_buf);
391 packet_flush(fd[1]);
392 return 1;
393 }
394
395 if (is_repository_shallow())
1a30f5a2 396 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
397 if (args->depth > 0)
398 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882 399 if (args->deepen_since) {
dddbad72 400 timestamp_t max_age = approxidate(args->deepen_since);
cb71f8bd 401 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
508ea882 402 }
a45a2600
NTND
403 if (args->deepen_not) {
404 int i;
405 for (i = 0; i < args->deepen_not->nr; i++) {
406 struct string_list_item *s = args->deepen_not->items + i;
407 packet_buf_write(&req_buf, "deepen-not %s", s->string);
408 }
409 }
745f7a8c
NTND
410 packet_buf_flush(&req_buf);
411 state_len = req_buf.len;
412
79891cb9 413 if (args->deepen) {
74543a04 414 char *line;
ae021d87 415 const char *arg;
1b283377 416 struct object_id oid;
745f7a8c
NTND
417
418 send_request(args, fd[1], &req_buf);
74543a04 419 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87 420 if (skip_prefix(line, "shallow ", &arg)) {
1b283377 421 if (get_oid_hex(arg, &oid))
1dd73e20 422 die(_("invalid shallow line: %s"), line);
e92b848c 423 register_shallow(&oid);
745f7a8c
NTND
424 continue;
425 }
ae021d87 426 if (skip_prefix(line, "unshallow ", &arg)) {
1b283377 427 if (get_oid_hex(arg, &oid))
1dd73e20 428 die(_("invalid unshallow line: %s"), line);
1b283377 429 if (!lookup_object(oid.hash))
1dd73e20 430 die(_("object not found: %s"), line);
745f7a8c 431 /* make sure that it is parsed as shallow */
c251c83d 432 if (!parse_object(&oid))
1dd73e20 433 die(_("error in object: %s"), line);
e92b848c 434 if (unregister_shallow(&oid))
1dd73e20 435 die(_("no shallow found: %s"), line);
745f7a8c
NTND
436 continue;
437 }
1dd73e20 438 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
439 }
440 } else if (!args->stateless_rpc)
441 send_request(args, fd[1], &req_buf);
442
443 if (!args->stateless_rpc) {
444 /* If we aren't using the stateless-rpc interface
445 * we don't need to retain the headers.
446 */
447 strbuf_setlen(&req_buf, 0);
448 state_len = 0;
449 }
450
451 flushes = 0;
452 retval = -1;
1b283377 453 while ((oid = get_rev())) {
454 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
455 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
456 in_vain++;
457 if (flush_at <= ++count) {
458 int ack;
459
460 packet_buf_flush(&req_buf);
461 send_request(args, fd[1], &req_buf);
462 strbuf_setlen(&req_buf, state_len);
463 flushes++;
464 flush_at = next_flush(args, count);
465
466 /*
467 * We keep one window "ahead" of the other side, and
468 * will wait for an ACK only on the next one
469 */
470 if (!args->stateless_rpc && count == INITIAL_FLUSH)
471 continue;
472
473 consume_shallow_list(args, fd[0]);
474 do {
1b283377 475 ack = get_ack(fd[0], result_oid);
0d789a5b 476 if (ack)
1dd73e20 477 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 478 ack, oid_to_hex(result_oid));
745f7a8c
NTND
479 switch (ack) {
480 case ACK:
481 flushes = 0;
482 multi_ack = 0;
483 retval = 0;
484 goto done;
485 case ACK_common:
486 case ACK_ready:
487 case ACK_continue: {
488 struct commit *commit =
bc83266a 489 lookup_commit(result_oid);
745f7a8c 490 if (!commit)
1b283377 491 die(_("invalid commit %s"), oid_to_hex(result_oid));
745f7a8c
NTND
492 if (args->stateless_rpc
493 && ack == ACK_common
494 && !(commit->object.flags & COMMON)) {
495 /* We need to replay the have for this object
496 * on the next RPC request so the peer knows
497 * it is in common with us.
498 */
1b283377 499 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
500 packet_buf_write(&req_buf, "have %s\n", hex);
501 state_len = req_buf.len;
06b3d386
JT
502 /*
503 * Reset in_vain because an ack
504 * for this commit has not been
505 * seen.
506 */
507 in_vain = 0;
508 } else if (!args->stateless_rpc
509 || ack != ACK_common)
510 in_vain = 0;
745f7a8c
NTND
511 mark_common(commit, 0, 1);
512 retval = 0;
745f7a8c
NTND
513 got_continue = 1;
514 if (ack == ACK_ready) {
099327b5 515 clear_prio_queue(&rev_list);
745f7a8c
NTND
516 got_ready = 1;
517 }
518 break;
519 }
520 }
521 } while (ack);
522 flushes--;
523 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 524 print_verbose(args, _("giving up"));
745f7a8c
NTND
525 break; /* give up */
526 }
527 }
528 }
529done:
530 if (!got_ready || !no_done) {
531 packet_buf_write(&req_buf, "done\n");
532 send_request(args, fd[1], &req_buf);
533 }
1dd73e20 534 print_verbose(args, _("done"));
745f7a8c
NTND
535 if (retval != 0) {
536 multi_ack = 0;
537 flushes++;
538 }
539 strbuf_release(&req_buf);
540
ff62eca7
NTND
541 if (!got_ready || !no_done)
542 consume_shallow_list(args, fd[0]);
745f7a8c 543 while (flushes || multi_ack) {
1b283377 544 int ack = get_ack(fd[0], result_oid);
745f7a8c 545 if (ack) {
1dd73e20 546 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 547 ack, oid_to_hex(result_oid));
745f7a8c
NTND
548 if (ack == ACK)
549 return 0;
550 multi_ack = 1;
551 continue;
552 }
553 flushes--;
554 }
555 /* it is no error to fetch into a completely empty repo */
556 return count ? retval : 0;
557}
558
559static struct commit_list *complete;
560
1b283377 561static int mark_complete(const struct object_id *oid)
745f7a8c 562{
c251c83d 563 struct object *o = parse_object(oid);
745f7a8c
NTND
564
565 while (o && o->type == OBJ_TAG) {
566 struct tag *t = (struct tag *) o;
567 if (!t->tagged)
568 break; /* broken repository */
569 o->flags |= COMPLETE;
c251c83d 570 o = parse_object(&t->tagged->oid);
745f7a8c
NTND
571 }
572 if (o && o->type == OBJ_COMMIT) {
573 struct commit *commit = (struct commit *)o;
574 if (!(commit->object.flags & COMPLETE)) {
575 commit->object.flags |= COMPLETE;
16445242 576 commit_list_insert(commit, &complete);
745f7a8c
NTND
577 }
578 }
579 return 0;
580}
581
f8ee4d85
MH
582static int mark_complete_oid(const char *refname, const struct object_id *oid,
583 int flag, void *cb_data)
584{
1b283377 585 return mark_complete(oid);
f8ee4d85
MH
586}
587
745f7a8c 588static void mark_recent_complete_commits(struct fetch_pack_args *args,
dddbad72 589 timestamp_t cutoff)
745f7a8c
NTND
590{
591 while (complete && cutoff <= complete->item->date) {
1dd73e20 592 print_verbose(args, _("Marking %s as complete"),
0d789a5b 593 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
594 pop_most_recent_commit(&complete, COMPLETE);
595 }
596}
597
fdb69d33
JT
598static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
599{
600 for (; refs; refs = refs->next)
601 oidset_insert(oids, &refs->old_oid);
602}
603
604static int tip_oids_contain(struct oidset *tip_oids,
605 struct ref *unmatched, struct ref *newlist,
606 const struct object_id *id)
607{
608 /*
609 * Note that this only looks at the ref lists the first time it's
610 * called. This works out in filter_refs() because even though it may
611 * add to "newlist" between calls, the additions will always be for
612 * oids that are already in the set.
613 */
9e6fabde 614 if (!tip_oids->map.map.tablesize) {
fdb69d33
JT
615 add_refs_to_oidset(tip_oids, unmatched);
616 add_refs_to_oidset(tip_oids, newlist);
617 }
618 return oidset_contains(tip_oids, id);
619}
620
745f7a8c 621static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
622 struct ref **refs,
623 struct ref **sought, int nr_sought)
745f7a8c
NTND
624{
625 struct ref *newlist = NULL;
626 struct ref **newtail = &newlist;
fdb69d33 627 struct ref *unmatched = NULL;
745f7a8c 628 struct ref *ref, *next;
fdb69d33 629 struct oidset tip_oids = OIDSET_INIT;
f2db854d 630 int i;
745f7a8c 631
f2db854d 632 i = 0;
745f7a8c
NTND
633 for (ref = *refs; ref; ref = next) {
634 int keep = 0;
635 next = ref->next;
f2db854d 636
50e19a83 637 if (starts_with(ref->name, "refs/") &&
4c224081 638 check_refname_format(ref->name, 0))
745f7a8c
NTND
639 ; /* trash */
640 else {
f2db854d
JH
641 while (i < nr_sought) {
642 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
643 if (cmp < 0)
644 break; /* definitely do not have it */
645 else if (cmp == 0) {
646 keep = 1; /* definitely have it */
d56583de 647 sought[i]->match_status = REF_MATCHED;
745f7a8c 648 }
f2db854d 649 i++;
745f7a8c
NTND
650 }
651 }
652
f2db854d 653 if (!keep && args->fetch_all &&
79891cb9 654 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
655 keep = 1;
656
657 if (keep) {
658 *newtail = ref;
659 ref->next = NULL;
660 newtail = &ref->next;
661 } else {
fdb69d33
JT
662 ref->next = unmatched;
663 unmatched = ref;
745f7a8c
NTND
664 }
665 }
666
6e7b66ee 667 /* Append unmatched requests to the list */
d56583de 668 for (i = 0; i < nr_sought; i++) {
1b283377 669 struct object_id oid;
670 const char *p;
b7916422 671
d56583de
MM
672 ref = sought[i];
673 if (ref->match_status != REF_NOT_MATCHED)
674 continue;
1b283377 675 if (parse_oid_hex(ref->name, &oid, &p) ||
676 *p != '\0' ||
677 oidcmp(&oid, &ref->old_oid))
d56583de 678 continue;
6e7b66ee 679
d56583de 680 if ((allow_unadvertised_object_request &
fdb69d33
JT
681 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
682 tip_oids_contain(&tip_oids, unmatched, newlist,
683 &ref->old_oid)) {
d56583de 684 ref->match_status = REF_MATCHED;
c3c17bf1
JK
685 *newtail = copy_ref(ref);
686 newtail = &(*newtail)->next;
d56583de
MM
687 } else {
688 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
689 }
690 }
fdb69d33
JT
691
692 oidset_clear(&tip_oids);
693 for (ref = unmatched; ref; ref = next) {
694 next = ref->next;
695 free(ref);
696 }
697
745f7a8c
NTND
698 *refs = newlist;
699}
700
41a078c6 701static void mark_alternate_complete(struct object *obj)
745f7a8c 702{
1b283377 703 mark_complete(&obj->oid);
745f7a8c
NTND
704}
705
706static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
707 struct ref **refs,
708 struct ref **sought, int nr_sought)
745f7a8c
NTND
709{
710 struct ref *ref;
711 int retval;
dddbad72 712 timestamp_t cutoff = 0;
745f7a8c
NTND
713
714 save_commit_buffer = 0;
715
716 for (ref = *refs; ref; ref = ref->next) {
717 struct object *o;
718
c291293b
JK
719 if (!has_object_file_with_flags(&ref->old_oid,
720 OBJECT_INFO_QUICK))
012a1bb5
JH
721 continue;
722
c251c83d 723 o = parse_object(&ref->old_oid);
745f7a8c
NTND
724 if (!o)
725 continue;
726
727 /* We already have it -- which may mean that we were
728 * in sync with the other side at some time after
729 * that (it is OK if we guess wrong here).
730 */
731 if (o->type == OBJ_COMMIT) {
732 struct commit *commit = (struct commit *)o;
733 if (!cutoff || cutoff < commit->date)
734 cutoff = commit->date;
735 }
736 }
737
79891cb9 738 if (!args->deepen) {
f8ee4d85 739 for_each_ref(mark_complete_oid, NULL);
41a078c6 740 for_each_cached_alternate(mark_alternate_complete);
16445242 741 commit_list_sort_by_date(&complete);
745f7a8c
NTND
742 if (cutoff)
743 mark_recent_complete_commits(args, cutoff);
744 }
745
746 /*
747 * Mark all complete remote refs as common refs.
748 * Don't mark them common yet; the server has to be told so first.
749 */
750 for (ref = *refs; ref; ref = ref->next) {
f4e54d02 751 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
745f7a8c
NTND
752 NULL, 0);
753
754 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
755 continue;
756
757 if (!(o->flags & SEEN)) {
758 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
759
760 mark_common((struct commit *)o, 1, 1);
761 }
762 }
763
f2db854d 764 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
765
766 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 767 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
768 struct object *o;
769
1b283377 770 o = lookup_object(remote->hash);
745f7a8c
NTND
771 if (!o || !(o->flags & COMPLETE)) {
772 retval = 0;
1b283377 773 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 774 ref->name);
745f7a8c
NTND
775 continue;
776 }
1b283377 777 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 778 ref->name);
745f7a8c
NTND
779 }
780 return retval;
781}
782
783static int sideband_demux(int in, int out, void *data)
784{
785 int *xd = data;
9ff18faf 786 int ret;
745f7a8c 787
9ff18faf 788 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
789 close(out);
790 return ret;
791}
792
793static int get_pack(struct fetch_pack_args *args,
794 int xd[2], char **pack_lockfile)
795{
796 struct async demux;
745f7a8c 797 int do_keep = args->keep_pack;
984a43b9
JK
798 const char *cmd_name;
799 struct pack_header header;
800 int pass_header = 0;
d3180279 801 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 802 int ret;
745f7a8c
NTND
803
804 memset(&demux, 0, sizeof(demux));
805 if (use_sideband) {
806 /* xd[] is talking with upload-pack; subprocess reads from
807 * xd[0], spits out band#2 to stderr, and feeds us band#1
808 * through demux->out.
809 */
810 demux.proc = sideband_demux;
811 demux.data = xd;
812 demux.out = -1;
df857572 813 demux.isolate_sigpipe = 1;
745f7a8c 814 if (start_async(&demux))
1dd73e20 815 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
816 }
817 else
818 demux.out = xd[0];
819
745f7a8c 820 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
821
822 if (read_pack_header(demux.out, &header))
1dd73e20 823 die(_("protocol error: bad pack header"));
984a43b9 824 pass_header = 1;
745f7a8c
NTND
825 if (ntohl(header.hdr_entries) < unpack_limit)
826 do_keep = 0;
827 else
828 do_keep = 1;
829 }
830
6035d6aa 831 if (alternate_shallow_file) {
984a43b9
JK
832 argv_array_push(&cmd.args, "--shallow-file");
833 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
834 }
835
745f7a8c
NTND
836 if (do_keep) {
837 if (pack_lockfile)
838 cmd.out = -1;
984a43b9
JK
839 cmd_name = "index-pack";
840 argv_array_push(&cmd.args, cmd_name);
841 argv_array_push(&cmd.args, "--stdin");
745f7a8c 842 if (!args->quiet && !args->no_progress)
984a43b9 843 argv_array_push(&cmd.args, "-v");
745f7a8c 844 if (args->use_thin_pack)
984a43b9 845 argv_array_push(&cmd.args, "--fix-thin");
745f7a8c 846 if (args->lock_pack || unpack_limit) {
da25bdb7 847 char hostname[HOST_NAME_MAX + 1];
5781a9a2 848 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
849 xsnprintf(hostname, sizeof(hostname), "localhost");
850 argv_array_pushf(&cmd.args,
851 "--keep=fetch-pack %"PRIuMAX " on %s",
852 (uintmax_t)getpid(), hostname);
745f7a8c 853 }
c6807a40 854 if (args->check_self_contained_and_connected)
984a43b9 855 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
745f7a8c
NTND
856 }
857 else {
984a43b9
JK
858 cmd_name = "unpack-objects";
859 argv_array_push(&cmd.args, cmd_name);
745f7a8c 860 if (args->quiet || args->no_progress)
984a43b9 861 argv_array_push(&cmd.args, "-q");
c6807a40 862 args->check_self_contained_and_connected = 0;
745f7a8c 863 }
984a43b9
JK
864
865 if (pass_header)
866 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
867 ntohl(header.hdr_version),
868 ntohl(header.hdr_entries));
745f7a8c
NTND
869 if (fetch_fsck_objects >= 0
870 ? fetch_fsck_objects
871 : transfer_fsck_objects >= 0
872 ? transfer_fsck_objects
873 : 0)
984a43b9 874 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
875
876 cmd.in = demux.out;
877 cmd.git_cmd = 1;
878 if (start_command(&cmd))
1dd73e20 879 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
880 if (do_keep && pack_lockfile) {
881 *pack_lockfile = index_pack_lockfile(cmd.out);
882 close(cmd.out);
883 }
884
37cb1dd6
JL
885 if (!use_sideband)
886 /* Closed by start_command() */
887 xd[0] = -1;
888
c6807a40
NTND
889 ret = finish_command(&cmd);
890 if (!ret || (args->check_self_contained_and_connected && ret == 1))
891 args->self_contained_and_connected =
892 args->check_self_contained_and_connected &&
893 ret == 0;
894 else
1dd73e20 895 die(_("%s failed"), cmd_name);
745f7a8c 896 if (use_sideband && finish_async(&demux))
1dd73e20 897 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
898 return 0;
899}
900
f2db854d
JH
901static int cmp_ref_by_name(const void *a_, const void *b_)
902{
903 const struct ref *a = *((const struct ref **)a_);
904 const struct ref *b = *((const struct ref **)b_);
905 return strcmp(a->name, b->name);
906}
907
745f7a8c
NTND
908static struct ref *do_fetch_pack(struct fetch_pack_args *args,
909 int fd[2],
910 const struct ref *orig_ref,
f2db854d 911 struct ref **sought, int nr_sought,
beea4152 912 struct shallow_info *si,
745f7a8c
NTND
913 char **pack_lockfile)
914{
915 struct ref *ref = copy_ref_list(orig_ref);
1b283377 916 struct object_id oid;
745f7a8c
NTND
917 const char *agent_feature;
918 int agent_len;
919
920 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 921 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 922
eb86a507 923 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 924 die(_("Server does not support shallow clients"));
a45a2600 925 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 926 args->deepen = 1;
745f7a8c 927 if (server_supports("multi_ack_detailed")) {
1dd73e20 928 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
929 multi_ack = 2;
930 if (server_supports("no-done")) {
1dd73e20 931 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
932 if (args->stateless_rpc)
933 no_done = 1;
934 }
935 }
936 else if (server_supports("multi_ack")) {
1dd73e20 937 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
938 multi_ack = 1;
939 }
940 if (server_supports("side-band-64k")) {
1dd73e20 941 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
942 use_sideband = 2;
943 }
944 else if (server_supports("side-band")) {
1dd73e20 945 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
946 use_sideband = 1;
947 }
6e7b66ee 948 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 949 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 950 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 951 }
68ee6289 952 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 953 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
954 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
955 }
745f7a8c
NTND
956 if (!server_supports("thin-pack"))
957 args->use_thin_pack = 0;
958 if (!server_supports("no-progress"))
959 args->no_progress = 0;
960 if (!server_supports("include-tag"))
961 args->include_tag = 0;
0d789a5b 962 if (server_supports("ofs-delta"))
1dd73e20 963 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 964 else
745f7a8c
NTND
965 prefer_ofs_delta = 0;
966
967 if ((agent_feature = server_feature_value("agent", &agent_len))) {
968 agent_supported = 1;
0d789a5b 969 if (agent_len)
1dd73e20 970 print_verbose(args, _("Server version is %.*s"),
0d789a5b 971 agent_len, agent_feature);
745f7a8c 972 }
508ea882
NTND
973 if (server_supports("deepen-since"))
974 deepen_since_ok = 1;
975 else if (args->deepen_since)
976 die(_("Server does not support --shallow-since"));
a45a2600
NTND
977 if (server_supports("deepen-not"))
978 deepen_not_ok = 1;
979 else if (args->deepen_not)
980 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
981 if (!server_supports("deepen-relative") && args->deepen_relative)
982 die(_("Server does not support --deepen"));
745f7a8c 983
f2db854d 984 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
985 packet_flush(fd[1]);
986 goto all_done;
987 }
1b283377 988 if (find_common(args, fd, &oid, ref) < 0)
745f7a8c
NTND
989 if (!args->keep_pack)
990 /* When cloning, it is not unusual to have
991 * no common commit.
992 */
1dd73e20 993 warning(_("no common commits"));
745f7a8c
NTND
994
995 if (args->stateless_rpc)
996 packet_flush(fd[1]);
79891cb9 997 if (args->deepen)
1a30f5a2
NTND
998 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
999 NULL);
4820a33b 1000 else if (si->nr_ours || si->nr_theirs)
beea4152 1001 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
1002 else
1003 alternate_shallow_file = NULL;
745f7a8c 1004 if (get_pack(args, fd, pack_lockfile))
1dd73e20 1005 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
1006
1007 all_done:
1008 return ref;
1009}
1010
f44af51d 1011static void fetch_pack_config(void)
745f7a8c 1012{
f44af51d
TA
1013 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1014 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1015 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1016 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1017 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 1018
f44af51d 1019 git_config(git_default_config, NULL);
745f7a8c
NTND
1020}
1021
745f7a8c
NTND
1022static void fetch_pack_setup(void)
1023{
1024 static int did_setup;
1025 if (did_setup)
1026 return;
f44af51d 1027 fetch_pack_config();
745f7a8c
NTND
1028 if (0 <= transfer_unpack_limit)
1029 unpack_limit = transfer_unpack_limit;
1030 else if (0 <= fetch_unpack_limit)
1031 unpack_limit = fetch_unpack_limit;
1032 did_setup = 1;
1033}
1034
f2db854d
JH
1035static int remove_duplicates_in_refs(struct ref **ref, int nr)
1036{
1037 struct string_list names = STRING_LIST_INIT_NODUP;
1038 int src, dst;
1039
1040 for (src = dst = 0; src < nr; src++) {
1041 struct string_list_item *item;
1042 item = string_list_insert(&names, ref[src]->name);
1043 if (item->util)
1044 continue; /* already have it */
1045 item->util = ref[src];
1046 if (src != dst)
1047 ref[dst] = ref[src];
1048 dst++;
1049 }
1050 for (src = dst; src < nr; src++)
1051 ref[src] = NULL;
1052 string_list_clear(&names, 0);
1053 return dst;
1054}
1055
beea4152 1056static void update_shallow(struct fetch_pack_args *args,
4820a33b 1057 struct ref **sought, int nr_sought,
beea4152 1058 struct shallow_info *si)
a796ccee 1059{
910650d2 1060 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1061 int *status;
beea4152
NTND
1062 int i;
1063
79891cb9 1064 if (args->deepen && alternate_shallow_file) {
a796ccee 1065 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 1066 unlink_or_warn(git_path_shallow());
a796ccee
NTND
1067 rollback_lock_file(&shallow_lock);
1068 } else
1069 commit_lock_file(&shallow_lock);
1070 return;
1071 }
beea4152
NTND
1072
1073 if (!si->shallow || !si->shallow->nr)
1074 return;
1075
beea4152
NTND
1076 if (args->cloning) {
1077 /*
1078 * remote is shallow, but this is a clone, there are
1079 * no objects in repo to worry about. Accept any
1080 * shallow points that exist in the pack (iow in repo
1081 * after get_pack() and reprepare_packed_git())
1082 */
910650d2 1083 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1084 struct object_id *oid = si->shallow->oid;
beea4152 1085 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1086 if (has_object_file(&oid[i]))
910650d2 1087 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1088 if (extra.nr) {
1089 setup_alternate_shallow(&shallow_lock,
1090 &alternate_shallow_file,
1091 &extra);
1092 commit_lock_file(&shallow_lock);
1093 }
910650d2 1094 oid_array_clear(&extra);
beea4152
NTND
1095 return;
1096 }
4820a33b
NTND
1097
1098 if (!si->nr_ours && !si->nr_theirs)
1099 return;
1100
1101 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1102 if (!si->nr_ours && !si->nr_theirs)
1103 return;
1104 for (i = 0; i < nr_sought; i++)
910650d2 1105 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1106 si->ref = &ref;
1107
48d25cae
NTND
1108 if (args->update_shallow) {
1109 /*
1110 * remote is also shallow, .git/shallow may be updated
1111 * so all refs can be accepted. Make sure we only add
1112 * shallow roots that are actually reachable from new
1113 * refs.
1114 */
910650d2 1115 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1116 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1117 assign_shallow_commits_to_refs(si, NULL, NULL);
1118 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1119 oid_array_clear(&ref);
48d25cae
NTND
1120 return;
1121 }
1122 for (i = 0; i < si->nr_ours; i++)
910650d2 1123 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1124 for (i = 0; i < si->nr_theirs; i++)
910650d2 1125 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1126 setup_alternate_shallow(&shallow_lock,
1127 &alternate_shallow_file,
1128 &extra);
1129 commit_lock_file(&shallow_lock);
910650d2 1130 oid_array_clear(&extra);
1131 oid_array_clear(&ref);
48d25cae
NTND
1132 return;
1133 }
1134
4820a33b
NTND
1135 /*
1136 * remote is also shallow, check what ref is safe to update
1137 * without updating .git/shallow
1138 */
1139 status = xcalloc(nr_sought, sizeof(*status));
1140 assign_shallow_commits_to_refs(si, NULL, status);
1141 if (si->nr_ours || si->nr_theirs) {
1142 for (i = 0; i < nr_sought; i++)
1143 if (status[i])
1144 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1145 }
1146 free(status);
910650d2 1147 oid_array_clear(&ref);
a796ccee
NTND
1148}
1149
745f7a8c
NTND
1150struct ref *fetch_pack(struct fetch_pack_args *args,
1151 int fd[], struct child_process *conn,
1152 const struct ref *ref,
1153 const char *dest,
f2db854d 1154 struct ref **sought, int nr_sought,
910650d2 1155 struct oid_array *shallow,
745f7a8c
NTND
1156 char **pack_lockfile)
1157{
745f7a8c 1158 struct ref *ref_cpy;
beea4152 1159 struct shallow_info si;
745f7a8c
NTND
1160
1161 fetch_pack_setup();
f2db854d
JH
1162 if (nr_sought)
1163 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1164
1165 if (!ref) {
1166 packet_flush(fd[1]);
1dd73e20 1167 die(_("no matching remote head"));
745f7a8c 1168 }
beea4152
NTND
1169 prepare_shallow_info(&si, shallow);
1170 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1171 &si, pack_lockfile);
745f7a8c 1172 reprepare_packed_git();
4820a33b 1173 update_shallow(args, sought, nr_sought, &si);
beea4152 1174 clear_shallow_info(&si);
745f7a8c
NTND
1175 return ref_cpy;
1176}
e860d96b
MM
1177
1178int report_unmatched_refs(struct ref **sought, int nr_sought)
1179{
1180 int i, ret = 0;
1181
1182 for (i = 0; i < nr_sought; i++) {
d56583de 1183 if (!sought[i])
e860d96b 1184 continue;
d56583de
MM
1185 switch (sought[i]->match_status) {
1186 case REF_MATCHED:
1187 continue;
1188 case REF_NOT_MATCHED:
1189 error(_("no such remote ref %s"), sought[i]->name);
1190 break;
1191 case REF_UNADVERTISED_NOT_ALLOWED:
1192 error(_("Server does not allow request for unadvertised object %s"),
1193 sought[i]->name);
1194 break;
1195 }
e860d96b
MM
1196 ret = 1;
1197 }
1198 return ret;
1199}