object: convert parse_object* to take struct object_id
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
697cc8ef 2#include "lockfile.h"
745f7a8c
NTND
3#include "refs.h"
4#include "pkt-line.h"
5#include "commit.h"
6#include "tag.h"
7#include "exec_cmd.h"
8#include "pack.h"
9#include "sideband.h"
10#include "fetch-pack.h"
11#include "remote.h"
12#include "run-command.h"
47a59185 13#include "connect.h"
745f7a8c
NTND
14#include "transport.h"
15#include "version.h"
099327b5 16#include "prio-queue.h"
beea4152 17#include "sha1-array.h"
745f7a8c
NTND
18
19static int transfer_unpack_limit = -1;
20static int fetch_unpack_limit = -1;
21static int unpack_limit = 100;
22static int prefer_ofs_delta = 1;
23static int no_done;
508ea882 24static int deepen_since_ok;
a45a2600 25static int deepen_not_ok;
745f7a8c
NTND
26static int fetch_fsck_objects = -1;
27static int transfer_fsck_objects = -1;
28static int agent_supported;
6035d6aa
NTND
29static struct lock_file shallow_lock;
30static const char *alternate_shallow_file;
745f7a8c 31
208acbfb 32/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
33#define COMPLETE (1U << 0)
34#define COMMON (1U << 1)
35#define COMMON_REF (1U << 2)
36#define SEEN (1U << 3)
37#define POPPED (1U << 4)
41a078c6 38#define ALTERNATE (1U << 5)
745f7a8c
NTND
39
40static int marked;
41
42/*
43 * After sending this many "have"s if we do not get any new ACK , we
44 * give up traversing our history.
45 */
46#define MAX_IN_VAIN 256
47
099327b5 48static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
49static int non_common_revs, multi_ack, use_sideband;
50/* Allow specifying sha1 if it is a ref tip. */
51#define ALLOW_TIP_SHA1 01
68ee6289
FM
52/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
53#define ALLOW_REACHABLE_SHA1 02
7199c093 54static unsigned int allow_unadvertised_object_request;
745f7a8c 55
0d789a5b
NTND
56__attribute__((format (printf, 2, 3)))
57static inline void print_verbose(const struct fetch_pack_args *args,
58 const char *fmt, ...)
59{
60 va_list params;
61
62 if (!args->verbose)
63 return;
64
65 va_start(params, fmt);
66 vfprintf(stderr, fmt, params);
67 va_end(params);
68 fputc('\n', stderr);
69}
70
41a078c6
JK
71struct alternate_object_cache {
72 struct object **items;
73 size_t nr, alloc;
74};
75
76static void cache_one_alternate(const char *refname,
77 const struct object_id *oid,
78 void *vcache)
79{
80 struct alternate_object_cache *cache = vcache;
c251c83d 81 struct object *obj = parse_object(oid);
41a078c6
JK
82
83 if (!obj || (obj->flags & ALTERNATE))
84 return;
85
86 obj->flags |= ALTERNATE;
87 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
88 cache->items[cache->nr++] = obj;
89}
90
91static void for_each_cached_alternate(void (*cb)(struct object *))
92{
93 static int initialized;
94 static struct alternate_object_cache cache;
95 size_t i;
96
97 if (!initialized) {
98 for_each_alternate_ref(cache_one_alternate, &cache);
99 initialized = 1;
100 }
101
102 for (i = 0; i < cache.nr; i++)
103 cb(cache.items[i]);
104}
105
745f7a8c
NTND
106static void rev_list_push(struct commit *commit, int mark)
107{
108 if (!(commit->object.flags & mark)) {
109 commit->object.flags |= mark;
110
0064053b
JK
111 if (parse_commit(commit))
112 return;
745f7a8c 113
099327b5 114 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
115
116 if (!(commit->object.flags & COMMON))
117 non_common_revs++;
118 }
119}
120
1b283377 121static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
745f7a8c 122{
c251c83d 123 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
124
125 if (o && o->type == OBJ_COMMIT)
126 rev_list_push((struct commit *)o, SEEN);
127
128 return 0;
129}
130
b1b49c6e
MH
131static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
132 int flag, void *cb_data)
745f7a8c 133{
1b283377 134 return rev_list_insert_ref(refname, oid);
b1b49c6e
MH
135}
136
c50fb6ce
MH
137static int clear_marks(const char *refname, const struct object_id *oid,
138 int flag, void *cb_data)
745f7a8c 139{
c251c83d 140 struct object *o = deref_tag(parse_object(oid), refname, 0);
745f7a8c
NTND
141
142 if (o && o->type == OBJ_COMMIT)
143 clear_commit_marks((struct commit *)o,
144 COMMON | COMMON_REF | SEEN | POPPED);
145 return 0;
146}
147
148/*
149 This function marks a rev and its ancestors as common.
150 In some cases, it is desirable to mark only the ancestors (for example
151 when only the server does not yet know that they are common).
152*/
153
154static void mark_common(struct commit *commit,
155 int ancestors_only, int dont_parse)
156{
157 if (commit != NULL && !(commit->object.flags & COMMON)) {
158 struct object *o = (struct object *)commit;
159
160 if (!ancestors_only)
161 o->flags |= COMMON;
162
163 if (!(o->flags & SEEN))
164 rev_list_push(commit, SEEN);
165 else {
166 struct commit_list *parents;
167
168 if (!ancestors_only && !(o->flags & POPPED))
169 non_common_revs--;
170 if (!o->parsed && !dont_parse)
171 if (parse_commit(commit))
172 return;
173
174 for (parents = commit->parents;
175 parents;
176 parents = parents->next)
177 mark_common(parents->item, 0, dont_parse);
178 }
179 }
180}
181
182/*
183 Get the next rev to send, ignoring the common.
184*/
185
1b283377 186static const struct object_id *get_rev(void)
745f7a8c
NTND
187{
188 struct commit *commit = NULL;
189
190 while (commit == NULL) {
191 unsigned int mark;
192 struct commit_list *parents;
193
099327b5 194 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
195 return NULL;
196
099327b5 197 commit = prio_queue_get(&rev_list);
0064053b 198 parse_commit(commit);
745f7a8c
NTND
199 parents = commit->parents;
200
201 commit->object.flags |= POPPED;
202 if (!(commit->object.flags & COMMON))
203 non_common_revs--;
204
205 if (commit->object.flags & COMMON) {
206 /* do not send "have", and ignore ancestors */
207 commit = NULL;
208 mark = COMMON | SEEN;
209 } else if (commit->object.flags & COMMON_REF)
210 /* send "have", and ignore ancestors */
211 mark = COMMON | SEEN;
212 else
213 /* send "have", also for its ancestors */
214 mark = SEEN;
215
216 while (parents) {
217 if (!(parents->item->object.flags & SEEN))
218 rev_list_push(parents->item, mark);
219 if (mark & COMMON)
220 mark_common(parents->item, 1, 0);
221 parents = parents->next;
222 }
745f7a8c
NTND
223 }
224
1b283377 225 return &commit->object.oid;
745f7a8c
NTND
226}
227
228enum ack_type {
229 NAK = 0,
230 ACK,
231 ACK_continue,
232 ACK_common,
233 ACK_ready
234};
235
236static void consume_shallow_list(struct fetch_pack_args *args, int fd)
237{
79891cb9 238 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
239 /* If we sent a depth we will get back "duplicate"
240 * shallow and unshallow commands every time there
241 * is a block of have lines exchanged.
242 */
74543a04
JK
243 char *line;
244 while ((line = packet_read_line(fd, NULL))) {
59556548 245 if (starts_with(line, "shallow "))
745f7a8c 246 continue;
59556548 247 if (starts_with(line, "unshallow "))
745f7a8c 248 continue;
1dd73e20 249 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
250 }
251 }
252}
253
1b283377 254static enum ack_type get_ack(int fd, struct object_id *result_oid)
745f7a8c 255{
74543a04
JK
256 int len;
257 char *line = packet_read_line(fd, &len);
82e56767 258 const char *arg;
745f7a8c
NTND
259
260 if (!len)
1dd73e20 261 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
262 if (!strcmp(line, "NAK"))
263 return NAK;
82e56767 264 if (skip_prefix(line, "ACK ", &arg)) {
1b283377 265 if (!get_oid_hex(arg, result_oid)) {
82e56767
JK
266 arg += 40;
267 len -= arg - line;
268 if (len < 1)
030e9dd6 269 return ACK;
82e56767 270 if (strstr(arg, "continue"))
745f7a8c 271 return ACK_continue;
82e56767 272 if (strstr(arg, "common"))
745f7a8c 273 return ACK_common;
82e56767 274 if (strstr(arg, "ready"))
745f7a8c
NTND
275 return ACK_ready;
276 return ACK;
277 }
278 }
8e2c7bef
JT
279 if (skip_prefix(line, "ERR ", &arg))
280 die(_("remote error: %s"), arg);
dfbfb9f3 281 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
282}
283
284static void send_request(struct fetch_pack_args *args,
285 int fd, struct strbuf *buf)
286{
287 if (args->stateless_rpc) {
288 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
289 packet_flush(fd);
290 } else
cdf4fb8e 291 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
292}
293
41a078c6 294static void insert_one_alternate_object(struct object *obj)
745f7a8c 295{
1b283377 296 rev_list_insert_ref(NULL, &obj->oid);
745f7a8c
NTND
297}
298
299#define INITIAL_FLUSH 16
300#define PIPESAFE_FLUSH 32
da470981 301#define LARGE_FLUSH 16384
745f7a8c
NTND
302
303static int next_flush(struct fetch_pack_args *args, int count)
304{
da470981
JT
305 if (args->stateless_rpc) {
306 if (count < LARGE_FLUSH)
307 count <<= 1;
308 else
309 count = count * 11 / 10;
310 } else {
311 if (count < PIPESAFE_FLUSH)
312 count <<= 1;
313 else
314 count += PIPESAFE_FLUSH;
315 }
745f7a8c
NTND
316 return count;
317}
318
319static int find_common(struct fetch_pack_args *args,
1b283377 320 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
321 struct ref *refs)
322{
323 int fetching;
324 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 325 const struct object_id *oid;
745f7a8c
NTND
326 unsigned in_vain = 0;
327 int got_continue = 0;
328 int got_ready = 0;
329 struct strbuf req_buf = STRBUF_INIT;
330 size_t state_len = 0;
331
332 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 333 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
334 if (marked)
335 for_each_ref(clear_marks, NULL);
336 marked = 1;
337
b1b49c6e 338 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 339 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
340
341 fetching = 0;
342 for ( ; refs ; refs = refs->next) {
1b283377 343 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
344 const char *remote_hex;
345 struct object *o;
346
347 /*
348 * If that object is complete (i.e. it is an ancestor of a
349 * local ref), we tell them we have it but do not have to
350 * tell them about its ancestors, which they already know
351 * about.
352 *
353 * We use lookup_object here because we are only
354 * interested in the case we *know* the object is
355 * reachable and we have already scanned it.
356 */
1b283377 357 if (((o = lookup_object(remote->hash)) != NULL) &&
745f7a8c
NTND
358 (o->flags & COMPLETE)) {
359 continue;
360 }
361
1b283377 362 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
363 if (!fetching) {
364 struct strbuf c = STRBUF_INIT;
365 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
366 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
367 if (no_done) strbuf_addstr(&c, " no-done");
368 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
369 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 370 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
371 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
372 if (args->no_progress) strbuf_addstr(&c, " no-progress");
373 if (args->include_tag) strbuf_addstr(&c, " include-tag");
374 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 375 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 376 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
377 if (agent_supported) strbuf_addf(&c, " agent=%s",
378 git_user_agent_sanitized());
379 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
380 strbuf_release(&c);
381 } else
382 packet_buf_write(&req_buf, "want %s\n", remote_hex);
383 fetching++;
384 }
385
386 if (!fetching) {
387 strbuf_release(&req_buf);
388 packet_flush(fd[1]);
389 return 1;
390 }
391
392 if (is_repository_shallow())
1a30f5a2 393 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
394 if (args->depth > 0)
395 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882
NTND
396 if (args->deepen_since) {
397 unsigned long max_age = approxidate(args->deepen_since);
398 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
399 }
a45a2600
NTND
400 if (args->deepen_not) {
401 int i;
402 for (i = 0; i < args->deepen_not->nr; i++) {
403 struct string_list_item *s = args->deepen_not->items + i;
404 packet_buf_write(&req_buf, "deepen-not %s", s->string);
405 }
406 }
745f7a8c
NTND
407 packet_buf_flush(&req_buf);
408 state_len = req_buf.len;
409
79891cb9 410 if (args->deepen) {
74543a04 411 char *line;
ae021d87 412 const char *arg;
1b283377 413 struct object_id oid;
745f7a8c
NTND
414
415 send_request(args, fd[1], &req_buf);
74543a04 416 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87 417 if (skip_prefix(line, "shallow ", &arg)) {
1b283377 418 if (get_oid_hex(arg, &oid))
1dd73e20 419 die(_("invalid shallow line: %s"), line);
e92b848c 420 register_shallow(&oid);
745f7a8c
NTND
421 continue;
422 }
ae021d87 423 if (skip_prefix(line, "unshallow ", &arg)) {
1b283377 424 if (get_oid_hex(arg, &oid))
1dd73e20 425 die(_("invalid unshallow line: %s"), line);
1b283377 426 if (!lookup_object(oid.hash))
1dd73e20 427 die(_("object not found: %s"), line);
745f7a8c 428 /* make sure that it is parsed as shallow */
c251c83d 429 if (!parse_object(&oid))
1dd73e20 430 die(_("error in object: %s"), line);
e92b848c 431 if (unregister_shallow(&oid))
1dd73e20 432 die(_("no shallow found: %s"), line);
745f7a8c
NTND
433 continue;
434 }
1dd73e20 435 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
436 }
437 } else if (!args->stateless_rpc)
438 send_request(args, fd[1], &req_buf);
439
440 if (!args->stateless_rpc) {
441 /* If we aren't using the stateless-rpc interface
442 * we don't need to retain the headers.
443 */
444 strbuf_setlen(&req_buf, 0);
445 state_len = 0;
446 }
447
448 flushes = 0;
449 retval = -1;
1b283377 450 while ((oid = get_rev())) {
451 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
452 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
453 in_vain++;
454 if (flush_at <= ++count) {
455 int ack;
456
457 packet_buf_flush(&req_buf);
458 send_request(args, fd[1], &req_buf);
459 strbuf_setlen(&req_buf, state_len);
460 flushes++;
461 flush_at = next_flush(args, count);
462
463 /*
464 * We keep one window "ahead" of the other side, and
465 * will wait for an ACK only on the next one
466 */
467 if (!args->stateless_rpc && count == INITIAL_FLUSH)
468 continue;
469
470 consume_shallow_list(args, fd[0]);
471 do {
1b283377 472 ack = get_ack(fd[0], result_oid);
0d789a5b 473 if (ack)
1dd73e20 474 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 475 ack, oid_to_hex(result_oid));
745f7a8c
NTND
476 switch (ack) {
477 case ACK:
478 flushes = 0;
479 multi_ack = 0;
480 retval = 0;
481 goto done;
482 case ACK_common:
483 case ACK_ready:
484 case ACK_continue: {
485 struct commit *commit =
bc83266a 486 lookup_commit(result_oid);
745f7a8c 487 if (!commit)
1b283377 488 die(_("invalid commit %s"), oid_to_hex(result_oid));
745f7a8c
NTND
489 if (args->stateless_rpc
490 && ack == ACK_common
491 && !(commit->object.flags & COMMON)) {
492 /* We need to replay the have for this object
493 * on the next RPC request so the peer knows
494 * it is in common with us.
495 */
1b283377 496 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
497 packet_buf_write(&req_buf, "have %s\n", hex);
498 state_len = req_buf.len;
06b3d386
JT
499 /*
500 * Reset in_vain because an ack
501 * for this commit has not been
502 * seen.
503 */
504 in_vain = 0;
505 } else if (!args->stateless_rpc
506 || ack != ACK_common)
507 in_vain = 0;
745f7a8c
NTND
508 mark_common(commit, 0, 1);
509 retval = 0;
745f7a8c
NTND
510 got_continue = 1;
511 if (ack == ACK_ready) {
099327b5 512 clear_prio_queue(&rev_list);
745f7a8c
NTND
513 got_ready = 1;
514 }
515 break;
516 }
517 }
518 } while (ack);
519 flushes--;
520 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 521 print_verbose(args, _("giving up"));
745f7a8c
NTND
522 break; /* give up */
523 }
524 }
525 }
526done:
527 if (!got_ready || !no_done) {
528 packet_buf_write(&req_buf, "done\n");
529 send_request(args, fd[1], &req_buf);
530 }
1dd73e20 531 print_verbose(args, _("done"));
745f7a8c
NTND
532 if (retval != 0) {
533 multi_ack = 0;
534 flushes++;
535 }
536 strbuf_release(&req_buf);
537
ff62eca7
NTND
538 if (!got_ready || !no_done)
539 consume_shallow_list(args, fd[0]);
745f7a8c 540 while (flushes || multi_ack) {
1b283377 541 int ack = get_ack(fd[0], result_oid);
745f7a8c 542 if (ack) {
1dd73e20 543 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 544 ack, oid_to_hex(result_oid));
745f7a8c
NTND
545 if (ack == ACK)
546 return 0;
547 multi_ack = 1;
548 continue;
549 }
550 flushes--;
551 }
552 /* it is no error to fetch into a completely empty repo */
553 return count ? retval : 0;
554}
555
556static struct commit_list *complete;
557
1b283377 558static int mark_complete(const struct object_id *oid)
745f7a8c 559{
c251c83d 560 struct object *o = parse_object(oid);
745f7a8c
NTND
561
562 while (o && o->type == OBJ_TAG) {
563 struct tag *t = (struct tag *) o;
564 if (!t->tagged)
565 break; /* broken repository */
566 o->flags |= COMPLETE;
c251c83d 567 o = parse_object(&t->tagged->oid);
745f7a8c
NTND
568 }
569 if (o && o->type == OBJ_COMMIT) {
570 struct commit *commit = (struct commit *)o;
571 if (!(commit->object.flags & COMPLETE)) {
572 commit->object.flags |= COMPLETE;
16445242 573 commit_list_insert(commit, &complete);
745f7a8c
NTND
574 }
575 }
576 return 0;
577}
578
f8ee4d85
MH
579static int mark_complete_oid(const char *refname, const struct object_id *oid,
580 int flag, void *cb_data)
581{
1b283377 582 return mark_complete(oid);
f8ee4d85
MH
583}
584
745f7a8c
NTND
585static void mark_recent_complete_commits(struct fetch_pack_args *args,
586 unsigned long cutoff)
587{
588 while (complete && cutoff <= complete->item->date) {
1dd73e20 589 print_verbose(args, _("Marking %s as complete"),
0d789a5b 590 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
591 pop_most_recent_commit(&complete, COMPLETE);
592 }
593}
594
745f7a8c 595static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
596 struct ref **refs,
597 struct ref **sought, int nr_sought)
745f7a8c
NTND
598{
599 struct ref *newlist = NULL;
600 struct ref **newtail = &newlist;
601 struct ref *ref, *next;
f2db854d 602 int i;
745f7a8c 603
f2db854d 604 i = 0;
745f7a8c
NTND
605 for (ref = *refs; ref; ref = next) {
606 int keep = 0;
607 next = ref->next;
f2db854d 608
50e19a83 609 if (starts_with(ref->name, "refs/") &&
4c224081 610 check_refname_format(ref->name, 0))
745f7a8c
NTND
611 ; /* trash */
612 else {
f2db854d
JH
613 while (i < nr_sought) {
614 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
615 if (cmp < 0)
616 break; /* definitely do not have it */
617 else if (cmp == 0) {
618 keep = 1; /* definitely have it */
d56583de 619 sought[i]->match_status = REF_MATCHED;
745f7a8c 620 }
f2db854d 621 i++;
745f7a8c
NTND
622 }
623 }
624
f2db854d 625 if (!keep && args->fetch_all &&
79891cb9 626 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
627 keep = 1;
628
629 if (keep) {
630 *newtail = ref;
631 ref->next = NULL;
632 newtail = &ref->next;
633 } else {
634 free(ref);
635 }
636 }
637
6e7b66ee 638 /* Append unmatched requests to the list */
d56583de 639 for (i = 0; i < nr_sought; i++) {
1b283377 640 struct object_id oid;
641 const char *p;
b7916422 642
d56583de
MM
643 ref = sought[i];
644 if (ref->match_status != REF_NOT_MATCHED)
645 continue;
1b283377 646 if (parse_oid_hex(ref->name, &oid, &p) ||
647 *p != '\0' ||
648 oidcmp(&oid, &ref->old_oid))
d56583de 649 continue;
6e7b66ee 650
d56583de
MM
651 if ((allow_unadvertised_object_request &
652 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
653 ref->match_status = REF_MATCHED;
c3c17bf1
JK
654 *newtail = copy_ref(ref);
655 newtail = &(*newtail)->next;
d56583de
MM
656 } else {
657 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
658 }
659 }
745f7a8c
NTND
660 *refs = newlist;
661}
662
41a078c6 663static void mark_alternate_complete(struct object *obj)
745f7a8c 664{
1b283377 665 mark_complete(&obj->oid);
745f7a8c
NTND
666}
667
668static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
669 struct ref **refs,
670 struct ref **sought, int nr_sought)
745f7a8c
NTND
671{
672 struct ref *ref;
673 int retval;
674 unsigned long cutoff = 0;
675
676 save_commit_buffer = 0;
677
678 for (ref = *refs; ref; ref = ref->next) {
679 struct object *o;
680
f4e54d02 681 if (!has_object_file(&ref->old_oid))
012a1bb5
JH
682 continue;
683
c251c83d 684 o = parse_object(&ref->old_oid);
745f7a8c
NTND
685 if (!o)
686 continue;
687
688 /* We already have it -- which may mean that we were
689 * in sync with the other side at some time after
690 * that (it is OK if we guess wrong here).
691 */
692 if (o->type == OBJ_COMMIT) {
693 struct commit *commit = (struct commit *)o;
694 if (!cutoff || cutoff < commit->date)
695 cutoff = commit->date;
696 }
697 }
698
79891cb9 699 if (!args->deepen) {
f8ee4d85 700 for_each_ref(mark_complete_oid, NULL);
41a078c6 701 for_each_cached_alternate(mark_alternate_complete);
16445242 702 commit_list_sort_by_date(&complete);
745f7a8c
NTND
703 if (cutoff)
704 mark_recent_complete_commits(args, cutoff);
705 }
706
707 /*
708 * Mark all complete remote refs as common refs.
709 * Don't mark them common yet; the server has to be told so first.
710 */
711 for (ref = *refs; ref; ref = ref->next) {
f4e54d02 712 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
745f7a8c
NTND
713 NULL, 0);
714
715 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
716 continue;
717
718 if (!(o->flags & SEEN)) {
719 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
720
721 mark_common((struct commit *)o, 1, 1);
722 }
723 }
724
f2db854d 725 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
726
727 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 728 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
729 struct object *o;
730
1b283377 731 o = lookup_object(remote->hash);
745f7a8c
NTND
732 if (!o || !(o->flags & COMPLETE)) {
733 retval = 0;
1b283377 734 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 735 ref->name);
745f7a8c
NTND
736 continue;
737 }
1b283377 738 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 739 ref->name);
745f7a8c
NTND
740 }
741 return retval;
742}
743
744static int sideband_demux(int in, int out, void *data)
745{
746 int *xd = data;
9ff18faf 747 int ret;
745f7a8c 748
9ff18faf 749 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
750 close(out);
751 return ret;
752}
753
754static int get_pack(struct fetch_pack_args *args,
755 int xd[2], char **pack_lockfile)
756{
757 struct async demux;
745f7a8c 758 int do_keep = args->keep_pack;
984a43b9
JK
759 const char *cmd_name;
760 struct pack_header header;
761 int pass_header = 0;
d3180279 762 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 763 int ret;
745f7a8c
NTND
764
765 memset(&demux, 0, sizeof(demux));
766 if (use_sideband) {
767 /* xd[] is talking with upload-pack; subprocess reads from
768 * xd[0], spits out band#2 to stderr, and feeds us band#1
769 * through demux->out.
770 */
771 demux.proc = sideband_demux;
772 demux.data = xd;
773 demux.out = -1;
df857572 774 demux.isolate_sigpipe = 1;
745f7a8c 775 if (start_async(&demux))
1dd73e20 776 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
777 }
778 else
779 demux.out = xd[0];
780
745f7a8c 781 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
782
783 if (read_pack_header(demux.out, &header))
1dd73e20 784 die(_("protocol error: bad pack header"));
984a43b9 785 pass_header = 1;
745f7a8c
NTND
786 if (ntohl(header.hdr_entries) < unpack_limit)
787 do_keep = 0;
788 else
789 do_keep = 1;
790 }
791
6035d6aa 792 if (alternate_shallow_file) {
984a43b9
JK
793 argv_array_push(&cmd.args, "--shallow-file");
794 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
795 }
796
745f7a8c
NTND
797 if (do_keep) {
798 if (pack_lockfile)
799 cmd.out = -1;
984a43b9
JK
800 cmd_name = "index-pack";
801 argv_array_push(&cmd.args, cmd_name);
802 argv_array_push(&cmd.args, "--stdin");
745f7a8c 803 if (!args->quiet && !args->no_progress)
984a43b9 804 argv_array_push(&cmd.args, "-v");
745f7a8c 805 if (args->use_thin_pack)
984a43b9 806 argv_array_push(&cmd.args, "--fix-thin");
745f7a8c 807 if (args->lock_pack || unpack_limit) {
da25bdb7 808 char hostname[HOST_NAME_MAX + 1];
5781a9a2 809 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
810 xsnprintf(hostname, sizeof(hostname), "localhost");
811 argv_array_pushf(&cmd.args,
812 "--keep=fetch-pack %"PRIuMAX " on %s",
813 (uintmax_t)getpid(), hostname);
745f7a8c 814 }
c6807a40 815 if (args->check_self_contained_and_connected)
984a43b9 816 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
745f7a8c
NTND
817 }
818 else {
984a43b9
JK
819 cmd_name = "unpack-objects";
820 argv_array_push(&cmd.args, cmd_name);
745f7a8c 821 if (args->quiet || args->no_progress)
984a43b9 822 argv_array_push(&cmd.args, "-q");
c6807a40 823 args->check_self_contained_and_connected = 0;
745f7a8c 824 }
984a43b9
JK
825
826 if (pass_header)
827 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
828 ntohl(header.hdr_version),
829 ntohl(header.hdr_entries));
745f7a8c
NTND
830 if (fetch_fsck_objects >= 0
831 ? fetch_fsck_objects
832 : transfer_fsck_objects >= 0
833 ? transfer_fsck_objects
834 : 0)
984a43b9 835 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
836
837 cmd.in = demux.out;
838 cmd.git_cmd = 1;
839 if (start_command(&cmd))
1dd73e20 840 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
841 if (do_keep && pack_lockfile) {
842 *pack_lockfile = index_pack_lockfile(cmd.out);
843 close(cmd.out);
844 }
845
37cb1dd6
JL
846 if (!use_sideband)
847 /* Closed by start_command() */
848 xd[0] = -1;
849
c6807a40
NTND
850 ret = finish_command(&cmd);
851 if (!ret || (args->check_self_contained_and_connected && ret == 1))
852 args->self_contained_and_connected =
853 args->check_self_contained_and_connected &&
854 ret == 0;
855 else
1dd73e20 856 die(_("%s failed"), cmd_name);
745f7a8c 857 if (use_sideband && finish_async(&demux))
1dd73e20 858 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
859 return 0;
860}
861
f2db854d
JH
862static int cmp_ref_by_name(const void *a_, const void *b_)
863{
864 const struct ref *a = *((const struct ref **)a_);
865 const struct ref *b = *((const struct ref **)b_);
866 return strcmp(a->name, b->name);
867}
868
745f7a8c
NTND
869static struct ref *do_fetch_pack(struct fetch_pack_args *args,
870 int fd[2],
871 const struct ref *orig_ref,
f2db854d 872 struct ref **sought, int nr_sought,
beea4152 873 struct shallow_info *si,
745f7a8c
NTND
874 char **pack_lockfile)
875{
876 struct ref *ref = copy_ref_list(orig_ref);
1b283377 877 struct object_id oid;
745f7a8c
NTND
878 const char *agent_feature;
879 int agent_len;
880
881 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 882 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 883
eb86a507 884 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 885 die(_("Server does not support shallow clients"));
a45a2600 886 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 887 args->deepen = 1;
745f7a8c 888 if (server_supports("multi_ack_detailed")) {
1dd73e20 889 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
890 multi_ack = 2;
891 if (server_supports("no-done")) {
1dd73e20 892 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
893 if (args->stateless_rpc)
894 no_done = 1;
895 }
896 }
897 else if (server_supports("multi_ack")) {
1dd73e20 898 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
899 multi_ack = 1;
900 }
901 if (server_supports("side-band-64k")) {
1dd73e20 902 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
903 use_sideband = 2;
904 }
905 else if (server_supports("side-band")) {
1dd73e20 906 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
907 use_sideband = 1;
908 }
6e7b66ee 909 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 910 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 911 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 912 }
68ee6289 913 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 914 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
915 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
916 }
745f7a8c
NTND
917 if (!server_supports("thin-pack"))
918 args->use_thin_pack = 0;
919 if (!server_supports("no-progress"))
920 args->no_progress = 0;
921 if (!server_supports("include-tag"))
922 args->include_tag = 0;
0d789a5b 923 if (server_supports("ofs-delta"))
1dd73e20 924 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 925 else
745f7a8c
NTND
926 prefer_ofs_delta = 0;
927
928 if ((agent_feature = server_feature_value("agent", &agent_len))) {
929 agent_supported = 1;
0d789a5b 930 if (agent_len)
1dd73e20 931 print_verbose(args, _("Server version is %.*s"),
0d789a5b 932 agent_len, agent_feature);
745f7a8c 933 }
508ea882
NTND
934 if (server_supports("deepen-since"))
935 deepen_since_ok = 1;
936 else if (args->deepen_since)
937 die(_("Server does not support --shallow-since"));
a45a2600
NTND
938 if (server_supports("deepen-not"))
939 deepen_not_ok = 1;
940 else if (args->deepen_not)
941 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
942 if (!server_supports("deepen-relative") && args->deepen_relative)
943 die(_("Server does not support --deepen"));
745f7a8c 944
f2db854d 945 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
946 packet_flush(fd[1]);
947 goto all_done;
948 }
1b283377 949 if (find_common(args, fd, &oid, ref) < 0)
745f7a8c
NTND
950 if (!args->keep_pack)
951 /* When cloning, it is not unusual to have
952 * no common commit.
953 */
1dd73e20 954 warning(_("no common commits"));
745f7a8c
NTND
955
956 if (args->stateless_rpc)
957 packet_flush(fd[1]);
79891cb9 958 if (args->deepen)
1a30f5a2
NTND
959 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
960 NULL);
4820a33b 961 else if (si->nr_ours || si->nr_theirs)
beea4152 962 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
963 else
964 alternate_shallow_file = NULL;
745f7a8c 965 if (get_pack(args, fd, pack_lockfile))
1dd73e20 966 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
967
968 all_done:
969 return ref;
970}
971
f44af51d 972static void fetch_pack_config(void)
745f7a8c 973{
f44af51d
TA
974 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
975 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
976 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
977 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
978 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 979
f44af51d 980 git_config(git_default_config, NULL);
745f7a8c
NTND
981}
982
745f7a8c
NTND
983static void fetch_pack_setup(void)
984{
985 static int did_setup;
986 if (did_setup)
987 return;
f44af51d 988 fetch_pack_config();
745f7a8c
NTND
989 if (0 <= transfer_unpack_limit)
990 unpack_limit = transfer_unpack_limit;
991 else if (0 <= fetch_unpack_limit)
992 unpack_limit = fetch_unpack_limit;
993 did_setup = 1;
994}
995
f2db854d
JH
996static int remove_duplicates_in_refs(struct ref **ref, int nr)
997{
998 struct string_list names = STRING_LIST_INIT_NODUP;
999 int src, dst;
1000
1001 for (src = dst = 0; src < nr; src++) {
1002 struct string_list_item *item;
1003 item = string_list_insert(&names, ref[src]->name);
1004 if (item->util)
1005 continue; /* already have it */
1006 item->util = ref[src];
1007 if (src != dst)
1008 ref[dst] = ref[src];
1009 dst++;
1010 }
1011 for (src = dst; src < nr; src++)
1012 ref[src] = NULL;
1013 string_list_clear(&names, 0);
1014 return dst;
1015}
1016
beea4152 1017static void update_shallow(struct fetch_pack_args *args,
4820a33b 1018 struct ref **sought, int nr_sought,
beea4152 1019 struct shallow_info *si)
a796ccee 1020{
910650d2 1021 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1022 int *status;
beea4152
NTND
1023 int i;
1024
79891cb9 1025 if (args->deepen && alternate_shallow_file) {
a796ccee 1026 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 1027 unlink_or_warn(git_path_shallow());
a796ccee
NTND
1028 rollback_lock_file(&shallow_lock);
1029 } else
1030 commit_lock_file(&shallow_lock);
1031 return;
1032 }
beea4152
NTND
1033
1034 if (!si->shallow || !si->shallow->nr)
1035 return;
1036
beea4152
NTND
1037 if (args->cloning) {
1038 /*
1039 * remote is shallow, but this is a clone, there are
1040 * no objects in repo to worry about. Accept any
1041 * shallow points that exist in the pack (iow in repo
1042 * after get_pack() and reprepare_packed_git())
1043 */
910650d2 1044 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1045 struct object_id *oid = si->shallow->oid;
beea4152 1046 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1047 if (has_object_file(&oid[i]))
910650d2 1048 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1049 if (extra.nr) {
1050 setup_alternate_shallow(&shallow_lock,
1051 &alternate_shallow_file,
1052 &extra);
1053 commit_lock_file(&shallow_lock);
1054 }
910650d2 1055 oid_array_clear(&extra);
beea4152
NTND
1056 return;
1057 }
4820a33b
NTND
1058
1059 if (!si->nr_ours && !si->nr_theirs)
1060 return;
1061
1062 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1063 if (!si->nr_ours && !si->nr_theirs)
1064 return;
1065 for (i = 0; i < nr_sought; i++)
910650d2 1066 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1067 si->ref = &ref;
1068
48d25cae
NTND
1069 if (args->update_shallow) {
1070 /*
1071 * remote is also shallow, .git/shallow may be updated
1072 * so all refs can be accepted. Make sure we only add
1073 * shallow roots that are actually reachable from new
1074 * refs.
1075 */
910650d2 1076 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1077 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1078 assign_shallow_commits_to_refs(si, NULL, NULL);
1079 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1080 oid_array_clear(&ref);
48d25cae
NTND
1081 return;
1082 }
1083 for (i = 0; i < si->nr_ours; i++)
910650d2 1084 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1085 for (i = 0; i < si->nr_theirs; i++)
910650d2 1086 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1087 setup_alternate_shallow(&shallow_lock,
1088 &alternate_shallow_file,
1089 &extra);
1090 commit_lock_file(&shallow_lock);
910650d2 1091 oid_array_clear(&extra);
1092 oid_array_clear(&ref);
48d25cae
NTND
1093 return;
1094 }
1095
4820a33b
NTND
1096 /*
1097 * remote is also shallow, check what ref is safe to update
1098 * without updating .git/shallow
1099 */
1100 status = xcalloc(nr_sought, sizeof(*status));
1101 assign_shallow_commits_to_refs(si, NULL, status);
1102 if (si->nr_ours || si->nr_theirs) {
1103 for (i = 0; i < nr_sought; i++)
1104 if (status[i])
1105 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1106 }
1107 free(status);
910650d2 1108 oid_array_clear(&ref);
a796ccee
NTND
1109}
1110
745f7a8c
NTND
1111struct ref *fetch_pack(struct fetch_pack_args *args,
1112 int fd[], struct child_process *conn,
1113 const struct ref *ref,
1114 const char *dest,
f2db854d 1115 struct ref **sought, int nr_sought,
910650d2 1116 struct oid_array *shallow,
745f7a8c
NTND
1117 char **pack_lockfile)
1118{
745f7a8c 1119 struct ref *ref_cpy;
beea4152 1120 struct shallow_info si;
745f7a8c
NTND
1121
1122 fetch_pack_setup();
f2db854d
JH
1123 if (nr_sought)
1124 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1125
1126 if (!ref) {
1127 packet_flush(fd[1]);
1dd73e20 1128 die(_("no matching remote head"));
745f7a8c 1129 }
beea4152
NTND
1130 prepare_shallow_info(&si, shallow);
1131 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1132 &si, pack_lockfile);
745f7a8c 1133 reprepare_packed_git();
4820a33b 1134 update_shallow(args, sought, nr_sought, &si);
beea4152 1135 clear_shallow_info(&si);
745f7a8c
NTND
1136 return ref_cpy;
1137}
e860d96b
MM
1138
1139int report_unmatched_refs(struct ref **sought, int nr_sought)
1140{
1141 int i, ret = 0;
1142
1143 for (i = 0; i < nr_sought; i++) {
d56583de 1144 if (!sought[i])
e860d96b 1145 continue;
d56583de
MM
1146 switch (sought[i]->match_status) {
1147 case REF_MATCHED:
1148 continue;
1149 case REF_NOT_MATCHED:
1150 error(_("no such remote ref %s"), sought[i]->name);
1151 break;
1152 case REF_UNADVERTISED_NOT_ALLOWED:
1153 error(_("Server does not allow request for unadvertised object %s"),
1154 sought[i]->name);
1155 break;
1156 }
e860d96b
MM
1157 ret = 1;
1158 }
1159 return ret;
1160}