Merge branch 'jt/fetch-pack-error-reporting'
[git/git.git] / fetch-pack.c
1 #include "cache.h"
2 #include "lockfile.h"
3 #include "refs.h"
4 #include "pkt-line.h"
5 #include "commit.h"
6 #include "tag.h"
7 #include "exec_cmd.h"
8 #include "pack.h"
9 #include "sideband.h"
10 #include "fetch-pack.h"
11 #include "remote.h"
12 #include "run-command.h"
13 #include "connect.h"
14 #include "transport.h"
15 #include "version.h"
16 #include "prio-queue.h"
17 #include "sha1-array.h"
18
19 static int transfer_unpack_limit = -1;
20 static int fetch_unpack_limit = -1;
21 static int unpack_limit = 100;
22 static int prefer_ofs_delta = 1;
23 static int no_done;
24 static int deepen_since_ok;
25 static int deepen_not_ok;
26 static int fetch_fsck_objects = -1;
27 static int transfer_fsck_objects = -1;
28 static int agent_supported;
29 static struct lock_file shallow_lock;
30 static const char *alternate_shallow_file;
31
32 /* Remember to update object flag allocation in object.h */
33 #define COMPLETE (1U << 0)
34 #define COMMON (1U << 1)
35 #define COMMON_REF (1U << 2)
36 #define SEEN (1U << 3)
37 #define POPPED (1U << 4)
38 #define ALTERNATE (1U << 5)
39
40 static int marked;
41
42 /*
43 * After sending this many "have"s if we do not get any new ACK , we
44 * give up traversing our history.
45 */
46 #define MAX_IN_VAIN 256
47
48 static struct prio_queue rev_list = { compare_commits_by_commit_date };
49 static int non_common_revs, multi_ack, use_sideband;
50 /* Allow specifying sha1 if it is a ref tip. */
51 #define ALLOW_TIP_SHA1 01
52 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
53 #define ALLOW_REACHABLE_SHA1 02
54 static unsigned int allow_unadvertised_object_request;
55
56 __attribute__((format (printf, 2, 3)))
57 static inline void print_verbose(const struct fetch_pack_args *args,
58 const char *fmt, ...)
59 {
60 va_list params;
61
62 if (!args->verbose)
63 return;
64
65 va_start(params, fmt);
66 vfprintf(stderr, fmt, params);
67 va_end(params);
68 fputc('\n', stderr);
69 }
70
71 struct alternate_object_cache {
72 struct object **items;
73 size_t nr, alloc;
74 };
75
76 static void cache_one_alternate(const char *refname,
77 const struct object_id *oid,
78 void *vcache)
79 {
80 struct alternate_object_cache *cache = vcache;
81 struct object *obj = parse_object(oid->hash);
82
83 if (!obj || (obj->flags & ALTERNATE))
84 return;
85
86 obj->flags |= ALTERNATE;
87 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
88 cache->items[cache->nr++] = obj;
89 }
90
91 static void for_each_cached_alternate(void (*cb)(struct object *))
92 {
93 static int initialized;
94 static struct alternate_object_cache cache;
95 size_t i;
96
97 if (!initialized) {
98 for_each_alternate_ref(cache_one_alternate, &cache);
99 initialized = 1;
100 }
101
102 for (i = 0; i < cache.nr; i++)
103 cb(cache.items[i]);
104 }
105
106 static void rev_list_push(struct commit *commit, int mark)
107 {
108 if (!(commit->object.flags & mark)) {
109 commit->object.flags |= mark;
110
111 if (parse_commit(commit))
112 return;
113
114 prio_queue_put(&rev_list, commit);
115
116 if (!(commit->object.flags & COMMON))
117 non_common_revs++;
118 }
119 }
120
121 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
122 {
123 struct object *o = deref_tag(parse_object(sha1), refname, 0);
124
125 if (o && o->type == OBJ_COMMIT)
126 rev_list_push((struct commit *)o, SEEN);
127
128 return 0;
129 }
130
131 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
132 int flag, void *cb_data)
133 {
134 return rev_list_insert_ref(refname, oid->hash);
135 }
136
137 static int clear_marks(const char *refname, const struct object_id *oid,
138 int flag, void *cb_data)
139 {
140 struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
141
142 if (o && o->type == OBJ_COMMIT)
143 clear_commit_marks((struct commit *)o,
144 COMMON | COMMON_REF | SEEN | POPPED);
145 return 0;
146 }
147
148 /*
149 This function marks a rev and its ancestors as common.
150 In some cases, it is desirable to mark only the ancestors (for example
151 when only the server does not yet know that they are common).
152 */
153
154 static void mark_common(struct commit *commit,
155 int ancestors_only, int dont_parse)
156 {
157 if (commit != NULL && !(commit->object.flags & COMMON)) {
158 struct object *o = (struct object *)commit;
159
160 if (!ancestors_only)
161 o->flags |= COMMON;
162
163 if (!(o->flags & SEEN))
164 rev_list_push(commit, SEEN);
165 else {
166 struct commit_list *parents;
167
168 if (!ancestors_only && !(o->flags & POPPED))
169 non_common_revs--;
170 if (!o->parsed && !dont_parse)
171 if (parse_commit(commit))
172 return;
173
174 for (parents = commit->parents;
175 parents;
176 parents = parents->next)
177 mark_common(parents->item, 0, dont_parse);
178 }
179 }
180 }
181
182 /*
183 Get the next rev to send, ignoring the common.
184 */
185
186 static const unsigned char *get_rev(void)
187 {
188 struct commit *commit = NULL;
189
190 while (commit == NULL) {
191 unsigned int mark;
192 struct commit_list *parents;
193
194 if (rev_list.nr == 0 || non_common_revs == 0)
195 return NULL;
196
197 commit = prio_queue_get(&rev_list);
198 parse_commit(commit);
199 parents = commit->parents;
200
201 commit->object.flags |= POPPED;
202 if (!(commit->object.flags & COMMON))
203 non_common_revs--;
204
205 if (commit->object.flags & COMMON) {
206 /* do not send "have", and ignore ancestors */
207 commit = NULL;
208 mark = COMMON | SEEN;
209 } else if (commit->object.flags & COMMON_REF)
210 /* send "have", and ignore ancestors */
211 mark = COMMON | SEEN;
212 else
213 /* send "have", also for its ancestors */
214 mark = SEEN;
215
216 while (parents) {
217 if (!(parents->item->object.flags & SEEN))
218 rev_list_push(parents->item, mark);
219 if (mark & COMMON)
220 mark_common(parents->item, 1, 0);
221 parents = parents->next;
222 }
223 }
224
225 return commit->object.oid.hash;
226 }
227
228 enum ack_type {
229 NAK = 0,
230 ACK,
231 ACK_continue,
232 ACK_common,
233 ACK_ready
234 };
235
236 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
237 {
238 if (args->stateless_rpc && args->deepen) {
239 /* If we sent a depth we will get back "duplicate"
240 * shallow and unshallow commands every time there
241 * is a block of have lines exchanged.
242 */
243 char *line;
244 while ((line = packet_read_line(fd, NULL))) {
245 if (starts_with(line, "shallow "))
246 continue;
247 if (starts_with(line, "unshallow "))
248 continue;
249 die(_("git fetch-pack: expected shallow list"));
250 }
251 }
252 }
253
254 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
255 {
256 int len;
257 char *line = packet_read_line(fd, &len);
258 const char *arg;
259
260 if (!len)
261 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
262 if (!strcmp(line, "NAK"))
263 return NAK;
264 if (skip_prefix(line, "ACK ", &arg)) {
265 if (!get_sha1_hex(arg, result_sha1)) {
266 arg += 40;
267 len -= arg - line;
268 if (len < 1)
269 return ACK;
270 if (strstr(arg, "continue"))
271 return ACK_continue;
272 if (strstr(arg, "common"))
273 return ACK_common;
274 if (strstr(arg, "ready"))
275 return ACK_ready;
276 return ACK;
277 }
278 }
279 if (skip_prefix(line, "ERR ", &arg))
280 die(_("remote error: %s"), arg);
281 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
282 }
283
284 static void send_request(struct fetch_pack_args *args,
285 int fd, struct strbuf *buf)
286 {
287 if (args->stateless_rpc) {
288 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
289 packet_flush(fd);
290 } else
291 write_or_die(fd, buf->buf, buf->len);
292 }
293
294 static void insert_one_alternate_object(struct object *obj)
295 {
296 rev_list_insert_ref(NULL, obj->oid.hash);
297 }
298
299 #define INITIAL_FLUSH 16
300 #define PIPESAFE_FLUSH 32
301 #define LARGE_FLUSH 16384
302
303 static int next_flush(struct fetch_pack_args *args, int count)
304 {
305 if (args->stateless_rpc) {
306 if (count < LARGE_FLUSH)
307 count <<= 1;
308 else
309 count = count * 11 / 10;
310 } else {
311 if (count < PIPESAFE_FLUSH)
312 count <<= 1;
313 else
314 count += PIPESAFE_FLUSH;
315 }
316 return count;
317 }
318
319 static int find_common(struct fetch_pack_args *args,
320 int fd[2], unsigned char *result_sha1,
321 struct ref *refs)
322 {
323 int fetching;
324 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
325 const unsigned char *sha1;
326 unsigned in_vain = 0;
327 int got_continue = 0;
328 int got_ready = 0;
329 struct strbuf req_buf = STRBUF_INIT;
330 size_t state_len = 0;
331
332 if (args->stateless_rpc && multi_ack == 1)
333 die(_("--stateless-rpc requires multi_ack_detailed"));
334 if (marked)
335 for_each_ref(clear_marks, NULL);
336 marked = 1;
337
338 for_each_ref(rev_list_insert_ref_oid, NULL);
339 for_each_cached_alternate(insert_one_alternate_object);
340
341 fetching = 0;
342 for ( ; refs ; refs = refs->next) {
343 unsigned char *remote = refs->old_oid.hash;
344 const char *remote_hex;
345 struct object *o;
346
347 /*
348 * If that object is complete (i.e. it is an ancestor of a
349 * local ref), we tell them we have it but do not have to
350 * tell them about its ancestors, which they already know
351 * about.
352 *
353 * We use lookup_object here because we are only
354 * interested in the case we *know* the object is
355 * reachable and we have already scanned it.
356 */
357 if (((o = lookup_object(remote)) != NULL) &&
358 (o->flags & COMPLETE)) {
359 continue;
360 }
361
362 remote_hex = sha1_to_hex(remote);
363 if (!fetching) {
364 struct strbuf c = STRBUF_INIT;
365 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
366 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
367 if (no_done) strbuf_addstr(&c, " no-done");
368 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
369 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
370 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
371 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
372 if (args->no_progress) strbuf_addstr(&c, " no-progress");
373 if (args->include_tag) strbuf_addstr(&c, " include-tag");
374 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
375 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
376 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
377 if (agent_supported) strbuf_addf(&c, " agent=%s",
378 git_user_agent_sanitized());
379 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
380 strbuf_release(&c);
381 } else
382 packet_buf_write(&req_buf, "want %s\n", remote_hex);
383 fetching++;
384 }
385
386 if (!fetching) {
387 strbuf_release(&req_buf);
388 packet_flush(fd[1]);
389 return 1;
390 }
391
392 if (is_repository_shallow())
393 write_shallow_commits(&req_buf, 1, NULL);
394 if (args->depth > 0)
395 packet_buf_write(&req_buf, "deepen %d", args->depth);
396 if (args->deepen_since) {
397 unsigned long max_age = approxidate(args->deepen_since);
398 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
399 }
400 if (args->deepen_not) {
401 int i;
402 for (i = 0; i < args->deepen_not->nr; i++) {
403 struct string_list_item *s = args->deepen_not->items + i;
404 packet_buf_write(&req_buf, "deepen-not %s", s->string);
405 }
406 }
407 packet_buf_flush(&req_buf);
408 state_len = req_buf.len;
409
410 if (args->deepen) {
411 char *line;
412 const char *arg;
413 unsigned char sha1[20];
414
415 send_request(args, fd[1], &req_buf);
416 while ((line = packet_read_line(fd[0], NULL))) {
417 if (skip_prefix(line, "shallow ", &arg)) {
418 if (get_sha1_hex(arg, sha1))
419 die(_("invalid shallow line: %s"), line);
420 register_shallow(sha1);
421 continue;
422 }
423 if (skip_prefix(line, "unshallow ", &arg)) {
424 if (get_sha1_hex(arg, sha1))
425 die(_("invalid unshallow line: %s"), line);
426 if (!lookup_object(sha1))
427 die(_("object not found: %s"), line);
428 /* make sure that it is parsed as shallow */
429 if (!parse_object(sha1))
430 die(_("error in object: %s"), line);
431 if (unregister_shallow(sha1))
432 die(_("no shallow found: %s"), line);
433 continue;
434 }
435 die(_("expected shallow/unshallow, got %s"), line);
436 }
437 } else if (!args->stateless_rpc)
438 send_request(args, fd[1], &req_buf);
439
440 if (!args->stateless_rpc) {
441 /* If we aren't using the stateless-rpc interface
442 * we don't need to retain the headers.
443 */
444 strbuf_setlen(&req_buf, 0);
445 state_len = 0;
446 }
447
448 flushes = 0;
449 retval = -1;
450 while ((sha1 = get_rev())) {
451 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
452 print_verbose(args, "have %s", sha1_to_hex(sha1));
453 in_vain++;
454 if (flush_at <= ++count) {
455 int ack;
456
457 packet_buf_flush(&req_buf);
458 send_request(args, fd[1], &req_buf);
459 strbuf_setlen(&req_buf, state_len);
460 flushes++;
461 flush_at = next_flush(args, count);
462
463 /*
464 * We keep one window "ahead" of the other side, and
465 * will wait for an ACK only on the next one
466 */
467 if (!args->stateless_rpc && count == INITIAL_FLUSH)
468 continue;
469
470 consume_shallow_list(args, fd[0]);
471 do {
472 ack = get_ack(fd[0], result_sha1);
473 if (ack)
474 print_verbose(args, _("got %s %d %s"), "ack",
475 ack, sha1_to_hex(result_sha1));
476 switch (ack) {
477 case ACK:
478 flushes = 0;
479 multi_ack = 0;
480 retval = 0;
481 goto done;
482 case ACK_common:
483 case ACK_ready:
484 case ACK_continue: {
485 struct commit *commit =
486 lookup_commit(result_sha1);
487 if (!commit)
488 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
489 if (args->stateless_rpc
490 && ack == ACK_common
491 && !(commit->object.flags & COMMON)) {
492 /* We need to replay the have for this object
493 * on the next RPC request so the peer knows
494 * it is in common with us.
495 */
496 const char *hex = sha1_to_hex(result_sha1);
497 packet_buf_write(&req_buf, "have %s\n", hex);
498 state_len = req_buf.len;
499 /*
500 * Reset in_vain because an ack
501 * for this commit has not been
502 * seen.
503 */
504 in_vain = 0;
505 } else if (!args->stateless_rpc
506 || ack != ACK_common)
507 in_vain = 0;
508 mark_common(commit, 0, 1);
509 retval = 0;
510 got_continue = 1;
511 if (ack == ACK_ready) {
512 clear_prio_queue(&rev_list);
513 got_ready = 1;
514 }
515 break;
516 }
517 }
518 } while (ack);
519 flushes--;
520 if (got_continue && MAX_IN_VAIN < in_vain) {
521 print_verbose(args, _("giving up"));
522 break; /* give up */
523 }
524 }
525 }
526 done:
527 if (!got_ready || !no_done) {
528 packet_buf_write(&req_buf, "done\n");
529 send_request(args, fd[1], &req_buf);
530 }
531 print_verbose(args, _("done"));
532 if (retval != 0) {
533 multi_ack = 0;
534 flushes++;
535 }
536 strbuf_release(&req_buf);
537
538 if (!got_ready || !no_done)
539 consume_shallow_list(args, fd[0]);
540 while (flushes || multi_ack) {
541 int ack = get_ack(fd[0], result_sha1);
542 if (ack) {
543 print_verbose(args, _("got %s (%d) %s"), "ack",
544 ack, sha1_to_hex(result_sha1));
545 if (ack == ACK)
546 return 0;
547 multi_ack = 1;
548 continue;
549 }
550 flushes--;
551 }
552 /* it is no error to fetch into a completely empty repo */
553 return count ? retval : 0;
554 }
555
556 static struct commit_list *complete;
557
558 static int mark_complete(const unsigned char *sha1)
559 {
560 struct object *o = parse_object(sha1);
561
562 while (o && o->type == OBJ_TAG) {
563 struct tag *t = (struct tag *) o;
564 if (!t->tagged)
565 break; /* broken repository */
566 o->flags |= COMPLETE;
567 o = parse_object(t->tagged->oid.hash);
568 }
569 if (o && o->type == OBJ_COMMIT) {
570 struct commit *commit = (struct commit *)o;
571 if (!(commit->object.flags & COMPLETE)) {
572 commit->object.flags |= COMPLETE;
573 commit_list_insert(commit, &complete);
574 }
575 }
576 return 0;
577 }
578
579 static int mark_complete_oid(const char *refname, const struct object_id *oid,
580 int flag, void *cb_data)
581 {
582 return mark_complete(oid->hash);
583 }
584
585 static void mark_recent_complete_commits(struct fetch_pack_args *args,
586 unsigned long cutoff)
587 {
588 while (complete && cutoff <= complete->item->date) {
589 print_verbose(args, _("Marking %s as complete"),
590 oid_to_hex(&complete->item->object.oid));
591 pop_most_recent_commit(&complete, COMPLETE);
592 }
593 }
594
595 static void filter_refs(struct fetch_pack_args *args,
596 struct ref **refs,
597 struct ref **sought, int nr_sought)
598 {
599 struct ref *newlist = NULL;
600 struct ref **newtail = &newlist;
601 struct ref *ref, *next;
602 int i;
603
604 i = 0;
605 for (ref = *refs; ref; ref = next) {
606 int keep = 0;
607 next = ref->next;
608
609 if (starts_with(ref->name, "refs/") &&
610 check_refname_format(ref->name, 0))
611 ; /* trash */
612 else {
613 while (i < nr_sought) {
614 int cmp = strcmp(ref->name, sought[i]->name);
615 if (cmp < 0)
616 break; /* definitely do not have it */
617 else if (cmp == 0) {
618 keep = 1; /* definitely have it */
619 sought[i]->match_status = REF_MATCHED;
620 }
621 i++;
622 }
623 }
624
625 if (!keep && args->fetch_all &&
626 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
627 keep = 1;
628
629 if (keep) {
630 *newtail = ref;
631 ref->next = NULL;
632 newtail = &ref->next;
633 } else {
634 free(ref);
635 }
636 }
637
638 /* Append unmatched requests to the list */
639 for (i = 0; i < nr_sought; i++) {
640 unsigned char sha1[20];
641
642 ref = sought[i];
643 if (ref->match_status != REF_NOT_MATCHED)
644 continue;
645 if (get_sha1_hex(ref->name, sha1) ||
646 ref->name[40] != '\0' ||
647 hashcmp(sha1, ref->old_oid.hash))
648 continue;
649
650 if ((allow_unadvertised_object_request &
651 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
652 ref->match_status = REF_MATCHED;
653 *newtail = copy_ref(ref);
654 newtail = &(*newtail)->next;
655 } else {
656 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
657 }
658 }
659 *refs = newlist;
660 }
661
662 static void mark_alternate_complete(struct object *obj)
663 {
664 mark_complete(obj->oid.hash);
665 }
666
667 static int everything_local(struct fetch_pack_args *args,
668 struct ref **refs,
669 struct ref **sought, int nr_sought)
670 {
671 struct ref *ref;
672 int retval;
673 unsigned long cutoff = 0;
674
675 save_commit_buffer = 0;
676
677 for (ref = *refs; ref; ref = ref->next) {
678 struct object *o;
679
680 if (!has_object_file(&ref->old_oid))
681 continue;
682
683 o = parse_object(ref->old_oid.hash);
684 if (!o)
685 continue;
686
687 /* We already have it -- which may mean that we were
688 * in sync with the other side at some time after
689 * that (it is OK if we guess wrong here).
690 */
691 if (o->type == OBJ_COMMIT) {
692 struct commit *commit = (struct commit *)o;
693 if (!cutoff || cutoff < commit->date)
694 cutoff = commit->date;
695 }
696 }
697
698 if (!args->deepen) {
699 for_each_ref(mark_complete_oid, NULL);
700 for_each_cached_alternate(mark_alternate_complete);
701 commit_list_sort_by_date(&complete);
702 if (cutoff)
703 mark_recent_complete_commits(args, cutoff);
704 }
705
706 /*
707 * Mark all complete remote refs as common refs.
708 * Don't mark them common yet; the server has to be told so first.
709 */
710 for (ref = *refs; ref; ref = ref->next) {
711 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
712 NULL, 0);
713
714 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
715 continue;
716
717 if (!(o->flags & SEEN)) {
718 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
719
720 mark_common((struct commit *)o, 1, 1);
721 }
722 }
723
724 filter_refs(args, refs, sought, nr_sought);
725
726 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
727 const unsigned char *remote = ref->old_oid.hash;
728 struct object *o;
729
730 o = lookup_object(remote);
731 if (!o || !(o->flags & COMPLETE)) {
732 retval = 0;
733 print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
734 ref->name);
735 continue;
736 }
737 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
738 ref->name);
739 }
740 return retval;
741 }
742
743 static int sideband_demux(int in, int out, void *data)
744 {
745 int *xd = data;
746 int ret;
747
748 ret = recv_sideband("fetch-pack", xd[0], out);
749 close(out);
750 return ret;
751 }
752
753 static int get_pack(struct fetch_pack_args *args,
754 int xd[2], char **pack_lockfile)
755 {
756 struct async demux;
757 int do_keep = args->keep_pack;
758 const char *cmd_name;
759 struct pack_header header;
760 int pass_header = 0;
761 struct child_process cmd = CHILD_PROCESS_INIT;
762 int ret;
763
764 memset(&demux, 0, sizeof(demux));
765 if (use_sideband) {
766 /* xd[] is talking with upload-pack; subprocess reads from
767 * xd[0], spits out band#2 to stderr, and feeds us band#1
768 * through demux->out.
769 */
770 demux.proc = sideband_demux;
771 demux.data = xd;
772 demux.out = -1;
773 demux.isolate_sigpipe = 1;
774 if (start_async(&demux))
775 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
776 }
777 else
778 demux.out = xd[0];
779
780 if (!args->keep_pack && unpack_limit) {
781
782 if (read_pack_header(demux.out, &header))
783 die(_("protocol error: bad pack header"));
784 pass_header = 1;
785 if (ntohl(header.hdr_entries) < unpack_limit)
786 do_keep = 0;
787 else
788 do_keep = 1;
789 }
790
791 if (alternate_shallow_file) {
792 argv_array_push(&cmd.args, "--shallow-file");
793 argv_array_push(&cmd.args, alternate_shallow_file);
794 }
795
796 if (do_keep) {
797 if (pack_lockfile)
798 cmd.out = -1;
799 cmd_name = "index-pack";
800 argv_array_push(&cmd.args, cmd_name);
801 argv_array_push(&cmd.args, "--stdin");
802 if (!args->quiet && !args->no_progress)
803 argv_array_push(&cmd.args, "-v");
804 if (args->use_thin_pack)
805 argv_array_push(&cmd.args, "--fix-thin");
806 if (args->lock_pack || unpack_limit) {
807 char hostname[256];
808 if (gethostname(hostname, sizeof(hostname)))
809 xsnprintf(hostname, sizeof(hostname), "localhost");
810 argv_array_pushf(&cmd.args,
811 "--keep=fetch-pack %"PRIuMAX " on %s",
812 (uintmax_t)getpid(), hostname);
813 }
814 if (args->check_self_contained_and_connected)
815 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
816 }
817 else {
818 cmd_name = "unpack-objects";
819 argv_array_push(&cmd.args, cmd_name);
820 if (args->quiet || args->no_progress)
821 argv_array_push(&cmd.args, "-q");
822 args->check_self_contained_and_connected = 0;
823 }
824
825 if (pass_header)
826 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
827 ntohl(header.hdr_version),
828 ntohl(header.hdr_entries));
829 if (fetch_fsck_objects >= 0
830 ? fetch_fsck_objects
831 : transfer_fsck_objects >= 0
832 ? transfer_fsck_objects
833 : 0)
834 argv_array_push(&cmd.args, "--strict");
835
836 cmd.in = demux.out;
837 cmd.git_cmd = 1;
838 if (start_command(&cmd))
839 die(_("fetch-pack: unable to fork off %s"), cmd_name);
840 if (do_keep && pack_lockfile) {
841 *pack_lockfile = index_pack_lockfile(cmd.out);
842 close(cmd.out);
843 }
844
845 if (!use_sideband)
846 /* Closed by start_command() */
847 xd[0] = -1;
848
849 ret = finish_command(&cmd);
850 if (!ret || (args->check_self_contained_and_connected && ret == 1))
851 args->self_contained_and_connected =
852 args->check_self_contained_and_connected &&
853 ret == 0;
854 else
855 die(_("%s failed"), cmd_name);
856 if (use_sideband && finish_async(&demux))
857 die(_("error in sideband demultiplexer"));
858 return 0;
859 }
860
861 static int cmp_ref_by_name(const void *a_, const void *b_)
862 {
863 const struct ref *a = *((const struct ref **)a_);
864 const struct ref *b = *((const struct ref **)b_);
865 return strcmp(a->name, b->name);
866 }
867
868 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
869 int fd[2],
870 const struct ref *orig_ref,
871 struct ref **sought, int nr_sought,
872 struct shallow_info *si,
873 char **pack_lockfile)
874 {
875 struct ref *ref = copy_ref_list(orig_ref);
876 unsigned char sha1[20];
877 const char *agent_feature;
878 int agent_len;
879
880 sort_ref_list(&ref, ref_compare_name);
881 QSORT(sought, nr_sought, cmp_ref_by_name);
882
883 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
884 die(_("Server does not support shallow clients"));
885 if (args->depth > 0 || args->deepen_since || args->deepen_not)
886 args->deepen = 1;
887 if (server_supports("multi_ack_detailed")) {
888 print_verbose(args, _("Server supports multi_ack_detailed"));
889 multi_ack = 2;
890 if (server_supports("no-done")) {
891 print_verbose(args, _("Server supports no-done"));
892 if (args->stateless_rpc)
893 no_done = 1;
894 }
895 }
896 else if (server_supports("multi_ack")) {
897 print_verbose(args, _("Server supports multi_ack"));
898 multi_ack = 1;
899 }
900 if (server_supports("side-band-64k")) {
901 print_verbose(args, _("Server supports side-band-64k"));
902 use_sideband = 2;
903 }
904 else if (server_supports("side-band")) {
905 print_verbose(args, _("Server supports side-band"));
906 use_sideband = 1;
907 }
908 if (server_supports("allow-tip-sha1-in-want")) {
909 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
910 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
911 }
912 if (server_supports("allow-reachable-sha1-in-want")) {
913 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
914 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
915 }
916 if (!server_supports("thin-pack"))
917 args->use_thin_pack = 0;
918 if (!server_supports("no-progress"))
919 args->no_progress = 0;
920 if (!server_supports("include-tag"))
921 args->include_tag = 0;
922 if (server_supports("ofs-delta"))
923 print_verbose(args, _("Server supports ofs-delta"));
924 else
925 prefer_ofs_delta = 0;
926
927 if ((agent_feature = server_feature_value("agent", &agent_len))) {
928 agent_supported = 1;
929 if (agent_len)
930 print_verbose(args, _("Server version is %.*s"),
931 agent_len, agent_feature);
932 }
933 if (server_supports("deepen-since"))
934 deepen_since_ok = 1;
935 else if (args->deepen_since)
936 die(_("Server does not support --shallow-since"));
937 if (server_supports("deepen-not"))
938 deepen_not_ok = 1;
939 else if (args->deepen_not)
940 die(_("Server does not support --shallow-exclude"));
941 if (!server_supports("deepen-relative") && args->deepen_relative)
942 die(_("Server does not support --deepen"));
943
944 if (everything_local(args, &ref, sought, nr_sought)) {
945 packet_flush(fd[1]);
946 goto all_done;
947 }
948 if (find_common(args, fd, sha1, ref) < 0)
949 if (!args->keep_pack)
950 /* When cloning, it is not unusual to have
951 * no common commit.
952 */
953 warning(_("no common commits"));
954
955 if (args->stateless_rpc)
956 packet_flush(fd[1]);
957 if (args->deepen)
958 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
959 NULL);
960 else if (si->nr_ours || si->nr_theirs)
961 alternate_shallow_file = setup_temporary_shallow(si->shallow);
962 else
963 alternate_shallow_file = NULL;
964 if (get_pack(args, fd, pack_lockfile))
965 die(_("git fetch-pack: fetch failed."));
966
967 all_done:
968 return ref;
969 }
970
971 static void fetch_pack_config(void)
972 {
973 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
974 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
975 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
976 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
977 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
978
979 git_config(git_default_config, NULL);
980 }
981
982 static void fetch_pack_setup(void)
983 {
984 static int did_setup;
985 if (did_setup)
986 return;
987 fetch_pack_config();
988 if (0 <= transfer_unpack_limit)
989 unpack_limit = transfer_unpack_limit;
990 else if (0 <= fetch_unpack_limit)
991 unpack_limit = fetch_unpack_limit;
992 did_setup = 1;
993 }
994
995 static int remove_duplicates_in_refs(struct ref **ref, int nr)
996 {
997 struct string_list names = STRING_LIST_INIT_NODUP;
998 int src, dst;
999
1000 for (src = dst = 0; src < nr; src++) {
1001 struct string_list_item *item;
1002 item = string_list_insert(&names, ref[src]->name);
1003 if (item->util)
1004 continue; /* already have it */
1005 item->util = ref[src];
1006 if (src != dst)
1007 ref[dst] = ref[src];
1008 dst++;
1009 }
1010 for (src = dst; src < nr; src++)
1011 ref[src] = NULL;
1012 string_list_clear(&names, 0);
1013 return dst;
1014 }
1015
1016 static void update_shallow(struct fetch_pack_args *args,
1017 struct ref **sought, int nr_sought,
1018 struct shallow_info *si)
1019 {
1020 struct oid_array ref = OID_ARRAY_INIT;
1021 int *status;
1022 int i;
1023
1024 if (args->deepen && alternate_shallow_file) {
1025 if (*alternate_shallow_file == '\0') { /* --unshallow */
1026 unlink_or_warn(git_path_shallow());
1027 rollback_lock_file(&shallow_lock);
1028 } else
1029 commit_lock_file(&shallow_lock);
1030 return;
1031 }
1032
1033 if (!si->shallow || !si->shallow->nr)
1034 return;
1035
1036 if (args->cloning) {
1037 /*
1038 * remote is shallow, but this is a clone, there are
1039 * no objects in repo to worry about. Accept any
1040 * shallow points that exist in the pack (iow in repo
1041 * after get_pack() and reprepare_packed_git())
1042 */
1043 struct oid_array extra = OID_ARRAY_INIT;
1044 struct object_id *oid = si->shallow->oid;
1045 for (i = 0; i < si->shallow->nr; i++)
1046 if (has_object_file(&oid[i]))
1047 oid_array_append(&extra, &oid[i]);
1048 if (extra.nr) {
1049 setup_alternate_shallow(&shallow_lock,
1050 &alternate_shallow_file,
1051 &extra);
1052 commit_lock_file(&shallow_lock);
1053 }
1054 oid_array_clear(&extra);
1055 return;
1056 }
1057
1058 if (!si->nr_ours && !si->nr_theirs)
1059 return;
1060
1061 remove_nonexistent_theirs_shallow(si);
1062 if (!si->nr_ours && !si->nr_theirs)
1063 return;
1064 for (i = 0; i < nr_sought; i++)
1065 oid_array_append(&ref, &sought[i]->old_oid);
1066 si->ref = &ref;
1067
1068 if (args->update_shallow) {
1069 /*
1070 * remote is also shallow, .git/shallow may be updated
1071 * so all refs can be accepted. Make sure we only add
1072 * shallow roots that are actually reachable from new
1073 * refs.
1074 */
1075 struct oid_array extra = OID_ARRAY_INIT;
1076 struct object_id *oid = si->shallow->oid;
1077 assign_shallow_commits_to_refs(si, NULL, NULL);
1078 if (!si->nr_ours && !si->nr_theirs) {
1079 oid_array_clear(&ref);
1080 return;
1081 }
1082 for (i = 0; i < si->nr_ours; i++)
1083 oid_array_append(&extra, &oid[si->ours[i]]);
1084 for (i = 0; i < si->nr_theirs; i++)
1085 oid_array_append(&extra, &oid[si->theirs[i]]);
1086 setup_alternate_shallow(&shallow_lock,
1087 &alternate_shallow_file,
1088 &extra);
1089 commit_lock_file(&shallow_lock);
1090 oid_array_clear(&extra);
1091 oid_array_clear(&ref);
1092 return;
1093 }
1094
1095 /*
1096 * remote is also shallow, check what ref is safe to update
1097 * without updating .git/shallow
1098 */
1099 status = xcalloc(nr_sought, sizeof(*status));
1100 assign_shallow_commits_to_refs(si, NULL, status);
1101 if (si->nr_ours || si->nr_theirs) {
1102 for (i = 0; i < nr_sought; i++)
1103 if (status[i])
1104 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1105 }
1106 free(status);
1107 oid_array_clear(&ref);
1108 }
1109
1110 struct ref *fetch_pack(struct fetch_pack_args *args,
1111 int fd[], struct child_process *conn,
1112 const struct ref *ref,
1113 const char *dest,
1114 struct ref **sought, int nr_sought,
1115 struct oid_array *shallow,
1116 char **pack_lockfile)
1117 {
1118 struct ref *ref_cpy;
1119 struct shallow_info si;
1120
1121 fetch_pack_setup();
1122 if (nr_sought)
1123 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1124
1125 if (!ref) {
1126 packet_flush(fd[1]);
1127 die(_("no matching remote head"));
1128 }
1129 prepare_shallow_info(&si, shallow);
1130 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1131 &si, pack_lockfile);
1132 reprepare_packed_git();
1133 update_shallow(args, sought, nr_sought, &si);
1134 clear_shallow_info(&si);
1135 return ref_cpy;
1136 }
1137
1138 int report_unmatched_refs(struct ref **sought, int nr_sought)
1139 {
1140 int i, ret = 0;
1141
1142 for (i = 0; i < nr_sought; i++) {
1143 if (!sought[i])
1144 continue;
1145 switch (sought[i]->match_status) {
1146 case REF_MATCHED:
1147 continue;
1148 case REF_NOT_MATCHED:
1149 error(_("no such remote ref %s"), sought[i]->name);
1150 break;
1151 case REF_UNADVERTISED_NOT_ALLOWED:
1152 error(_("Server does not allow request for unadvertised object %s"),
1153 sought[i]->name);
1154 break;
1155 }
1156 ret = 1;
1157 }
1158 return ret;
1159 }