Merge branch 'ti/fetch-everything-local-optim'
[git/git.git] / fetch-pack.c
1 #include "cache.h"
2 #include "config.h"
3 #include "lockfile.h"
4 #include "refs.h"
5 #include "pkt-line.h"
6 #include "commit.h"
7 #include "tag.h"
8 #include "exec_cmd.h"
9 #include "pack.h"
10 #include "sideband.h"
11 #include "fetch-pack.h"
12 #include "remote.h"
13 #include "run-command.h"
14 #include "connect.h"
15 #include "transport.h"
16 #include "version.h"
17 #include "prio-queue.h"
18 #include "sha1-array.h"
19 #include "oidset.h"
20 #include "packfile.h"
21
22 static int transfer_unpack_limit = -1;
23 static int fetch_unpack_limit = -1;
24 static int unpack_limit = 100;
25 static int prefer_ofs_delta = 1;
26 static int no_done;
27 static int deepen_since_ok;
28 static int deepen_not_ok;
29 static int fetch_fsck_objects = -1;
30 static int transfer_fsck_objects = -1;
31 static int agent_supported;
32 static int server_supports_filtering;
33 static struct lock_file shallow_lock;
34 static const char *alternate_shallow_file;
35
36 /* Remember to update object flag allocation in object.h */
37 #define COMPLETE (1U << 0)
38 #define COMMON (1U << 1)
39 #define COMMON_REF (1U << 2)
40 #define SEEN (1U << 3)
41 #define POPPED (1U << 4)
42 #define ALTERNATE (1U << 5)
43
44 static int marked;
45
46 /*
47 * After sending this many "have"s if we do not get any new ACK , we
48 * give up traversing our history.
49 */
50 #define MAX_IN_VAIN 256
51
52 static struct prio_queue rev_list = { compare_commits_by_commit_date };
53 static int non_common_revs, multi_ack, use_sideband;
54 /* Allow specifying sha1 if it is a ref tip. */
55 #define ALLOW_TIP_SHA1 01
56 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
57 #define ALLOW_REACHABLE_SHA1 02
58 static unsigned int allow_unadvertised_object_request;
59
60 __attribute__((format (printf, 2, 3)))
61 static inline void print_verbose(const struct fetch_pack_args *args,
62 const char *fmt, ...)
63 {
64 va_list params;
65
66 if (!args->verbose)
67 return;
68
69 va_start(params, fmt);
70 vfprintf(stderr, fmt, params);
71 va_end(params);
72 fputc('\n', stderr);
73 }
74
75 struct alternate_object_cache {
76 struct object **items;
77 size_t nr, alloc;
78 };
79
80 static void cache_one_alternate(const char *refname,
81 const struct object_id *oid,
82 void *vcache)
83 {
84 struct alternate_object_cache *cache = vcache;
85 struct object *obj = parse_object(oid);
86
87 if (!obj || (obj->flags & ALTERNATE))
88 return;
89
90 obj->flags |= ALTERNATE;
91 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
92 cache->items[cache->nr++] = obj;
93 }
94
95 static void for_each_cached_alternate(void (*cb)(struct object *))
96 {
97 static int initialized;
98 static struct alternate_object_cache cache;
99 size_t i;
100
101 if (!initialized) {
102 for_each_alternate_ref(cache_one_alternate, &cache);
103 initialized = 1;
104 }
105
106 for (i = 0; i < cache.nr; i++)
107 cb(cache.items[i]);
108 }
109
110 static void rev_list_push(struct commit *commit, int mark)
111 {
112 if (!(commit->object.flags & mark)) {
113 commit->object.flags |= mark;
114
115 if (parse_commit(commit))
116 return;
117
118 prio_queue_put(&rev_list, commit);
119
120 if (!(commit->object.flags & COMMON))
121 non_common_revs++;
122 }
123 }
124
125 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
126 {
127 struct object *o = deref_tag(parse_object(oid), refname, 0);
128
129 if (o && o->type == OBJ_COMMIT)
130 rev_list_push((struct commit *)o, SEEN);
131
132 return 0;
133 }
134
135 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
136 int flag, void *cb_data)
137 {
138 return rev_list_insert_ref(refname, oid);
139 }
140
141 static int clear_marks(const char *refname, const struct object_id *oid,
142 int flag, void *cb_data)
143 {
144 struct object *o = deref_tag(parse_object(oid), refname, 0);
145
146 if (o && o->type == OBJ_COMMIT)
147 clear_commit_marks((struct commit *)o,
148 COMMON | COMMON_REF | SEEN | POPPED);
149 return 0;
150 }
151
152 /*
153 This function marks a rev and its ancestors as common.
154 In some cases, it is desirable to mark only the ancestors (for example
155 when only the server does not yet know that they are common).
156 */
157
158 static void mark_common(struct commit *commit,
159 int ancestors_only, int dont_parse)
160 {
161 if (commit != NULL && !(commit->object.flags & COMMON)) {
162 struct object *o = (struct object *)commit;
163
164 if (!ancestors_only)
165 o->flags |= COMMON;
166
167 if (!(o->flags & SEEN))
168 rev_list_push(commit, SEEN);
169 else {
170 struct commit_list *parents;
171
172 if (!ancestors_only && !(o->flags & POPPED))
173 non_common_revs--;
174 if (!o->parsed && !dont_parse)
175 if (parse_commit(commit))
176 return;
177
178 for (parents = commit->parents;
179 parents;
180 parents = parents->next)
181 mark_common(parents->item, 0, dont_parse);
182 }
183 }
184 }
185
186 /*
187 Get the next rev to send, ignoring the common.
188 */
189
190 static const struct object_id *get_rev(void)
191 {
192 struct commit *commit = NULL;
193
194 while (commit == NULL) {
195 unsigned int mark;
196 struct commit_list *parents;
197
198 if (rev_list.nr == 0 || non_common_revs == 0)
199 return NULL;
200
201 commit = prio_queue_get(&rev_list);
202 parse_commit(commit);
203 parents = commit->parents;
204
205 commit->object.flags |= POPPED;
206 if (!(commit->object.flags & COMMON))
207 non_common_revs--;
208
209 if (commit->object.flags & COMMON) {
210 /* do not send "have", and ignore ancestors */
211 commit = NULL;
212 mark = COMMON | SEEN;
213 } else if (commit->object.flags & COMMON_REF)
214 /* send "have", and ignore ancestors */
215 mark = COMMON | SEEN;
216 else
217 /* send "have", also for its ancestors */
218 mark = SEEN;
219
220 while (parents) {
221 if (!(parents->item->object.flags & SEEN))
222 rev_list_push(parents->item, mark);
223 if (mark & COMMON)
224 mark_common(parents->item, 1, 0);
225 parents = parents->next;
226 }
227 }
228
229 return &commit->object.oid;
230 }
231
232 enum ack_type {
233 NAK = 0,
234 ACK,
235 ACK_continue,
236 ACK_common,
237 ACK_ready
238 };
239
240 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
241 {
242 if (args->stateless_rpc && args->deepen) {
243 /* If we sent a depth we will get back "duplicate"
244 * shallow and unshallow commands every time there
245 * is a block of have lines exchanged.
246 */
247 char *line;
248 while ((line = packet_read_line(fd, NULL))) {
249 if (starts_with(line, "shallow "))
250 continue;
251 if (starts_with(line, "unshallow "))
252 continue;
253 die(_("git fetch-pack: expected shallow list"));
254 }
255 }
256 }
257
258 static enum ack_type get_ack(int fd, struct object_id *result_oid)
259 {
260 int len;
261 char *line = packet_read_line(fd, &len);
262 const char *arg;
263
264 if (!line)
265 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
266 if (!strcmp(line, "NAK"))
267 return NAK;
268 if (skip_prefix(line, "ACK ", &arg)) {
269 if (!get_oid_hex(arg, result_oid)) {
270 arg += 40;
271 len -= arg - line;
272 if (len < 1)
273 return ACK;
274 if (strstr(arg, "continue"))
275 return ACK_continue;
276 if (strstr(arg, "common"))
277 return ACK_common;
278 if (strstr(arg, "ready"))
279 return ACK_ready;
280 return ACK;
281 }
282 }
283 if (skip_prefix(line, "ERR ", &arg))
284 die(_("remote error: %s"), arg);
285 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
286 }
287
288 static void send_request(struct fetch_pack_args *args,
289 int fd, struct strbuf *buf)
290 {
291 if (args->stateless_rpc) {
292 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
293 packet_flush(fd);
294 } else
295 write_or_die(fd, buf->buf, buf->len);
296 }
297
298 static void insert_one_alternate_object(struct object *obj)
299 {
300 rev_list_insert_ref(NULL, &obj->oid);
301 }
302
303 #define INITIAL_FLUSH 16
304 #define PIPESAFE_FLUSH 32
305 #define LARGE_FLUSH 16384
306
307 static int next_flush(struct fetch_pack_args *args, int count)
308 {
309 if (args->stateless_rpc) {
310 if (count < LARGE_FLUSH)
311 count <<= 1;
312 else
313 count = count * 11 / 10;
314 } else {
315 if (count < PIPESAFE_FLUSH)
316 count <<= 1;
317 else
318 count += PIPESAFE_FLUSH;
319 }
320 return count;
321 }
322
323 static int find_common(struct fetch_pack_args *args,
324 int fd[2], struct object_id *result_oid,
325 struct ref *refs)
326 {
327 int fetching;
328 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
329 const struct object_id *oid;
330 unsigned in_vain = 0;
331 int got_continue = 0;
332 int got_ready = 0;
333 struct strbuf req_buf = STRBUF_INIT;
334 size_t state_len = 0;
335
336 if (args->stateless_rpc && multi_ack == 1)
337 die(_("--stateless-rpc requires multi_ack_detailed"));
338 if (marked)
339 for_each_ref(clear_marks, NULL);
340 marked = 1;
341
342 for_each_ref(rev_list_insert_ref_oid, NULL);
343 for_each_cached_alternate(insert_one_alternate_object);
344
345 fetching = 0;
346 for ( ; refs ; refs = refs->next) {
347 struct object_id *remote = &refs->old_oid;
348 const char *remote_hex;
349 struct object *o;
350
351 /*
352 * If that object is complete (i.e. it is an ancestor of a
353 * local ref), we tell them we have it but do not have to
354 * tell them about its ancestors, which they already know
355 * about.
356 *
357 * We use lookup_object here because we are only
358 * interested in the case we *know* the object is
359 * reachable and we have already scanned it.
360 */
361 if (((o = lookup_object(remote->hash)) != NULL) &&
362 (o->flags & COMPLETE)) {
363 continue;
364 }
365
366 remote_hex = oid_to_hex(remote);
367 if (!fetching) {
368 struct strbuf c = STRBUF_INIT;
369 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
370 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
371 if (no_done) strbuf_addstr(&c, " no-done");
372 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
373 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
374 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
375 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
376 if (args->no_progress) strbuf_addstr(&c, " no-progress");
377 if (args->include_tag) strbuf_addstr(&c, " include-tag");
378 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
379 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
380 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
381 if (agent_supported) strbuf_addf(&c, " agent=%s",
382 git_user_agent_sanitized());
383 if (args->filter_options.choice)
384 strbuf_addstr(&c, " filter");
385 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
386 strbuf_release(&c);
387 } else
388 packet_buf_write(&req_buf, "want %s\n", remote_hex);
389 fetching++;
390 }
391
392 if (!fetching) {
393 strbuf_release(&req_buf);
394 packet_flush(fd[1]);
395 return 1;
396 }
397
398 if (is_repository_shallow())
399 write_shallow_commits(&req_buf, 1, NULL);
400 if (args->depth > 0)
401 packet_buf_write(&req_buf, "deepen %d", args->depth);
402 if (args->deepen_since) {
403 timestamp_t max_age = approxidate(args->deepen_since);
404 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
405 }
406 if (args->deepen_not) {
407 int i;
408 for (i = 0; i < args->deepen_not->nr; i++) {
409 struct string_list_item *s = args->deepen_not->items + i;
410 packet_buf_write(&req_buf, "deepen-not %s", s->string);
411 }
412 }
413 if (server_supports_filtering && args->filter_options.choice)
414 packet_buf_write(&req_buf, "filter %s",
415 args->filter_options.filter_spec);
416 packet_buf_flush(&req_buf);
417 state_len = req_buf.len;
418
419 if (args->deepen) {
420 char *line;
421 const char *arg;
422 struct object_id oid;
423
424 send_request(args, fd[1], &req_buf);
425 while ((line = packet_read_line(fd[0], NULL))) {
426 if (skip_prefix(line, "shallow ", &arg)) {
427 if (get_oid_hex(arg, &oid))
428 die(_("invalid shallow line: %s"), line);
429 register_shallow(&oid);
430 continue;
431 }
432 if (skip_prefix(line, "unshallow ", &arg)) {
433 if (get_oid_hex(arg, &oid))
434 die(_("invalid unshallow line: %s"), line);
435 if (!lookup_object(oid.hash))
436 die(_("object not found: %s"), line);
437 /* make sure that it is parsed as shallow */
438 if (!parse_object(&oid))
439 die(_("error in object: %s"), line);
440 if (unregister_shallow(&oid))
441 die(_("no shallow found: %s"), line);
442 continue;
443 }
444 die(_("expected shallow/unshallow, got %s"), line);
445 }
446 } else if (!args->stateless_rpc)
447 send_request(args, fd[1], &req_buf);
448
449 if (!args->stateless_rpc) {
450 /* If we aren't using the stateless-rpc interface
451 * we don't need to retain the headers.
452 */
453 strbuf_setlen(&req_buf, 0);
454 state_len = 0;
455 }
456
457 flushes = 0;
458 retval = -1;
459 if (args->no_dependents)
460 goto done;
461 while ((oid = get_rev())) {
462 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
463 print_verbose(args, "have %s", oid_to_hex(oid));
464 in_vain++;
465 if (flush_at <= ++count) {
466 int ack;
467
468 packet_buf_flush(&req_buf);
469 send_request(args, fd[1], &req_buf);
470 strbuf_setlen(&req_buf, state_len);
471 flushes++;
472 flush_at = next_flush(args, count);
473
474 /*
475 * We keep one window "ahead" of the other side, and
476 * will wait for an ACK only on the next one
477 */
478 if (!args->stateless_rpc && count == INITIAL_FLUSH)
479 continue;
480
481 consume_shallow_list(args, fd[0]);
482 do {
483 ack = get_ack(fd[0], result_oid);
484 if (ack)
485 print_verbose(args, _("got %s %d %s"), "ack",
486 ack, oid_to_hex(result_oid));
487 switch (ack) {
488 case ACK:
489 flushes = 0;
490 multi_ack = 0;
491 retval = 0;
492 goto done;
493 case ACK_common:
494 case ACK_ready:
495 case ACK_continue: {
496 struct commit *commit =
497 lookup_commit(result_oid);
498 if (!commit)
499 die(_("invalid commit %s"), oid_to_hex(result_oid));
500 if (args->stateless_rpc
501 && ack == ACK_common
502 && !(commit->object.flags & COMMON)) {
503 /* We need to replay the have for this object
504 * on the next RPC request so the peer knows
505 * it is in common with us.
506 */
507 const char *hex = oid_to_hex(result_oid);
508 packet_buf_write(&req_buf, "have %s\n", hex);
509 state_len = req_buf.len;
510 /*
511 * Reset in_vain because an ack
512 * for this commit has not been
513 * seen.
514 */
515 in_vain = 0;
516 } else if (!args->stateless_rpc
517 || ack != ACK_common)
518 in_vain = 0;
519 mark_common(commit, 0, 1);
520 retval = 0;
521 got_continue = 1;
522 if (ack == ACK_ready) {
523 clear_prio_queue(&rev_list);
524 got_ready = 1;
525 }
526 break;
527 }
528 }
529 } while (ack);
530 flushes--;
531 if (got_continue && MAX_IN_VAIN < in_vain) {
532 print_verbose(args, _("giving up"));
533 break; /* give up */
534 }
535 }
536 }
537 done:
538 if (!got_ready || !no_done) {
539 packet_buf_write(&req_buf, "done\n");
540 send_request(args, fd[1], &req_buf);
541 }
542 print_verbose(args, _("done"));
543 if (retval != 0) {
544 multi_ack = 0;
545 flushes++;
546 }
547 strbuf_release(&req_buf);
548
549 if (!got_ready || !no_done)
550 consume_shallow_list(args, fd[0]);
551 while (flushes || multi_ack) {
552 int ack = get_ack(fd[0], result_oid);
553 if (ack) {
554 print_verbose(args, _("got %s (%d) %s"), "ack",
555 ack, oid_to_hex(result_oid));
556 if (ack == ACK)
557 return 0;
558 multi_ack = 1;
559 continue;
560 }
561 flushes--;
562 }
563 /* it is no error to fetch into a completely empty repo */
564 return count ? retval : 0;
565 }
566
567 static struct commit_list *complete;
568
569 static int mark_complete(const struct object_id *oid)
570 {
571 struct object *o = parse_object(oid);
572
573 while (o && o->type == OBJ_TAG) {
574 struct tag *t = (struct tag *) o;
575 if (!t->tagged)
576 break; /* broken repository */
577 o->flags |= COMPLETE;
578 o = parse_object(&t->tagged->oid);
579 }
580 if (o && o->type == OBJ_COMMIT) {
581 struct commit *commit = (struct commit *)o;
582 if (!(commit->object.flags & COMPLETE)) {
583 commit->object.flags |= COMPLETE;
584 commit_list_insert(commit, &complete);
585 }
586 }
587 return 0;
588 }
589
590 static int mark_complete_oid(const char *refname, const struct object_id *oid,
591 int flag, void *cb_data)
592 {
593 return mark_complete(oid);
594 }
595
596 static void mark_recent_complete_commits(struct fetch_pack_args *args,
597 timestamp_t cutoff)
598 {
599 while (complete && cutoff <= complete->item->date) {
600 print_verbose(args, _("Marking %s as complete"),
601 oid_to_hex(&complete->item->object.oid));
602 pop_most_recent_commit(&complete, COMPLETE);
603 }
604 }
605
606 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
607 {
608 for (; refs; refs = refs->next)
609 oidset_insert(oids, &refs->old_oid);
610 }
611
612 static int tip_oids_contain(struct oidset *tip_oids,
613 struct ref *unmatched, struct ref *newlist,
614 const struct object_id *id)
615 {
616 /*
617 * Note that this only looks at the ref lists the first time it's
618 * called. This works out in filter_refs() because even though it may
619 * add to "newlist" between calls, the additions will always be for
620 * oids that are already in the set.
621 */
622 if (!tip_oids->map.map.tablesize) {
623 add_refs_to_oidset(tip_oids, unmatched);
624 add_refs_to_oidset(tip_oids, newlist);
625 }
626 return oidset_contains(tip_oids, id);
627 }
628
629 static void filter_refs(struct fetch_pack_args *args,
630 struct ref **refs,
631 struct ref **sought, int nr_sought)
632 {
633 struct ref *newlist = NULL;
634 struct ref **newtail = &newlist;
635 struct ref *unmatched = NULL;
636 struct ref *ref, *next;
637 struct oidset tip_oids = OIDSET_INIT;
638 int i;
639
640 i = 0;
641 for (ref = *refs; ref; ref = next) {
642 int keep = 0;
643 next = ref->next;
644
645 if (starts_with(ref->name, "refs/") &&
646 check_refname_format(ref->name, 0))
647 ; /* trash */
648 else {
649 while (i < nr_sought) {
650 int cmp = strcmp(ref->name, sought[i]->name);
651 if (cmp < 0)
652 break; /* definitely do not have it */
653 else if (cmp == 0) {
654 keep = 1; /* definitely have it */
655 sought[i]->match_status = REF_MATCHED;
656 }
657 i++;
658 }
659 }
660
661 if (!keep && args->fetch_all &&
662 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
663 keep = 1;
664
665 if (keep) {
666 *newtail = ref;
667 ref->next = NULL;
668 newtail = &ref->next;
669 } else {
670 ref->next = unmatched;
671 unmatched = ref;
672 }
673 }
674
675 /* Append unmatched requests to the list */
676 for (i = 0; i < nr_sought; i++) {
677 struct object_id oid;
678 const char *p;
679
680 ref = sought[i];
681 if (ref->match_status != REF_NOT_MATCHED)
682 continue;
683 if (parse_oid_hex(ref->name, &oid, &p) ||
684 *p != '\0' ||
685 oidcmp(&oid, &ref->old_oid))
686 continue;
687
688 if ((allow_unadvertised_object_request &
689 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
690 tip_oids_contain(&tip_oids, unmatched, newlist,
691 &ref->old_oid)) {
692 ref->match_status = REF_MATCHED;
693 *newtail = copy_ref(ref);
694 newtail = &(*newtail)->next;
695 } else {
696 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
697 }
698 }
699
700 oidset_clear(&tip_oids);
701 for (ref = unmatched; ref; ref = next) {
702 next = ref->next;
703 free(ref);
704 }
705
706 *refs = newlist;
707 }
708
709 static void mark_alternate_complete(struct object *obj)
710 {
711 mark_complete(&obj->oid);
712 }
713
714 struct loose_object_iter {
715 struct oidset *loose_object_set;
716 struct ref *refs;
717 };
718
719 /*
720 * If the number of refs is not larger than the number of loose objects,
721 * this function stops inserting.
722 */
723 static int add_loose_objects_to_set(const struct object_id *oid,
724 const char *path,
725 void *data)
726 {
727 struct loose_object_iter *iter = data;
728 oidset_insert(iter->loose_object_set, oid);
729 if (iter->refs == NULL)
730 return 1;
731
732 iter->refs = iter->refs->next;
733 return 0;
734 }
735
736 static int everything_local(struct fetch_pack_args *args,
737 struct ref **refs,
738 struct ref **sought, int nr_sought)
739 {
740 struct ref *ref;
741 int retval;
742 int old_save_commit_buffer = save_commit_buffer;
743 timestamp_t cutoff = 0;
744 struct oidset loose_oid_set = OIDSET_INIT;
745 int use_oidset = 0;
746 struct loose_object_iter iter = {&loose_oid_set, *refs};
747
748 /* Enumerate all loose objects or know refs are not so many. */
749 use_oidset = !for_each_loose_object(add_loose_objects_to_set,
750 &iter, 0);
751
752 save_commit_buffer = 0;
753
754 for (ref = *refs; ref; ref = ref->next) {
755 struct object *o;
756 unsigned int flags = OBJECT_INFO_QUICK;
757
758 if (use_oidset &&
759 !oidset_contains(&loose_oid_set, &ref->old_oid)) {
760 /*
761 * I know this does not exist in the loose form,
762 * so check if it exists in a non-loose form.
763 */
764 flags |= OBJECT_INFO_IGNORE_LOOSE;
765 }
766
767 if (!has_object_file_with_flags(&ref->old_oid, flags))
768 continue;
769 o = parse_object(&ref->old_oid);
770 if (!o)
771 continue;
772
773 /* We already have it -- which may mean that we were
774 * in sync with the other side at some time after
775 * that (it is OK if we guess wrong here).
776 */
777 if (o->type == OBJ_COMMIT) {
778 struct commit *commit = (struct commit *)o;
779 if (!cutoff || cutoff < commit->date)
780 cutoff = commit->date;
781 }
782 }
783
784 oidset_clear(&loose_oid_set);
785
786 if (!args->no_dependents) {
787 if (!args->deepen) {
788 for_each_ref(mark_complete_oid, NULL);
789 for_each_cached_alternate(mark_alternate_complete);
790 commit_list_sort_by_date(&complete);
791 if (cutoff)
792 mark_recent_complete_commits(args, cutoff);
793 }
794
795 /*
796 * Mark all complete remote refs as common refs.
797 * Don't mark them common yet; the server has to be told so first.
798 */
799 for (ref = *refs; ref; ref = ref->next) {
800 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
801 NULL, 0);
802
803 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
804 continue;
805
806 if (!(o->flags & SEEN)) {
807 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
808
809 mark_common((struct commit *)o, 1, 1);
810 }
811 }
812 }
813
814 filter_refs(args, refs, sought, nr_sought);
815
816 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
817 const struct object_id *remote = &ref->old_oid;
818 struct object *o;
819
820 o = lookup_object(remote->hash);
821 if (!o || !(o->flags & COMPLETE)) {
822 retval = 0;
823 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
824 ref->name);
825 continue;
826 }
827 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
828 ref->name);
829 }
830
831 save_commit_buffer = old_save_commit_buffer;
832
833 return retval;
834 }
835
836 static int sideband_demux(int in, int out, void *data)
837 {
838 int *xd = data;
839 int ret;
840
841 ret = recv_sideband("fetch-pack", xd[0], out);
842 close(out);
843 return ret;
844 }
845
846 static int get_pack(struct fetch_pack_args *args,
847 int xd[2], char **pack_lockfile)
848 {
849 struct async demux;
850 int do_keep = args->keep_pack;
851 const char *cmd_name;
852 struct pack_header header;
853 int pass_header = 0;
854 struct child_process cmd = CHILD_PROCESS_INIT;
855 int ret;
856
857 memset(&demux, 0, sizeof(demux));
858 if (use_sideband) {
859 /* xd[] is talking with upload-pack; subprocess reads from
860 * xd[0], spits out band#2 to stderr, and feeds us band#1
861 * through demux->out.
862 */
863 demux.proc = sideband_demux;
864 demux.data = xd;
865 demux.out = -1;
866 demux.isolate_sigpipe = 1;
867 if (start_async(&demux))
868 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
869 }
870 else
871 demux.out = xd[0];
872
873 if (!args->keep_pack && unpack_limit) {
874
875 if (read_pack_header(demux.out, &header))
876 die(_("protocol error: bad pack header"));
877 pass_header = 1;
878 if (ntohl(header.hdr_entries) < unpack_limit)
879 do_keep = 0;
880 else
881 do_keep = 1;
882 }
883
884 if (alternate_shallow_file) {
885 argv_array_push(&cmd.args, "--shallow-file");
886 argv_array_push(&cmd.args, alternate_shallow_file);
887 }
888
889 if (do_keep || args->from_promisor) {
890 if (pack_lockfile)
891 cmd.out = -1;
892 cmd_name = "index-pack";
893 argv_array_push(&cmd.args, cmd_name);
894 argv_array_push(&cmd.args, "--stdin");
895 if (!args->quiet && !args->no_progress)
896 argv_array_push(&cmd.args, "-v");
897 if (args->use_thin_pack)
898 argv_array_push(&cmd.args, "--fix-thin");
899 if (do_keep && (args->lock_pack || unpack_limit)) {
900 char hostname[HOST_NAME_MAX + 1];
901 if (xgethostname(hostname, sizeof(hostname)))
902 xsnprintf(hostname, sizeof(hostname), "localhost");
903 argv_array_pushf(&cmd.args,
904 "--keep=fetch-pack %"PRIuMAX " on %s",
905 (uintmax_t)getpid(), hostname);
906 }
907 if (args->check_self_contained_and_connected)
908 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
909 if (args->from_promisor)
910 argv_array_push(&cmd.args, "--promisor");
911 }
912 else {
913 cmd_name = "unpack-objects";
914 argv_array_push(&cmd.args, cmd_name);
915 if (args->quiet || args->no_progress)
916 argv_array_push(&cmd.args, "-q");
917 args->check_self_contained_and_connected = 0;
918 }
919
920 if (pass_header)
921 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
922 ntohl(header.hdr_version),
923 ntohl(header.hdr_entries));
924 if (fetch_fsck_objects >= 0
925 ? fetch_fsck_objects
926 : transfer_fsck_objects >= 0
927 ? transfer_fsck_objects
928 : 0) {
929 if (args->from_promisor)
930 /*
931 * We cannot use --strict in index-pack because it
932 * checks both broken objects and links, but we only
933 * want to check for broken objects.
934 */
935 argv_array_push(&cmd.args, "--fsck-objects");
936 else
937 argv_array_push(&cmd.args, "--strict");
938 }
939
940 cmd.in = demux.out;
941 cmd.git_cmd = 1;
942 if (start_command(&cmd))
943 die(_("fetch-pack: unable to fork off %s"), cmd_name);
944 if (do_keep && pack_lockfile) {
945 *pack_lockfile = index_pack_lockfile(cmd.out);
946 close(cmd.out);
947 }
948
949 if (!use_sideband)
950 /* Closed by start_command() */
951 xd[0] = -1;
952
953 ret = finish_command(&cmd);
954 if (!ret || (args->check_self_contained_and_connected && ret == 1))
955 args->self_contained_and_connected =
956 args->check_self_contained_and_connected &&
957 ret == 0;
958 else
959 die(_("%s failed"), cmd_name);
960 if (use_sideband && finish_async(&demux))
961 die(_("error in sideband demultiplexer"));
962 return 0;
963 }
964
965 static int cmp_ref_by_name(const void *a_, const void *b_)
966 {
967 const struct ref *a = *((const struct ref **)a_);
968 const struct ref *b = *((const struct ref **)b_);
969 return strcmp(a->name, b->name);
970 }
971
972 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
973 int fd[2],
974 const struct ref *orig_ref,
975 struct ref **sought, int nr_sought,
976 struct shallow_info *si,
977 char **pack_lockfile)
978 {
979 struct ref *ref = copy_ref_list(orig_ref);
980 struct object_id oid;
981 const char *agent_feature;
982 int agent_len;
983
984 sort_ref_list(&ref, ref_compare_name);
985 QSORT(sought, nr_sought, cmp_ref_by_name);
986
987 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
988 die(_("Server does not support shallow clients"));
989 if (args->depth > 0 || args->deepen_since || args->deepen_not)
990 args->deepen = 1;
991 if (server_supports("multi_ack_detailed")) {
992 print_verbose(args, _("Server supports multi_ack_detailed"));
993 multi_ack = 2;
994 if (server_supports("no-done")) {
995 print_verbose(args, _("Server supports no-done"));
996 if (args->stateless_rpc)
997 no_done = 1;
998 }
999 }
1000 else if (server_supports("multi_ack")) {
1001 print_verbose(args, _("Server supports multi_ack"));
1002 multi_ack = 1;
1003 }
1004 if (server_supports("side-band-64k")) {
1005 print_verbose(args, _("Server supports side-band-64k"));
1006 use_sideband = 2;
1007 }
1008 else if (server_supports("side-band")) {
1009 print_verbose(args, _("Server supports side-band"));
1010 use_sideband = 1;
1011 }
1012 if (server_supports("allow-tip-sha1-in-want")) {
1013 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1014 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1015 }
1016 if (server_supports("allow-reachable-sha1-in-want")) {
1017 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1018 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1019 }
1020 if (!server_supports("thin-pack"))
1021 args->use_thin_pack = 0;
1022 if (!server_supports("no-progress"))
1023 args->no_progress = 0;
1024 if (!server_supports("include-tag"))
1025 args->include_tag = 0;
1026 if (server_supports("ofs-delta"))
1027 print_verbose(args, _("Server supports ofs-delta"));
1028 else
1029 prefer_ofs_delta = 0;
1030
1031 if (server_supports("filter")) {
1032 server_supports_filtering = 1;
1033 print_verbose(args, _("Server supports filter"));
1034 } else if (args->filter_options.choice) {
1035 warning("filtering not recognized by server, ignoring");
1036 }
1037
1038 if ((agent_feature = server_feature_value("agent", &agent_len))) {
1039 agent_supported = 1;
1040 if (agent_len)
1041 print_verbose(args, _("Server version is %.*s"),
1042 agent_len, agent_feature);
1043 }
1044 if (server_supports("deepen-since"))
1045 deepen_since_ok = 1;
1046 else if (args->deepen_since)
1047 die(_("Server does not support --shallow-since"));
1048 if (server_supports("deepen-not"))
1049 deepen_not_ok = 1;
1050 else if (args->deepen_not)
1051 die(_("Server does not support --shallow-exclude"));
1052 if (!server_supports("deepen-relative") && args->deepen_relative)
1053 die(_("Server does not support --deepen"));
1054
1055 if (everything_local(args, &ref, sought, nr_sought)) {
1056 packet_flush(fd[1]);
1057 goto all_done;
1058 }
1059 if (find_common(args, fd, &oid, ref) < 0)
1060 if (!args->keep_pack)
1061 /* When cloning, it is not unusual to have
1062 * no common commit.
1063 */
1064 warning(_("no common commits"));
1065
1066 if (args->stateless_rpc)
1067 packet_flush(fd[1]);
1068 if (args->deepen)
1069 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1070 NULL);
1071 else if (si->nr_ours || si->nr_theirs)
1072 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1073 else
1074 alternate_shallow_file = NULL;
1075 if (get_pack(args, fd, pack_lockfile))
1076 die(_("git fetch-pack: fetch failed."));
1077
1078 all_done:
1079 return ref;
1080 }
1081
1082 static void fetch_pack_config(void)
1083 {
1084 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1085 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1086 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1087 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1088 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1089
1090 git_config(git_default_config, NULL);
1091 }
1092
1093 static void fetch_pack_setup(void)
1094 {
1095 static int did_setup;
1096 if (did_setup)
1097 return;
1098 fetch_pack_config();
1099 if (0 <= transfer_unpack_limit)
1100 unpack_limit = transfer_unpack_limit;
1101 else if (0 <= fetch_unpack_limit)
1102 unpack_limit = fetch_unpack_limit;
1103 did_setup = 1;
1104 }
1105
1106 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1107 {
1108 struct string_list names = STRING_LIST_INIT_NODUP;
1109 int src, dst;
1110
1111 for (src = dst = 0; src < nr; src++) {
1112 struct string_list_item *item;
1113 item = string_list_insert(&names, ref[src]->name);
1114 if (item->util)
1115 continue; /* already have it */
1116 item->util = ref[src];
1117 if (src != dst)
1118 ref[dst] = ref[src];
1119 dst++;
1120 }
1121 for (src = dst; src < nr; src++)
1122 ref[src] = NULL;
1123 string_list_clear(&names, 0);
1124 return dst;
1125 }
1126
1127 static void update_shallow(struct fetch_pack_args *args,
1128 struct ref **sought, int nr_sought,
1129 struct shallow_info *si)
1130 {
1131 struct oid_array ref = OID_ARRAY_INIT;
1132 int *status;
1133 int i;
1134
1135 if (args->deepen && alternate_shallow_file) {
1136 if (*alternate_shallow_file == '\0') { /* --unshallow */
1137 unlink_or_warn(git_path_shallow());
1138 rollback_lock_file(&shallow_lock);
1139 } else
1140 commit_lock_file(&shallow_lock);
1141 return;
1142 }
1143
1144 if (!si->shallow || !si->shallow->nr)
1145 return;
1146
1147 if (args->cloning) {
1148 /*
1149 * remote is shallow, but this is a clone, there are
1150 * no objects in repo to worry about. Accept any
1151 * shallow points that exist in the pack (iow in repo
1152 * after get_pack() and reprepare_packed_git())
1153 */
1154 struct oid_array extra = OID_ARRAY_INIT;
1155 struct object_id *oid = si->shallow->oid;
1156 for (i = 0; i < si->shallow->nr; i++)
1157 if (has_object_file(&oid[i]))
1158 oid_array_append(&extra, &oid[i]);
1159 if (extra.nr) {
1160 setup_alternate_shallow(&shallow_lock,
1161 &alternate_shallow_file,
1162 &extra);
1163 commit_lock_file(&shallow_lock);
1164 }
1165 oid_array_clear(&extra);
1166 return;
1167 }
1168
1169 if (!si->nr_ours && !si->nr_theirs)
1170 return;
1171
1172 remove_nonexistent_theirs_shallow(si);
1173 if (!si->nr_ours && !si->nr_theirs)
1174 return;
1175 for (i = 0; i < nr_sought; i++)
1176 oid_array_append(&ref, &sought[i]->old_oid);
1177 si->ref = &ref;
1178
1179 if (args->update_shallow) {
1180 /*
1181 * remote is also shallow, .git/shallow may be updated
1182 * so all refs can be accepted. Make sure we only add
1183 * shallow roots that are actually reachable from new
1184 * refs.
1185 */
1186 struct oid_array extra = OID_ARRAY_INIT;
1187 struct object_id *oid = si->shallow->oid;
1188 assign_shallow_commits_to_refs(si, NULL, NULL);
1189 if (!si->nr_ours && !si->nr_theirs) {
1190 oid_array_clear(&ref);
1191 return;
1192 }
1193 for (i = 0; i < si->nr_ours; i++)
1194 oid_array_append(&extra, &oid[si->ours[i]]);
1195 for (i = 0; i < si->nr_theirs; i++)
1196 oid_array_append(&extra, &oid[si->theirs[i]]);
1197 setup_alternate_shallow(&shallow_lock,
1198 &alternate_shallow_file,
1199 &extra);
1200 commit_lock_file(&shallow_lock);
1201 oid_array_clear(&extra);
1202 oid_array_clear(&ref);
1203 return;
1204 }
1205
1206 /*
1207 * remote is also shallow, check what ref is safe to update
1208 * without updating .git/shallow
1209 */
1210 status = xcalloc(nr_sought, sizeof(*status));
1211 assign_shallow_commits_to_refs(si, NULL, status);
1212 if (si->nr_ours || si->nr_theirs) {
1213 for (i = 0; i < nr_sought; i++)
1214 if (status[i])
1215 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1216 }
1217 free(status);
1218 oid_array_clear(&ref);
1219 }
1220
1221 struct ref *fetch_pack(struct fetch_pack_args *args,
1222 int fd[], struct child_process *conn,
1223 const struct ref *ref,
1224 const char *dest,
1225 struct ref **sought, int nr_sought,
1226 struct oid_array *shallow,
1227 char **pack_lockfile)
1228 {
1229 struct ref *ref_cpy;
1230 struct shallow_info si;
1231
1232 fetch_pack_setup();
1233 if (nr_sought)
1234 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1235
1236 if (!ref) {
1237 packet_flush(fd[1]);
1238 die(_("no matching remote head"));
1239 }
1240 prepare_shallow_info(&si, shallow);
1241 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1242 &si, pack_lockfile);
1243 reprepare_packed_git();
1244 update_shallow(args, sought, nr_sought, &si);
1245 clear_shallow_info(&si);
1246 return ref_cpy;
1247 }
1248
1249 int report_unmatched_refs(struct ref **sought, int nr_sought)
1250 {
1251 int i, ret = 0;
1252
1253 for (i = 0; i < nr_sought; i++) {
1254 if (!sought[i])
1255 continue;
1256 switch (sought[i]->match_status) {
1257 case REF_MATCHED:
1258 continue;
1259 case REF_NOT_MATCHED:
1260 error(_("no such remote ref %s"), sought[i]->name);
1261 break;
1262 case REF_UNADVERTISED_NOT_ALLOWED:
1263 error(_("Server does not allow request for unadvertised object %s"),
1264 sought[i]->name);
1265 break;
1266 }
1267 ret = 1;
1268 }
1269 return ret;
1270 }