Merging a handful of topics before -rc2
[git/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
697cc8ef 2#include "lockfile.h"
745f7a8c
NTND
3#include "refs.h"
4#include "pkt-line.h"
5#include "commit.h"
6#include "tag.h"
7#include "exec_cmd.h"
8#include "pack.h"
9#include "sideband.h"
10#include "fetch-pack.h"
11#include "remote.h"
12#include "run-command.h"
47a59185 13#include "connect.h"
745f7a8c
NTND
14#include "transport.h"
15#include "version.h"
099327b5 16#include "prio-queue.h"
beea4152 17#include "sha1-array.h"
745f7a8c
NTND
18
19static int transfer_unpack_limit = -1;
20static int fetch_unpack_limit = -1;
21static int unpack_limit = 100;
22static int prefer_ofs_delta = 1;
23static int no_done;
508ea882 24static int deepen_since_ok;
a45a2600 25static int deepen_not_ok;
745f7a8c
NTND
26static int fetch_fsck_objects = -1;
27static int transfer_fsck_objects = -1;
28static int agent_supported;
6035d6aa
NTND
29static struct lock_file shallow_lock;
30static const char *alternate_shallow_file;
745f7a8c 31
208acbfb 32/* Remember to update object flag allocation in object.h */
745f7a8c
NTND
33#define COMPLETE (1U << 0)
34#define COMMON (1U << 1)
35#define COMMON_REF (1U << 2)
36#define SEEN (1U << 3)
37#define POPPED (1U << 4)
41a078c6 38#define ALTERNATE (1U << 5)
745f7a8c
NTND
39
40static int marked;
41
42/*
43 * After sending this many "have"s if we do not get any new ACK , we
44 * give up traversing our history.
45 */
46#define MAX_IN_VAIN 256
47
099327b5 48static struct prio_queue rev_list = { compare_commits_by_commit_date };
7199c093
FM
49static int non_common_revs, multi_ack, use_sideband;
50/* Allow specifying sha1 if it is a ref tip. */
51#define ALLOW_TIP_SHA1 01
68ee6289
FM
52/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
53#define ALLOW_REACHABLE_SHA1 02
7199c093 54static unsigned int allow_unadvertised_object_request;
745f7a8c 55
0d789a5b
NTND
56__attribute__((format (printf, 2, 3)))
57static inline void print_verbose(const struct fetch_pack_args *args,
58 const char *fmt, ...)
59{
60 va_list params;
61
62 if (!args->verbose)
63 return;
64
65 va_start(params, fmt);
66 vfprintf(stderr, fmt, params);
67 va_end(params);
68 fputc('\n', stderr);
69}
70
41a078c6
JK
71struct alternate_object_cache {
72 struct object **items;
73 size_t nr, alloc;
74};
75
76static void cache_one_alternate(const char *refname,
77 const struct object_id *oid,
78 void *vcache)
79{
80 struct alternate_object_cache *cache = vcache;
81 struct object *obj = parse_object(oid->hash);
82
83 if (!obj || (obj->flags & ALTERNATE))
84 return;
85
86 obj->flags |= ALTERNATE;
87 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
88 cache->items[cache->nr++] = obj;
89}
90
91static void for_each_cached_alternate(void (*cb)(struct object *))
92{
93 static int initialized;
94 static struct alternate_object_cache cache;
95 size_t i;
96
97 if (!initialized) {
98 for_each_alternate_ref(cache_one_alternate, &cache);
99 initialized = 1;
100 }
101
102 for (i = 0; i < cache.nr; i++)
103 cb(cache.items[i]);
104}
105
745f7a8c
NTND
106static void rev_list_push(struct commit *commit, int mark)
107{
108 if (!(commit->object.flags & mark)) {
109 commit->object.flags |= mark;
110
0064053b
JK
111 if (parse_commit(commit))
112 return;
745f7a8c 113
099327b5 114 prio_queue_put(&rev_list, commit);
745f7a8c
NTND
115
116 if (!(commit->object.flags & COMMON))
117 non_common_revs++;
118 }
119}
120
c38cd1c8 121static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
745f7a8c
NTND
122{
123 struct object *o = deref_tag(parse_object(sha1), refname, 0);
124
125 if (o && o->type == OBJ_COMMIT)
126 rev_list_push((struct commit *)o, SEEN);
127
128 return 0;
129}
130
b1b49c6e
MH
131static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
132 int flag, void *cb_data)
745f7a8c 133{
c38cd1c8 134 return rev_list_insert_ref(refname, oid->hash);
b1b49c6e
MH
135}
136
c50fb6ce
MH
137static int clear_marks(const char *refname, const struct object_id *oid,
138 int flag, void *cb_data)
745f7a8c 139{
c50fb6ce 140 struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
745f7a8c
NTND
141
142 if (o && o->type == OBJ_COMMIT)
143 clear_commit_marks((struct commit *)o,
144 COMMON | COMMON_REF | SEEN | POPPED);
145 return 0;
146}
147
148/*
149 This function marks a rev and its ancestors as common.
150 In some cases, it is desirable to mark only the ancestors (for example
151 when only the server does not yet know that they are common).
152*/
153
154static void mark_common(struct commit *commit,
155 int ancestors_only, int dont_parse)
156{
157 if (commit != NULL && !(commit->object.flags & COMMON)) {
158 struct object *o = (struct object *)commit;
159
160 if (!ancestors_only)
161 o->flags |= COMMON;
162
163 if (!(o->flags & SEEN))
164 rev_list_push(commit, SEEN);
165 else {
166 struct commit_list *parents;
167
168 if (!ancestors_only && !(o->flags & POPPED))
169 non_common_revs--;
170 if (!o->parsed && !dont_parse)
171 if (parse_commit(commit))
172 return;
173
174 for (parents = commit->parents;
175 parents;
176 parents = parents->next)
177 mark_common(parents->item, 0, dont_parse);
178 }
179 }
180}
181
182/*
183 Get the next rev to send, ignoring the common.
184*/
185
186static const unsigned char *get_rev(void)
187{
188 struct commit *commit = NULL;
189
190 while (commit == NULL) {
191 unsigned int mark;
192 struct commit_list *parents;
193
099327b5 194 if (rev_list.nr == 0 || non_common_revs == 0)
745f7a8c
NTND
195 return NULL;
196
099327b5 197 commit = prio_queue_get(&rev_list);
0064053b 198 parse_commit(commit);
745f7a8c
NTND
199 parents = commit->parents;
200
201 commit->object.flags |= POPPED;
202 if (!(commit->object.flags & COMMON))
203 non_common_revs--;
204
205 if (commit->object.flags & COMMON) {
206 /* do not send "have", and ignore ancestors */
207 commit = NULL;
208 mark = COMMON | SEEN;
209 } else if (commit->object.flags & COMMON_REF)
210 /* send "have", and ignore ancestors */
211 mark = COMMON | SEEN;
212 else
213 /* send "have", also for its ancestors */
214 mark = SEEN;
215
216 while (parents) {
217 if (!(parents->item->object.flags & SEEN))
218 rev_list_push(parents->item, mark);
219 if (mark & COMMON)
220 mark_common(parents->item, 1, 0);
221 parents = parents->next;
222 }
745f7a8c
NTND
223 }
224
ed1c9977 225 return commit->object.oid.hash;
745f7a8c
NTND
226}
227
228enum ack_type {
229 NAK = 0,
230 ACK,
231 ACK_continue,
232 ACK_common,
233 ACK_ready
234};
235
236static void consume_shallow_list(struct fetch_pack_args *args, int fd)
237{
79891cb9 238 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
239 /* If we sent a depth we will get back "duplicate"
240 * shallow and unshallow commands every time there
241 * is a block of have lines exchanged.
242 */
74543a04
JK
243 char *line;
244 while ((line = packet_read_line(fd, NULL))) {
59556548 245 if (starts_with(line, "shallow "))
745f7a8c 246 continue;
59556548 247 if (starts_with(line, "unshallow "))
745f7a8c 248 continue;
1dd73e20 249 die(_("git fetch-pack: expected shallow list"));
745f7a8c
NTND
250 }
251 }
252}
253
745f7a8c
NTND
254static enum ack_type get_ack(int fd, unsigned char *result_sha1)
255{
74543a04
JK
256 int len;
257 char *line = packet_read_line(fd, &len);
82e56767 258 const char *arg;
745f7a8c
NTND
259
260 if (!len)
1dd73e20 261 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
745f7a8c
NTND
262 if (!strcmp(line, "NAK"))
263 return NAK;
82e56767
JK
264 if (skip_prefix(line, "ACK ", &arg)) {
265 if (!get_sha1_hex(arg, result_sha1)) {
266 arg += 40;
267 len -= arg - line;
268 if (len < 1)
030e9dd6 269 return ACK;
82e56767 270 if (strstr(arg, "continue"))
745f7a8c 271 return ACK_continue;
82e56767 272 if (strstr(arg, "common"))
745f7a8c 273 return ACK_common;
82e56767 274 if (strstr(arg, "ready"))
745f7a8c
NTND
275 return ACK_ready;
276 return ACK;
277 }
278 }
8e2c7bef
JT
279 if (skip_prefix(line, "ERR ", &arg))
280 die(_("remote error: %s"), arg);
dfbfb9f3 281 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
745f7a8c
NTND
282}
283
284static void send_request(struct fetch_pack_args *args,
285 int fd, struct strbuf *buf)
286{
287 if (args->stateless_rpc) {
288 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
289 packet_flush(fd);
290 } else
cdf4fb8e 291 write_or_die(fd, buf->buf, buf->len);
745f7a8c
NTND
292}
293
41a078c6 294static void insert_one_alternate_object(struct object *obj)
745f7a8c 295{
41a078c6 296 rev_list_insert_ref(NULL, obj->oid.hash);
745f7a8c
NTND
297}
298
299#define INITIAL_FLUSH 16
300#define PIPESAFE_FLUSH 32
da470981 301#define LARGE_FLUSH 16384
745f7a8c
NTND
302
303static int next_flush(struct fetch_pack_args *args, int count)
304{
da470981
JT
305 if (args->stateless_rpc) {
306 if (count < LARGE_FLUSH)
307 count <<= 1;
308 else
309 count = count * 11 / 10;
310 } else {
311 if (count < PIPESAFE_FLUSH)
312 count <<= 1;
313 else
314 count += PIPESAFE_FLUSH;
315 }
745f7a8c
NTND
316 return count;
317}
318
319static int find_common(struct fetch_pack_args *args,
320 int fd[2], unsigned char *result_sha1,
321 struct ref *refs)
322{
323 int fetching;
324 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
325 const unsigned char *sha1;
326 unsigned in_vain = 0;
327 int got_continue = 0;
328 int got_ready = 0;
329 struct strbuf req_buf = STRBUF_INIT;
330 size_t state_len = 0;
331
332 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 333 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c
NTND
334 if (marked)
335 for_each_ref(clear_marks, NULL);
336 marked = 1;
337
b1b49c6e 338 for_each_ref(rev_list_insert_ref_oid, NULL);
41a078c6 339 for_each_cached_alternate(insert_one_alternate_object);
745f7a8c
NTND
340
341 fetching = 0;
342 for ( ; refs ; refs = refs->next) {
f4e54d02 343 unsigned char *remote = refs->old_oid.hash;
745f7a8c
NTND
344 const char *remote_hex;
345 struct object *o;
346
347 /*
348 * If that object is complete (i.e. it is an ancestor of a
349 * local ref), we tell them we have it but do not have to
350 * tell them about its ancestors, which they already know
351 * about.
352 *
353 * We use lookup_object here because we are only
354 * interested in the case we *know* the object is
355 * reachable and we have already scanned it.
356 */
357 if (((o = lookup_object(remote)) != NULL) &&
358 (o->flags & COMPLETE)) {
359 continue;
360 }
361
362 remote_hex = sha1_to_hex(remote);
363 if (!fetching) {
364 struct strbuf c = STRBUF_INIT;
365 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
366 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
367 if (no_done) strbuf_addstr(&c, " no-done");
368 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
369 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 370 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
371 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
372 if (args->no_progress) strbuf_addstr(&c, " no-progress");
373 if (args->include_tag) strbuf_addstr(&c, " include-tag");
374 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 375 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 376 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
377 if (agent_supported) strbuf_addf(&c, " agent=%s",
378 git_user_agent_sanitized());
379 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
380 strbuf_release(&c);
381 } else
382 packet_buf_write(&req_buf, "want %s\n", remote_hex);
383 fetching++;
384 }
385
386 if (!fetching) {
387 strbuf_release(&req_buf);
388 packet_flush(fd[1]);
389 return 1;
390 }
391
392 if (is_repository_shallow())
1a30f5a2 393 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
394 if (args->depth > 0)
395 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882
NTND
396 if (args->deepen_since) {
397 unsigned long max_age = approxidate(args->deepen_since);
398 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
399 }
a45a2600
NTND
400 if (args->deepen_not) {
401 int i;
402 for (i = 0; i < args->deepen_not->nr; i++) {
403 struct string_list_item *s = args->deepen_not->items + i;
404 packet_buf_write(&req_buf, "deepen-not %s", s->string);
405 }
406 }
745f7a8c
NTND
407 packet_buf_flush(&req_buf);
408 state_len = req_buf.len;
409
79891cb9 410 if (args->deepen) {
74543a04 411 char *line;
ae021d87 412 const char *arg;
745f7a8c
NTND
413 unsigned char sha1[20];
414
415 send_request(args, fd[1], &req_buf);
74543a04 416 while ((line = packet_read_line(fd[0], NULL))) {
ae021d87
JK
417 if (skip_prefix(line, "shallow ", &arg)) {
418 if (get_sha1_hex(arg, sha1))
1dd73e20 419 die(_("invalid shallow line: %s"), line);
745f7a8c
NTND
420 register_shallow(sha1);
421 continue;
422 }
ae021d87
JK
423 if (skip_prefix(line, "unshallow ", &arg)) {
424 if (get_sha1_hex(arg, sha1))
1dd73e20 425 die(_("invalid unshallow line: %s"), line);
745f7a8c 426 if (!lookup_object(sha1))
1dd73e20 427 die(_("object not found: %s"), line);
745f7a8c
NTND
428 /* make sure that it is parsed as shallow */
429 if (!parse_object(sha1))
1dd73e20 430 die(_("error in object: %s"), line);
745f7a8c 431 if (unregister_shallow(sha1))
1dd73e20 432 die(_("no shallow found: %s"), line);
745f7a8c
NTND
433 continue;
434 }
1dd73e20 435 die(_("expected shallow/unshallow, got %s"), line);
745f7a8c
NTND
436 }
437 } else if (!args->stateless_rpc)
438 send_request(args, fd[1], &req_buf);
439
440 if (!args->stateless_rpc) {
441 /* If we aren't using the stateless-rpc interface
442 * we don't need to retain the headers.
443 */
444 strbuf_setlen(&req_buf, 0);
445 state_len = 0;
446 }
447
448 flushes = 0;
449 retval = -1;
450 while ((sha1 = get_rev())) {
451 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
0d789a5b 452 print_verbose(args, "have %s", sha1_to_hex(sha1));
745f7a8c
NTND
453 in_vain++;
454 if (flush_at <= ++count) {
455 int ack;
456
457 packet_buf_flush(&req_buf);
458 send_request(args, fd[1], &req_buf);
459 strbuf_setlen(&req_buf, state_len);
460 flushes++;
461 flush_at = next_flush(args, count);
462
463 /*
464 * We keep one window "ahead" of the other side, and
465 * will wait for an ACK only on the next one
466 */
467 if (!args->stateless_rpc && count == INITIAL_FLUSH)
468 continue;
469
470 consume_shallow_list(args, fd[0]);
471 do {
472 ack = get_ack(fd[0], result_sha1);
0d789a5b 473 if (ack)
1dd73e20
NTND
474 print_verbose(args, _("got %s %d %s"), "ack",
475 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
476 switch (ack) {
477 case ACK:
478 flushes = 0;
479 multi_ack = 0;
480 retval = 0;
481 goto done;
482 case ACK_common:
483 case ACK_ready:
484 case ACK_continue: {
485 struct commit *commit =
486 lookup_commit(result_sha1);
487 if (!commit)
1dd73e20 488 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
745f7a8c
NTND
489 if (args->stateless_rpc
490 && ack == ACK_common
491 && !(commit->object.flags & COMMON)) {
492 /* We need to replay the have for this object
493 * on the next RPC request so the peer knows
494 * it is in common with us.
495 */
496 const char *hex = sha1_to_hex(result_sha1);
497 packet_buf_write(&req_buf, "have %s\n", hex);
498 state_len = req_buf.len;
06b3d386
JT
499 /*
500 * Reset in_vain because an ack
501 * for this commit has not been
502 * seen.
503 */
504 in_vain = 0;
505 } else if (!args->stateless_rpc
506 || ack != ACK_common)
507 in_vain = 0;
745f7a8c
NTND
508 mark_common(commit, 0, 1);
509 retval = 0;
745f7a8c
NTND
510 got_continue = 1;
511 if (ack == ACK_ready) {
099327b5 512 clear_prio_queue(&rev_list);
745f7a8c
NTND
513 got_ready = 1;
514 }
515 break;
516 }
517 }
518 } while (ack);
519 flushes--;
520 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 521 print_verbose(args, _("giving up"));
745f7a8c
NTND
522 break; /* give up */
523 }
524 }
525 }
526done:
527 if (!got_ready || !no_done) {
528 packet_buf_write(&req_buf, "done\n");
529 send_request(args, fd[1], &req_buf);
530 }
1dd73e20 531 print_verbose(args, _("done"));
745f7a8c
NTND
532 if (retval != 0) {
533 multi_ack = 0;
534 flushes++;
535 }
536 strbuf_release(&req_buf);
537
ff62eca7
NTND
538 if (!got_ready || !no_done)
539 consume_shallow_list(args, fd[0]);
745f7a8c
NTND
540 while (flushes || multi_ack) {
541 int ack = get_ack(fd[0], result_sha1);
542 if (ack) {
1dd73e20
NTND
543 print_verbose(args, _("got %s (%d) %s"), "ack",
544 ack, sha1_to_hex(result_sha1));
745f7a8c
NTND
545 if (ack == ACK)
546 return 0;
547 multi_ack = 1;
548 continue;
549 }
550 flushes--;
551 }
552 /* it is no error to fetch into a completely empty repo */
553 return count ? retval : 0;
554}
555
556static struct commit_list *complete;
557
6e20a51a 558static int mark_complete(const unsigned char *sha1)
745f7a8c
NTND
559{
560 struct object *o = parse_object(sha1);
561
562 while (o && o->type == OBJ_TAG) {
563 struct tag *t = (struct tag *) o;
564 if (!t->tagged)
565 break; /* broken repository */
566 o->flags |= COMPLETE;
ed1c9977 567 o = parse_object(t->tagged->oid.hash);
745f7a8c
NTND
568 }
569 if (o && o->type == OBJ_COMMIT) {
570 struct commit *commit = (struct commit *)o;
571 if (!(commit->object.flags & COMPLETE)) {
572 commit->object.flags |= COMPLETE;
16445242 573 commit_list_insert(commit, &complete);
745f7a8c
NTND
574 }
575 }
576 return 0;
577}
578
f8ee4d85
MH
579static int mark_complete_oid(const char *refname, const struct object_id *oid,
580 int flag, void *cb_data)
581{
6e20a51a 582 return mark_complete(oid->hash);
f8ee4d85
MH
583}
584
745f7a8c
NTND
585static void mark_recent_complete_commits(struct fetch_pack_args *args,
586 unsigned long cutoff)
587{
588 while (complete && cutoff <= complete->item->date) {
1dd73e20 589 print_verbose(args, _("Marking %s as complete"),
0d789a5b 590 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
591 pop_most_recent_commit(&complete, COMPLETE);
592 }
593}
594
745f7a8c 595static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
596 struct ref **refs,
597 struct ref **sought, int nr_sought)
745f7a8c
NTND
598{
599 struct ref *newlist = NULL;
600 struct ref **newtail = &newlist;
601 struct ref *ref, *next;
f2db854d 602 int i;
745f7a8c 603
f2db854d 604 i = 0;
745f7a8c
NTND
605 for (ref = *refs; ref; ref = next) {
606 int keep = 0;
607 next = ref->next;
f2db854d 608
50e19a83 609 if (starts_with(ref->name, "refs/") &&
4c224081 610 check_refname_format(ref->name, 0))
745f7a8c
NTND
611 ; /* trash */
612 else {
f2db854d
JH
613 while (i < nr_sought) {
614 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
615 if (cmp < 0)
616 break; /* definitely do not have it */
617 else if (cmp == 0) {
618 keep = 1; /* definitely have it */
d56583de 619 sought[i]->match_status = REF_MATCHED;
745f7a8c 620 }
f2db854d 621 i++;
745f7a8c
NTND
622 }
623 }
624
f2db854d 625 if (!keep && args->fetch_all &&
79891cb9 626 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
745f7a8c
NTND
627 keep = 1;
628
629 if (keep) {
630 *newtail = ref;
631 ref->next = NULL;
632 newtail = &ref->next;
633 } else {
634 free(ref);
635 }
636 }
637
6e7b66ee 638 /* Append unmatched requests to the list */
d56583de
MM
639 for (i = 0; i < nr_sought; i++) {
640 unsigned char sha1[20];
b7916422 641
d56583de
MM
642 ref = sought[i];
643 if (ref->match_status != REF_NOT_MATCHED)
644 continue;
645 if (get_sha1_hex(ref->name, sha1) ||
646 ref->name[40] != '\0' ||
647 hashcmp(sha1, ref->old_oid.hash))
648 continue;
6e7b66ee 649
d56583de
MM
650 if ((allow_unadvertised_object_request &
651 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
652 ref->match_status = REF_MATCHED;
c3c17bf1
JK
653 *newtail = copy_ref(ref);
654 newtail = &(*newtail)->next;
d56583de
MM
655 } else {
656 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
657 }
658 }
745f7a8c
NTND
659 *refs = newlist;
660}
661
41a078c6 662static void mark_alternate_complete(struct object *obj)
745f7a8c 663{
41a078c6 664 mark_complete(obj->oid.hash);
745f7a8c
NTND
665}
666
667static int everything_local(struct fetch_pack_args *args,
f2db854d
JH
668 struct ref **refs,
669 struct ref **sought, int nr_sought)
745f7a8c
NTND
670{
671 struct ref *ref;
672 int retval;
673 unsigned long cutoff = 0;
674
675 save_commit_buffer = 0;
676
677 for (ref = *refs; ref; ref = ref->next) {
678 struct object *o;
679
f4e54d02 680 if (!has_object_file(&ref->old_oid))
012a1bb5
JH
681 continue;
682
f4e54d02 683 o = parse_object(ref->old_oid.hash);
745f7a8c
NTND
684 if (!o)
685 continue;
686
687 /* We already have it -- which may mean that we were
688 * in sync with the other side at some time after
689 * that (it is OK if we guess wrong here).
690 */
691 if (o->type == OBJ_COMMIT) {
692 struct commit *commit = (struct commit *)o;
693 if (!cutoff || cutoff < commit->date)
694 cutoff = commit->date;
695 }
696 }
697
79891cb9 698 if (!args->deepen) {
f8ee4d85 699 for_each_ref(mark_complete_oid, NULL);
41a078c6 700 for_each_cached_alternate(mark_alternate_complete);
16445242 701 commit_list_sort_by_date(&complete);
745f7a8c
NTND
702 if (cutoff)
703 mark_recent_complete_commits(args, cutoff);
704 }
705
706 /*
707 * Mark all complete remote refs as common refs.
708 * Don't mark them common yet; the server has to be told so first.
709 */
710 for (ref = *refs; ref; ref = ref->next) {
f4e54d02 711 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
745f7a8c
NTND
712 NULL, 0);
713
714 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
715 continue;
716
717 if (!(o->flags & SEEN)) {
718 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
719
720 mark_common((struct commit *)o, 1, 1);
721 }
722 }
723
f2db854d 724 filter_refs(args, refs, sought, nr_sought);
745f7a8c
NTND
725
726 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
f4e54d02 727 const unsigned char *remote = ref->old_oid.hash;
745f7a8c
NTND
728 struct object *o;
729
730 o = lookup_object(remote);
731 if (!o || !(o->flags & COMPLETE)) {
732 retval = 0;
0d789a5b
NTND
733 print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
734 ref->name);
745f7a8c
NTND
735 continue;
736 }
1dd73e20 737 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
0d789a5b 738 ref->name);
745f7a8c
NTND
739 }
740 return retval;
741}
742
743static int sideband_demux(int in, int out, void *data)
744{
745 int *xd = data;
9ff18faf 746 int ret;
745f7a8c 747
9ff18faf 748 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
749 close(out);
750 return ret;
751}
752
753static int get_pack(struct fetch_pack_args *args,
754 int xd[2], char **pack_lockfile)
755{
756 struct async demux;
745f7a8c 757 int do_keep = args->keep_pack;
984a43b9
JK
758 const char *cmd_name;
759 struct pack_header header;
760 int pass_header = 0;
d3180279 761 struct child_process cmd = CHILD_PROCESS_INIT;
c6807a40 762 int ret;
745f7a8c
NTND
763
764 memset(&demux, 0, sizeof(demux));
765 if (use_sideband) {
766 /* xd[] is talking with upload-pack; subprocess reads from
767 * xd[0], spits out band#2 to stderr, and feeds us band#1
768 * through demux->out.
769 */
770 demux.proc = sideband_demux;
771 demux.data = xd;
772 demux.out = -1;
df857572 773 demux.isolate_sigpipe = 1;
745f7a8c 774 if (start_async(&demux))
1dd73e20 775 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
776 }
777 else
778 demux.out = xd[0];
779
745f7a8c 780 if (!args->keep_pack && unpack_limit) {
745f7a8c
NTND
781
782 if (read_pack_header(demux.out, &header))
1dd73e20 783 die(_("protocol error: bad pack header"));
984a43b9 784 pass_header = 1;
745f7a8c
NTND
785 if (ntohl(header.hdr_entries) < unpack_limit)
786 do_keep = 0;
787 else
788 do_keep = 1;
789 }
790
6035d6aa 791 if (alternate_shallow_file) {
984a43b9
JK
792 argv_array_push(&cmd.args, "--shallow-file");
793 argv_array_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
794 }
795
745f7a8c
NTND
796 if (do_keep) {
797 if (pack_lockfile)
798 cmd.out = -1;
984a43b9
JK
799 cmd_name = "index-pack";
800 argv_array_push(&cmd.args, cmd_name);
801 argv_array_push(&cmd.args, "--stdin");
745f7a8c 802 if (!args->quiet && !args->no_progress)
984a43b9 803 argv_array_push(&cmd.args, "-v");
745f7a8c 804 if (args->use_thin_pack)
984a43b9 805 argv_array_push(&cmd.args, "--fix-thin");
745f7a8c 806 if (args->lock_pack || unpack_limit) {
da25bdb7 807 char hostname[HOST_NAME_MAX + 1];
5781a9a2 808 if (xgethostname(hostname, sizeof(hostname)))
984a43b9
JK
809 xsnprintf(hostname, sizeof(hostname), "localhost");
810 argv_array_pushf(&cmd.args,
811 "--keep=fetch-pack %"PRIuMAX " on %s",
812 (uintmax_t)getpid(), hostname);
745f7a8c 813 }
c6807a40 814 if (args->check_self_contained_and_connected)
984a43b9 815 argv_array_push(&cmd.args, "--check-self-contained-and-connected");
745f7a8c
NTND
816 }
817 else {
984a43b9
JK
818 cmd_name = "unpack-objects";
819 argv_array_push(&cmd.args, cmd_name);
745f7a8c 820 if (args->quiet || args->no_progress)
984a43b9 821 argv_array_push(&cmd.args, "-q");
c6807a40 822 args->check_self_contained_and_connected = 0;
745f7a8c 823 }
984a43b9
JK
824
825 if (pass_header)
826 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
827 ntohl(header.hdr_version),
828 ntohl(header.hdr_entries));
745f7a8c
NTND
829 if (fetch_fsck_objects >= 0
830 ? fetch_fsck_objects
831 : transfer_fsck_objects >= 0
832 ? transfer_fsck_objects
833 : 0)
984a43b9 834 argv_array_push(&cmd.args, "--strict");
745f7a8c
NTND
835
836 cmd.in = demux.out;
837 cmd.git_cmd = 1;
838 if (start_command(&cmd))
1dd73e20 839 die(_("fetch-pack: unable to fork off %s"), cmd_name);
745f7a8c
NTND
840 if (do_keep && pack_lockfile) {
841 *pack_lockfile = index_pack_lockfile(cmd.out);
842 close(cmd.out);
843 }
844
37cb1dd6
JL
845 if (!use_sideband)
846 /* Closed by start_command() */
847 xd[0] = -1;
848
c6807a40
NTND
849 ret = finish_command(&cmd);
850 if (!ret || (args->check_self_contained_and_connected && ret == 1))
851 args->self_contained_and_connected =
852 args->check_self_contained_and_connected &&
853 ret == 0;
854 else
1dd73e20 855 die(_("%s failed"), cmd_name);
745f7a8c 856 if (use_sideband && finish_async(&demux))
1dd73e20 857 die(_("error in sideband demultiplexer"));
745f7a8c
NTND
858 return 0;
859}
860
f2db854d
JH
861static int cmp_ref_by_name(const void *a_, const void *b_)
862{
863 const struct ref *a = *((const struct ref **)a_);
864 const struct ref *b = *((const struct ref **)b_);
865 return strcmp(a->name, b->name);
866}
867
745f7a8c
NTND
868static struct ref *do_fetch_pack(struct fetch_pack_args *args,
869 int fd[2],
870 const struct ref *orig_ref,
f2db854d 871 struct ref **sought, int nr_sought,
beea4152 872 struct shallow_info *si,
745f7a8c
NTND
873 char **pack_lockfile)
874{
875 struct ref *ref = copy_ref_list(orig_ref);
876 unsigned char sha1[20];
877 const char *agent_feature;
878 int agent_len;
879
880 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 881 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 882
eb86a507 883 if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1dd73e20 884 die(_("Server does not support shallow clients"));
a45a2600 885 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 886 args->deepen = 1;
745f7a8c 887 if (server_supports("multi_ack_detailed")) {
1dd73e20 888 print_verbose(args, _("Server supports multi_ack_detailed"));
745f7a8c
NTND
889 multi_ack = 2;
890 if (server_supports("no-done")) {
1dd73e20 891 print_verbose(args, _("Server supports no-done"));
745f7a8c
NTND
892 if (args->stateless_rpc)
893 no_done = 1;
894 }
895 }
896 else if (server_supports("multi_ack")) {
1dd73e20 897 print_verbose(args, _("Server supports multi_ack"));
745f7a8c
NTND
898 multi_ack = 1;
899 }
900 if (server_supports("side-band-64k")) {
1dd73e20 901 print_verbose(args, _("Server supports side-band-64k"));
745f7a8c
NTND
902 use_sideband = 2;
903 }
904 else if (server_supports("side-band")) {
1dd73e20 905 print_verbose(args, _("Server supports side-band"));
745f7a8c
NTND
906 use_sideband = 1;
907 }
6e7b66ee 908 if (server_supports("allow-tip-sha1-in-want")) {
1dd73e20 909 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
7199c093 910 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 911 }
68ee6289 912 if (server_supports("allow-reachable-sha1-in-want")) {
1dd73e20 913 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
68ee6289
FM
914 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
915 }
745f7a8c
NTND
916 if (!server_supports("thin-pack"))
917 args->use_thin_pack = 0;
918 if (!server_supports("no-progress"))
919 args->no_progress = 0;
920 if (!server_supports("include-tag"))
921 args->include_tag = 0;
0d789a5b 922 if (server_supports("ofs-delta"))
1dd73e20 923 print_verbose(args, _("Server supports ofs-delta"));
0d789a5b 924 else
745f7a8c
NTND
925 prefer_ofs_delta = 0;
926
927 if ((agent_feature = server_feature_value("agent", &agent_len))) {
928 agent_supported = 1;
0d789a5b 929 if (agent_len)
1dd73e20 930 print_verbose(args, _("Server version is %.*s"),
0d789a5b 931 agent_len, agent_feature);
745f7a8c 932 }
508ea882
NTND
933 if (server_supports("deepen-since"))
934 deepen_since_ok = 1;
935 else if (args->deepen_since)
936 die(_("Server does not support --shallow-since"));
a45a2600
NTND
937 if (server_supports("deepen-not"))
938 deepen_not_ok = 1;
939 else if (args->deepen_not)
940 die(_("Server does not support --shallow-exclude"));
cccf74e2
NTND
941 if (!server_supports("deepen-relative") && args->deepen_relative)
942 die(_("Server does not support --deepen"));
745f7a8c 943
f2db854d 944 if (everything_local(args, &ref, sought, nr_sought)) {
745f7a8c
NTND
945 packet_flush(fd[1]);
946 goto all_done;
947 }
948 if (find_common(args, fd, sha1, ref) < 0)
949 if (!args->keep_pack)
950 /* When cloning, it is not unusual to have
951 * no common commit.
952 */
1dd73e20 953 warning(_("no common commits"));
745f7a8c
NTND
954
955 if (args->stateless_rpc)
956 packet_flush(fd[1]);
79891cb9 957 if (args->deepen)
1a30f5a2
NTND
958 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
959 NULL);
4820a33b 960 else if (si->nr_ours || si->nr_theirs)
beea4152 961 alternate_shallow_file = setup_temporary_shallow(si->shallow);
6da8bdcb
NTND
962 else
963 alternate_shallow_file = NULL;
745f7a8c 964 if (get_pack(args, fd, pack_lockfile))
1dd73e20 965 die(_("git fetch-pack: fetch failed."));
745f7a8c
NTND
966
967 all_done:
968 return ref;
969}
970
f44af51d 971static void fetch_pack_config(void)
745f7a8c 972{
f44af51d
TA
973 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
974 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
975 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
976 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
977 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
745f7a8c 978
f44af51d 979 git_config(git_default_config, NULL);
745f7a8c
NTND
980}
981
745f7a8c
NTND
982static void fetch_pack_setup(void)
983{
984 static int did_setup;
985 if (did_setup)
986 return;
f44af51d 987 fetch_pack_config();
745f7a8c
NTND
988 if (0 <= transfer_unpack_limit)
989 unpack_limit = transfer_unpack_limit;
990 else if (0 <= fetch_unpack_limit)
991 unpack_limit = fetch_unpack_limit;
992 did_setup = 1;
993}
994
f2db854d
JH
995static int remove_duplicates_in_refs(struct ref **ref, int nr)
996{
997 struct string_list names = STRING_LIST_INIT_NODUP;
998 int src, dst;
999
1000 for (src = dst = 0; src < nr; src++) {
1001 struct string_list_item *item;
1002 item = string_list_insert(&names, ref[src]->name);
1003 if (item->util)
1004 continue; /* already have it */
1005 item->util = ref[src];
1006 if (src != dst)
1007 ref[dst] = ref[src];
1008 dst++;
1009 }
1010 for (src = dst; src < nr; src++)
1011 ref[src] = NULL;
1012 string_list_clear(&names, 0);
1013 return dst;
1014}
1015
beea4152 1016static void update_shallow(struct fetch_pack_args *args,
4820a33b 1017 struct ref **sought, int nr_sought,
beea4152 1018 struct shallow_info *si)
a796ccee 1019{
910650d2 1020 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1021 int *status;
beea4152
NTND
1022 int i;
1023
79891cb9 1024 if (args->deepen && alternate_shallow_file) {
a796ccee 1025 if (*alternate_shallow_file == '\0') { /* --unshallow */
f932729c 1026 unlink_or_warn(git_path_shallow());
a796ccee
NTND
1027 rollback_lock_file(&shallow_lock);
1028 } else
1029 commit_lock_file(&shallow_lock);
1030 return;
1031 }
beea4152
NTND
1032
1033 if (!si->shallow || !si->shallow->nr)
1034 return;
1035
beea4152
NTND
1036 if (args->cloning) {
1037 /*
1038 * remote is shallow, but this is a clone, there are
1039 * no objects in repo to worry about. Accept any
1040 * shallow points that exist in the pack (iow in repo
1041 * after get_pack() and reprepare_packed_git())
1042 */
910650d2 1043 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1044 struct object_id *oid = si->shallow->oid;
beea4152 1045 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1046 if (has_object_file(&oid[i]))
910650d2 1047 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1048 if (extra.nr) {
1049 setup_alternate_shallow(&shallow_lock,
1050 &alternate_shallow_file,
1051 &extra);
1052 commit_lock_file(&shallow_lock);
1053 }
910650d2 1054 oid_array_clear(&extra);
beea4152
NTND
1055 return;
1056 }
4820a33b
NTND
1057
1058 if (!si->nr_ours && !si->nr_theirs)
1059 return;
1060
1061 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1062 if (!si->nr_ours && !si->nr_theirs)
1063 return;
1064 for (i = 0; i < nr_sought; i++)
910650d2 1065 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1066 si->ref = &ref;
1067
48d25cae
NTND
1068 if (args->update_shallow) {
1069 /*
1070 * remote is also shallow, .git/shallow may be updated
1071 * so all refs can be accepted. Make sure we only add
1072 * shallow roots that are actually reachable from new
1073 * refs.
1074 */
910650d2 1075 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1076 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1077 assign_shallow_commits_to_refs(si, NULL, NULL);
1078 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1079 oid_array_clear(&ref);
48d25cae
NTND
1080 return;
1081 }
1082 for (i = 0; i < si->nr_ours; i++)
910650d2 1083 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1084 for (i = 0; i < si->nr_theirs; i++)
910650d2 1085 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1086 setup_alternate_shallow(&shallow_lock,
1087 &alternate_shallow_file,
1088 &extra);
1089 commit_lock_file(&shallow_lock);
910650d2 1090 oid_array_clear(&extra);
1091 oid_array_clear(&ref);
48d25cae
NTND
1092 return;
1093 }
1094
4820a33b
NTND
1095 /*
1096 * remote is also shallow, check what ref is safe to update
1097 * without updating .git/shallow
1098 */
1099 status = xcalloc(nr_sought, sizeof(*status));
1100 assign_shallow_commits_to_refs(si, NULL, status);
1101 if (si->nr_ours || si->nr_theirs) {
1102 for (i = 0; i < nr_sought; i++)
1103 if (status[i])
1104 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1105 }
1106 free(status);
910650d2 1107 oid_array_clear(&ref);
a796ccee
NTND
1108}
1109
745f7a8c
NTND
1110struct ref *fetch_pack(struct fetch_pack_args *args,
1111 int fd[], struct child_process *conn,
1112 const struct ref *ref,
1113 const char *dest,
f2db854d 1114 struct ref **sought, int nr_sought,
910650d2 1115 struct oid_array *shallow,
745f7a8c
NTND
1116 char **pack_lockfile)
1117{
745f7a8c 1118 struct ref *ref_cpy;
beea4152 1119 struct shallow_info si;
745f7a8c
NTND
1120
1121 fetch_pack_setup();
f2db854d
JH
1122 if (nr_sought)
1123 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c
NTND
1124
1125 if (!ref) {
1126 packet_flush(fd[1]);
1dd73e20 1127 die(_("no matching remote head"));
745f7a8c 1128 }
beea4152
NTND
1129 prepare_shallow_info(&si, shallow);
1130 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1131 &si, pack_lockfile);
745f7a8c 1132 reprepare_packed_git();
4820a33b 1133 update_shallow(args, sought, nr_sought, &si);
beea4152 1134 clear_shallow_info(&si);
745f7a8c
NTND
1135 return ref_cpy;
1136}
e860d96b
MM
1137
1138int report_unmatched_refs(struct ref **sought, int nr_sought)
1139{
1140 int i, ret = 0;
1141
1142 for (i = 0; i < nr_sought; i++) {
d56583de 1143 if (!sought[i])
e860d96b 1144 continue;
d56583de
MM
1145 switch (sought[i]->match_status) {
1146 case REF_MATCHED:
1147 continue;
1148 case REF_NOT_MATCHED:
1149 error(_("no such remote ref %s"), sought[i]->name);
1150 break;
1151 case REF_UNADVERTISED_NOT_ALLOWED:
1152 error(_("Server does not allow request for unadvertised object %s"),
1153 sought[i]->name);
1154 break;
1155 }
e860d96b
MM
1156 ret = 1;
1157 }
1158 return ret;
1159}