Don't die in git-http-fetch when fetching packs.
[git/git.git] / http-fetch.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
271421cd 3#include "pack.h"
215a7ad1 4#include "fetch.h"
29508e1e 5#include "http.h"
7baa3e86 6
49a0f240
NH
7#define PREV_BUF_SIZE 4096
8#define RANGE_HEADER_SIZE 30
9
96f1e58f 10static int commits_on_stdin;
8e29f6a0 11
acc075a8 12static int got_alternates = -1;
96f1e58f 13static int corrupt_object_found;
1d389ab6 14
1db69b57 15static struct curl_slist *no_pragma_header;
6eb7ed54 16
b3661567
DB
17struct alt_base
18{
8e29f6a0 19 const char *base;
8d9fbe57 20 int path_len;
b3661567
DB
21 int got_indices;
22 struct packed_git *packs;
23 struct alt_base *next;
24};
25
96f1e58f 26static struct alt_base *alt;
6eb7ed54 27
e388ab74 28enum object_request_state {
1d389ab6
NH
29 WAITING,
30 ABORTED,
31 ACTIVE,
32 COMPLETE,
33};
6eb7ed54 34
e388ab74 35struct object_request
1d389ab6
NH
36{
37 unsigned char sha1[20];
38 struct alt_base *repo;
39 char *url;
40 char filename[PATH_MAX];
41 char tmpfile[PATH_MAX];
42 int local;
e388ab74 43 enum object_request_state state;
1d389ab6
NH
44 CURLcode curl_result;
45 char errorstr[CURL_ERROR_SIZE];
46 long http_code;
47 unsigned char real_sha1[20];
48 SHA_CTX c;
49 z_stream stream;
50 int zret;
51 int rename;
52 struct active_request_slot *slot;
e388ab74 53 struct object_request *next;
1d389ab6
NH
54};
55
e388ab74 56struct alternates_request {
8e29f6a0 57 const char *base;
acc075a8
NH
58 char *url;
59 struct buffer *buffer;
60 struct active_request_slot *slot;
61 int http_specific;
62};
63
96f1e58f 64static struct object_request *object_queue_head;
bc8f2652 65
182005b9
DB
66static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
67 void *data)
68{
bf0f910d 69 unsigned char expn[4096];
6eb7ed54
DB
70 size_t size = eltsize * nmemb;
71 int posn = 0;
e388ab74 72 struct object_request *obj_req = (struct object_request *)data;
6eb7ed54 73 do {
93822c22 74 ssize_t retval = xwrite(obj_req->local,
1d7f171c 75 (char *) ptr + posn, size - posn);
6eb7ed54
DB
76 if (retval < 0)
77 return posn;
78 posn += retval;
79 } while (posn < size);
80
e388ab74
NH
81 obj_req->stream.avail_in = size;
82 obj_req->stream.next_in = ptr;
6eb7ed54 83 do {
e388ab74
NH
84 obj_req->stream.next_out = expn;
85 obj_req->stream.avail_out = sizeof(expn);
86 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
87 SHA1_Update(&obj_req->c, expn,
88 sizeof(expn) - obj_req->stream.avail_out);
89 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
1d389ab6 90 data_received++;
6eb7ed54
DB
91 return size;
92}
93
be4a015b
JH
94static int missing__target(int code, int result)
95{
96 return /* file:// URL -- do we ever use one??? */
97 (result == CURLE_FILE_COULDNT_READ_FILE) ||
98 /* http:// and https:// URL */
4adffc7b
JH
99 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
100 /* ftp:// URL */
101 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
be4a015b
JH
102 ;
103}
104
105#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
106
8e29f6a0 107static void fetch_alternates(const char *base);
1d389ab6 108
29508e1e 109static void process_object_response(void *callback_data);
1d389ab6 110
e388ab74 111static void start_object_request(struct object_request *obj_req)
1d389ab6 112{
e388ab74 113 char *hex = sha1_to_hex(obj_req->sha1);
1d389ab6
NH
114 char prevfile[PATH_MAX];
115 char *url;
116 char *posn;
117 int prevlocal;
118 unsigned char prev_buf[PREV_BUF_SIZE];
119 ssize_t prev_read = 0;
120 long prev_posn = 0;
121 char range[RANGE_HEADER_SIZE];
122 struct curl_slist *range_header = NULL;
123 struct active_request_slot *slot;
124
e388ab74 125 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
1d389ab6 126 unlink(prevfile);
e388ab74
NH
127 rename(obj_req->tmpfile, prevfile);
128 unlink(obj_req->tmpfile);
1d389ab6 129
e388ab74
NH
130 if (obj_req->local != -1)
131 error("fd leakage in start: %d", obj_req->local);
132 obj_req->local = open(obj_req->tmpfile,
1d389ab6 133 O_WRONLY | O_CREAT | O_EXCL, 0666);
b721e01f
JH
134 /* This could have failed due to the "lazy directory creation";
135 * try to mkdir the last path component.
136 */
e388ab74
NH
137 if (obj_req->local < 0 && errno == ENOENT) {
138 char *dir = strrchr(obj_req->tmpfile, '/');
b721e01f
JH
139 if (dir) {
140 *dir = 0;
e388ab74 141 mkdir(obj_req->tmpfile, 0777);
b721e01f
JH
142 *dir = '/';
143 }
e388ab74 144 obj_req->local = open(obj_req->tmpfile,
b721e01f
JH
145 O_WRONLY | O_CREAT | O_EXCL, 0666);
146 }
147
e388ab74
NH
148 if (obj_req->local < 0) {
149 obj_req->state = ABORTED;
bd2afde8 150 error("Couldn't create temporary file %s for %s: %s",
e388ab74 151 obj_req->tmpfile, obj_req->filename, strerror(errno));
1d389ab6
NH
152 return;
153 }
154
e388ab74 155 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
1d389ab6 156
e388ab74 157 inflateInit(&obj_req->stream);
1d389ab6 158
e388ab74 159 SHA1_Init(&obj_req->c);
1d389ab6 160
e388ab74
NH
161 url = xmalloc(strlen(obj_req->repo->base) + 50);
162 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
163 strcpy(url, obj_req->repo->base);
164 posn = url + strlen(obj_req->repo->base);
1d389ab6
NH
165 strcpy(posn, "objects/");
166 posn += 8;
167 memcpy(posn, hex, 2);
168 posn += 2;
169 *(posn++) = '/';
170 strcpy(posn, hex + 2);
e388ab74 171 strcpy(obj_req->url, url);
1d389ab6
NH
172
173 /* If a previous temp file is present, process what was already
174 fetched. */
175 prevlocal = open(prevfile, O_RDONLY);
176 if (prevlocal != -1) {
177 do {
93d26e4c 178 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1d389ab6
NH
179 if (prev_read>0) {
180 if (fwrite_sha1_file(prev_buf,
181 1,
182 prev_read,
e388ab74 183 obj_req) == prev_read) {
1d389ab6
NH
184 prev_posn += prev_read;
185 } else {
186 prev_read = -1;
187 }
188 }
189 } while (prev_read > 0);
190 close(prevlocal);
191 }
192 unlink(prevfile);
193
194 /* Reset inflate/SHA1 if there was an error reading the previous temp
195 file; also rewind to the beginning of the local file. */
196 if (prev_read == -1) {
e388ab74
NH
197 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
198 inflateInit(&obj_req->stream);
199 SHA1_Init(&obj_req->c);
1d389ab6
NH
200 if (prev_posn>0) {
201 prev_posn = 0;
e388ab74
NH
202 lseek(obj_req->local, SEEK_SET, 0);
203 ftruncate(obj_req->local, 0);
1d389ab6
NH
204 }
205 }
206
207 slot = get_active_slot();
29508e1e 208 slot->callback_func = process_object_response;
e388ab74
NH
209 slot->callback_data = obj_req;
210 obj_req->slot = slot;
29508e1e 211
e388ab74 212 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
1d389ab6 213 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
e388ab74 214 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
1d389ab6 215 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 216 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
217
218 /* If we have successfully processed data from a previous fetch
219 attempt, only fetch the data we don't already have. */
220 if (prev_posn>0) {
221 if (get_verbosely)
222 fprintf(stderr,
223 "Resuming fetch of object %s at byte %ld\n",
224 hex, prev_posn);
225 sprintf(range, "Range: bytes=%ld-", prev_posn);
226 range_header = curl_slist_append(range_header, range);
227 curl_easy_setopt(slot->curl,
228 CURLOPT_HTTPHEADER, range_header);
229 }
230
a7a8d378 231 /* Try to get the request started, abort the request on error */
e388ab74 232 obj_req->state = ACTIVE;
1d389ab6 233 if (!start_active_slot(slot)) {
e388ab74
NH
234 obj_req->state = ABORTED;
235 obj_req->slot = NULL;
236 close(obj_req->local); obj_req->local = -1;
237 free(obj_req->url);
238 return;
1d389ab6 239 }
1d389ab6
NH
240}
241
e388ab74 242static void finish_object_request(struct object_request *obj_req)
1d389ab6 243{
50496b21
NH
244 struct stat st;
245
e388ab74
NH
246 fchmod(obj_req->local, 0444);
247 close(obj_req->local); obj_req->local = -1;
1d389ab6 248
e388ab74 249 if (obj_req->http_code == 416) {
1d389ab6 250 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
e388ab74
NH
251 } else if (obj_req->curl_result != CURLE_OK) {
252 if (stat(obj_req->tmpfile, &st) == 0)
50496b21 253 if (st.st_size == 0)
e388ab74 254 unlink(obj_req->tmpfile);
1d389ab6
NH
255 return;
256 }
257
e388ab74
NH
258 inflateEnd(&obj_req->stream);
259 SHA1_Final(obj_req->real_sha1, &obj_req->c);
260 if (obj_req->zret != Z_STREAM_END) {
261 unlink(obj_req->tmpfile);
1d389ab6
NH
262 return;
263 }
a89fccd2 264 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
e388ab74 265 unlink(obj_req->tmpfile);
1d389ab6
NH
266 return;
267 }
e388ab74
NH
268 obj_req->rename =
269 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
1d389ab6 270
e388ab74
NH
271 if (obj_req->rename == 0)
272 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
1d389ab6
NH
273}
274
29508e1e
NH
275static void process_object_response(void *callback_data)
276{
e388ab74
NH
277 struct object_request *obj_req =
278 (struct object_request *)callback_data;
29508e1e 279
e388ab74
NH
280 obj_req->curl_result = obj_req->slot->curl_result;
281 obj_req->http_code = obj_req->slot->http_code;
282 obj_req->slot = NULL;
283 obj_req->state = COMPLETE;
29508e1e
NH
284
285 /* Use alternates if necessary */
be4a015b 286 if (missing_target(obj_req)) {
29508e1e 287 fetch_alternates(alt->base);
e388ab74
NH
288 if (obj_req->repo->next != NULL) {
289 obj_req->repo =
290 obj_req->repo->next;
291 close(obj_req->local);
292 obj_req->local = -1;
293 start_object_request(obj_req);
29508e1e
NH
294 return;
295 }
296 }
297
e388ab74 298 finish_object_request(obj_req);
29508e1e
NH
299}
300
e388ab74 301static void release_object_request(struct object_request *obj_req)
1d389ab6 302{
e388ab74 303 struct object_request *entry = object_queue_head;
1d389ab6 304
e388ab74
NH
305 if (obj_req->local != -1)
306 error("fd leakage in release: %d", obj_req->local);
307 if (obj_req == object_queue_head) {
308 object_queue_head = obj_req->next;
1d389ab6 309 } else {
e388ab74 310 while (entry->next != NULL && entry->next != obj_req)
1d389ab6 311 entry = entry->next;
e388ab74 312 if (entry->next == obj_req)
1d389ab6
NH
313 entry->next = entry->next->next;
314 }
315
e388ab74
NH
316 free(obj_req->url);
317 free(obj_req);
1d389ab6
NH
318}
319
a7a8d378 320#ifdef USE_CURL_MULTI
29508e1e 321void fill_active_slots(void)
1d389ab6 322{
e388ab74 323 struct object_request *obj_req = object_queue_head;
f1a906a3 324 struct active_request_slot *slot = active_queue_head;
1d389ab6
NH
325 int num_transfers;
326
e388ab74
NH
327 while (active_requests < max_requests && obj_req != NULL) {
328 if (obj_req->state == WAITING) {
329 if (has_sha1_file(obj_req->sha1))
09db444f 330 obj_req->state = COMPLETE;
11f0dafe 331 else
e388ab74 332 start_object_request(obj_req);
1d389ab6
NH
333 curl_multi_perform(curlm, &num_transfers);
334 }
e388ab74 335 obj_req = obj_req->next;
1d389ab6 336 }
f1a906a3
NH
337
338 while (slot != NULL) {
339 if (!slot->in_use && slot->curl != NULL) {
340 curl_easy_cleanup(slot->curl);
341 slot->curl = NULL;
342 }
343 slot = slot->next;
8fcf7f9a 344 }
1d389ab6 345}
a7a8d378 346#endif
1d389ab6
NH
347
348void prefetch(unsigned char *sha1)
349{
e388ab74
NH
350 struct object_request *newreq;
351 struct object_request *tail;
1d389ab6
NH
352 char *filename = sha1_file_name(sha1);
353
354 newreq = xmalloc(sizeof(*newreq));
e702496e 355 hashcpy(newreq->sha1, sha1);
1d389ab6
NH
356 newreq->repo = alt;
357 newreq->url = NULL;
358 newreq->local = -1;
359 newreq->state = WAITING;
360 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
361 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
362 "%s.temp", filename);
e8dff6ba 363 newreq->slot = NULL;
1d389ab6
NH
364 newreq->next = NULL;
365
e388ab74
NH
366 if (object_queue_head == NULL) {
367 object_queue_head = newreq;
1d389ab6 368 } else {
e388ab74 369 tail = object_queue_head;
1d389ab6
NH
370 while (tail->next != NULL) {
371 tail = tail->next;
372 }
373 tail->next = newreq;
374 }
29508e1e 375
a7a8d378 376#ifdef USE_CURL_MULTI
29508e1e
NH
377 fill_active_slots();
378 step_active_slots();
a7a8d378 379#endif
1d389ab6
NH
380}
381
b3661567 382static int fetch_index(struct alt_base *repo, unsigned char *sha1)
182005b9 383{
1d389ab6 384 char *hex = sha1_to_hex(sha1);
182005b9
DB
385 char *filename;
386 char *url;
49a0f240 387 char tmpfile[PATH_MAX];
49a0f240
NH
388 long prev_posn = 0;
389 char range[RANGE_HEADER_SIZE];
390 struct curl_slist *range_header = NULL;
182005b9
DB
391
392 FILE *indexfile;
1d389ab6 393 struct active_request_slot *slot;
cb754fdf 394 struct slot_results results;
182005b9
DB
395
396 if (has_pack_index(sha1))
397 return 0;
398
399 if (get_verbosely)
1d389ab6 400 fprintf(stderr, "Getting index for pack %s\n", hex);
8fcf7f9a 401
b3661567 402 url = xmalloc(strlen(repo->base) + 64);
1d389ab6 403 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
8fcf7f9a 404
182005b9 405 filename = sha1_pack_index_name(sha1);
49a0f240
NH
406 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
407 indexfile = fopen(tmpfile, "a");
182005b9
DB
408 if (!indexfile)
409 return error("Unable to open local file %s for pack index",
410 filename);
411
1d389ab6 412 slot = get_active_slot();
c8568e13 413 slot->results = &results;
1d389ab6
NH
414 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
415 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
416 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 417 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
418 slot->local = indexfile;
419
49a0f240
NH
420 /* If there is data present from a previous transfer attempt,
421 resume where it left off */
422 prev_posn = ftell(indexfile);
423 if (prev_posn>0) {
424 if (get_verbosely)
425 fprintf(stderr,
426 "Resuming fetch of index for pack %s at byte %ld\n",
1d389ab6 427 hex, prev_posn);
49a0f240
NH
428 sprintf(range, "Range: bytes=%ld-", prev_posn);
429 range_header = curl_slist_append(range_header, range);
1d389ab6 430 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
431 }
432
1d389ab6
NH
433 if (start_active_slot(slot)) {
434 run_active_slot(slot);
c8568e13 435 if (results.curl_result != CURLE_OK) {
1d389ab6
NH
436 fclose(indexfile);
437 return error("Unable to get pack index %s\n%s", url,
438 curl_errorstr);
439 }
440 } else {
313c4714 441 fclose(indexfile);
1d389ab6 442 return error("Unable to start request");
182005b9
DB
443 }
444
445 fclose(indexfile);
49a0f240 446
b721e01f 447 return move_temp_to_file(tmpfile, filename);
182005b9
DB
448}
449
b3661567 450static int setup_index(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
451{
452 struct packed_git *new_pack;
453 if (has_pack_file(sha1))
a9486b02 454 return 0; /* don't list this as something we can get */
182005b9 455
b3661567 456 if (fetch_index(repo, sha1))
182005b9
DB
457 return -1;
458
459 new_pack = parse_pack_index(sha1);
b3661567
DB
460 new_pack->next = repo->packs;
461 repo->packs = new_pack;
182005b9
DB
462 return 0;
463}
464
e388ab74 465static void process_alternates_response(void *callback_data)
b3661567 466{
e388ab74
NH
467 struct alternates_request *alt_req =
468 (struct alternates_request *)callback_data;
acc075a8 469 struct active_request_slot *slot = alt_req->slot;
1d389ab6 470 struct alt_base *tail = alt;
8e29f6a0 471 const char *base = alt_req->base;
bc8f2652 472 static const char null_byte = '\0';
acc075a8
NH
473 char *data;
474 int i = 0;
1d389ab6 475
acc075a8
NH
476 if (alt_req->http_specific) {
477 if (slot->curl_result != CURLE_OK ||
478 !alt_req->buffer->posn) {
479
480 /* Try reusing the slot to get non-http alternates */
481 alt_req->http_specific = 0;
482 sprintf(alt_req->url, "%s/objects/info/alternates",
483 base);
484 curl_easy_setopt(slot->curl, CURLOPT_URL,
485 alt_req->url);
486 active_requests++;
487 slot->in_use = 1;
c9826473
NH
488 if (slot->finished != NULL)
489 (*slot->finished) = 0;
a3f583cb 490 if (!start_active_slot(slot)) {
acc075a8 491 got_alternates = -1;
29508e1e 492 slot->in_use = 0;
c9826473
NH
493 if (slot->finished != NULL)
494 (*slot->finished) = 1;
1d389ab6 495 }
a3f583cb 496 return;
b3661567 497 }
acc075a8 498 } else if (slot->curl_result != CURLE_OK) {
be4a015b 499 if (!missing_target(slot)) {
acc075a8
NH
500 got_alternates = -1;
501 return;
502 }
b3661567
DB
503 }
504
29508e1e 505 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
acc075a8
NH
506 alt_req->buffer->posn--;
507 data = alt_req->buffer->buffer;
1b0c1e67 508
acc075a8 509 while (i < alt_req->buffer->posn) {
b3661567 510 int posn = i;
acc075a8 511 while (posn < alt_req->buffer->posn && data[posn] != '\n')
b3661567
DB
512 posn++;
513 if (data[posn] == '\n') {
1b0c1e67
DB
514 int okay = 0;
515 int serverlen = 0;
516 struct alt_base *newalt;
517 char *target = NULL;
8d9fbe57 518 char *path;
b3661567 519 if (data[i] == '/') {
5df1e0d0
JH
520 /* This counts
521 * http://git.host/pub/scm/linux.git/
522 * -----------here^
523 * so memcpy(dst, base, serverlen) will
524 * copy up to "...git.host".
525 */
526 const char *colon_ss = strstr(base,"://");
527 if (colon_ss) {
528 serverlen = (strchr(colon_ss + 3, '/')
529 - base);
530 okay = 1;
531 }
1b0c1e67 532 } else if (!memcmp(data + i, "../", 3)) {
5df1e0d0
JH
533 /* Relative URL; chop the corresponding
534 * number of subpath from base (and ../
535 * from data), and concatenate the result.
536 *
537 * The code first drops ../ from data, and
538 * then drops one ../ from data and one path
539 * from base. IOW, one extra ../ is dropped
540 * from data than path is dropped from base.
541 *
542 * This is not wrong. The alternate in
543 * http://git.host/pub/scm/linux.git/
544 * to borrow from
545 * http://git.host/pub/scm/linus.git/
546 * is ../../linus.git/objects/. You need
547 * two ../../ to borrow from your direct
548 * neighbour.
549 */
1b0c1e67
DB
550 i += 3;
551 serverlen = strlen(base);
8fcf7f9a 552 while (i + 2 < posn &&
1b0c1e67
DB
553 !memcmp(data + i, "../", 3)) {
554 do {
555 serverlen--;
556 } while (serverlen &&
557 base[serverlen - 1] != '/');
558 i += 3;
559 }
a9486b02 560 /* If the server got removed, give up. */
8fcf7f9a 561 okay = strchr(base, ':') - base + 3 <
1b0c1e67 562 serverlen;
acc075a8 563 } else if (alt_req->http_specific) {
1b0c1e67
DB
564 char *colon = strchr(data + i, ':');
565 char *slash = strchr(data + i, '/');
566 if (colon && slash && colon < data + posn &&
567 slash < data + posn && colon < slash) {
568 okay = 1;
569 }
570 }
5df1e0d0 571 /* skip "objects\n" at end */
1b0c1e67
DB
572 if (okay) {
573 target = xmalloc(serverlen + posn - i - 6);
5df1e0d0
JH
574 memcpy(target, base, serverlen);
575 memcpy(target + serverlen, data + i,
576 posn - i - 7);
577 target[serverlen + posn - i - 7] = 0;
b3661567 578 if (get_verbosely)
8fcf7f9a 579 fprintf(stderr,
b3661567
DB
580 "Also look at %s\n", target);
581 newalt = xmalloc(sizeof(*newalt));
1d389ab6 582 newalt->next = NULL;
b3661567
DB
583 newalt->base = target;
584 newalt->got_indices = 0;
585 newalt->packs = NULL;
8d9fbe57
NH
586 path = strstr(target, "//");
587 if (path) {
ef9e58c8 588 path = strchr(path+2, '/');
8d9fbe57
NH
589 if (path)
590 newalt->path_len = strlen(path);
591 }
592
1d389ab6
NH
593 while (tail->next != NULL)
594 tail = tail->next;
595 tail->next = newalt;
b3661567
DB
596 }
597 }
598 i = posn + 1;
599 }
bc8f2652 600
f7eb290f 601 got_alternates = 1;
acc075a8
NH
602}
603
8e29f6a0 604static void fetch_alternates(const char *base)
acc075a8
NH
605{
606 struct buffer buffer;
607 char *url;
608 char *data;
609 struct active_request_slot *slot;
cb754fdf 610 struct alternates_request alt_req;
acc075a8
NH
611
612 /* If another request has already started fetching alternates,
613 wait for them to arrive and return to processing this request's
614 curl message */
29508e1e 615#ifdef USE_CURL_MULTI
acc075a8 616 while (got_alternates == 0) {
29508e1e 617 step_active_slots();
acc075a8 618 }
29508e1e 619#endif
acc075a8
NH
620
621 /* Nothing to do if they've already been fetched */
622 if (got_alternates == 1)
623 return;
624
625 /* Start the fetch */
626 got_alternates = 0;
627
628 data = xmalloc(4096);
629 buffer.size = 4096;
630 buffer.posn = 0;
631 buffer.buffer = data;
632
633 if (get_verbosely)
634 fprintf(stderr, "Getting alternates list for %s\n", base);
8fcf7f9a 635
acc075a8
NH
636 url = xmalloc(strlen(base) + 31);
637 sprintf(url, "%s/objects/info/http-alternates", base);
638
639 /* Use a callback to process the result, since another request
640 may fail and need to have alternates loaded before continuing */
641 slot = get_active_slot();
e388ab74 642 slot->callback_func = process_alternates_response;
acc075a8
NH
643 slot->callback_data = &alt_req;
644
645 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 646 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
acc075a8
NH
647 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
648
649 alt_req.base = base;
650 alt_req.url = url;
651 alt_req.buffer = &buffer;
652 alt_req.http_specific = 1;
653 alt_req.slot = slot;
654
655 if (start_active_slot(slot))
656 run_active_slot(slot);
657 else
658 got_alternates = -1;
659
660 free(data);
661 free(url);
b3661567
DB
662}
663
664static int fetch_indices(struct alt_base *repo)
182005b9
DB
665{
666 unsigned char sha1[20];
667 char *url;
668 struct buffer buffer;
669 char *data;
670 int i = 0;
671
1d389ab6 672 struct active_request_slot *slot;
cb754fdf 673 struct slot_results results;
1d389ab6 674
b3661567 675 if (repo->got_indices)
182005b9
DB
676 return 0;
677
678 data = xmalloc(4096);
679 buffer.size = 4096;
680 buffer.posn = 0;
681 buffer.buffer = data;
682
683 if (get_verbosely)
6fd72e39 684 fprintf(stderr, "Getting pack list for %s\n", repo->base);
8fcf7f9a 685
b3661567
DB
686 url = xmalloc(strlen(repo->base) + 21);
687 sprintf(url, "%s/objects/info/packs", repo->base);
182005b9 688
1d389ab6 689 slot = get_active_slot();
c8568e13 690 slot->results = &results;
1d389ab6 691 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 692 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1d389ab6
NH
693 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
694 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
695 if (start_active_slot(slot)) {
696 run_active_slot(slot);
c8568e13 697 if (results.curl_result != CURLE_OK) {
be4a015b 698 if (missing_target(&results)) {
5e3a7691
NH
699 repo->got_indices = 1;
700 free(buffer.buffer);
701 return 0;
702 } else {
703 repo->got_indices = 0;
704 free(buffer.buffer);
705 return error("%s", curl_errorstr);
706 }
bc8f2652 707 }
1d389ab6 708 } else {
5e3a7691 709 repo->got_indices = 0;
bc8f2652 710 free(buffer.buffer);
1d389ab6
NH
711 return error("Unable to start request");
712 }
182005b9 713
bc8f2652 714 data = buffer.buffer;
b3661567 715 while (i < buffer.posn) {
182005b9
DB
716 switch (data[i]) {
717 case 'P':
718 i++;
455c161c 719 if (i + 52 <= buffer.posn &&
182005b9
DB
720 !strncmp(data + i, " pack-", 6) &&
721 !strncmp(data + i + 46, ".pack\n", 6)) {
722 get_sha1_hex(data + i + 6, sha1);
b3661567 723 setup_index(repo, sha1);
182005b9
DB
724 i += 51;
725 break;
726 }
727 default:
455c161c 728 while (i < buffer.posn && data[i] != '\n')
182005b9
DB
729 i++;
730 }
731 i++;
b3661567 732 }
182005b9 733
bc8f2652 734 free(buffer.buffer);
b3661567 735 repo->got_indices = 1;
182005b9
DB
736 return 0;
737}
738
b3661567 739static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
740{
741 char *url;
742 struct packed_git *target;
743 struct packed_git **lst;
744 FILE *packfile;
745 char *filename;
49a0f240
NH
746 char tmpfile[PATH_MAX];
747 int ret;
748 long prev_posn = 0;
749 char range[RANGE_HEADER_SIZE];
750 struct curl_slist *range_header = NULL;
1d389ab6
NH
751
752 struct active_request_slot *slot;
cb754fdf 753 struct slot_results results;
182005b9 754
b3661567 755 if (fetch_indices(repo))
182005b9 756 return -1;
b3661567 757 target = find_sha1_pack(sha1, repo->packs);
182005b9 758 if (!target)
b3661567 759 return -1;
182005b9
DB
760
761 if (get_verbosely) {
762 fprintf(stderr, "Getting pack %s\n",
763 sha1_to_hex(target->sha1));
764 fprintf(stderr, " which contains %s\n",
765 sha1_to_hex(sha1));
766 }
767
b3661567 768 url = xmalloc(strlen(repo->base) + 65);
182005b9 769 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 770 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
771
772 filename = sha1_pack_name(target->sha1);
49a0f240
NH
773 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
774 packfile = fopen(tmpfile, "a");
182005b9
DB
775 if (!packfile)
776 return error("Unable to open local file %s for pack",
777 filename);
778
1d389ab6 779 slot = get_active_slot();
c8568e13 780 slot->results = &results;
1d389ab6
NH
781 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
782 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
783 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 784 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 785 slot->local = packfile;
1ddea77e 786
49a0f240
NH
787 /* If there is data present from a previous transfer attempt,
788 resume where it left off */
789 prev_posn = ftell(packfile);
790 if (prev_posn>0) {
791 if (get_verbosely)
792 fprintf(stderr,
793 "Resuming fetch of pack %s at byte %ld\n",
794 sha1_to_hex(target->sha1), prev_posn);
795 sprintf(range, "Range: bytes=%ld-", prev_posn);
796 range_header = curl_slist_append(range_header, range);
1d389ab6 797 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
798 }
799
1d389ab6
NH
800 if (start_active_slot(slot)) {
801 run_active_slot(slot);
c8568e13 802 if (results.curl_result != CURLE_OK) {
1d389ab6
NH
803 fclose(packfile);
804 return error("Unable to get pack file %s\n%s", url,
805 curl_errorstr);
806 }
807 } else {
313c4714 808 fclose(packfile);
1d389ab6 809 return error("Unable to start request");
182005b9
DB
810 }
811
1c23d794 812 target->pack_size = ftell(packfile);
182005b9
DB
813 fclose(packfile);
814
b721e01f 815 ret = move_temp_to_file(tmpfile, filename);
49a0f240 816 if (ret)
b721e01f 817 return ret;
49a0f240 818
b3661567 819 lst = &repo->packs;
182005b9
DB
820 while (*lst != target)
821 lst = &((*lst)->next);
822 *lst = (*lst)->next;
823
271421cd
JH
824 if (verify_pack(target, 0))
825 return -1;
182005b9
DB
826 install_packed_git(target);
827
828 return 0;
829}
830
53f31389
MW
831static void abort_object_request(struct object_request *obj_req)
832{
833 if (obj_req->local >= 0) {
834 close(obj_req->local);
835 obj_req->local = -1;
836 }
837 unlink(obj_req->tmpfile);
838 if (obj_req->slot) {
839 release_active_slot(obj_req->slot);
840 obj_req->slot = NULL;
841 }
842 release_object_request(obj_req);
843}
844
a7928f8e 845static int fetch_object(struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
846{
847 char *hex = sha1_to_hex(sha1);
29508e1e 848 int ret = 0;
e388ab74 849 struct object_request *obj_req = object_queue_head;
1d389ab6 850
a89fccd2 851 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
e388ab74
NH
852 obj_req = obj_req->next;
853 if (obj_req == NULL)
1d389ab6
NH
854 return error("Couldn't find request for %s in the queue", hex);
855
e388ab74 856 if (has_sha1_file(obj_req->sha1)) {
53f31389 857 abort_object_request(obj_req);
11f0dafe
NH
858 return 0;
859 }
860
a7a8d378 861#ifdef USE_CURL_MULTI
e388ab74 862 while (obj_req->state == WAITING) {
29508e1e 863 step_active_slots();
1d389ab6 864 }
a7a8d378 865#else
e388ab74 866 start_object_request(obj_req);
a7a8d378 867#endif
6eb7ed54 868
e388ab74
NH
869 while (obj_req->state == ACTIVE) {
870 run_active_slot(obj_req->slot);
a7a8d378 871 }
e388ab74
NH
872 if (obj_req->local != -1) {
873 close(obj_req->local); obj_req->local = -1;
313c4714 874 }
6eb7ed54 875
e388ab74 876 if (obj_req->state == ABORTED) {
29508e1e 877 ret = error("Request for %s aborted", hex);
e388ab74
NH
878 } else if (obj_req->curl_result != CURLE_OK &&
879 obj_req->http_code != 416) {
be4a015b 880 if (missing_target(obj_req))
e2029eb9
PB
881 ret = -1; /* Be silent, it is probably in a pack. */
882 else
883 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
e388ab74
NH
884 obj_req->errorstr, obj_req->curl_result,
885 obj_req->http_code, hex);
886 } else if (obj_req->zret != Z_STREAM_END) {
bb528079 887 corrupt_object_found++;
bd2afde8 888 ret = error("File %s (%s) corrupt", hex, obj_req->url);
a89fccd2 889 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
bd2afde8 890 ret = error("File %s has bad hash", hex);
e388ab74 891 } else if (obj_req->rename < 0) {
7b934ec0
MW
892 ret = error("unable to write sha1 filename %s",
893 obj_req->filename);
6eb7ed54 894 }
49a0f240 895
e388ab74 896 release_object_request(obj_req);
29508e1e 897 return ret;
6eb7ed54
DB
898}
899
b3661567
DB
900int fetch(unsigned char *sha1)
901{
902 struct alt_base *altbase = alt;
1d389ab6
NH
903
904 if (!fetch_object(altbase, sha1))
905 return 0;
b3661567 906 while (altbase) {
b3661567
DB
907 if (!fetch_pack(altbase, sha1))
908 return 0;
f7eb290f 909 fetch_alternates(alt->base);
b3661567
DB
910 altbase = altbase->next;
911 }
bd2afde8 912 return error("Unable to find %s under %s", sha1_to_hex(sha1),
1d389ab6 913 alt->base);
b3661567
DB
914}
915
94fa447a
JH
916static inline int needs_quote(int ch)
917{
cfd432e6
FF
918 if (((ch >= 'A') && (ch <= 'Z'))
919 || ((ch >= 'a') && (ch <= 'z'))
920 || ((ch >= '0') && (ch <= '9'))
921 || (ch == '/')
922 || (ch == '-')
923 || (ch == '.'))
94fa447a 924 return 0;
cfd432e6 925 return 1;
94fa447a
JH
926}
927
928static inline int hex(int v)
929{
930 if (v < 10) return '0' + v;
931 else return 'A' + v - 10;
932}
933
934static char *quote_ref_url(const char *base, const char *ref)
935{
936 const char *cp;
937 char *dp, *qref;
938 int len, baselen, ch;
939
940 baselen = strlen(base);
941 len = baselen + 6; /* "refs/" + NUL */
942 for (cp = ref; (ch = *cp) != 0; cp++, len++)
943 if (needs_quote(ch))
944 len += 2; /* extra two hex plus replacement % */
945 qref = xmalloc(len);
946 memcpy(qref, base, baselen);
947 memcpy(qref + baselen, "refs/", 5);
948 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
949 if (needs_quote(ch)) {
950 *dp++ = '%';
951 *dp++ = hex((ch >> 4) & 0xF);
952 *dp++ = hex(ch & 0xF);
953 }
954 else
955 *dp++ = ch;
956 }
957 *dp = 0;
958
959 return qref;
960}
961
cd541a68
DB
962int fetch_ref(char *ref, unsigned char *sha1)
963{
94fa447a 964 char *url;
fa3e0655
DB
965 char hex[42];
966 struct buffer buffer;
8e29f6a0 967 const char *base = alt->base;
1d389ab6 968 struct active_request_slot *slot;
cb754fdf 969 struct slot_results results;
fa3e0655
DB
970 buffer.size = 41;
971 buffer.posn = 0;
972 buffer.buffer = hex;
973 hex[41] = '\0';
8fcf7f9a 974
94fa447a 975 url = quote_ref_url(base, ref);
1d389ab6 976 slot = get_active_slot();
c8568e13 977 slot->results = &results;
1d389ab6
NH
978 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
979 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
980 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
981 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
982 if (start_active_slot(slot)) {
983 run_active_slot(slot);
c8568e13 984 if (results.curl_result != CURLE_OK)
1d389ab6
NH
985 return error("Couldn't get %s for %s\n%s",
986 url, ref, curl_errorstr);
987 } else {
988 return error("Unable to start request");
989 }
fa3e0655
DB
990
991 hex[40] = '\0';
992 get_sha1_hex(hex, sha1);
993 return 0;
cd541a68
DB
994}
995
8e29f6a0 996int main(int argc, const char **argv)
6eb7ed54 997{
8e29f6a0
PB
998 int commits;
999 const char **write_ref = NULL;
1000 char **commit_id;
1001 const char *url;
8d9fbe57 1002 char *path;
6eb7ed54 1003 int arg = 1;
7b9ae53e 1004 int rc = 0;
6eb7ed54 1005
19c45881 1006 setup_ident();
5a327713 1007 setup_git_directory();
d0740d92 1008 git_config(git_default_config);
5a327713 1009
6eb7ed54
DB
1010 while (arg < argc && argv[arg][0] == '-') {
1011 if (argv[arg][1] == 't') {
4250a5e5 1012 get_tree = 1;
6eb7ed54 1013 } else if (argv[arg][1] == 'c') {
4250a5e5 1014 get_history = 1;
6eb7ed54 1015 } else if (argv[arg][1] == 'a') {
4250a5e5
DB
1016 get_all = 1;
1017 get_tree = 1;
1018 get_history = 1;
e78d9772
JH
1019 } else if (argv[arg][1] == 'v') {
1020 get_verbosely = 1;
fa3e0655 1021 } else if (argv[arg][1] == 'w') {
8e29f6a0 1022 write_ref = &argv[arg + 1];
fa3e0655 1023 arg++;
820eca68
DB
1024 } else if (!strcmp(argv[arg], "--recover")) {
1025 get_recover = 1;
8e29f6a0
PB
1026 } else if (!strcmp(argv[arg], "--stdin")) {
1027 commits_on_stdin = 1;
6eb7ed54
DB
1028 }
1029 arg++;
1030 }
8e29f6a0
PB
1031 if (argc < arg + 2 - commits_on_stdin) {
1032 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
6eb7ed54
DB
1033 return 1;
1034 }
8e29f6a0
PB
1035 if (commits_on_stdin) {
1036 commits = pull_targets_stdin(&commit_id, &write_ref);
1037 } else {
1038 commit_id = (char **) &argv[arg++];
1039 commits = 1;
1040 }
1041 url = argv[arg];
6eb7ed54 1042
29508e1e 1043 http_init();
d402d556 1044
1db69b57 1045 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
3dcb90f5 1046
b3661567
DB
1047 alt = xmalloc(sizeof(*alt));
1048 alt->base = url;
1049 alt->got_indices = 0;
1050 alt->packs = NULL;
1051 alt->next = NULL;
8d9fbe57
NH
1052 path = strstr(url, "//");
1053 if (path) {
ef9e58c8 1054 path = strchr(path+2, '/');
8d9fbe57
NH
1055 if (path)
1056 alt->path_len = strlen(path);
1057 }
6eb7ed54 1058
8e29f6a0 1059 if (pull(commits, commit_id, write_ref, url))
7b9ae53e 1060 rc = 1;
6eb7ed54 1061
29508e1e
NH
1062 http_cleanup();
1063
07001f95
SE
1064 curl_slist_free_all(no_pragma_header);
1065
8e29f6a0
PB
1066 if (commits_on_stdin)
1067 pull_targets_free(commits, commit_id, write_ref);
1068
bb528079
JH
1069 if (corrupt_object_found) {
1070 fprintf(stderr,
1071"Some loose object were found to be corrupt, but they might be just\n"
1072"a false '404 Not Found' error message sent with incorrect HTTP\n"
1073"status code. Suggest running git fsck-objects.\n");
1074 }
7b9ae53e 1075 return rc;
6eb7ed54 1076}