Isolate shared HTTP request functionality
[git/git.git] / http-fetch.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
271421cd 3#include "pack.h"
215a7ad1 4#include "fetch.h"
29508e1e 5#include "http.h"
7baa3e86 6
49a0f240
NH
7#define PREV_BUF_SIZE 4096
8#define RANGE_HEADER_SIZE 30
9
acc075a8 10static int got_alternates = -1;
1d389ab6 11
1db69b57 12static struct curl_slist *no_pragma_header;
6eb7ed54 13
b3661567
DB
14struct alt_base
15{
16 char *base;
17 int got_indices;
18 struct packed_git *packs;
19 struct alt_base *next;
20};
21
a7928f8e 22static struct alt_base *alt = NULL;
6eb7ed54 23
1d389ab6
NH
24enum transfer_state {
25 WAITING,
26 ABORTED,
27 ACTIVE,
28 COMPLETE,
29};
6eb7ed54 30
1d389ab6
NH
31struct transfer_request
32{
33 unsigned char sha1[20];
34 struct alt_base *repo;
35 char *url;
36 char filename[PATH_MAX];
37 char tmpfile[PATH_MAX];
38 int local;
39 enum transfer_state state;
40 CURLcode curl_result;
41 char errorstr[CURL_ERROR_SIZE];
42 long http_code;
43 unsigned char real_sha1[20];
44 SHA_CTX c;
45 z_stream stream;
46 int zret;
47 int rename;
48 struct active_request_slot *slot;
49 struct transfer_request *next;
50};
51
acc075a8
NH
52struct alt_request {
53 char *base;
54 char *url;
55 struct buffer *buffer;
56 struct active_request_slot *slot;
57 int http_specific;
58};
59
1d389ab6 60static struct transfer_request *request_queue_head = NULL;
bc8f2652 61
182005b9
DB
62static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
63 void *data)
64{
bf0f910d 65 unsigned char expn[4096];
6eb7ed54
DB
66 size_t size = eltsize * nmemb;
67 int posn = 0;
1d389ab6 68 struct transfer_request *request = (struct transfer_request *)data;
6eb7ed54 69 do {
1d389ab6
NH
70 ssize_t retval = write(request->local,
71 ptr + posn, size - posn);
6eb7ed54
DB
72 if (retval < 0)
73 return posn;
74 posn += retval;
75 } while (posn < size);
76
1d389ab6
NH
77 request->stream.avail_in = size;
78 request->stream.next_in = ptr;
6eb7ed54 79 do {
1d389ab6
NH
80 request->stream.next_out = expn;
81 request->stream.avail_out = sizeof(expn);
82 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
83 SHA1_Update(&request->c, expn,
84 sizeof(expn) - request->stream.avail_out);
85 } while (request->stream.avail_in && request->zret == Z_OK);
86 data_received++;
6eb7ed54
DB
87 return size;
88}
89
acc075a8 90static void fetch_alternates(char *base);
1d389ab6 91
29508e1e 92static void process_object_response(void *callback_data);
1d389ab6 93
f5dce806 94static void start_request(struct transfer_request *request)
1d389ab6
NH
95{
96 char *hex = sha1_to_hex(request->sha1);
97 char prevfile[PATH_MAX];
98 char *url;
99 char *posn;
100 int prevlocal;
101 unsigned char prev_buf[PREV_BUF_SIZE];
102 ssize_t prev_read = 0;
103 long prev_posn = 0;
104 char range[RANGE_HEADER_SIZE];
105 struct curl_slist *range_header = NULL;
106 struct active_request_slot *slot;
107
108 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
109 unlink(prevfile);
110 rename(request->tmpfile, prevfile);
111 unlink(request->tmpfile);
112
313c4714
PB
113 if (request->local != -1)
114 error("fd leakage in start: %d", request->local);
1d389ab6
NH
115 request->local = open(request->tmpfile,
116 O_WRONLY | O_CREAT | O_EXCL, 0666);
b721e01f
JH
117 /* This could have failed due to the "lazy directory creation";
118 * try to mkdir the last path component.
119 */
120 if (request->local < 0 && errno == ENOENT) {
121 char *dir = strrchr(request->tmpfile, '/');
122 if (dir) {
123 *dir = 0;
124 mkdir(request->tmpfile, 0777);
125 *dir = '/';
126 }
127 request->local = open(request->tmpfile,
128 O_WRONLY | O_CREAT | O_EXCL, 0666);
129 }
130
1d389ab6
NH
131 if (request->local < 0) {
132 request->state = ABORTED;
133 error("Couldn't create temporary file %s for %s: %s\n",
134 request->tmpfile, request->filename, strerror(errno));
135 return;
136 }
137
138 memset(&request->stream, 0, sizeof(request->stream));
139
140 inflateInit(&request->stream);
141
142 SHA1_Init(&request->c);
143
144 url = xmalloc(strlen(request->repo->base) + 50);
145 request->url = xmalloc(strlen(request->repo->base) + 50);
146 strcpy(url, request->repo->base);
147 posn = url + strlen(request->repo->base);
148 strcpy(posn, "objects/");
149 posn += 8;
150 memcpy(posn, hex, 2);
151 posn += 2;
152 *(posn++) = '/';
153 strcpy(posn, hex + 2);
154 strcpy(request->url, url);
155
156 /* If a previous temp file is present, process what was already
157 fetched. */
158 prevlocal = open(prevfile, O_RDONLY);
159 if (prevlocal != -1) {
160 do {
161 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
162 if (prev_read>0) {
163 if (fwrite_sha1_file(prev_buf,
164 1,
165 prev_read,
166 request) == prev_read) {
167 prev_posn += prev_read;
168 } else {
169 prev_read = -1;
170 }
171 }
172 } while (prev_read > 0);
173 close(prevlocal);
174 }
175 unlink(prevfile);
176
177 /* Reset inflate/SHA1 if there was an error reading the previous temp
178 file; also rewind to the beginning of the local file. */
179 if (prev_read == -1) {
180 memset(&request->stream, 0, sizeof(request->stream));
181 inflateInit(&request->stream);
182 SHA1_Init(&request->c);
183 if (prev_posn>0) {
184 prev_posn = 0;
185 lseek(request->local, SEEK_SET, 0);
186 ftruncate(request->local, 0);
187 }
188 }
189
190 slot = get_active_slot();
29508e1e
NH
191 slot->callback_func = process_object_response;
192 slot->callback_data = request;
193 request->slot = slot;
194
1d389ab6
NH
195 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
196 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
197 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
198 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 199 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
200
201 /* If we have successfully processed data from a previous fetch
202 attempt, only fetch the data we don't already have. */
203 if (prev_posn>0) {
204 if (get_verbosely)
205 fprintf(stderr,
206 "Resuming fetch of object %s at byte %ld\n",
207 hex, prev_posn);
208 sprintf(range, "Range: bytes=%ld-", prev_posn);
209 range_header = curl_slist_append(range_header, range);
210 curl_easy_setopt(slot->curl,
211 CURLOPT_HTTPHEADER, range_header);
212 }
213
a7a8d378 214 /* Try to get the request started, abort the request on error */
29508e1e 215 request->state = ACTIVE;
1d389ab6
NH
216 if (!start_active_slot(slot)) {
217 request->state = ABORTED;
29508e1e 218 request->slot = NULL;
313c4714 219 close(request->local); request->local = -1;
1d389ab6 220 free(request->url);
1d389ab6 221 }
1d389ab6
NH
222}
223
f5dce806 224static void finish_request(struct transfer_request *request)
1d389ab6 225{
50496b21
NH
226 struct stat st;
227
1d389ab6 228 fchmod(request->local, 0444);
313c4714 229 close(request->local); request->local = -1;
1d389ab6
NH
230
231 if (request->http_code == 416) {
232 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
233 } else if (request->curl_result != CURLE_OK) {
50496b21
NH
234 if (stat(request->tmpfile, &st) == 0)
235 if (st.st_size == 0)
236 unlink(request->tmpfile);
1d389ab6
NH
237 return;
238 }
239
240 inflateEnd(&request->stream);
241 SHA1_Final(request->real_sha1, &request->c);
242 if (request->zret != Z_STREAM_END) {
243 unlink(request->tmpfile);
244 return;
245 }
246 if (memcmp(request->sha1, request->real_sha1, 20)) {
247 unlink(request->tmpfile);
248 return;
249 }
250 request->rename =
b721e01f 251 move_temp_to_file(request->tmpfile, request->filename);
1d389ab6
NH
252
253 if (request->rename == 0)
254 pull_say("got %s\n", sha1_to_hex(request->sha1));
255}
256
29508e1e
NH
257static void process_object_response(void *callback_data)
258{
259 struct transfer_request *request =
260 (struct transfer_request *)callback_data;
261
262 request->curl_result = request->slot->curl_result;
263 request->http_code = request->slot->http_code;
264 request->slot = NULL;
265 request->state = COMPLETE;
266
267 /* Use alternates if necessary */
268 if (request->http_code == 404) {
269 fetch_alternates(alt->base);
270 if (request->repo->next != NULL) {
271 request->repo =
272 request->repo->next;
273 close(request->local);
274 request->local = -1;
275 start_request(request);
276 return;
277 }
278 }
279
280 finish_request(request);
281}
282
f5dce806 283static void release_request(struct transfer_request *request)
1d389ab6
NH
284{
285 struct transfer_request *entry = request_queue_head;
286
313c4714
PB
287 if (request->local != -1)
288 error("fd leakage in release: %d", request->local);
1d389ab6
NH
289 if (request == request_queue_head) {
290 request_queue_head = request->next;
291 } else {
292 while (entry->next != NULL && entry->next != request)
293 entry = entry->next;
294 if (entry->next == request)
295 entry->next = entry->next->next;
296 }
297
298 free(request->url);
299 free(request);
300}
301
a7a8d378 302#ifdef USE_CURL_MULTI
29508e1e 303void fill_active_slots(void)
1d389ab6
NH
304{
305 struct transfer_request *request = request_queue_head;
f1a906a3 306 struct active_request_slot *slot = active_queue_head;
1d389ab6
NH
307 int num_transfers;
308
309 while (active_requests < max_requests && request != NULL) {
310 if (request->state == WAITING) {
11f0dafe
NH
311 if (has_sha1_file(request->sha1))
312 release_request(request);
313 else
314 start_request(request);
1d389ab6
NH
315 curl_multi_perform(curlm, &num_transfers);
316 }
317 request = request->next;
318 }
f1a906a3
NH
319
320 while (slot != NULL) {
321 if (!slot->in_use && slot->curl != NULL) {
322 curl_easy_cleanup(slot->curl);
323 slot->curl = NULL;
324 }
325 slot = slot->next;
326 }
1d389ab6 327}
a7a8d378 328#endif
1d389ab6
NH
329
330void prefetch(unsigned char *sha1)
331{
332 struct transfer_request *newreq;
333 struct transfer_request *tail;
334 char *filename = sha1_file_name(sha1);
335
336 newreq = xmalloc(sizeof(*newreq));
337 memcpy(newreq->sha1, sha1, 20);
338 newreq->repo = alt;
339 newreq->url = NULL;
340 newreq->local = -1;
341 newreq->state = WAITING;
342 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
343 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
344 "%s.temp", filename);
345 newreq->next = NULL;
346
347 if (request_queue_head == NULL) {
348 request_queue_head = newreq;
349 } else {
350 tail = request_queue_head;
351 while (tail->next != NULL) {
352 tail = tail->next;
353 }
354 tail->next = newreq;
355 }
29508e1e 356
a7a8d378 357#ifdef USE_CURL_MULTI
29508e1e
NH
358 fill_active_slots();
359 step_active_slots();
a7a8d378 360#endif
1d389ab6
NH
361}
362
b3661567 363static int fetch_index(struct alt_base *repo, unsigned char *sha1)
182005b9 364{
1d389ab6 365 char *hex = sha1_to_hex(sha1);
182005b9
DB
366 char *filename;
367 char *url;
49a0f240 368 char tmpfile[PATH_MAX];
49a0f240
NH
369 long prev_posn = 0;
370 char range[RANGE_HEADER_SIZE];
371 struct curl_slist *range_header = NULL;
182005b9
DB
372
373 FILE *indexfile;
1d389ab6 374 struct active_request_slot *slot;
182005b9
DB
375
376 if (has_pack_index(sha1))
377 return 0;
378
379 if (get_verbosely)
1d389ab6 380 fprintf(stderr, "Getting index for pack %s\n", hex);
182005b9 381
b3661567 382 url = xmalloc(strlen(repo->base) + 64);
1d389ab6 383 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
182005b9
DB
384
385 filename = sha1_pack_index_name(sha1);
49a0f240
NH
386 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
387 indexfile = fopen(tmpfile, "a");
182005b9
DB
388 if (!indexfile)
389 return error("Unable to open local file %s for pack index",
390 filename);
391
1d389ab6
NH
392 slot = get_active_slot();
393 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
394 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
395 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 396 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
397 slot->local = indexfile;
398
49a0f240
NH
399 /* If there is data present from a previous transfer attempt,
400 resume where it left off */
401 prev_posn = ftell(indexfile);
402 if (prev_posn>0) {
403 if (get_verbosely)
404 fprintf(stderr,
405 "Resuming fetch of index for pack %s at byte %ld\n",
1d389ab6 406 hex, prev_posn);
49a0f240
NH
407 sprintf(range, "Range: bytes=%ld-", prev_posn);
408 range_header = curl_slist_append(range_header, range);
1d389ab6 409 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
410 }
411
1d389ab6
NH
412 if (start_active_slot(slot)) {
413 run_active_slot(slot);
414 if (slot->curl_result != CURLE_OK) {
415 fclose(indexfile);
416 return error("Unable to get pack index %s\n%s", url,
417 curl_errorstr);
418 }
419 } else {
313c4714 420 fclose(indexfile);
1d389ab6 421 return error("Unable to start request");
182005b9
DB
422 }
423
424 fclose(indexfile);
49a0f240 425
b721e01f 426 return move_temp_to_file(tmpfile, filename);
182005b9
DB
427}
428
b3661567 429static int setup_index(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
430{
431 struct packed_git *new_pack;
432 if (has_pack_file(sha1))
433 return 0; // don't list this as something we can get
434
b3661567 435 if (fetch_index(repo, sha1))
182005b9
DB
436 return -1;
437
438 new_pack = parse_pack_index(sha1);
b3661567
DB
439 new_pack->next = repo->packs;
440 repo->packs = new_pack;
182005b9
DB
441 return 0;
442}
443
acc075a8 444static void process_alternates(void *callback_data)
b3661567 445{
acc075a8
NH
446 struct alt_request *alt_req = (struct alt_request *)callback_data;
447 struct active_request_slot *slot = alt_req->slot;
1d389ab6 448 struct alt_base *tail = alt;
acc075a8 449 char *base = alt_req->base;
bc8f2652 450 static const char null_byte = '\0';
acc075a8
NH
451 char *data;
452 int i = 0;
1d389ab6 453
acc075a8
NH
454 if (alt_req->http_specific) {
455 if (slot->curl_result != CURLE_OK ||
456 !alt_req->buffer->posn) {
457
458 /* Try reusing the slot to get non-http alternates */
459 alt_req->http_specific = 0;
460 sprintf(alt_req->url, "%s/objects/info/alternates",
461 base);
462 curl_easy_setopt(slot->curl, CURLOPT_URL,
463 alt_req->url);
464 active_requests++;
465 slot->in_use = 1;
1d389ab6 466 if (start_active_slot(slot)) {
acc075a8
NH
467 return;
468 } else {
469 got_alternates = -1;
29508e1e 470 slot->in_use = 0;
acc075a8 471 return;
1d389ab6 472 }
b3661567 473 }
acc075a8
NH
474 } else if (slot->curl_result != CURLE_OK) {
475 if (slot->http_code != 404) {
476 got_alternates = -1;
477 return;
478 }
b3661567
DB
479 }
480
29508e1e 481 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
acc075a8
NH
482 alt_req->buffer->posn--;
483 data = alt_req->buffer->buffer;
1b0c1e67 484
acc075a8 485 while (i < alt_req->buffer->posn) {
b3661567 486 int posn = i;
acc075a8 487 while (posn < alt_req->buffer->posn && data[posn] != '\n')
b3661567
DB
488 posn++;
489 if (data[posn] == '\n') {
1b0c1e67
DB
490 int okay = 0;
491 int serverlen = 0;
492 struct alt_base *newalt;
493 char *target = NULL;
b3661567 494 if (data[i] == '/') {
1b0c1e67
DB
495 serverlen = strchr(base + 8, '/') - base;
496 okay = 1;
497 } else if (!memcmp(data + i, "../", 3)) {
498 i += 3;
499 serverlen = strlen(base);
500 while (i + 2 < posn &&
501 !memcmp(data + i, "../", 3)) {
502 do {
503 serverlen--;
504 } while (serverlen &&
505 base[serverlen - 1] != '/');
506 i += 3;
507 }
508 // If the server got removed, give up.
509 okay = strchr(base, ':') - base + 3 <
510 serverlen;
acc075a8 511 } else if (alt_req->http_specific) {
1b0c1e67
DB
512 char *colon = strchr(data + i, ':');
513 char *slash = strchr(data + i, '/');
514 if (colon && slash && colon < data + posn &&
515 slash < data + posn && colon < slash) {
516 okay = 1;
517 }
518 }
519 // skip 'objects' at end
520 if (okay) {
521 target = xmalloc(serverlen + posn - i - 6);
b3661567
DB
522 strncpy(target, base, serverlen);
523 strncpy(target + serverlen, data + i,
524 posn - i - 7);
525 target[serverlen + posn - i - 7] = '\0';
526 if (get_verbosely)
527 fprintf(stderr,
528 "Also look at %s\n", target);
529 newalt = xmalloc(sizeof(*newalt));
1d389ab6 530 newalt->next = NULL;
b3661567
DB
531 newalt->base = target;
532 newalt->got_indices = 0;
533 newalt->packs = NULL;
1d389ab6
NH
534 while (tail->next != NULL)
535 tail = tail->next;
536 tail->next = newalt;
b3661567
DB
537 }
538 }
539 i = posn + 1;
540 }
bc8f2652 541
f7eb290f 542 got_alternates = 1;
acc075a8
NH
543}
544
545static void fetch_alternates(char *base)
546{
547 struct buffer buffer;
548 char *url;
549 char *data;
550 struct active_request_slot *slot;
551 static struct alt_request alt_req;
acc075a8
NH
552
553 /* If another request has already started fetching alternates,
554 wait for them to arrive and return to processing this request's
555 curl message */
29508e1e 556#ifdef USE_CURL_MULTI
acc075a8 557 while (got_alternates == 0) {
29508e1e 558 step_active_slots();
acc075a8 559 }
29508e1e 560#endif
acc075a8
NH
561
562 /* Nothing to do if they've already been fetched */
563 if (got_alternates == 1)
564 return;
565
566 /* Start the fetch */
567 got_alternates = 0;
568
569 data = xmalloc(4096);
570 buffer.size = 4096;
571 buffer.posn = 0;
572 buffer.buffer = data;
573
574 if (get_verbosely)
575 fprintf(stderr, "Getting alternates list for %s\n", base);
576
577 url = xmalloc(strlen(base) + 31);
578 sprintf(url, "%s/objects/info/http-alternates", base);
579
580 /* Use a callback to process the result, since another request
581 may fail and need to have alternates loaded before continuing */
582 slot = get_active_slot();
583 slot->callback_func = process_alternates;
584 slot->callback_data = &alt_req;
585
586 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 587 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
acc075a8
NH
588 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
589
590 alt_req.base = base;
591 alt_req.url = url;
592 alt_req.buffer = &buffer;
593 alt_req.http_specific = 1;
594 alt_req.slot = slot;
595
596 if (start_active_slot(slot))
597 run_active_slot(slot);
598 else
599 got_alternates = -1;
600
601 free(data);
602 free(url);
b3661567
DB
603}
604
605static int fetch_indices(struct alt_base *repo)
182005b9
DB
606{
607 unsigned char sha1[20];
608 char *url;
609 struct buffer buffer;
610 char *data;
611 int i = 0;
612
1d389ab6
NH
613 struct active_request_slot *slot;
614
b3661567 615 if (repo->got_indices)
182005b9
DB
616 return 0;
617
618 data = xmalloc(4096);
619 buffer.size = 4096;
620 buffer.posn = 0;
621 buffer.buffer = data;
622
623 if (get_verbosely)
6fd72e39 624 fprintf(stderr, "Getting pack list for %s\n", repo->base);
182005b9 625
b3661567
DB
626 url = xmalloc(strlen(repo->base) + 21);
627 sprintf(url, "%s/objects/info/packs", repo->base);
182005b9 628
1d389ab6
NH
629 slot = get_active_slot();
630 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 631 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1d389ab6
NH
632 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
633 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
634 if (start_active_slot(slot)) {
635 run_active_slot(slot);
bc8f2652
SV
636 if (slot->curl_result != CURLE_OK) {
637 free(buffer.buffer);
1d389ab6 638 return error("%s", curl_errorstr);
bc8f2652 639 }
1d389ab6 640 } else {
bc8f2652 641 free(buffer.buffer);
1d389ab6
NH
642 return error("Unable to start request");
643 }
182005b9 644
bc8f2652 645 data = buffer.buffer;
b3661567 646 while (i < buffer.posn) {
182005b9
DB
647 switch (data[i]) {
648 case 'P':
649 i++;
650 if (i + 52 < buffer.posn &&
651 !strncmp(data + i, " pack-", 6) &&
652 !strncmp(data + i + 46, ".pack\n", 6)) {
653 get_sha1_hex(data + i + 6, sha1);
b3661567 654 setup_index(repo, sha1);
182005b9
DB
655 i += 51;
656 break;
657 }
658 default:
659 while (data[i] != '\n')
660 i++;
661 }
662 i++;
b3661567 663 }
182005b9 664
bc8f2652 665 free(buffer.buffer);
b3661567 666 repo->got_indices = 1;
182005b9
DB
667 return 0;
668}
669
b3661567 670static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
671{
672 char *url;
673 struct packed_git *target;
674 struct packed_git **lst;
675 FILE *packfile;
676 char *filename;
49a0f240
NH
677 char tmpfile[PATH_MAX];
678 int ret;
679 long prev_posn = 0;
680 char range[RANGE_HEADER_SIZE];
681 struct curl_slist *range_header = NULL;
1d389ab6
NH
682
683 struct active_request_slot *slot;
182005b9 684
b3661567 685 if (fetch_indices(repo))
182005b9 686 return -1;
b3661567 687 target = find_sha1_pack(sha1, repo->packs);
182005b9 688 if (!target)
b3661567 689 return -1;
182005b9
DB
690
691 if (get_verbosely) {
692 fprintf(stderr, "Getting pack %s\n",
693 sha1_to_hex(target->sha1));
694 fprintf(stderr, " which contains %s\n",
695 sha1_to_hex(sha1));
696 }
697
b3661567 698 url = xmalloc(strlen(repo->base) + 65);
182005b9 699 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 700 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
701
702 filename = sha1_pack_name(target->sha1);
49a0f240
NH
703 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
704 packfile = fopen(tmpfile, "a");
182005b9
DB
705 if (!packfile)
706 return error("Unable to open local file %s for pack",
707 filename);
708
1d389ab6
NH
709 slot = get_active_slot();
710 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
711 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
712 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 713 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 714 slot->local = packfile;
1ddea77e 715
49a0f240
NH
716 /* If there is data present from a previous transfer attempt,
717 resume where it left off */
718 prev_posn = ftell(packfile);
719 if (prev_posn>0) {
720 if (get_verbosely)
721 fprintf(stderr,
722 "Resuming fetch of pack %s at byte %ld\n",
723 sha1_to_hex(target->sha1), prev_posn);
724 sprintf(range, "Range: bytes=%ld-", prev_posn);
725 range_header = curl_slist_append(range_header, range);
1d389ab6 726 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
727 }
728
1d389ab6
NH
729 if (start_active_slot(slot)) {
730 run_active_slot(slot);
731 if (slot->curl_result != CURLE_OK) {
732 fclose(packfile);
733 return error("Unable to get pack file %s\n%s", url,
734 curl_errorstr);
735 }
736 } else {
313c4714 737 fclose(packfile);
1d389ab6 738 return error("Unable to start request");
182005b9
DB
739 }
740
741 fclose(packfile);
742
b721e01f 743 ret = move_temp_to_file(tmpfile, filename);
49a0f240 744 if (ret)
b721e01f 745 return ret;
49a0f240 746
b3661567 747 lst = &repo->packs;
182005b9
DB
748 while (*lst != target)
749 lst = &((*lst)->next);
750 *lst = (*lst)->next;
751
271421cd
JH
752 if (verify_pack(target, 0))
753 return -1;
182005b9
DB
754 install_packed_git(target);
755
756 return 0;
757}
758
a7928f8e 759static int fetch_object(struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
760{
761 char *hex = sha1_to_hex(sha1);
29508e1e 762 int ret = 0;
1d389ab6 763 struct transfer_request *request = request_queue_head;
1d389ab6
NH
764
765 while (request != NULL && memcmp(request->sha1, sha1, 20))
766 request = request->next;
767 if (request == NULL)
768 return error("Couldn't find request for %s in the queue", hex);
769
11f0dafe
NH
770 if (has_sha1_file(request->sha1)) {
771 release_request(request);
772 return 0;
773 }
774
a7a8d378 775#ifdef USE_CURL_MULTI
1d389ab6 776 while (request->state == WAITING) {
29508e1e 777 step_active_slots();
1d389ab6 778 }
a7a8d378
NH
779#else
780 start_request(request);
781#endif
6eb7ed54 782
a7a8d378 783 while (request->state == ACTIVE) {
1d389ab6 784 run_active_slot(request->slot);
a7a8d378 785 }
313c4714
PB
786 if (request->local != -1) {
787 close(request->local); request->local = -1;
788 }
6eb7ed54 789
1d389ab6 790 if (request->state == ABORTED) {
29508e1e
NH
791 ret = error("Request for %s aborted", hex);
792 } else if (request->curl_result != CURLE_OK &&
793 request->http_code != 416) {
e2029eb9
PB
794 if (request->http_code == 404)
795 ret = -1; /* Be silent, it is probably in a pack. */
796 else
797 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
798 request->errorstr, request->curl_result,
799 request->http_code, hex);
29508e1e 800 } else if (request->zret != Z_STREAM_END) {
1d389ab6 801 ret = error("File %s (%s) corrupt\n", hex, request->url);
29508e1e
NH
802 } else if (memcmp(request->sha1, request->real_sha1, 20)) {
803 ret = error("File %s has bad hash\n", hex);
804 } else if (request->rename < 0) {
1d389ab6
NH
805 ret = error("unable to write sha1 filename %s: %s",
806 request->filename,
807 strerror(request->rename));
6eb7ed54 808 }
49a0f240 809
1d389ab6 810 release_request(request);
29508e1e 811 return ret;
6eb7ed54
DB
812}
813
b3661567
DB
814int fetch(unsigned char *sha1)
815{
816 struct alt_base *altbase = alt;
1d389ab6
NH
817
818 if (!fetch_object(altbase, sha1))
819 return 0;
b3661567 820 while (altbase) {
b3661567
DB
821 if (!fetch_pack(altbase, sha1))
822 return 0;
f7eb290f 823 fetch_alternates(alt->base);
b3661567
DB
824 altbase = altbase->next;
825 }
826 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1d389ab6 827 alt->base);
b3661567
DB
828}
829
94fa447a
JH
830static inline int needs_quote(int ch)
831{
832 switch (ch) {
833 case '/': case '-': case '.':
834 case 'A'...'Z': case 'a'...'z': case '0'...'9':
835 return 0;
836 default:
837 return 1;
838 }
839}
840
841static inline int hex(int v)
842{
843 if (v < 10) return '0' + v;
844 else return 'A' + v - 10;
845}
846
847static char *quote_ref_url(const char *base, const char *ref)
848{
849 const char *cp;
850 char *dp, *qref;
851 int len, baselen, ch;
852
853 baselen = strlen(base);
854 len = baselen + 6; /* "refs/" + NUL */
855 for (cp = ref; (ch = *cp) != 0; cp++, len++)
856 if (needs_quote(ch))
857 len += 2; /* extra two hex plus replacement % */
858 qref = xmalloc(len);
859 memcpy(qref, base, baselen);
860 memcpy(qref + baselen, "refs/", 5);
861 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
862 if (needs_quote(ch)) {
863 *dp++ = '%';
864 *dp++ = hex((ch >> 4) & 0xF);
865 *dp++ = hex(ch & 0xF);
866 }
867 else
868 *dp++ = ch;
869 }
870 *dp = 0;
871
872 return qref;
873}
874
cd541a68
DB
875int fetch_ref(char *ref, unsigned char *sha1)
876{
94fa447a 877 char *url;
fa3e0655
DB
878 char hex[42];
879 struct buffer buffer;
1d389ab6
NH
880 char *base = alt->base;
881 struct active_request_slot *slot;
fa3e0655
DB
882 buffer.size = 41;
883 buffer.posn = 0;
884 buffer.buffer = hex;
885 hex[41] = '\0';
886
94fa447a 887 url = quote_ref_url(base, ref);
1d389ab6
NH
888 slot = get_active_slot();
889 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
890 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
891 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
892 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
893 if (start_active_slot(slot)) {
894 run_active_slot(slot);
895 if (slot->curl_result != CURLE_OK)
896 return error("Couldn't get %s for %s\n%s",
897 url, ref, curl_errorstr);
898 } else {
899 return error("Unable to start request");
900 }
fa3e0655
DB
901
902 hex[40] = '\0';
903 get_sha1_hex(hex, sha1);
904 return 0;
cd541a68
DB
905}
906
6eb7ed54
DB
907int main(int argc, char **argv)
908{
909 char *commit_id;
910 char *url;
911 int arg = 1;
7b9ae53e 912 int rc = 0;
6eb7ed54
DB
913
914 while (arg < argc && argv[arg][0] == '-') {
915 if (argv[arg][1] == 't') {
4250a5e5 916 get_tree = 1;
6eb7ed54 917 } else if (argv[arg][1] == 'c') {
4250a5e5 918 get_history = 1;
6eb7ed54 919 } else if (argv[arg][1] == 'a') {
4250a5e5
DB
920 get_all = 1;
921 get_tree = 1;
922 get_history = 1;
e78d9772
JH
923 } else if (argv[arg][1] == 'v') {
924 get_verbosely = 1;
fa3e0655
DB
925 } else if (argv[arg][1] == 'w') {
926 write_ref = argv[arg + 1];
927 arg++;
820eca68
DB
928 } else if (!strcmp(argv[arg], "--recover")) {
929 get_recover = 1;
6eb7ed54
DB
930 }
931 arg++;
932 }
933 if (argc < arg + 2) {
215a7ad1 934 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
6eb7ed54
DB
935 return 1;
936 }
937 commit_id = argv[arg];
938 url = argv[arg + 1];
939
29508e1e 940 http_init();
d402d556 941
1db69b57 942 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
3dcb90f5 943
b3661567
DB
944 alt = xmalloc(sizeof(*alt));
945 alt->base = url;
946 alt->got_indices = 0;
947 alt->packs = NULL;
948 alt->next = NULL;
6eb7ed54 949
4250a5e5 950 if (pull(commit_id))
7b9ae53e 951 rc = 1;
6eb7ed54 952
1db69b57 953 curl_slist_free_all(no_pragma_header);
29508e1e
NH
954
955 http_cleanup();
956
7b9ae53e 957 return rc;
6eb7ed54 958}