Set the parallel HTTP request limit via an environment variable
[git/git.git] / http-fetch.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
271421cd 3#include "pack.h"
215a7ad1 4#include "fetch.h"
4250a5e5 5
6eb7ed54
DB
6#include <curl/curl.h>
7#include <curl/easy.h>
8
a7a8d378
NH
9#if LIBCURL_VERSION_NUM >= 0x070908
10#define USE_CURL_MULTI
1d389ab6 11#define DEFAULT_MAX_REQUESTS 5
a7a8d378 12#endif
1d389ab6 13
4a30976e
JS
14#if LIBCURL_VERSION_NUM < 0x070704
15#define curl_global_cleanup() do { /* nothing */ } while(0)
16#endif
17#if LIBCURL_VERSION_NUM < 0x070800
18#define curl_global_init(a) do { /* nothing */ } while(0)
19#endif
4a30976e 20
49a0f240
NH
21#define PREV_BUF_SIZE 4096
22#define RANGE_HEADER_SIZE 30
23
1d389ab6
NH
24static int active_requests = 0;
25static int data_received;
26
a7a8d378
NH
27#ifdef USE_CURL_MULTI
28static int max_requests = DEFAULT_MAX_REQUESTS;
1d389ab6 29static CURLM *curlm;
a7a8d378 30#endif
1d389ab6 31static CURL *curl_default;
1db69b57 32static struct curl_slist *no_pragma_header;
49a0f240 33static struct curl_slist *no_range_header;
1ddea77e 34static char curl_errorstr[CURL_ERROR_SIZE];
6eb7ed54 35
b3661567
DB
36struct alt_base
37{
38 char *base;
39 int got_indices;
40 struct packed_git *packs;
41 struct alt_base *next;
42};
43
a7928f8e 44static struct alt_base *alt = NULL;
6eb7ed54 45
1d389ab6
NH
46enum transfer_state {
47 WAITING,
48 ABORTED,
49 ACTIVE,
50 COMPLETE,
51};
6eb7ed54 52
1d389ab6
NH
53struct transfer_request
54{
55 unsigned char sha1[20];
56 struct alt_base *repo;
57 char *url;
58 char filename[PATH_MAX];
59 char tmpfile[PATH_MAX];
60 int local;
61 enum transfer_state state;
62 CURLcode curl_result;
63 char errorstr[CURL_ERROR_SIZE];
64 long http_code;
65 unsigned char real_sha1[20];
66 SHA_CTX c;
67 z_stream stream;
68 int zret;
69 int rename;
70 struct active_request_slot *slot;
71 struct transfer_request *next;
72};
73
74struct active_request_slot
75{
76 CURL *curl;
77 FILE *local;
78 int in_use;
79 int done;
80 CURLcode curl_result;
81 struct active_request_slot *next;
82};
83
84static struct transfer_request *request_queue_head = NULL;
85static struct active_request_slot *active_queue_head = NULL;
6eb7ed54 86
3dcb90f5 87static int curl_ssl_verify;
5acb6de1
NH
88static char *ssl_cert;
89static char *ssl_key;
90static char *ssl_capath;
91static char *ssl_cainfo;
3dcb90f5 92
fa3e0655
DB
93struct buffer
94{
95 size_t posn;
96 size_t size;
97 void *buffer;
98};
99
100static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
182005b9
DB
101 struct buffer *buffer)
102{
fa3e0655
DB
103 size_t size = eltsize * nmemb;
104 if (size > buffer->size - buffer->posn)
105 size = buffer->size - buffer->posn;
106 memcpy(buffer->buffer + buffer->posn, ptr, size);
107 buffer->posn += size;
1d389ab6 108 data_received++;
fa3e0655
DB
109 return size;
110}
111
182005b9
DB
112static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
113 void *data)
114{
bf0f910d 115 unsigned char expn[4096];
6eb7ed54
DB
116 size_t size = eltsize * nmemb;
117 int posn = 0;
1d389ab6 118 struct transfer_request *request = (struct transfer_request *)data;
6eb7ed54 119 do {
1d389ab6
NH
120 ssize_t retval = write(request->local,
121 ptr + posn, size - posn);
6eb7ed54
DB
122 if (retval < 0)
123 return posn;
124 posn += retval;
125 } while (posn < size);
126
1d389ab6
NH
127 request->stream.avail_in = size;
128 request->stream.next_in = ptr;
6eb7ed54 129 do {
1d389ab6
NH
130 request->stream.next_out = expn;
131 request->stream.avail_out = sizeof(expn);
132 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
133 SHA1_Update(&request->c, expn,
134 sizeof(expn) - request->stream.avail_out);
135 } while (request->stream.avail_in && request->zret == Z_OK);
136 data_received++;
6eb7ed54
DB
137 return size;
138}
139
49a0f240
NH
140int relink_or_rename(char *old, char *new) {
141 int ret;
142
143 ret = link(old, new);
144 if (ret < 0) {
145 /* Same Coda hack as in write_sha1_file(sha1_file.c) */
146 ret = errno;
147 if (ret == EXDEV && !rename(old, new))
148 return 0;
149 }
150 unlink(old);
151 if (ret) {
152 if (ret != EEXIST)
153 return ret;
154 }
155
156 return 0;
157}
158
a7a8d378 159#ifdef USE_CURL_MULTI
1d389ab6
NH
160void process_curl_messages();
161void process_request_queue();
a7a8d378 162#endif
1d389ab6
NH
163
164struct active_request_slot *get_active_slot()
165{
166 struct active_request_slot *slot = active_queue_head;
167 struct active_request_slot *newslot;
a7a8d378
NH
168
169#ifdef USE_CURL_MULTI
1d389ab6
NH
170 int num_transfers;
171
172 /* Wait for a slot to open up if the queue is full */
173 while (active_requests >= max_requests) {
174 curl_multi_perform(curlm, &num_transfers);
175 if (num_transfers < active_requests) {
176 process_curl_messages();
177 }
178 }
a7a8d378 179#endif
1d389ab6
NH
180
181 while (slot != NULL && slot->in_use) {
182 slot = slot->next;
183 }
184 if (slot == NULL) {
185 newslot = xmalloc(sizeof(*newslot));
186 newslot->curl = curl_easy_duphandle(curl_default);
187 newslot->in_use = 0;
188 newslot->next = NULL;
189
190 slot = active_queue_head;
191 if (slot == NULL) {
192 active_queue_head = newslot;
193 } else {
194 while (slot->next != NULL) {
195 slot = slot->next;
196 }
197 slot->next = newslot;
198 }
199 slot = newslot;
200 }
201
202 active_requests++;
203 slot->in_use = 1;
204 slot->done = 0;
205 slot->local = NULL;
206 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
207 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
208 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
209
210 return slot;
211}
212
213int start_active_slot(struct active_request_slot *slot)
214{
a7a8d378 215#ifdef USE_CURL_MULTI
1d389ab6
NH
216 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
217
218 if (curlm_result != CURLM_OK &&
219 curlm_result != CURLM_CALL_MULTI_PERFORM) {
220 active_requests--;
221 slot->in_use = 0;
222 return 0;
223 }
a7a8d378 224#endif
1d389ab6
NH
225 return 1;
226}
227
228void run_active_slot(struct active_request_slot *slot)
229{
a7a8d378 230#ifdef USE_CURL_MULTI
1d389ab6
NH
231 int num_transfers;
232 long last_pos = 0;
233 long current_pos;
234 fd_set readfds;
235 fd_set writefds;
236 fd_set excfds;
237 int max_fd;
238 struct timeval select_timeout;
239 CURLMcode curlm_result;
240
241 while (!slot->done) {
242 data_received = 0;
243 do {
244 curlm_result = curl_multi_perform(curlm,
245 &num_transfers);
246 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
247 if (num_transfers < active_requests) {
248 process_curl_messages();
249 process_request_queue();
250 }
251
252 if (!data_received && slot->local != NULL) {
253 current_pos = ftell(slot->local);
254 if (current_pos > last_pos)
255 data_received++;
256 last_pos = current_pos;
257 }
258
259 if (!slot->done && !data_received) {
260 max_fd = 0;
261 FD_ZERO(&readfds);
262 FD_ZERO(&writefds);
263 FD_ZERO(&excfds);
264 select_timeout.tv_sec = 0;
265 select_timeout.tv_usec = 50000;
266 select(max_fd, &readfds, &writefds,
267 &excfds, &select_timeout);
268 }
269 }
a7a8d378
NH
270#else
271 slot->curl_result = curl_easy_perform(slot->curl);
272 active_requests--;
273#endif
1d389ab6
NH
274}
275
276void start_request(struct transfer_request *request)
277{
278 char *hex = sha1_to_hex(request->sha1);
279 char prevfile[PATH_MAX];
280 char *url;
281 char *posn;
282 int prevlocal;
283 unsigned char prev_buf[PREV_BUF_SIZE];
284 ssize_t prev_read = 0;
285 long prev_posn = 0;
286 char range[RANGE_HEADER_SIZE];
287 struct curl_slist *range_header = NULL;
288 struct active_request_slot *slot;
289
290 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
291 unlink(prevfile);
292 rename(request->tmpfile, prevfile);
293 unlink(request->tmpfile);
294
295 request->local = open(request->tmpfile,
296 O_WRONLY | O_CREAT | O_EXCL, 0666);
297 if (request->local < 0) {
298 request->state = ABORTED;
299 error("Couldn't create temporary file %s for %s: %s\n",
300 request->tmpfile, request->filename, strerror(errno));
301 return;
302 }
303
304 memset(&request->stream, 0, sizeof(request->stream));
305
306 inflateInit(&request->stream);
307
308 SHA1_Init(&request->c);
309
310 url = xmalloc(strlen(request->repo->base) + 50);
311 request->url = xmalloc(strlen(request->repo->base) + 50);
312 strcpy(url, request->repo->base);
313 posn = url + strlen(request->repo->base);
314 strcpy(posn, "objects/");
315 posn += 8;
316 memcpy(posn, hex, 2);
317 posn += 2;
318 *(posn++) = '/';
319 strcpy(posn, hex + 2);
320 strcpy(request->url, url);
321
322 /* If a previous temp file is present, process what was already
323 fetched. */
324 prevlocal = open(prevfile, O_RDONLY);
325 if (prevlocal != -1) {
326 do {
327 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
328 if (prev_read>0) {
329 if (fwrite_sha1_file(prev_buf,
330 1,
331 prev_read,
332 request) == prev_read) {
333 prev_posn += prev_read;
334 } else {
335 prev_read = -1;
336 }
337 }
338 } while (prev_read > 0);
339 close(prevlocal);
340 }
341 unlink(prevfile);
342
343 /* Reset inflate/SHA1 if there was an error reading the previous temp
344 file; also rewind to the beginning of the local file. */
345 if (prev_read == -1) {
346 memset(&request->stream, 0, sizeof(request->stream));
347 inflateInit(&request->stream);
348 SHA1_Init(&request->c);
349 if (prev_posn>0) {
350 prev_posn = 0;
351 lseek(request->local, SEEK_SET, 0);
352 ftruncate(request->local, 0);
353 }
354 }
355
356 slot = get_active_slot();
357 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
358 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
359 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
360 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
361
362 /* If we have successfully processed data from a previous fetch
363 attempt, only fetch the data we don't already have. */
364 if (prev_posn>0) {
365 if (get_verbosely)
366 fprintf(stderr,
367 "Resuming fetch of object %s at byte %ld\n",
368 hex, prev_posn);
369 sprintf(range, "Range: bytes=%ld-", prev_posn);
370 range_header = curl_slist_append(range_header, range);
371 curl_easy_setopt(slot->curl,
372 CURLOPT_HTTPHEADER, range_header);
373 }
374
a7a8d378 375 /* Try to get the request started, abort the request on error */
1d389ab6
NH
376 if (!start_active_slot(slot)) {
377 request->state = ABORTED;
378 close(request->local);
379 free(request->url);
380 return;
381 }
382
383 request->slot = slot;
384 request->state = ACTIVE;
385}
386
387void finish_request(struct transfer_request *request)
388{
389 fchmod(request->local, 0444);
390 close(request->local);
391
392 if (request->http_code == 416) {
393 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
394 } else if (request->curl_result != CURLE_OK) {
395 return;
396 }
397
398 inflateEnd(&request->stream);
399 SHA1_Final(request->real_sha1, &request->c);
400 if (request->zret != Z_STREAM_END) {
401 unlink(request->tmpfile);
402 return;
403 }
404 if (memcmp(request->sha1, request->real_sha1, 20)) {
405 unlink(request->tmpfile);
406 return;
407 }
408 request->rename =
409 relink_or_rename(request->tmpfile, request->filename);
410
411 if (request->rename == 0)
412 pull_say("got %s\n", sha1_to_hex(request->sha1));
413}
414
415void release_request(struct transfer_request *request)
416{
417 struct transfer_request *entry = request_queue_head;
418
419 if (request == request_queue_head) {
420 request_queue_head = request->next;
421 } else {
422 while (entry->next != NULL && entry->next != request)
423 entry = entry->next;
424 if (entry->next == request)
425 entry->next = entry->next->next;
426 }
427
428 free(request->url);
429 free(request);
430}
431
a7a8d378 432#ifdef USE_CURL_MULTI
1d389ab6
NH
433void process_curl_messages()
434{
435 int num_messages;
436 struct active_request_slot *slot;
437 struct transfer_request *request = NULL;
438 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
439
440 while (curl_message != NULL) {
441 if (curl_message->msg == CURLMSG_DONE) {
442 slot = active_queue_head;
443 while (slot != NULL &&
444 slot->curl != curl_message->easy_handle)
445 slot = slot->next;
446 if (slot != NULL) {
447 curl_multi_remove_handle(curlm, slot->curl);
448 active_requests--;
449 slot->done = 1;
450 slot->in_use = 0;
451 slot->curl_result = curl_message->data.result;
452 request = request_queue_head;
453 while (request != NULL &&
454 request->slot != slot)
455 request = request->next;
456 } else {
457 fprintf(stderr, "Received DONE message for unknown request!\n");
458 }
459 if (request != NULL) {
460 request->curl_result =
461 curl_message->data.result;
462 curl_easy_getinfo(slot->curl,
463 CURLINFO_HTTP_CODE,
464 &request->http_code);
465 request->slot = NULL;
466
467 /* Use alternates if necessary */
468 if (request->http_code == 404 &&
469 request->repo->next != NULL) {
470 request->repo = request->repo->next;
471 start_request(request);
472 } else {
473 finish_request(request);
474 request->state = COMPLETE;
475 }
476 }
477 } else {
478 fprintf(stderr, "Unknown CURL message received: %d\n",
479 (int)curl_message->msg);
480 }
481 curl_message = curl_multi_info_read(curlm, &num_messages);
482 }
483}
484
485void process_request_queue()
486{
487 struct transfer_request *request = request_queue_head;
488 int num_transfers;
489
490 while (active_requests < max_requests && request != NULL) {
491 if (request->state == WAITING) {
492 start_request(request);
493 curl_multi_perform(curlm, &num_transfers);
494 }
495 request = request->next;
496 }
497}
a7a8d378 498#endif
1d389ab6
NH
499
500void prefetch(unsigned char *sha1)
501{
502 struct transfer_request *newreq;
503 struct transfer_request *tail;
504 char *filename = sha1_file_name(sha1);
505
506 newreq = xmalloc(sizeof(*newreq));
507 memcpy(newreq->sha1, sha1, 20);
508 newreq->repo = alt;
509 newreq->url = NULL;
510 newreq->local = -1;
511 newreq->state = WAITING;
512 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
513 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
514 "%s.temp", filename);
515 newreq->next = NULL;
516
517 if (request_queue_head == NULL) {
518 request_queue_head = newreq;
519 } else {
520 tail = request_queue_head;
521 while (tail->next != NULL) {
522 tail = tail->next;
523 }
524 tail->next = newreq;
525 }
a7a8d378 526#ifdef USE_CURL_MULTI
1d389ab6
NH
527 process_request_queue();
528 process_curl_messages();
a7a8d378 529#endif
1d389ab6
NH
530}
531
b3661567 532static int got_alternates = 0;
182005b9 533
b3661567 534static int fetch_index(struct alt_base *repo, unsigned char *sha1)
182005b9 535{
1d389ab6 536 char *hex = sha1_to_hex(sha1);
182005b9
DB
537 char *filename;
538 char *url;
49a0f240
NH
539 char tmpfile[PATH_MAX];
540 int ret;
541 long prev_posn = 0;
542 char range[RANGE_HEADER_SIZE];
543 struct curl_slist *range_header = NULL;
182005b9
DB
544
545 FILE *indexfile;
1d389ab6 546 struct active_request_slot *slot;
182005b9
DB
547
548 if (has_pack_index(sha1))
549 return 0;
550
551 if (get_verbosely)
1d389ab6 552 fprintf(stderr, "Getting index for pack %s\n", hex);
182005b9 553
b3661567 554 url = xmalloc(strlen(repo->base) + 64);
1d389ab6 555 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
182005b9
DB
556
557 filename = sha1_pack_index_name(sha1);
49a0f240
NH
558 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
559 indexfile = fopen(tmpfile, "a");
182005b9
DB
560 if (!indexfile)
561 return error("Unable to open local file %s for pack index",
562 filename);
563
1d389ab6
NH
564 slot = get_active_slot();
565 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
566 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
567 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
568 slot->local = indexfile;
569
49a0f240
NH
570 /* If there is data present from a previous transfer attempt,
571 resume where it left off */
572 prev_posn = ftell(indexfile);
573 if (prev_posn>0) {
574 if (get_verbosely)
575 fprintf(stderr,
576 "Resuming fetch of index for pack %s at byte %ld\n",
1d389ab6 577 hex, prev_posn);
49a0f240
NH
578 sprintf(range, "Range: bytes=%ld-", prev_posn);
579 range_header = curl_slist_append(range_header, range);
1d389ab6 580 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
581 }
582
1d389ab6
NH
583 if (start_active_slot(slot)) {
584 run_active_slot(slot);
585 if (slot->curl_result != CURLE_OK) {
586 fclose(indexfile);
587 return error("Unable to get pack index %s\n%s", url,
588 curl_errorstr);
589 }
590 } else {
591 return error("Unable to start request");
182005b9
DB
592 }
593
594 fclose(indexfile);
49a0f240
NH
595
596 ret = relink_or_rename(tmpfile, filename);
597 if (ret)
598 return error("unable to write index filename %s: %s",
599 filename, strerror(ret));
600
182005b9
DB
601 return 0;
602}
603
b3661567 604static int setup_index(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
605{
606 struct packed_git *new_pack;
607 if (has_pack_file(sha1))
608 return 0; // don't list this as something we can get
609
b3661567 610 if (fetch_index(repo, sha1))
182005b9
DB
611 return -1;
612
613 new_pack = parse_pack_index(sha1);
b3661567
DB
614 new_pack->next = repo->packs;
615 repo->packs = new_pack;
182005b9
DB
616 return 0;
617}
618
b3661567
DB
619static int fetch_alternates(char *base)
620{
621 int ret = 0;
622 struct buffer buffer;
623 char *url;
624 char *data;
625 int i = 0;
1b0c1e67 626 int http_specific = 1;
1d389ab6
NH
627 struct alt_base *tail = alt;
628
629 struct active_request_slot *slot;
b3661567
DB
630 if (got_alternates)
631 return 0;
632 data = xmalloc(4096);
1b0c1e67 633 buffer.size = 4095;
b3661567
DB
634 buffer.posn = 0;
635 buffer.buffer = data;
636
637 if (get_verbosely)
638 fprintf(stderr, "Getting alternates list\n");
639
640 url = xmalloc(strlen(base) + 31);
641 sprintf(url, "%s/objects/info/http-alternates", base);
642
1d389ab6
NH
643 slot = get_active_slot();
644 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
645 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
646 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
647 if (start_active_slot(slot)) {
648 run_active_slot(slot);
649 if (slot->curl_result != CURLE_OK || !buffer.posn) {
650 http_specific = 0;
651
652 sprintf(url, "%s/objects/info/alternates", base);
653
654 slot = get_active_slot();
655 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
656 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
657 fwrite_buffer);
658 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
659 if (start_active_slot(slot)) {
660 run_active_slot(slot);
661 if (slot->curl_result != CURLE_OK) {
662 return 0;
663 }
664 }
b3661567 665 }
1d389ab6
NH
666 } else {
667 return 0;
b3661567
DB
668 }
669
1b0c1e67
DB
670 data[buffer.posn] = '\0';
671
b3661567
DB
672 while (i < buffer.posn) {
673 int posn = i;
674 while (posn < buffer.posn && data[posn] != '\n')
675 posn++;
676 if (data[posn] == '\n') {
1b0c1e67
DB
677 int okay = 0;
678 int serverlen = 0;
679 struct alt_base *newalt;
680 char *target = NULL;
b3661567 681 if (data[i] == '/') {
1b0c1e67
DB
682 serverlen = strchr(base + 8, '/') - base;
683 okay = 1;
684 } else if (!memcmp(data + i, "../", 3)) {
685 i += 3;
686 serverlen = strlen(base);
687 while (i + 2 < posn &&
688 !memcmp(data + i, "../", 3)) {
689 do {
690 serverlen--;
691 } while (serverlen &&
692 base[serverlen - 1] != '/');
693 i += 3;
694 }
695 // If the server got removed, give up.
696 okay = strchr(base, ':') - base + 3 <
697 serverlen;
698 } else if (http_specific) {
699 char *colon = strchr(data + i, ':');
700 char *slash = strchr(data + i, '/');
701 if (colon && slash && colon < data + posn &&
702 slash < data + posn && colon < slash) {
703 okay = 1;
704 }
705 }
706 // skip 'objects' at end
707 if (okay) {
708 target = xmalloc(serverlen + posn - i - 6);
b3661567
DB
709 strncpy(target, base, serverlen);
710 strncpy(target + serverlen, data + i,
711 posn - i - 7);
712 target[serverlen + posn - i - 7] = '\0';
713 if (get_verbosely)
714 fprintf(stderr,
715 "Also look at %s\n", target);
716 newalt = xmalloc(sizeof(*newalt));
1d389ab6 717 newalt->next = NULL;
b3661567
DB
718 newalt->base = target;
719 newalt->got_indices = 0;
720 newalt->packs = NULL;
1d389ab6
NH
721 while (tail->next != NULL)
722 tail = tail->next;
723 tail->next = newalt;
b3661567
DB
724 ret++;
725 }
726 }
727 i = posn + 1;
728 }
729 got_alternates = 1;
730
731 return ret;
732}
733
734static int fetch_indices(struct alt_base *repo)
182005b9
DB
735{
736 unsigned char sha1[20];
737 char *url;
738 struct buffer buffer;
739 char *data;
740 int i = 0;
741
1d389ab6
NH
742 struct active_request_slot *slot;
743
b3661567 744 if (repo->got_indices)
182005b9
DB
745 return 0;
746
747 data = xmalloc(4096);
748 buffer.size = 4096;
749 buffer.posn = 0;
750 buffer.buffer = data;
751
752 if (get_verbosely)
753 fprintf(stderr, "Getting pack list\n");
754
b3661567
DB
755 url = xmalloc(strlen(repo->base) + 21);
756 sprintf(url, "%s/objects/info/packs", repo->base);
182005b9 757
1d389ab6
NH
758 slot = get_active_slot();
759 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
760 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
761 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
762 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
763 if (start_active_slot(slot)) {
764 run_active_slot(slot);
765 if (slot->curl_result != CURLE_OK)
766 return error("%s", curl_errorstr);
767 } else {
768 return error("Unable to start request");
769 }
182005b9 770
b3661567 771 while (i < buffer.posn) {
182005b9
DB
772 switch (data[i]) {
773 case 'P':
774 i++;
775 if (i + 52 < buffer.posn &&
776 !strncmp(data + i, " pack-", 6) &&
777 !strncmp(data + i + 46, ".pack\n", 6)) {
778 get_sha1_hex(data + i + 6, sha1);
b3661567 779 setup_index(repo, sha1);
182005b9
DB
780 i += 51;
781 break;
782 }
783 default:
784 while (data[i] != '\n')
785 i++;
786 }
787 i++;
b3661567 788 }
182005b9 789
b3661567 790 repo->got_indices = 1;
182005b9
DB
791 return 0;
792}
793
b3661567 794static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
795{
796 char *url;
797 struct packed_git *target;
798 struct packed_git **lst;
799 FILE *packfile;
800 char *filename;
49a0f240
NH
801 char tmpfile[PATH_MAX];
802 int ret;
803 long prev_posn = 0;
804 char range[RANGE_HEADER_SIZE];
805 struct curl_slist *range_header = NULL;
1d389ab6
NH
806
807 struct active_request_slot *slot;
182005b9 808
b3661567 809 if (fetch_indices(repo))
182005b9 810 return -1;
b3661567 811 target = find_sha1_pack(sha1, repo->packs);
182005b9 812 if (!target)
b3661567 813 return -1;
182005b9
DB
814
815 if (get_verbosely) {
816 fprintf(stderr, "Getting pack %s\n",
817 sha1_to_hex(target->sha1));
818 fprintf(stderr, " which contains %s\n",
819 sha1_to_hex(sha1));
820 }
821
b3661567 822 url = xmalloc(strlen(repo->base) + 65);
182005b9 823 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 824 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
825
826 filename = sha1_pack_name(target->sha1);
49a0f240
NH
827 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
828 packfile = fopen(tmpfile, "a");
182005b9
DB
829 if (!packfile)
830 return error("Unable to open local file %s for pack",
831 filename);
832
1d389ab6
NH
833 slot = get_active_slot();
834 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
835 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
836 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
837 slot->local = packfile;
1ddea77e 838
49a0f240
NH
839 /* If there is data present from a previous transfer attempt,
840 resume where it left off */
841 prev_posn = ftell(packfile);
842 if (prev_posn>0) {
843 if (get_verbosely)
844 fprintf(stderr,
845 "Resuming fetch of pack %s at byte %ld\n",
846 sha1_to_hex(target->sha1), prev_posn);
847 sprintf(range, "Range: bytes=%ld-", prev_posn);
848 range_header = curl_slist_append(range_header, range);
1d389ab6 849 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
850 }
851
1d389ab6
NH
852 if (start_active_slot(slot)) {
853 run_active_slot(slot);
854 if (slot->curl_result != CURLE_OK) {
855 fclose(packfile);
856 return error("Unable to get pack file %s\n%s", url,
857 curl_errorstr);
858 }
859 } else {
860 return error("Unable to start request");
182005b9
DB
861 }
862
863 fclose(packfile);
864
49a0f240
NH
865 ret = relink_or_rename(tmpfile, filename);
866 if (ret)
867 return error("unable to write pack filename %s: %s",
868 filename, strerror(ret));
869
b3661567 870 lst = &repo->packs;
182005b9
DB
871 while (*lst != target)
872 lst = &((*lst)->next);
873 *lst = (*lst)->next;
874
271421cd
JH
875 if (verify_pack(target, 0))
876 return -1;
182005b9
DB
877 install_packed_git(target);
878
879 return 0;
880}
881
a7928f8e 882static int fetch_object(struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
883{
884 char *hex = sha1_to_hex(sha1);
09d92083 885 int ret;
1d389ab6 886 struct transfer_request *request = request_queue_head;
1d389ab6
NH
887
888 while (request != NULL && memcmp(request->sha1, sha1, 20))
889 request = request->next;
890 if (request == NULL)
891 return error("Couldn't find request for %s in the queue", hex);
892
a7a8d378
NH
893#ifdef USE_CURL_MULTI
894 int num_transfers;
1d389ab6
NH
895 while (request->state == WAITING) {
896 curl_multi_perform(curlm, &num_transfers);
897 if (num_transfers < active_requests) {
898 process_curl_messages();
899 process_request_queue();
900 }
901 }
a7a8d378
NH
902#else
903 start_request(request);
904#endif
6eb7ed54 905
a7a8d378 906 while (request->state == ACTIVE) {
1d389ab6 907 run_active_slot(request->slot);
a7a8d378
NH
908#ifndef USE_CURL_MULTI
909 request->curl_result = request->slot->curl_result;
910 curl_easy_getinfo(request->slot->curl,
911 CURLINFO_HTTP_CODE,
912 &request->http_code);
913 request->slot = NULL;
914
915 /* Use alternates if necessary */
916 if (request->http_code == 404 &&
917 request->repo->next != NULL) {
918 request->repo = request->repo->next;
919 start_request(request);
920 } else {
921 finish_request(request);
922 request->state = COMPLETE;
923 }
924#endif
925 }
6eb7ed54 926
1d389ab6
NH
927 if (request->state == ABORTED) {
928 release_request(request);
929 return error("Request for %s aborted", hex);
49a0f240 930 }
49a0f240 931
1d389ab6
NH
932 if (request->curl_result != CURLE_OK && request->http_code != 416) {
933 ret = error("%s", request->errorstr);
934 release_request(request);
935 return ret;
49a0f240
NH
936 }
937
1d389ab6
NH
938 if (request->zret != Z_STREAM_END) {
939 ret = error("File %s (%s) corrupt\n", hex, request->url);
940 release_request(request);
941 return ret;
49a0f240
NH
942 }
943
1d389ab6
NH
944 if (memcmp(request->sha1, request->real_sha1, 20)) {
945 release_request(request);
946 return error("File %s has bad hash\n", hex);
182005b9 947 }
6eb7ed54 948
1d389ab6
NH
949 if (request->rename < 0) {
950 ret = error("unable to write sha1 filename %s: %s",
951 request->filename,
952 strerror(request->rename));
953 release_request(request);
954 return ret;
6eb7ed54 955 }
49a0f240 956
1d389ab6 957 release_request(request);
6eb7ed54
DB
958 return 0;
959}
960
b3661567
DB
961int fetch(unsigned char *sha1)
962{
963 struct alt_base *altbase = alt;
1d389ab6
NH
964
965 if (!fetch_object(altbase, sha1))
966 return 0;
b3661567 967 while (altbase) {
b3661567
DB
968 if (!fetch_pack(altbase, sha1))
969 return 0;
b3661567
DB
970 altbase = altbase->next;
971 }
972 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1d389ab6 973 alt->base);
b3661567
DB
974}
975
cd541a68
DB
976int fetch_ref(char *ref, unsigned char *sha1)
977{
fa3e0655
DB
978 char *url, *posn;
979 char hex[42];
980 struct buffer buffer;
1d389ab6
NH
981 char *base = alt->base;
982 struct active_request_slot *slot;
fa3e0655
DB
983 buffer.size = 41;
984 buffer.posn = 0;
985 buffer.buffer = hex;
986 hex[41] = '\0';
987
fa3e0655
DB
988 url = xmalloc(strlen(base) + 6 + strlen(ref));
989 strcpy(url, base);
990 posn = url + strlen(base);
991 strcpy(posn, "refs/");
992 posn += 5;
993 strcpy(posn, ref);
994
1d389ab6
NH
995 slot = get_active_slot();
996 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
997 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
998 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
999 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1000 if (start_active_slot(slot)) {
1001 run_active_slot(slot);
1002 if (slot->curl_result != CURLE_OK)
1003 return error("Couldn't get %s for %s\n%s",
1004 url, ref, curl_errorstr);
1005 } else {
1006 return error("Unable to start request");
1007 }
fa3e0655
DB
1008
1009 hex[40] = '\0';
1010 get_sha1_hex(hex, sha1);
1011 return 0;
cd541a68
DB
1012}
1013
6eb7ed54
DB
1014int main(int argc, char **argv)
1015{
1016 char *commit_id;
1017 char *url;
1018 int arg = 1;
1d389ab6 1019 struct active_request_slot *slot;
6eb7ed54
DB
1020
1021 while (arg < argc && argv[arg][0] == '-') {
1022 if (argv[arg][1] == 't') {
4250a5e5 1023 get_tree = 1;
6eb7ed54 1024 } else if (argv[arg][1] == 'c') {
4250a5e5 1025 get_history = 1;
6eb7ed54 1026 } else if (argv[arg][1] == 'a') {
4250a5e5
DB
1027 get_all = 1;
1028 get_tree = 1;
1029 get_history = 1;
e78d9772
JH
1030 } else if (argv[arg][1] == 'v') {
1031 get_verbosely = 1;
fa3e0655
DB
1032 } else if (argv[arg][1] == 'w') {
1033 write_ref = argv[arg + 1];
1034 arg++;
820eca68
DB
1035 } else if (!strcmp(argv[arg], "--recover")) {
1036 get_recover = 1;
6eb7ed54
DB
1037 }
1038 arg++;
1039 }
1040 if (argc < arg + 2) {
215a7ad1 1041 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
6eb7ed54
DB
1042 return 1;
1043 }
1044 commit_id = argv[arg];
1045 url = argv[arg + 1];
1046
6eb7ed54
DB
1047 curl_global_init(CURL_GLOBAL_ALL);
1048
a7a8d378 1049#ifdef USE_CURL_MULTI
38079239
NH
1050 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1051 if (http_max_requests != NULL)
1052 max_requests = atoi(http_max_requests);
1053 if (max_requests < 1)
1054 max_requests = DEFAULT_MAX_REQUESTS;
1055
1d389ab6
NH
1056 curlm = curl_multi_init();
1057 if (curlm == NULL) {
1058 fprintf(stderr, "Error creating curl multi handle.\n");
1059 return 1;
1060 }
a7a8d378 1061#endif
1db69b57 1062 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
49a0f240 1063 no_range_header = curl_slist_append(no_range_header, "Range:");
6eb7ed54 1064
1d389ab6
NH
1065 curl_default = curl_easy_init();
1066
a9ab586a 1067 curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1;
1d389ab6 1068 curl_easy_setopt(curl_default, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
9f6cf65e 1069#if LIBCURL_VERSION_NUM >= 0x070907
1d389ab6 1070 curl_easy_setopt(curl_default, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
9f6cf65e 1071#endif
3dcb90f5 1072
5acb6de1 1073 if ((ssl_cert = getenv("GIT_SSL_CERT")) != NULL) {
1d389ab6 1074 curl_easy_setopt(curl_default, CURLOPT_SSLCERT, ssl_cert);
5acb6de1 1075 }
7d167feb 1076#if LIBCURL_VERSION_NUM >= 0x070902
5acb6de1 1077 if ((ssl_key = getenv("GIT_SSL_KEY")) != NULL) {
1d389ab6 1078 curl_easy_setopt(curl_default, CURLOPT_SSLKEY, ssl_key);
5acb6de1 1079 }
7d167feb 1080#endif
5acb6de1
NH
1081#if LIBCURL_VERSION_NUM >= 0x070908
1082 if ((ssl_capath = getenv("GIT_SSL_CAPATH")) != NULL) {
1d389ab6 1083 curl_easy_setopt(curl_default, CURLOPT_CAPATH, ssl_capath);
5acb6de1
NH
1084 }
1085#endif
1086 if ((ssl_cainfo = getenv("GIT_SSL_CAINFO")) != NULL) {
1d389ab6 1087 curl_easy_setopt(curl_default, CURLOPT_CAINFO, ssl_cainfo);
5acb6de1 1088 }
1d389ab6 1089 curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
5acb6de1 1090
b3661567
DB
1091 alt = xmalloc(sizeof(*alt));
1092 alt->base = url;
1093 alt->got_indices = 0;
1094 alt->packs = NULL;
1095 alt->next = NULL;
1d389ab6 1096 fetch_alternates(alt->base);
6eb7ed54 1097
4250a5e5 1098 if (pull(commit_id))
6eb7ed54
DB
1099 return 1;
1100
1db69b57 1101 curl_slist_free_all(no_pragma_header);
1d389ab6
NH
1102 curl_slist_free_all(no_range_header);
1103 curl_easy_cleanup(curl_default);
1104 slot = active_queue_head;
1105 while (slot != NULL) {
1106 curl_easy_cleanup(slot->curl);
1107 slot = slot->next;
1108 }
a7a8d378 1109#ifdef USE_CURL_MULTI
1d389ab6 1110 curl_multi_cleanup(curlm);
a7a8d378 1111#endif
6eb7ed54
DB
1112 curl_global_cleanup();
1113 return 0;
1114}