Don't die in git-http-fetch when fetching packs.
[git/git.git] / http-fetch.c
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
5 #include "http.h"
6
7 #define PREV_BUF_SIZE 4096
8 #define RANGE_HEADER_SIZE 30
9
10 static int commits_on_stdin;
11
12 static int got_alternates = -1;
13 static int corrupt_object_found;
14
15 static struct curl_slist *no_pragma_header;
16
17 struct alt_base
18 {
19 const char *base;
20 int path_len;
21 int got_indices;
22 struct packed_git *packs;
23 struct alt_base *next;
24 };
25
26 static struct alt_base *alt;
27
28 enum object_request_state {
29 WAITING,
30 ABORTED,
31 ACTIVE,
32 COMPLETE,
33 };
34
35 struct object_request
36 {
37 unsigned char sha1[20];
38 struct alt_base *repo;
39 char *url;
40 char filename[PATH_MAX];
41 char tmpfile[PATH_MAX];
42 int local;
43 enum object_request_state state;
44 CURLcode curl_result;
45 char errorstr[CURL_ERROR_SIZE];
46 long http_code;
47 unsigned char real_sha1[20];
48 SHA_CTX c;
49 z_stream stream;
50 int zret;
51 int rename;
52 struct active_request_slot *slot;
53 struct object_request *next;
54 };
55
56 struct alternates_request {
57 const char *base;
58 char *url;
59 struct buffer *buffer;
60 struct active_request_slot *slot;
61 int http_specific;
62 };
63
64 static struct object_request *object_queue_head;
65
66 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
67 void *data)
68 {
69 unsigned char expn[4096];
70 size_t size = eltsize * nmemb;
71 int posn = 0;
72 struct object_request *obj_req = (struct object_request *)data;
73 do {
74 ssize_t retval = xwrite(obj_req->local,
75 (char *) ptr + posn, size - posn);
76 if (retval < 0)
77 return posn;
78 posn += retval;
79 } while (posn < size);
80
81 obj_req->stream.avail_in = size;
82 obj_req->stream.next_in = ptr;
83 do {
84 obj_req->stream.next_out = expn;
85 obj_req->stream.avail_out = sizeof(expn);
86 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
87 SHA1_Update(&obj_req->c, expn,
88 sizeof(expn) - obj_req->stream.avail_out);
89 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
90 data_received++;
91 return size;
92 }
93
94 static int missing__target(int code, int result)
95 {
96 return /* file:// URL -- do we ever use one??? */
97 (result == CURLE_FILE_COULDNT_READ_FILE) ||
98 /* http:// and https:// URL */
99 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
100 /* ftp:// URL */
101 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
102 ;
103 }
104
105 #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
106
107 static void fetch_alternates(const char *base);
108
109 static void process_object_response(void *callback_data);
110
111 static void start_object_request(struct object_request *obj_req)
112 {
113 char *hex = sha1_to_hex(obj_req->sha1);
114 char prevfile[PATH_MAX];
115 char *url;
116 char *posn;
117 int prevlocal;
118 unsigned char prev_buf[PREV_BUF_SIZE];
119 ssize_t prev_read = 0;
120 long prev_posn = 0;
121 char range[RANGE_HEADER_SIZE];
122 struct curl_slist *range_header = NULL;
123 struct active_request_slot *slot;
124
125 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
126 unlink(prevfile);
127 rename(obj_req->tmpfile, prevfile);
128 unlink(obj_req->tmpfile);
129
130 if (obj_req->local != -1)
131 error("fd leakage in start: %d", obj_req->local);
132 obj_req->local = open(obj_req->tmpfile,
133 O_WRONLY | O_CREAT | O_EXCL, 0666);
134 /* This could have failed due to the "lazy directory creation";
135 * try to mkdir the last path component.
136 */
137 if (obj_req->local < 0 && errno == ENOENT) {
138 char *dir = strrchr(obj_req->tmpfile, '/');
139 if (dir) {
140 *dir = 0;
141 mkdir(obj_req->tmpfile, 0777);
142 *dir = '/';
143 }
144 obj_req->local = open(obj_req->tmpfile,
145 O_WRONLY | O_CREAT | O_EXCL, 0666);
146 }
147
148 if (obj_req->local < 0) {
149 obj_req->state = ABORTED;
150 error("Couldn't create temporary file %s for %s: %s",
151 obj_req->tmpfile, obj_req->filename, strerror(errno));
152 return;
153 }
154
155 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
156
157 inflateInit(&obj_req->stream);
158
159 SHA1_Init(&obj_req->c);
160
161 url = xmalloc(strlen(obj_req->repo->base) + 50);
162 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
163 strcpy(url, obj_req->repo->base);
164 posn = url + strlen(obj_req->repo->base);
165 strcpy(posn, "objects/");
166 posn += 8;
167 memcpy(posn, hex, 2);
168 posn += 2;
169 *(posn++) = '/';
170 strcpy(posn, hex + 2);
171 strcpy(obj_req->url, url);
172
173 /* If a previous temp file is present, process what was already
174 fetched. */
175 prevlocal = open(prevfile, O_RDONLY);
176 if (prevlocal != -1) {
177 do {
178 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
179 if (prev_read>0) {
180 if (fwrite_sha1_file(prev_buf,
181 1,
182 prev_read,
183 obj_req) == prev_read) {
184 prev_posn += prev_read;
185 } else {
186 prev_read = -1;
187 }
188 }
189 } while (prev_read > 0);
190 close(prevlocal);
191 }
192 unlink(prevfile);
193
194 /* Reset inflate/SHA1 if there was an error reading the previous temp
195 file; also rewind to the beginning of the local file. */
196 if (prev_read == -1) {
197 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
198 inflateInit(&obj_req->stream);
199 SHA1_Init(&obj_req->c);
200 if (prev_posn>0) {
201 prev_posn = 0;
202 lseek(obj_req->local, SEEK_SET, 0);
203 ftruncate(obj_req->local, 0);
204 }
205 }
206
207 slot = get_active_slot();
208 slot->callback_func = process_object_response;
209 slot->callback_data = obj_req;
210 obj_req->slot = slot;
211
212 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
213 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
214 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
215 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
216 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
217
218 /* If we have successfully processed data from a previous fetch
219 attempt, only fetch the data we don't already have. */
220 if (prev_posn>0) {
221 if (get_verbosely)
222 fprintf(stderr,
223 "Resuming fetch of object %s at byte %ld\n",
224 hex, prev_posn);
225 sprintf(range, "Range: bytes=%ld-", prev_posn);
226 range_header = curl_slist_append(range_header, range);
227 curl_easy_setopt(slot->curl,
228 CURLOPT_HTTPHEADER, range_header);
229 }
230
231 /* Try to get the request started, abort the request on error */
232 obj_req->state = ACTIVE;
233 if (!start_active_slot(slot)) {
234 obj_req->state = ABORTED;
235 obj_req->slot = NULL;
236 close(obj_req->local); obj_req->local = -1;
237 free(obj_req->url);
238 return;
239 }
240 }
241
242 static void finish_object_request(struct object_request *obj_req)
243 {
244 struct stat st;
245
246 fchmod(obj_req->local, 0444);
247 close(obj_req->local); obj_req->local = -1;
248
249 if (obj_req->http_code == 416) {
250 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
251 } else if (obj_req->curl_result != CURLE_OK) {
252 if (stat(obj_req->tmpfile, &st) == 0)
253 if (st.st_size == 0)
254 unlink(obj_req->tmpfile);
255 return;
256 }
257
258 inflateEnd(&obj_req->stream);
259 SHA1_Final(obj_req->real_sha1, &obj_req->c);
260 if (obj_req->zret != Z_STREAM_END) {
261 unlink(obj_req->tmpfile);
262 return;
263 }
264 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
265 unlink(obj_req->tmpfile);
266 return;
267 }
268 obj_req->rename =
269 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
270
271 if (obj_req->rename == 0)
272 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
273 }
274
275 static void process_object_response(void *callback_data)
276 {
277 struct object_request *obj_req =
278 (struct object_request *)callback_data;
279
280 obj_req->curl_result = obj_req->slot->curl_result;
281 obj_req->http_code = obj_req->slot->http_code;
282 obj_req->slot = NULL;
283 obj_req->state = COMPLETE;
284
285 /* Use alternates if necessary */
286 if (missing_target(obj_req)) {
287 fetch_alternates(alt->base);
288 if (obj_req->repo->next != NULL) {
289 obj_req->repo =
290 obj_req->repo->next;
291 close(obj_req->local);
292 obj_req->local = -1;
293 start_object_request(obj_req);
294 return;
295 }
296 }
297
298 finish_object_request(obj_req);
299 }
300
301 static void release_object_request(struct object_request *obj_req)
302 {
303 struct object_request *entry = object_queue_head;
304
305 if (obj_req->local != -1)
306 error("fd leakage in release: %d", obj_req->local);
307 if (obj_req == object_queue_head) {
308 object_queue_head = obj_req->next;
309 } else {
310 while (entry->next != NULL && entry->next != obj_req)
311 entry = entry->next;
312 if (entry->next == obj_req)
313 entry->next = entry->next->next;
314 }
315
316 free(obj_req->url);
317 free(obj_req);
318 }
319
320 #ifdef USE_CURL_MULTI
321 void fill_active_slots(void)
322 {
323 struct object_request *obj_req = object_queue_head;
324 struct active_request_slot *slot = active_queue_head;
325 int num_transfers;
326
327 while (active_requests < max_requests && obj_req != NULL) {
328 if (obj_req->state == WAITING) {
329 if (has_sha1_file(obj_req->sha1))
330 obj_req->state = COMPLETE;
331 else
332 start_object_request(obj_req);
333 curl_multi_perform(curlm, &num_transfers);
334 }
335 obj_req = obj_req->next;
336 }
337
338 while (slot != NULL) {
339 if (!slot->in_use && slot->curl != NULL) {
340 curl_easy_cleanup(slot->curl);
341 slot->curl = NULL;
342 }
343 slot = slot->next;
344 }
345 }
346 #endif
347
348 void prefetch(unsigned char *sha1)
349 {
350 struct object_request *newreq;
351 struct object_request *tail;
352 char *filename = sha1_file_name(sha1);
353
354 newreq = xmalloc(sizeof(*newreq));
355 hashcpy(newreq->sha1, sha1);
356 newreq->repo = alt;
357 newreq->url = NULL;
358 newreq->local = -1;
359 newreq->state = WAITING;
360 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
361 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
362 "%s.temp", filename);
363 newreq->slot = NULL;
364 newreq->next = NULL;
365
366 if (object_queue_head == NULL) {
367 object_queue_head = newreq;
368 } else {
369 tail = object_queue_head;
370 while (tail->next != NULL) {
371 tail = tail->next;
372 }
373 tail->next = newreq;
374 }
375
376 #ifdef USE_CURL_MULTI
377 fill_active_slots();
378 step_active_slots();
379 #endif
380 }
381
382 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
383 {
384 char *hex = sha1_to_hex(sha1);
385 char *filename;
386 char *url;
387 char tmpfile[PATH_MAX];
388 long prev_posn = 0;
389 char range[RANGE_HEADER_SIZE];
390 struct curl_slist *range_header = NULL;
391
392 FILE *indexfile;
393 struct active_request_slot *slot;
394 struct slot_results results;
395
396 if (has_pack_index(sha1))
397 return 0;
398
399 if (get_verbosely)
400 fprintf(stderr, "Getting index for pack %s\n", hex);
401
402 url = xmalloc(strlen(repo->base) + 64);
403 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
404
405 filename = sha1_pack_index_name(sha1);
406 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
407 indexfile = fopen(tmpfile, "a");
408 if (!indexfile)
409 return error("Unable to open local file %s for pack index",
410 filename);
411
412 slot = get_active_slot();
413 slot->results = &results;
414 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
415 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
416 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
417 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
418 slot->local = indexfile;
419
420 /* If there is data present from a previous transfer attempt,
421 resume where it left off */
422 prev_posn = ftell(indexfile);
423 if (prev_posn>0) {
424 if (get_verbosely)
425 fprintf(stderr,
426 "Resuming fetch of index for pack %s at byte %ld\n",
427 hex, prev_posn);
428 sprintf(range, "Range: bytes=%ld-", prev_posn);
429 range_header = curl_slist_append(range_header, range);
430 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
431 }
432
433 if (start_active_slot(slot)) {
434 run_active_slot(slot);
435 if (results.curl_result != CURLE_OK) {
436 fclose(indexfile);
437 return error("Unable to get pack index %s\n%s", url,
438 curl_errorstr);
439 }
440 } else {
441 fclose(indexfile);
442 return error("Unable to start request");
443 }
444
445 fclose(indexfile);
446
447 return move_temp_to_file(tmpfile, filename);
448 }
449
450 static int setup_index(struct alt_base *repo, unsigned char *sha1)
451 {
452 struct packed_git *new_pack;
453 if (has_pack_file(sha1))
454 return 0; /* don't list this as something we can get */
455
456 if (fetch_index(repo, sha1))
457 return -1;
458
459 new_pack = parse_pack_index(sha1);
460 new_pack->next = repo->packs;
461 repo->packs = new_pack;
462 return 0;
463 }
464
465 static void process_alternates_response(void *callback_data)
466 {
467 struct alternates_request *alt_req =
468 (struct alternates_request *)callback_data;
469 struct active_request_slot *slot = alt_req->slot;
470 struct alt_base *tail = alt;
471 const char *base = alt_req->base;
472 static const char null_byte = '\0';
473 char *data;
474 int i = 0;
475
476 if (alt_req->http_specific) {
477 if (slot->curl_result != CURLE_OK ||
478 !alt_req->buffer->posn) {
479
480 /* Try reusing the slot to get non-http alternates */
481 alt_req->http_specific = 0;
482 sprintf(alt_req->url, "%s/objects/info/alternates",
483 base);
484 curl_easy_setopt(slot->curl, CURLOPT_URL,
485 alt_req->url);
486 active_requests++;
487 slot->in_use = 1;
488 if (slot->finished != NULL)
489 (*slot->finished) = 0;
490 if (!start_active_slot(slot)) {
491 got_alternates = -1;
492 slot->in_use = 0;
493 if (slot->finished != NULL)
494 (*slot->finished) = 1;
495 }
496 return;
497 }
498 } else if (slot->curl_result != CURLE_OK) {
499 if (!missing_target(slot)) {
500 got_alternates = -1;
501 return;
502 }
503 }
504
505 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
506 alt_req->buffer->posn--;
507 data = alt_req->buffer->buffer;
508
509 while (i < alt_req->buffer->posn) {
510 int posn = i;
511 while (posn < alt_req->buffer->posn && data[posn] != '\n')
512 posn++;
513 if (data[posn] == '\n') {
514 int okay = 0;
515 int serverlen = 0;
516 struct alt_base *newalt;
517 char *target = NULL;
518 char *path;
519 if (data[i] == '/') {
520 /* This counts
521 * http://git.host/pub/scm/linux.git/
522 * -----------here^
523 * so memcpy(dst, base, serverlen) will
524 * copy up to "...git.host".
525 */
526 const char *colon_ss = strstr(base,"://");
527 if (colon_ss) {
528 serverlen = (strchr(colon_ss + 3, '/')
529 - base);
530 okay = 1;
531 }
532 } else if (!memcmp(data + i, "../", 3)) {
533 /* Relative URL; chop the corresponding
534 * number of subpath from base (and ../
535 * from data), and concatenate the result.
536 *
537 * The code first drops ../ from data, and
538 * then drops one ../ from data and one path
539 * from base. IOW, one extra ../ is dropped
540 * from data than path is dropped from base.
541 *
542 * This is not wrong. The alternate in
543 * http://git.host/pub/scm/linux.git/
544 * to borrow from
545 * http://git.host/pub/scm/linus.git/
546 * is ../../linus.git/objects/. You need
547 * two ../../ to borrow from your direct
548 * neighbour.
549 */
550 i += 3;
551 serverlen = strlen(base);
552 while (i + 2 < posn &&
553 !memcmp(data + i, "../", 3)) {
554 do {
555 serverlen--;
556 } while (serverlen &&
557 base[serverlen - 1] != '/');
558 i += 3;
559 }
560 /* If the server got removed, give up. */
561 okay = strchr(base, ':') - base + 3 <
562 serverlen;
563 } else if (alt_req->http_specific) {
564 char *colon = strchr(data + i, ':');
565 char *slash = strchr(data + i, '/');
566 if (colon && slash && colon < data + posn &&
567 slash < data + posn && colon < slash) {
568 okay = 1;
569 }
570 }
571 /* skip "objects\n" at end */
572 if (okay) {
573 target = xmalloc(serverlen + posn - i - 6);
574 memcpy(target, base, serverlen);
575 memcpy(target + serverlen, data + i,
576 posn - i - 7);
577 target[serverlen + posn - i - 7] = 0;
578 if (get_verbosely)
579 fprintf(stderr,
580 "Also look at %s\n", target);
581 newalt = xmalloc(sizeof(*newalt));
582 newalt->next = NULL;
583 newalt->base = target;
584 newalt->got_indices = 0;
585 newalt->packs = NULL;
586 path = strstr(target, "//");
587 if (path) {
588 path = strchr(path+2, '/');
589 if (path)
590 newalt->path_len = strlen(path);
591 }
592
593 while (tail->next != NULL)
594 tail = tail->next;
595 tail->next = newalt;
596 }
597 }
598 i = posn + 1;
599 }
600
601 got_alternates = 1;
602 }
603
604 static void fetch_alternates(const char *base)
605 {
606 struct buffer buffer;
607 char *url;
608 char *data;
609 struct active_request_slot *slot;
610 struct alternates_request alt_req;
611
612 /* If another request has already started fetching alternates,
613 wait for them to arrive and return to processing this request's
614 curl message */
615 #ifdef USE_CURL_MULTI
616 while (got_alternates == 0) {
617 step_active_slots();
618 }
619 #endif
620
621 /* Nothing to do if they've already been fetched */
622 if (got_alternates == 1)
623 return;
624
625 /* Start the fetch */
626 got_alternates = 0;
627
628 data = xmalloc(4096);
629 buffer.size = 4096;
630 buffer.posn = 0;
631 buffer.buffer = data;
632
633 if (get_verbosely)
634 fprintf(stderr, "Getting alternates list for %s\n", base);
635
636 url = xmalloc(strlen(base) + 31);
637 sprintf(url, "%s/objects/info/http-alternates", base);
638
639 /* Use a callback to process the result, since another request
640 may fail and need to have alternates loaded before continuing */
641 slot = get_active_slot();
642 slot->callback_func = process_alternates_response;
643 slot->callback_data = &alt_req;
644
645 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
646 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
647 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
648
649 alt_req.base = base;
650 alt_req.url = url;
651 alt_req.buffer = &buffer;
652 alt_req.http_specific = 1;
653 alt_req.slot = slot;
654
655 if (start_active_slot(slot))
656 run_active_slot(slot);
657 else
658 got_alternates = -1;
659
660 free(data);
661 free(url);
662 }
663
664 static int fetch_indices(struct alt_base *repo)
665 {
666 unsigned char sha1[20];
667 char *url;
668 struct buffer buffer;
669 char *data;
670 int i = 0;
671
672 struct active_request_slot *slot;
673 struct slot_results results;
674
675 if (repo->got_indices)
676 return 0;
677
678 data = xmalloc(4096);
679 buffer.size = 4096;
680 buffer.posn = 0;
681 buffer.buffer = data;
682
683 if (get_verbosely)
684 fprintf(stderr, "Getting pack list for %s\n", repo->base);
685
686 url = xmalloc(strlen(repo->base) + 21);
687 sprintf(url, "%s/objects/info/packs", repo->base);
688
689 slot = get_active_slot();
690 slot->results = &results;
691 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
692 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
693 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
694 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
695 if (start_active_slot(slot)) {
696 run_active_slot(slot);
697 if (results.curl_result != CURLE_OK) {
698 if (missing_target(&results)) {
699 repo->got_indices = 1;
700 free(buffer.buffer);
701 return 0;
702 } else {
703 repo->got_indices = 0;
704 free(buffer.buffer);
705 return error("%s", curl_errorstr);
706 }
707 }
708 } else {
709 repo->got_indices = 0;
710 free(buffer.buffer);
711 return error("Unable to start request");
712 }
713
714 data = buffer.buffer;
715 while (i < buffer.posn) {
716 switch (data[i]) {
717 case 'P':
718 i++;
719 if (i + 52 <= buffer.posn &&
720 !strncmp(data + i, " pack-", 6) &&
721 !strncmp(data + i + 46, ".pack\n", 6)) {
722 get_sha1_hex(data + i + 6, sha1);
723 setup_index(repo, sha1);
724 i += 51;
725 break;
726 }
727 default:
728 while (i < buffer.posn && data[i] != '\n')
729 i++;
730 }
731 i++;
732 }
733
734 free(buffer.buffer);
735 repo->got_indices = 1;
736 return 0;
737 }
738
739 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
740 {
741 char *url;
742 struct packed_git *target;
743 struct packed_git **lst;
744 FILE *packfile;
745 char *filename;
746 char tmpfile[PATH_MAX];
747 int ret;
748 long prev_posn = 0;
749 char range[RANGE_HEADER_SIZE];
750 struct curl_slist *range_header = NULL;
751
752 struct active_request_slot *slot;
753 struct slot_results results;
754
755 if (fetch_indices(repo))
756 return -1;
757 target = find_sha1_pack(sha1, repo->packs);
758 if (!target)
759 return -1;
760
761 if (get_verbosely) {
762 fprintf(stderr, "Getting pack %s\n",
763 sha1_to_hex(target->sha1));
764 fprintf(stderr, " which contains %s\n",
765 sha1_to_hex(sha1));
766 }
767
768 url = xmalloc(strlen(repo->base) + 65);
769 sprintf(url, "%s/objects/pack/pack-%s.pack",
770 repo->base, sha1_to_hex(target->sha1));
771
772 filename = sha1_pack_name(target->sha1);
773 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
774 packfile = fopen(tmpfile, "a");
775 if (!packfile)
776 return error("Unable to open local file %s for pack",
777 filename);
778
779 slot = get_active_slot();
780 slot->results = &results;
781 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
782 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
783 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
784 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
785 slot->local = packfile;
786
787 /* If there is data present from a previous transfer attempt,
788 resume where it left off */
789 prev_posn = ftell(packfile);
790 if (prev_posn>0) {
791 if (get_verbosely)
792 fprintf(stderr,
793 "Resuming fetch of pack %s at byte %ld\n",
794 sha1_to_hex(target->sha1), prev_posn);
795 sprintf(range, "Range: bytes=%ld-", prev_posn);
796 range_header = curl_slist_append(range_header, range);
797 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
798 }
799
800 if (start_active_slot(slot)) {
801 run_active_slot(slot);
802 if (results.curl_result != CURLE_OK) {
803 fclose(packfile);
804 return error("Unable to get pack file %s\n%s", url,
805 curl_errorstr);
806 }
807 } else {
808 fclose(packfile);
809 return error("Unable to start request");
810 }
811
812 target->pack_size = ftell(packfile);
813 fclose(packfile);
814
815 ret = move_temp_to_file(tmpfile, filename);
816 if (ret)
817 return ret;
818
819 lst = &repo->packs;
820 while (*lst != target)
821 lst = &((*lst)->next);
822 *lst = (*lst)->next;
823
824 if (verify_pack(target, 0))
825 return -1;
826 install_packed_git(target);
827
828 return 0;
829 }
830
831 static void abort_object_request(struct object_request *obj_req)
832 {
833 if (obj_req->local >= 0) {
834 close(obj_req->local);
835 obj_req->local = -1;
836 }
837 unlink(obj_req->tmpfile);
838 if (obj_req->slot) {
839 release_active_slot(obj_req->slot);
840 obj_req->slot = NULL;
841 }
842 release_object_request(obj_req);
843 }
844
845 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
846 {
847 char *hex = sha1_to_hex(sha1);
848 int ret = 0;
849 struct object_request *obj_req = object_queue_head;
850
851 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
852 obj_req = obj_req->next;
853 if (obj_req == NULL)
854 return error("Couldn't find request for %s in the queue", hex);
855
856 if (has_sha1_file(obj_req->sha1)) {
857 abort_object_request(obj_req);
858 return 0;
859 }
860
861 #ifdef USE_CURL_MULTI
862 while (obj_req->state == WAITING) {
863 step_active_slots();
864 }
865 #else
866 start_object_request(obj_req);
867 #endif
868
869 while (obj_req->state == ACTIVE) {
870 run_active_slot(obj_req->slot);
871 }
872 if (obj_req->local != -1) {
873 close(obj_req->local); obj_req->local = -1;
874 }
875
876 if (obj_req->state == ABORTED) {
877 ret = error("Request for %s aborted", hex);
878 } else if (obj_req->curl_result != CURLE_OK &&
879 obj_req->http_code != 416) {
880 if (missing_target(obj_req))
881 ret = -1; /* Be silent, it is probably in a pack. */
882 else
883 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
884 obj_req->errorstr, obj_req->curl_result,
885 obj_req->http_code, hex);
886 } else if (obj_req->zret != Z_STREAM_END) {
887 corrupt_object_found++;
888 ret = error("File %s (%s) corrupt", hex, obj_req->url);
889 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
890 ret = error("File %s has bad hash", hex);
891 } else if (obj_req->rename < 0) {
892 ret = error("unable to write sha1 filename %s",
893 obj_req->filename);
894 }
895
896 release_object_request(obj_req);
897 return ret;
898 }
899
900 int fetch(unsigned char *sha1)
901 {
902 struct alt_base *altbase = alt;
903
904 if (!fetch_object(altbase, sha1))
905 return 0;
906 while (altbase) {
907 if (!fetch_pack(altbase, sha1))
908 return 0;
909 fetch_alternates(alt->base);
910 altbase = altbase->next;
911 }
912 return error("Unable to find %s under %s", sha1_to_hex(sha1),
913 alt->base);
914 }
915
916 static inline int needs_quote(int ch)
917 {
918 if (((ch >= 'A') && (ch <= 'Z'))
919 || ((ch >= 'a') && (ch <= 'z'))
920 || ((ch >= '0') && (ch <= '9'))
921 || (ch == '/')
922 || (ch == '-')
923 || (ch == '.'))
924 return 0;
925 return 1;
926 }
927
928 static inline int hex(int v)
929 {
930 if (v < 10) return '0' + v;
931 else return 'A' + v - 10;
932 }
933
934 static char *quote_ref_url(const char *base, const char *ref)
935 {
936 const char *cp;
937 char *dp, *qref;
938 int len, baselen, ch;
939
940 baselen = strlen(base);
941 len = baselen + 6; /* "refs/" + NUL */
942 for (cp = ref; (ch = *cp) != 0; cp++, len++)
943 if (needs_quote(ch))
944 len += 2; /* extra two hex plus replacement % */
945 qref = xmalloc(len);
946 memcpy(qref, base, baselen);
947 memcpy(qref + baselen, "refs/", 5);
948 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
949 if (needs_quote(ch)) {
950 *dp++ = '%';
951 *dp++ = hex((ch >> 4) & 0xF);
952 *dp++ = hex(ch & 0xF);
953 }
954 else
955 *dp++ = ch;
956 }
957 *dp = 0;
958
959 return qref;
960 }
961
962 int fetch_ref(char *ref, unsigned char *sha1)
963 {
964 char *url;
965 char hex[42];
966 struct buffer buffer;
967 const char *base = alt->base;
968 struct active_request_slot *slot;
969 struct slot_results results;
970 buffer.size = 41;
971 buffer.posn = 0;
972 buffer.buffer = hex;
973 hex[41] = '\0';
974
975 url = quote_ref_url(base, ref);
976 slot = get_active_slot();
977 slot->results = &results;
978 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
979 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
980 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
981 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
982 if (start_active_slot(slot)) {
983 run_active_slot(slot);
984 if (results.curl_result != CURLE_OK)
985 return error("Couldn't get %s for %s\n%s",
986 url, ref, curl_errorstr);
987 } else {
988 return error("Unable to start request");
989 }
990
991 hex[40] = '\0';
992 get_sha1_hex(hex, sha1);
993 return 0;
994 }
995
996 int main(int argc, const char **argv)
997 {
998 int commits;
999 const char **write_ref = NULL;
1000 char **commit_id;
1001 const char *url;
1002 char *path;
1003 int arg = 1;
1004 int rc = 0;
1005
1006 setup_ident();
1007 setup_git_directory();
1008 git_config(git_default_config);
1009
1010 while (arg < argc && argv[arg][0] == '-') {
1011 if (argv[arg][1] == 't') {
1012 get_tree = 1;
1013 } else if (argv[arg][1] == 'c') {
1014 get_history = 1;
1015 } else if (argv[arg][1] == 'a') {
1016 get_all = 1;
1017 get_tree = 1;
1018 get_history = 1;
1019 } else if (argv[arg][1] == 'v') {
1020 get_verbosely = 1;
1021 } else if (argv[arg][1] == 'w') {
1022 write_ref = &argv[arg + 1];
1023 arg++;
1024 } else if (!strcmp(argv[arg], "--recover")) {
1025 get_recover = 1;
1026 } else if (!strcmp(argv[arg], "--stdin")) {
1027 commits_on_stdin = 1;
1028 }
1029 arg++;
1030 }
1031 if (argc < arg + 2 - commits_on_stdin) {
1032 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1033 return 1;
1034 }
1035 if (commits_on_stdin) {
1036 commits = pull_targets_stdin(&commit_id, &write_ref);
1037 } else {
1038 commit_id = (char **) &argv[arg++];
1039 commits = 1;
1040 }
1041 url = argv[arg];
1042
1043 http_init();
1044
1045 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1046
1047 alt = xmalloc(sizeof(*alt));
1048 alt->base = url;
1049 alt->got_indices = 0;
1050 alt->packs = NULL;
1051 alt->next = NULL;
1052 path = strstr(url, "//");
1053 if (path) {
1054 path = strchr(path+2, '/');
1055 if (path)
1056 alt->path_len = strlen(path);
1057 }
1058
1059 if (pull(commits, commit_id, write_ref, url))
1060 rc = 1;
1061
1062 http_cleanup();
1063
1064 curl_slist_free_all(no_pragma_header);
1065
1066 if (commits_on_stdin)
1067 pull_targets_free(commits, commit_id, write_ref);
1068
1069 if (corrupt_object_found) {
1070 fprintf(stderr,
1071 "Some loose object were found to be corrupt, but they might be just\n"
1072 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1073 "status code. Suggest running git fsck-objects.\n");
1074 }
1075 return rc;
1076 }