Merge branch 'jc/web-blame'
[git/git.git] / http-fetch.c
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
5 #include "http.h"
6
7 #define PREV_BUF_SIZE 4096
8 #define RANGE_HEADER_SIZE 30
9
10 static int commits_on_stdin;
11
12 static int got_alternates = -1;
13 static int corrupt_object_found;
14
15 static struct curl_slist *no_pragma_header;
16
17 struct alt_base
18 {
19 const char *base;
20 int path_len;
21 int got_indices;
22 struct packed_git *packs;
23 struct alt_base *next;
24 };
25
26 static struct alt_base *alt;
27
28 enum object_request_state {
29 WAITING,
30 ABORTED,
31 ACTIVE,
32 COMPLETE,
33 };
34
35 struct object_request
36 {
37 unsigned char sha1[20];
38 struct alt_base *repo;
39 char *url;
40 char filename[PATH_MAX];
41 char tmpfile[PATH_MAX];
42 int local;
43 enum object_request_state state;
44 CURLcode curl_result;
45 char errorstr[CURL_ERROR_SIZE];
46 long http_code;
47 unsigned char real_sha1[20];
48 SHA_CTX c;
49 z_stream stream;
50 int zret;
51 int rename;
52 struct active_request_slot *slot;
53 struct object_request *next;
54 };
55
56 struct alternates_request {
57 const char *base;
58 char *url;
59 struct buffer *buffer;
60 struct active_request_slot *slot;
61 int http_specific;
62 };
63
64 static struct object_request *object_queue_head;
65
66 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
67 void *data)
68 {
69 unsigned char expn[4096];
70 size_t size = eltsize * nmemb;
71 int posn = 0;
72 struct object_request *obj_req = (struct object_request *)data;
73 do {
74 ssize_t retval = write(obj_req->local,
75 (char *) ptr + posn, size - posn);
76 if (retval < 0)
77 return posn;
78 posn += retval;
79 } while (posn < size);
80
81 obj_req->stream.avail_in = size;
82 obj_req->stream.next_in = ptr;
83 do {
84 obj_req->stream.next_out = expn;
85 obj_req->stream.avail_out = sizeof(expn);
86 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
87 SHA1_Update(&obj_req->c, expn,
88 sizeof(expn) - obj_req->stream.avail_out);
89 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
90 data_received++;
91 return size;
92 }
93
94 static int missing__target(int code, int result)
95 {
96 return /* file:// URL -- do we ever use one??? */
97 (result == CURLE_FILE_COULDNT_READ_FILE) ||
98 /* http:// and https:// URL */
99 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
100 /* ftp:// URL */
101 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
102 ;
103 }
104
105 #define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
106
107 static void fetch_alternates(const char *base);
108
109 static void process_object_response(void *callback_data);
110
111 static void start_object_request(struct object_request *obj_req)
112 {
113 char *hex = sha1_to_hex(obj_req->sha1);
114 char prevfile[PATH_MAX];
115 char *url;
116 char *posn;
117 int prevlocal;
118 unsigned char prev_buf[PREV_BUF_SIZE];
119 ssize_t prev_read = 0;
120 long prev_posn = 0;
121 char range[RANGE_HEADER_SIZE];
122 struct curl_slist *range_header = NULL;
123 struct active_request_slot *slot;
124
125 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
126 unlink(prevfile);
127 rename(obj_req->tmpfile, prevfile);
128 unlink(obj_req->tmpfile);
129
130 if (obj_req->local != -1)
131 error("fd leakage in start: %d", obj_req->local);
132 obj_req->local = open(obj_req->tmpfile,
133 O_WRONLY | O_CREAT | O_EXCL, 0666);
134 /* This could have failed due to the "lazy directory creation";
135 * try to mkdir the last path component.
136 */
137 if (obj_req->local < 0 && errno == ENOENT) {
138 char *dir = strrchr(obj_req->tmpfile, '/');
139 if (dir) {
140 *dir = 0;
141 mkdir(obj_req->tmpfile, 0777);
142 *dir = '/';
143 }
144 obj_req->local = open(obj_req->tmpfile,
145 O_WRONLY | O_CREAT | O_EXCL, 0666);
146 }
147
148 if (obj_req->local < 0) {
149 obj_req->state = ABORTED;
150 error("Couldn't create temporary file %s for %s: %s",
151 obj_req->tmpfile, obj_req->filename, strerror(errno));
152 return;
153 }
154
155 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
156
157 inflateInit(&obj_req->stream);
158
159 SHA1_Init(&obj_req->c);
160
161 url = xmalloc(strlen(obj_req->repo->base) + 50);
162 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
163 strcpy(url, obj_req->repo->base);
164 posn = url + strlen(obj_req->repo->base);
165 strcpy(posn, "objects/");
166 posn += 8;
167 memcpy(posn, hex, 2);
168 posn += 2;
169 *(posn++) = '/';
170 strcpy(posn, hex + 2);
171 strcpy(obj_req->url, url);
172
173 /* If a previous temp file is present, process what was already
174 fetched. */
175 prevlocal = open(prevfile, O_RDONLY);
176 if (prevlocal != -1) {
177 do {
178 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
179 if (prev_read>0) {
180 if (fwrite_sha1_file(prev_buf,
181 1,
182 prev_read,
183 obj_req) == prev_read) {
184 prev_posn += prev_read;
185 } else {
186 prev_read = -1;
187 }
188 }
189 } while (prev_read > 0);
190 close(prevlocal);
191 }
192 unlink(prevfile);
193
194 /* Reset inflate/SHA1 if there was an error reading the previous temp
195 file; also rewind to the beginning of the local file. */
196 if (prev_read == -1) {
197 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
198 inflateInit(&obj_req->stream);
199 SHA1_Init(&obj_req->c);
200 if (prev_posn>0) {
201 prev_posn = 0;
202 lseek(obj_req->local, SEEK_SET, 0);
203 ftruncate(obj_req->local, 0);
204 }
205 }
206
207 slot = get_active_slot();
208 slot->callback_func = process_object_response;
209 slot->callback_data = obj_req;
210 obj_req->slot = slot;
211
212 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
213 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
214 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
215 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
216 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
217
218 /* If we have successfully processed data from a previous fetch
219 attempt, only fetch the data we don't already have. */
220 if (prev_posn>0) {
221 if (get_verbosely)
222 fprintf(stderr,
223 "Resuming fetch of object %s at byte %ld\n",
224 hex, prev_posn);
225 sprintf(range, "Range: bytes=%ld-", prev_posn);
226 range_header = curl_slist_append(range_header, range);
227 curl_easy_setopt(slot->curl,
228 CURLOPT_HTTPHEADER, range_header);
229 }
230
231 /* Try to get the request started, abort the request on error */
232 obj_req->state = ACTIVE;
233 if (!start_active_slot(slot)) {
234 obj_req->state = ABORTED;
235 obj_req->slot = NULL;
236 close(obj_req->local); obj_req->local = -1;
237 free(obj_req->url);
238 return;
239 }
240 }
241
242 static void finish_object_request(struct object_request *obj_req)
243 {
244 struct stat st;
245
246 fchmod(obj_req->local, 0444);
247 close(obj_req->local); obj_req->local = -1;
248
249 if (obj_req->http_code == 416) {
250 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
251 } else if (obj_req->curl_result != CURLE_OK) {
252 if (stat(obj_req->tmpfile, &st) == 0)
253 if (st.st_size == 0)
254 unlink(obj_req->tmpfile);
255 return;
256 }
257
258 inflateEnd(&obj_req->stream);
259 SHA1_Final(obj_req->real_sha1, &obj_req->c);
260 if (obj_req->zret != Z_STREAM_END) {
261 unlink(obj_req->tmpfile);
262 return;
263 }
264 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
265 unlink(obj_req->tmpfile);
266 return;
267 }
268 obj_req->rename =
269 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
270
271 if (obj_req->rename == 0)
272 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
273 }
274
275 static void process_object_response(void *callback_data)
276 {
277 struct object_request *obj_req =
278 (struct object_request *)callback_data;
279
280 obj_req->curl_result = obj_req->slot->curl_result;
281 obj_req->http_code = obj_req->slot->http_code;
282 obj_req->slot = NULL;
283 obj_req->state = COMPLETE;
284
285 /* Use alternates if necessary */
286 if (missing_target(obj_req)) {
287 fetch_alternates(alt->base);
288 if (obj_req->repo->next != NULL) {
289 obj_req->repo =
290 obj_req->repo->next;
291 close(obj_req->local);
292 obj_req->local = -1;
293 start_object_request(obj_req);
294 return;
295 }
296 }
297
298 finish_object_request(obj_req);
299 }
300
301 static void release_object_request(struct object_request *obj_req)
302 {
303 struct object_request *entry = object_queue_head;
304
305 if (obj_req->local != -1)
306 error("fd leakage in release: %d", obj_req->local);
307 if (obj_req == object_queue_head) {
308 object_queue_head = obj_req->next;
309 } else {
310 while (entry->next != NULL && entry->next != obj_req)
311 entry = entry->next;
312 if (entry->next == obj_req)
313 entry->next = entry->next->next;
314 }
315
316 free(obj_req->url);
317 free(obj_req);
318 }
319
320 #ifdef USE_CURL_MULTI
321 void fill_active_slots(void)
322 {
323 struct object_request *obj_req = object_queue_head;
324 struct active_request_slot *slot = active_queue_head;
325 int num_transfers;
326
327 while (active_requests < max_requests && obj_req != NULL) {
328 if (obj_req->state == WAITING) {
329 if (has_sha1_file(obj_req->sha1))
330 obj_req->state = COMPLETE;
331 else
332 start_object_request(obj_req);
333 curl_multi_perform(curlm, &num_transfers);
334 }
335 obj_req = obj_req->next;
336 }
337
338 while (slot != NULL) {
339 if (!slot->in_use && slot->curl != NULL) {
340 curl_easy_cleanup(slot->curl);
341 slot->curl = NULL;
342 }
343 slot = slot->next;
344 }
345 }
346 #endif
347
348 void prefetch(unsigned char *sha1)
349 {
350 struct object_request *newreq;
351 struct object_request *tail;
352 char *filename = sha1_file_name(sha1);
353
354 newreq = xmalloc(sizeof(*newreq));
355 hashcpy(newreq->sha1, sha1);
356 newreq->repo = alt;
357 newreq->url = NULL;
358 newreq->local = -1;
359 newreq->state = WAITING;
360 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
361 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
362 "%s.temp", filename);
363 newreq->slot = NULL;
364 newreq->next = NULL;
365
366 if (object_queue_head == NULL) {
367 object_queue_head = newreq;
368 } else {
369 tail = object_queue_head;
370 while (tail->next != NULL) {
371 tail = tail->next;
372 }
373 tail->next = newreq;
374 }
375
376 #ifdef USE_CURL_MULTI
377 fill_active_slots();
378 step_active_slots();
379 #endif
380 }
381
382 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
383 {
384 char *hex = sha1_to_hex(sha1);
385 char *filename;
386 char *url;
387 char tmpfile[PATH_MAX];
388 long prev_posn = 0;
389 char range[RANGE_HEADER_SIZE];
390 struct curl_slist *range_header = NULL;
391
392 FILE *indexfile;
393 struct active_request_slot *slot;
394 struct slot_results results;
395
396 if (has_pack_index(sha1))
397 return 0;
398
399 if (get_verbosely)
400 fprintf(stderr, "Getting index for pack %s\n", hex);
401
402 url = xmalloc(strlen(repo->base) + 64);
403 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
404
405 filename = sha1_pack_index_name(sha1);
406 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
407 indexfile = fopen(tmpfile, "a");
408 if (!indexfile)
409 return error("Unable to open local file %s for pack index",
410 filename);
411
412 slot = get_active_slot();
413 slot->results = &results;
414 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
415 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
416 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
417 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
418 slot->local = indexfile;
419
420 /* If there is data present from a previous transfer attempt,
421 resume where it left off */
422 prev_posn = ftell(indexfile);
423 if (prev_posn>0) {
424 if (get_verbosely)
425 fprintf(stderr,
426 "Resuming fetch of index for pack %s at byte %ld\n",
427 hex, prev_posn);
428 sprintf(range, "Range: bytes=%ld-", prev_posn);
429 range_header = curl_slist_append(range_header, range);
430 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
431 }
432
433 if (start_active_slot(slot)) {
434 run_active_slot(slot);
435 if (results.curl_result != CURLE_OK) {
436 fclose(indexfile);
437 return error("Unable to get pack index %s\n%s", url,
438 curl_errorstr);
439 }
440 } else {
441 fclose(indexfile);
442 return error("Unable to start request");
443 }
444
445 fclose(indexfile);
446
447 return move_temp_to_file(tmpfile, filename);
448 }
449
450 static int setup_index(struct alt_base *repo, unsigned char *sha1)
451 {
452 struct packed_git *new_pack;
453 if (has_pack_file(sha1))
454 return 0; /* don't list this as something we can get */
455
456 if (fetch_index(repo, sha1))
457 return -1;
458
459 new_pack = parse_pack_index(sha1);
460 new_pack->next = repo->packs;
461 repo->packs = new_pack;
462 return 0;
463 }
464
465 static void process_alternates_response(void *callback_data)
466 {
467 struct alternates_request *alt_req =
468 (struct alternates_request *)callback_data;
469 struct active_request_slot *slot = alt_req->slot;
470 struct alt_base *tail = alt;
471 const char *base = alt_req->base;
472 static const char null_byte = '\0';
473 char *data;
474 int i = 0;
475
476 if (alt_req->http_specific) {
477 if (slot->curl_result != CURLE_OK ||
478 !alt_req->buffer->posn) {
479
480 /* Try reusing the slot to get non-http alternates */
481 alt_req->http_specific = 0;
482 sprintf(alt_req->url, "%s/objects/info/alternates",
483 base);
484 curl_easy_setopt(slot->curl, CURLOPT_URL,
485 alt_req->url);
486 active_requests++;
487 slot->in_use = 1;
488 if (slot->finished != NULL)
489 (*slot->finished) = 0;
490 if (!start_active_slot(slot)) {
491 got_alternates = -1;
492 slot->in_use = 0;
493 if (slot->finished != NULL)
494 (*slot->finished) = 1;
495 }
496 return;
497 }
498 } else if (slot->curl_result != CURLE_OK) {
499 if (!missing_target(slot)) {
500 got_alternates = -1;
501 return;
502 }
503 }
504
505 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
506 alt_req->buffer->posn--;
507 data = alt_req->buffer->buffer;
508
509 while (i < alt_req->buffer->posn) {
510 int posn = i;
511 while (posn < alt_req->buffer->posn && data[posn] != '\n')
512 posn++;
513 if (data[posn] == '\n') {
514 int okay = 0;
515 int serverlen = 0;
516 struct alt_base *newalt;
517 char *target = NULL;
518 char *path;
519 if (data[i] == '/') {
520 /* This counts
521 * http://git.host/pub/scm/linux.git/
522 * -----------here^
523 * so memcpy(dst, base, serverlen) will
524 * copy up to "...git.host".
525 */
526 const char *colon_ss = strstr(base,"://");
527 if (colon_ss) {
528 serverlen = (strchr(colon_ss + 3, '/')
529 - base);
530 okay = 1;
531 }
532 } else if (!memcmp(data + i, "../", 3)) {
533 /* Relative URL; chop the corresponding
534 * number of subpath from base (and ../
535 * from data), and concatenate the result.
536 *
537 * The code first drops ../ from data, and
538 * then drops one ../ from data and one path
539 * from base. IOW, one extra ../ is dropped
540 * from data than path is dropped from base.
541 *
542 * This is not wrong. The alternate in
543 * http://git.host/pub/scm/linux.git/
544 * to borrow from
545 * http://git.host/pub/scm/linus.git/
546 * is ../../linus.git/objects/. You need
547 * two ../../ to borrow from your direct
548 * neighbour.
549 */
550 i += 3;
551 serverlen = strlen(base);
552 while (i + 2 < posn &&
553 !memcmp(data + i, "../", 3)) {
554 do {
555 serverlen--;
556 } while (serverlen &&
557 base[serverlen - 1] != '/');
558 i += 3;
559 }
560 /* If the server got removed, give up. */
561 okay = strchr(base, ':') - base + 3 <
562 serverlen;
563 } else if (alt_req->http_specific) {
564 char *colon = strchr(data + i, ':');
565 char *slash = strchr(data + i, '/');
566 if (colon && slash && colon < data + posn &&
567 slash < data + posn && colon < slash) {
568 okay = 1;
569 }
570 }
571 /* skip "objects\n" at end */
572 if (okay) {
573 target = xmalloc(serverlen + posn - i - 6);
574 memcpy(target, base, serverlen);
575 memcpy(target + serverlen, data + i,
576 posn - i - 7);
577 target[serverlen + posn - i - 7] = 0;
578 if (get_verbosely)
579 fprintf(stderr,
580 "Also look at %s\n", target);
581 newalt = xmalloc(sizeof(*newalt));
582 newalt->next = NULL;
583 newalt->base = target;
584 newalt->got_indices = 0;
585 newalt->packs = NULL;
586 path = strstr(target, "//");
587 if (path) {
588 path = strchr(path+2, '/');
589 if (path)
590 newalt->path_len = strlen(path);
591 }
592
593 while (tail->next != NULL)
594 tail = tail->next;
595 tail->next = newalt;
596 }
597 }
598 i = posn + 1;
599 }
600
601 got_alternates = 1;
602 }
603
604 static void fetch_alternates(const char *base)
605 {
606 struct buffer buffer;
607 char *url;
608 char *data;
609 struct active_request_slot *slot;
610 struct alternates_request alt_req;
611
612 /* If another request has already started fetching alternates,
613 wait for them to arrive and return to processing this request's
614 curl message */
615 #ifdef USE_CURL_MULTI
616 while (got_alternates == 0) {
617 step_active_slots();
618 }
619 #endif
620
621 /* Nothing to do if they've already been fetched */
622 if (got_alternates == 1)
623 return;
624
625 /* Start the fetch */
626 got_alternates = 0;
627
628 data = xmalloc(4096);
629 buffer.size = 4096;
630 buffer.posn = 0;
631 buffer.buffer = data;
632
633 if (get_verbosely)
634 fprintf(stderr, "Getting alternates list for %s\n", base);
635
636 url = xmalloc(strlen(base) + 31);
637 sprintf(url, "%s/objects/info/http-alternates", base);
638
639 /* Use a callback to process the result, since another request
640 may fail and need to have alternates loaded before continuing */
641 slot = get_active_slot();
642 slot->callback_func = process_alternates_response;
643 slot->callback_data = &alt_req;
644
645 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
646 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
647 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
648
649 alt_req.base = base;
650 alt_req.url = url;
651 alt_req.buffer = &buffer;
652 alt_req.http_specific = 1;
653 alt_req.slot = slot;
654
655 if (start_active_slot(slot))
656 run_active_slot(slot);
657 else
658 got_alternates = -1;
659
660 free(data);
661 free(url);
662 }
663
664 static int fetch_indices(struct alt_base *repo)
665 {
666 unsigned char sha1[20];
667 char *url;
668 struct buffer buffer;
669 char *data;
670 int i = 0;
671
672 struct active_request_slot *slot;
673 struct slot_results results;
674
675 if (repo->got_indices)
676 return 0;
677
678 data = xmalloc(4096);
679 buffer.size = 4096;
680 buffer.posn = 0;
681 buffer.buffer = data;
682
683 if (get_verbosely)
684 fprintf(stderr, "Getting pack list for %s\n", repo->base);
685
686 url = xmalloc(strlen(repo->base) + 21);
687 sprintf(url, "%s/objects/info/packs", repo->base);
688
689 slot = get_active_slot();
690 slot->results = &results;
691 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
692 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
693 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
694 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
695 if (start_active_slot(slot)) {
696 run_active_slot(slot);
697 if (results.curl_result != CURLE_OK) {
698 if (missing_target(&results)) {
699 repo->got_indices = 1;
700 free(buffer.buffer);
701 return 0;
702 } else {
703 repo->got_indices = 0;
704 free(buffer.buffer);
705 return error("%s", curl_errorstr);
706 }
707 }
708 } else {
709 repo->got_indices = 0;
710 free(buffer.buffer);
711 return error("Unable to start request");
712 }
713
714 data = buffer.buffer;
715 while (i < buffer.posn) {
716 switch (data[i]) {
717 case 'P':
718 i++;
719 if (i + 52 <= buffer.posn &&
720 !strncmp(data + i, " pack-", 6) &&
721 !strncmp(data + i + 46, ".pack\n", 6)) {
722 get_sha1_hex(data + i + 6, sha1);
723 setup_index(repo, sha1);
724 i += 51;
725 break;
726 }
727 default:
728 while (i < buffer.posn && data[i] != '\n')
729 i++;
730 }
731 i++;
732 }
733
734 free(buffer.buffer);
735 repo->got_indices = 1;
736 return 0;
737 }
738
739 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
740 {
741 char *url;
742 struct packed_git *target;
743 struct packed_git **lst;
744 FILE *packfile;
745 char *filename;
746 char tmpfile[PATH_MAX];
747 int ret;
748 long prev_posn = 0;
749 char range[RANGE_HEADER_SIZE];
750 struct curl_slist *range_header = NULL;
751
752 struct active_request_slot *slot;
753 struct slot_results results;
754
755 if (fetch_indices(repo))
756 return -1;
757 target = find_sha1_pack(sha1, repo->packs);
758 if (!target)
759 return -1;
760
761 if (get_verbosely) {
762 fprintf(stderr, "Getting pack %s\n",
763 sha1_to_hex(target->sha1));
764 fprintf(stderr, " which contains %s\n",
765 sha1_to_hex(sha1));
766 }
767
768 url = xmalloc(strlen(repo->base) + 65);
769 sprintf(url, "%s/objects/pack/pack-%s.pack",
770 repo->base, sha1_to_hex(target->sha1));
771
772 filename = sha1_pack_name(target->sha1);
773 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
774 packfile = fopen(tmpfile, "a");
775 if (!packfile)
776 return error("Unable to open local file %s for pack",
777 filename);
778
779 slot = get_active_slot();
780 slot->results = &results;
781 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
782 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
783 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
784 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
785 slot->local = packfile;
786
787 /* If there is data present from a previous transfer attempt,
788 resume where it left off */
789 prev_posn = ftell(packfile);
790 if (prev_posn>0) {
791 if (get_verbosely)
792 fprintf(stderr,
793 "Resuming fetch of pack %s at byte %ld\n",
794 sha1_to_hex(target->sha1), prev_posn);
795 sprintf(range, "Range: bytes=%ld-", prev_posn);
796 range_header = curl_slist_append(range_header, range);
797 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
798 }
799
800 if (start_active_slot(slot)) {
801 run_active_slot(slot);
802 if (results.curl_result != CURLE_OK) {
803 fclose(packfile);
804 return error("Unable to get pack file %s\n%s", url,
805 curl_errorstr);
806 }
807 } else {
808 fclose(packfile);
809 return error("Unable to start request");
810 }
811
812 fclose(packfile);
813
814 ret = move_temp_to_file(tmpfile, filename);
815 if (ret)
816 return ret;
817
818 lst = &repo->packs;
819 while (*lst != target)
820 lst = &((*lst)->next);
821 *lst = (*lst)->next;
822
823 if (verify_pack(target, 0))
824 return -1;
825 install_packed_git(target);
826
827 return 0;
828 }
829
830 static void abort_object_request(struct object_request *obj_req)
831 {
832 if (obj_req->local >= 0) {
833 close(obj_req->local);
834 obj_req->local = -1;
835 }
836 unlink(obj_req->tmpfile);
837 if (obj_req->slot) {
838 release_active_slot(obj_req->slot);
839 obj_req->slot = NULL;
840 }
841 release_object_request(obj_req);
842 }
843
844 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
845 {
846 char *hex = sha1_to_hex(sha1);
847 int ret = 0;
848 struct object_request *obj_req = object_queue_head;
849
850 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
851 obj_req = obj_req->next;
852 if (obj_req == NULL)
853 return error("Couldn't find request for %s in the queue", hex);
854
855 if (has_sha1_file(obj_req->sha1)) {
856 abort_object_request(obj_req);
857 return 0;
858 }
859
860 #ifdef USE_CURL_MULTI
861 while (obj_req->state == WAITING) {
862 step_active_slots();
863 }
864 #else
865 start_object_request(obj_req);
866 #endif
867
868 while (obj_req->state == ACTIVE) {
869 run_active_slot(obj_req->slot);
870 }
871 if (obj_req->local != -1) {
872 close(obj_req->local); obj_req->local = -1;
873 }
874
875 if (obj_req->state == ABORTED) {
876 ret = error("Request for %s aborted", hex);
877 } else if (obj_req->curl_result != CURLE_OK &&
878 obj_req->http_code != 416) {
879 if (missing_target(obj_req))
880 ret = -1; /* Be silent, it is probably in a pack. */
881 else
882 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
883 obj_req->errorstr, obj_req->curl_result,
884 obj_req->http_code, hex);
885 } else if (obj_req->zret != Z_STREAM_END) {
886 corrupt_object_found++;
887 ret = error("File %s (%s) corrupt", hex, obj_req->url);
888 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
889 ret = error("File %s has bad hash", hex);
890 } else if (obj_req->rename < 0) {
891 ret = error("unable to write sha1 filename %s",
892 obj_req->filename);
893 }
894
895 release_object_request(obj_req);
896 return ret;
897 }
898
899 int fetch(unsigned char *sha1)
900 {
901 struct alt_base *altbase = alt;
902
903 if (!fetch_object(altbase, sha1))
904 return 0;
905 while (altbase) {
906 if (!fetch_pack(altbase, sha1))
907 return 0;
908 fetch_alternates(alt->base);
909 altbase = altbase->next;
910 }
911 return error("Unable to find %s under %s", sha1_to_hex(sha1),
912 alt->base);
913 }
914
915 static inline int needs_quote(int ch)
916 {
917 if (((ch >= 'A') && (ch <= 'Z'))
918 || ((ch >= 'a') && (ch <= 'z'))
919 || ((ch >= '0') && (ch <= '9'))
920 || (ch == '/')
921 || (ch == '-')
922 || (ch == '.'))
923 return 0;
924 return 1;
925 }
926
927 static inline int hex(int v)
928 {
929 if (v < 10) return '0' + v;
930 else return 'A' + v - 10;
931 }
932
933 static char *quote_ref_url(const char *base, const char *ref)
934 {
935 const char *cp;
936 char *dp, *qref;
937 int len, baselen, ch;
938
939 baselen = strlen(base);
940 len = baselen + 6; /* "refs/" + NUL */
941 for (cp = ref; (ch = *cp) != 0; cp++, len++)
942 if (needs_quote(ch))
943 len += 2; /* extra two hex plus replacement % */
944 qref = xmalloc(len);
945 memcpy(qref, base, baselen);
946 memcpy(qref + baselen, "refs/", 5);
947 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
948 if (needs_quote(ch)) {
949 *dp++ = '%';
950 *dp++ = hex((ch >> 4) & 0xF);
951 *dp++ = hex(ch & 0xF);
952 }
953 else
954 *dp++ = ch;
955 }
956 *dp = 0;
957
958 return qref;
959 }
960
961 int fetch_ref(char *ref, unsigned char *sha1)
962 {
963 char *url;
964 char hex[42];
965 struct buffer buffer;
966 const char *base = alt->base;
967 struct active_request_slot *slot;
968 struct slot_results results;
969 buffer.size = 41;
970 buffer.posn = 0;
971 buffer.buffer = hex;
972 hex[41] = '\0';
973
974 url = quote_ref_url(base, ref);
975 slot = get_active_slot();
976 slot->results = &results;
977 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
978 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
979 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
980 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
981 if (start_active_slot(slot)) {
982 run_active_slot(slot);
983 if (results.curl_result != CURLE_OK)
984 return error("Couldn't get %s for %s\n%s",
985 url, ref, curl_errorstr);
986 } else {
987 return error("Unable to start request");
988 }
989
990 hex[40] = '\0';
991 get_sha1_hex(hex, sha1);
992 return 0;
993 }
994
995 int main(int argc, const char **argv)
996 {
997 int commits;
998 const char **write_ref = NULL;
999 char **commit_id;
1000 const char *url;
1001 char *path;
1002 int arg = 1;
1003 int rc = 0;
1004
1005 setup_ident();
1006 setup_git_directory();
1007 git_config(git_default_config);
1008
1009 while (arg < argc && argv[arg][0] == '-') {
1010 if (argv[arg][1] == 't') {
1011 get_tree = 1;
1012 } else if (argv[arg][1] == 'c') {
1013 get_history = 1;
1014 } else if (argv[arg][1] == 'a') {
1015 get_all = 1;
1016 get_tree = 1;
1017 get_history = 1;
1018 } else if (argv[arg][1] == 'v') {
1019 get_verbosely = 1;
1020 } else if (argv[arg][1] == 'w') {
1021 write_ref = &argv[arg + 1];
1022 arg++;
1023 } else if (!strcmp(argv[arg], "--recover")) {
1024 get_recover = 1;
1025 } else if (!strcmp(argv[arg], "--stdin")) {
1026 commits_on_stdin = 1;
1027 }
1028 arg++;
1029 }
1030 if (argc < arg + 2 - commits_on_stdin) {
1031 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1032 return 1;
1033 }
1034 if (commits_on_stdin) {
1035 commits = pull_targets_stdin(&commit_id, &write_ref);
1036 } else {
1037 commit_id = (char **) &argv[arg++];
1038 commits = 1;
1039 }
1040 url = argv[arg];
1041
1042 http_init();
1043
1044 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1045
1046 alt = xmalloc(sizeof(*alt));
1047 alt->base = url;
1048 alt->got_indices = 0;
1049 alt->packs = NULL;
1050 alt->next = NULL;
1051 path = strstr(url, "//");
1052 if (path) {
1053 path = strchr(path+2, '/');
1054 if (path)
1055 alt->path_len = strlen(path);
1056 }
1057
1058 if (pull(commits, commit_id, write_ref, url))
1059 rc = 1;
1060
1061 http_cleanup();
1062
1063 curl_slist_free_all(no_pragma_header);
1064
1065 if (commits_on_stdin)
1066 pull_targets_free(commits, commit_id, write_ref);
1067
1068 if (corrupt_object_found) {
1069 fprintf(stderr,
1070 "Some loose object were found to be corrupt, but they might be just\n"
1071 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1072 "status code. Suggest running git fsck-objects.\n");
1073 }
1074 return rc;
1075 }