Merge branch 'ew/rebase'
[git/git.git] / http-fetch.c
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
5 #include "http.h"
6
7 #ifndef NO_EXPAT
8 #include <expat.h>
9
10 /* Definitions for DAV requests */
11 #define DAV_PROPFIND "PROPFIND"
12 #define DAV_PROPFIND_RESP ".multistatus.response"
13 #define DAV_PROPFIND_NAME ".multistatus.response.href"
14 #define DAV_PROPFIND_COLLECTION ".multistatus.response.propstat.prop.resourcetype.collection"
15 #define PROPFIND_ALL_REQUEST "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<D:propfind xmlns:D=\"DAV:\">\n<D:allprop/>\n</D:propfind>"
16
17 /* Definitions for processing XML DAV responses */
18 #ifndef XML_STATUS_OK
19 enum XML_Status {
20 XML_STATUS_OK = 1,
21 XML_STATUS_ERROR = 0
22 };
23 #define XML_STATUS_OK 1
24 #define XML_STATUS_ERROR 0
25 #endif
26
27 /* Flags that control remote_ls processing */
28 #define PROCESS_FILES (1u << 0)
29 #define PROCESS_DIRS (1u << 1)
30 #define RECURSIVE (1u << 2)
31
32 /* Flags that remote_ls passes to callback functions */
33 #define IS_DIR (1u << 0)
34 #endif
35
36 #define PREV_BUF_SIZE 4096
37 #define RANGE_HEADER_SIZE 30
38
39 static int got_alternates = -1;
40 static int corrupt_object_found = 0;
41
42 static struct curl_slist *no_pragma_header;
43
44 struct alt_base
45 {
46 char *base;
47 int path_len;
48 int got_indices;
49 struct packed_git *packs;
50 struct alt_base *next;
51 };
52
53 static struct alt_base *alt = NULL;
54
55 enum object_request_state {
56 WAITING,
57 ABORTED,
58 ACTIVE,
59 COMPLETE,
60 };
61
62 struct object_request
63 {
64 unsigned char sha1[20];
65 struct alt_base *repo;
66 char *url;
67 char filename[PATH_MAX];
68 char tmpfile[PATH_MAX];
69 int local;
70 enum object_request_state state;
71 CURLcode curl_result;
72 char errorstr[CURL_ERROR_SIZE];
73 long http_code;
74 unsigned char real_sha1[20];
75 SHA_CTX c;
76 z_stream stream;
77 int zret;
78 int rename;
79 struct active_request_slot *slot;
80 struct object_request *next;
81 };
82
83 struct alternates_request {
84 char *base;
85 char *url;
86 struct buffer *buffer;
87 struct active_request_slot *slot;
88 int http_specific;
89 };
90
91 #ifndef NO_EXPAT
92 struct xml_ctx
93 {
94 char *name;
95 int len;
96 char *cdata;
97 void (*userFunc)(struct xml_ctx *ctx, int tag_closed);
98 void *userData;
99 };
100
101 struct remote_ls_ctx
102 {
103 struct alt_base *repo;
104 char *path;
105 void (*userFunc)(struct remote_ls_ctx *ls);
106 void *userData;
107 int flags;
108 char *dentry_name;
109 int dentry_flags;
110 int rc;
111 struct remote_ls_ctx *parent;
112 };
113 #endif
114
115 static struct object_request *object_queue_head = NULL;
116
117 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
118 void *data)
119 {
120 unsigned char expn[4096];
121 size_t size = eltsize * nmemb;
122 int posn = 0;
123 struct object_request *obj_req = (struct object_request *)data;
124 do {
125 ssize_t retval = write(obj_req->local,
126 (char *) ptr + posn, size - posn);
127 if (retval < 0)
128 return posn;
129 posn += retval;
130 } while (posn < size);
131
132 obj_req->stream.avail_in = size;
133 obj_req->stream.next_in = ptr;
134 do {
135 obj_req->stream.next_out = expn;
136 obj_req->stream.avail_out = sizeof(expn);
137 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
138 SHA1_Update(&obj_req->c, expn,
139 sizeof(expn) - obj_req->stream.avail_out);
140 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
141 data_received++;
142 return size;
143 }
144
145 static void fetch_alternates(char *base);
146
147 static void process_object_response(void *callback_data);
148
149 static void start_object_request(struct object_request *obj_req)
150 {
151 char *hex = sha1_to_hex(obj_req->sha1);
152 char prevfile[PATH_MAX];
153 char *url;
154 char *posn;
155 int prevlocal;
156 unsigned char prev_buf[PREV_BUF_SIZE];
157 ssize_t prev_read = 0;
158 long prev_posn = 0;
159 char range[RANGE_HEADER_SIZE];
160 struct curl_slist *range_header = NULL;
161 struct active_request_slot *slot;
162
163 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
164 unlink(prevfile);
165 rename(obj_req->tmpfile, prevfile);
166 unlink(obj_req->tmpfile);
167
168 if (obj_req->local != -1)
169 error("fd leakage in start: %d", obj_req->local);
170 obj_req->local = open(obj_req->tmpfile,
171 O_WRONLY | O_CREAT | O_EXCL, 0666);
172 /* This could have failed due to the "lazy directory creation";
173 * try to mkdir the last path component.
174 */
175 if (obj_req->local < 0 && errno == ENOENT) {
176 char *dir = strrchr(obj_req->tmpfile, '/');
177 if (dir) {
178 *dir = 0;
179 mkdir(obj_req->tmpfile, 0777);
180 *dir = '/';
181 }
182 obj_req->local = open(obj_req->tmpfile,
183 O_WRONLY | O_CREAT | O_EXCL, 0666);
184 }
185
186 if (obj_req->local < 0) {
187 obj_req->state = ABORTED;
188 error("Couldn't create temporary file %s for %s: %s",
189 obj_req->tmpfile, obj_req->filename, strerror(errno));
190 return;
191 }
192
193 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
194
195 inflateInit(&obj_req->stream);
196
197 SHA1_Init(&obj_req->c);
198
199 url = xmalloc(strlen(obj_req->repo->base) + 50);
200 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
201 strcpy(url, obj_req->repo->base);
202 posn = url + strlen(obj_req->repo->base);
203 strcpy(posn, "objects/");
204 posn += 8;
205 memcpy(posn, hex, 2);
206 posn += 2;
207 *(posn++) = '/';
208 strcpy(posn, hex + 2);
209 strcpy(obj_req->url, url);
210
211 /* If a previous temp file is present, process what was already
212 fetched. */
213 prevlocal = open(prevfile, O_RDONLY);
214 if (prevlocal != -1) {
215 do {
216 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
217 if (prev_read>0) {
218 if (fwrite_sha1_file(prev_buf,
219 1,
220 prev_read,
221 obj_req) == prev_read) {
222 prev_posn += prev_read;
223 } else {
224 prev_read = -1;
225 }
226 }
227 } while (prev_read > 0);
228 close(prevlocal);
229 }
230 unlink(prevfile);
231
232 /* Reset inflate/SHA1 if there was an error reading the previous temp
233 file; also rewind to the beginning of the local file. */
234 if (prev_read == -1) {
235 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
236 inflateInit(&obj_req->stream);
237 SHA1_Init(&obj_req->c);
238 if (prev_posn>0) {
239 prev_posn = 0;
240 lseek(obj_req->local, SEEK_SET, 0);
241 ftruncate(obj_req->local, 0);
242 }
243 }
244
245 slot = get_active_slot();
246 slot->callback_func = process_object_response;
247 slot->callback_data = obj_req;
248 obj_req->slot = slot;
249
250 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
251 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
252 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
253 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
254 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
255
256 /* If we have successfully processed data from a previous fetch
257 attempt, only fetch the data we don't already have. */
258 if (prev_posn>0) {
259 if (get_verbosely)
260 fprintf(stderr,
261 "Resuming fetch of object %s at byte %ld\n",
262 hex, prev_posn);
263 sprintf(range, "Range: bytes=%ld-", prev_posn);
264 range_header = curl_slist_append(range_header, range);
265 curl_easy_setopt(slot->curl,
266 CURLOPT_HTTPHEADER, range_header);
267 }
268
269 /* Try to get the request started, abort the request on error */
270 obj_req->state = ACTIVE;
271 if (!start_active_slot(slot)) {
272 obj_req->state = ABORTED;
273 obj_req->slot = NULL;
274 close(obj_req->local); obj_req->local = -1;
275 free(obj_req->url);
276 return;
277 }
278 }
279
280 static void finish_object_request(struct object_request *obj_req)
281 {
282 struct stat st;
283
284 fchmod(obj_req->local, 0444);
285 close(obj_req->local); obj_req->local = -1;
286
287 if (obj_req->http_code == 416) {
288 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
289 } else if (obj_req->curl_result != CURLE_OK) {
290 if (stat(obj_req->tmpfile, &st) == 0)
291 if (st.st_size == 0)
292 unlink(obj_req->tmpfile);
293 return;
294 }
295
296 inflateEnd(&obj_req->stream);
297 SHA1_Final(obj_req->real_sha1, &obj_req->c);
298 if (obj_req->zret != Z_STREAM_END) {
299 unlink(obj_req->tmpfile);
300 return;
301 }
302 if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
303 unlink(obj_req->tmpfile);
304 return;
305 }
306 obj_req->rename =
307 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
308
309 if (obj_req->rename == 0)
310 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
311 }
312
313 static void process_object_response(void *callback_data)
314 {
315 struct object_request *obj_req =
316 (struct object_request *)callback_data;
317
318 obj_req->curl_result = obj_req->slot->curl_result;
319 obj_req->http_code = obj_req->slot->http_code;
320 obj_req->slot = NULL;
321 obj_req->state = COMPLETE;
322
323 /* Use alternates if necessary */
324 if (obj_req->http_code == 404 ||
325 obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE) {
326 fetch_alternates(alt->base);
327 if (obj_req->repo->next != NULL) {
328 obj_req->repo =
329 obj_req->repo->next;
330 close(obj_req->local);
331 obj_req->local = -1;
332 start_object_request(obj_req);
333 return;
334 }
335 }
336
337 finish_object_request(obj_req);
338 }
339
340 static void release_object_request(struct object_request *obj_req)
341 {
342 struct object_request *entry = object_queue_head;
343
344 if (obj_req->local != -1)
345 error("fd leakage in release: %d", obj_req->local);
346 if (obj_req == object_queue_head) {
347 object_queue_head = obj_req->next;
348 } else {
349 while (entry->next != NULL && entry->next != obj_req)
350 entry = entry->next;
351 if (entry->next == obj_req)
352 entry->next = entry->next->next;
353 }
354
355 free(obj_req->url);
356 free(obj_req);
357 }
358
359 #ifdef USE_CURL_MULTI
360 void fill_active_slots(void)
361 {
362 struct object_request *obj_req = object_queue_head;
363 struct active_request_slot *slot = active_queue_head;
364 int num_transfers;
365
366 while (active_requests < max_requests && obj_req != NULL) {
367 if (obj_req->state == WAITING) {
368 if (has_sha1_file(obj_req->sha1))
369 obj_req->state = COMPLETE;
370 else
371 start_object_request(obj_req);
372 curl_multi_perform(curlm, &num_transfers);
373 }
374 obj_req = obj_req->next;
375 }
376
377 while (slot != NULL) {
378 if (!slot->in_use && slot->curl != NULL) {
379 curl_easy_cleanup(slot->curl);
380 slot->curl = NULL;
381 }
382 slot = slot->next;
383 }
384 }
385 #endif
386
387 void prefetch(unsigned char *sha1)
388 {
389 struct object_request *newreq;
390 struct object_request *tail;
391 char *filename = sha1_file_name(sha1);
392
393 newreq = xmalloc(sizeof(*newreq));
394 memcpy(newreq->sha1, sha1, 20);
395 newreq->repo = alt;
396 newreq->url = NULL;
397 newreq->local = -1;
398 newreq->state = WAITING;
399 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
400 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
401 "%s.temp", filename);
402 newreq->slot = NULL;
403 newreq->next = NULL;
404
405 if (object_queue_head == NULL) {
406 object_queue_head = newreq;
407 } else {
408 tail = object_queue_head;
409 while (tail->next != NULL) {
410 tail = tail->next;
411 }
412 tail->next = newreq;
413 }
414
415 #ifdef USE_CURL_MULTI
416 fill_active_slots();
417 step_active_slots();
418 #endif
419 }
420
421 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
422 {
423 char *hex = sha1_to_hex(sha1);
424 char *filename;
425 char *url;
426 char tmpfile[PATH_MAX];
427 long prev_posn = 0;
428 char range[RANGE_HEADER_SIZE];
429 struct curl_slist *range_header = NULL;
430
431 FILE *indexfile;
432 struct active_request_slot *slot;
433 struct slot_results results;
434
435 if (has_pack_index(sha1))
436 return 0;
437
438 if (get_verbosely)
439 fprintf(stderr, "Getting index for pack %s\n", hex);
440
441 url = xmalloc(strlen(repo->base) + 64);
442 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
443
444 filename = sha1_pack_index_name(sha1);
445 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
446 indexfile = fopen(tmpfile, "a");
447 if (!indexfile)
448 return error("Unable to open local file %s for pack index",
449 filename);
450
451 slot = get_active_slot();
452 slot->results = &results;
453 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
454 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
455 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
456 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
457 slot->local = indexfile;
458
459 /* If there is data present from a previous transfer attempt,
460 resume where it left off */
461 prev_posn = ftell(indexfile);
462 if (prev_posn>0) {
463 if (get_verbosely)
464 fprintf(stderr,
465 "Resuming fetch of index for pack %s at byte %ld\n",
466 hex, prev_posn);
467 sprintf(range, "Range: bytes=%ld-", prev_posn);
468 range_header = curl_slist_append(range_header, range);
469 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
470 }
471
472 if (start_active_slot(slot)) {
473 run_active_slot(slot);
474 if (results.curl_result != CURLE_OK) {
475 fclose(indexfile);
476 return error("Unable to get pack index %s\n%s", url,
477 curl_errorstr);
478 }
479 } else {
480 fclose(indexfile);
481 return error("Unable to start request");
482 }
483
484 fclose(indexfile);
485
486 return move_temp_to_file(tmpfile, filename);
487 }
488
489 static int setup_index(struct alt_base *repo, unsigned char *sha1)
490 {
491 struct packed_git *new_pack;
492 if (has_pack_file(sha1))
493 return 0; // don't list this as something we can get
494
495 if (fetch_index(repo, sha1))
496 return -1;
497
498 new_pack = parse_pack_index(sha1);
499 new_pack->next = repo->packs;
500 repo->packs = new_pack;
501 return 0;
502 }
503
504 static void process_alternates_response(void *callback_data)
505 {
506 struct alternates_request *alt_req =
507 (struct alternates_request *)callback_data;
508 struct active_request_slot *slot = alt_req->slot;
509 struct alt_base *tail = alt;
510 char *base = alt_req->base;
511 static const char null_byte = '\0';
512 char *data;
513 int i = 0;
514
515 if (alt_req->http_specific) {
516 if (slot->curl_result != CURLE_OK ||
517 !alt_req->buffer->posn) {
518
519 /* Try reusing the slot to get non-http alternates */
520 alt_req->http_specific = 0;
521 sprintf(alt_req->url, "%s/objects/info/alternates",
522 base);
523 curl_easy_setopt(slot->curl, CURLOPT_URL,
524 alt_req->url);
525 active_requests++;
526 slot->in_use = 1;
527 if (slot->finished != NULL)
528 (*slot->finished) = 0;
529 if (!start_active_slot(slot)) {
530 got_alternates = -1;
531 slot->in_use = 0;
532 if (slot->finished != NULL)
533 (*slot->finished) = 1;
534 }
535 return;
536 }
537 } else if (slot->curl_result != CURLE_OK) {
538 if (slot->http_code != 404 &&
539 slot->curl_result != CURLE_FILE_COULDNT_READ_FILE) {
540 got_alternates = -1;
541 return;
542 }
543 }
544
545 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
546 alt_req->buffer->posn--;
547 data = alt_req->buffer->buffer;
548
549 while (i < alt_req->buffer->posn) {
550 int posn = i;
551 while (posn < alt_req->buffer->posn && data[posn] != '\n')
552 posn++;
553 if (data[posn] == '\n') {
554 int okay = 0;
555 int serverlen = 0;
556 struct alt_base *newalt;
557 char *target = NULL;
558 char *path;
559 if (data[i] == '/') {
560 serverlen = strchr(base + 8, '/') - base;
561 okay = 1;
562 } else if (!memcmp(data + i, "../", 3)) {
563 i += 3;
564 serverlen = strlen(base);
565 while (i + 2 < posn &&
566 !memcmp(data + i, "../", 3)) {
567 do {
568 serverlen--;
569 } while (serverlen &&
570 base[serverlen - 1] != '/');
571 i += 3;
572 }
573 // If the server got removed, give up.
574 okay = strchr(base, ':') - base + 3 <
575 serverlen;
576 } else if (alt_req->http_specific) {
577 char *colon = strchr(data + i, ':');
578 char *slash = strchr(data + i, '/');
579 if (colon && slash && colon < data + posn &&
580 slash < data + posn && colon < slash) {
581 okay = 1;
582 }
583 }
584 // skip 'objects' at end
585 if (okay) {
586 target = xmalloc(serverlen + posn - i - 6);
587 safe_strncpy(target, base, serverlen);
588 safe_strncpy(target + serverlen, data + i, posn - i - 6);
589 if (get_verbosely)
590 fprintf(stderr,
591 "Also look at %s\n", target);
592 newalt = xmalloc(sizeof(*newalt));
593 newalt->next = NULL;
594 newalt->base = target;
595 newalt->got_indices = 0;
596 newalt->packs = NULL;
597 path = strstr(target, "//");
598 if (path) {
599 path = strchr(path+2, '/');
600 if (path)
601 newalt->path_len = strlen(path);
602 }
603
604 while (tail->next != NULL)
605 tail = tail->next;
606 tail->next = newalt;
607 }
608 }
609 i = posn + 1;
610 }
611
612 got_alternates = 1;
613 }
614
615 static void fetch_alternates(char *base)
616 {
617 struct buffer buffer;
618 char *url;
619 char *data;
620 struct active_request_slot *slot;
621 struct alternates_request alt_req;
622
623 /* If another request has already started fetching alternates,
624 wait for them to arrive and return to processing this request's
625 curl message */
626 #ifdef USE_CURL_MULTI
627 while (got_alternates == 0) {
628 step_active_slots();
629 }
630 #endif
631
632 /* Nothing to do if they've already been fetched */
633 if (got_alternates == 1)
634 return;
635
636 /* Start the fetch */
637 got_alternates = 0;
638
639 data = xmalloc(4096);
640 buffer.size = 4096;
641 buffer.posn = 0;
642 buffer.buffer = data;
643
644 if (get_verbosely)
645 fprintf(stderr, "Getting alternates list for %s\n", base);
646
647 url = xmalloc(strlen(base) + 31);
648 sprintf(url, "%s/objects/info/http-alternates", base);
649
650 /* Use a callback to process the result, since another request
651 may fail and need to have alternates loaded before continuing */
652 slot = get_active_slot();
653 slot->callback_func = process_alternates_response;
654 slot->callback_data = &alt_req;
655
656 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
657 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
658 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
659
660 alt_req.base = base;
661 alt_req.url = url;
662 alt_req.buffer = &buffer;
663 alt_req.http_specific = 1;
664 alt_req.slot = slot;
665
666 if (start_active_slot(slot))
667 run_active_slot(slot);
668 else
669 got_alternates = -1;
670
671 free(data);
672 free(url);
673 }
674
675 #ifndef NO_EXPAT
676 static void
677 xml_start_tag(void *userData, const char *name, const char **atts)
678 {
679 struct xml_ctx *ctx = (struct xml_ctx *)userData;
680 const char *c = strchr(name, ':');
681 int new_len;
682
683 if (c == NULL)
684 c = name;
685 else
686 c++;
687
688 new_len = strlen(ctx->name) + strlen(c) + 2;
689
690 if (new_len > ctx->len) {
691 ctx->name = xrealloc(ctx->name, new_len);
692 ctx->len = new_len;
693 }
694 strcat(ctx->name, ".");
695 strcat(ctx->name, c);
696
697 if (ctx->cdata) {
698 free(ctx->cdata);
699 ctx->cdata = NULL;
700 }
701
702 ctx->userFunc(ctx, 0);
703 }
704
705 static void
706 xml_end_tag(void *userData, const char *name)
707 {
708 struct xml_ctx *ctx = (struct xml_ctx *)userData;
709 const char *c = strchr(name, ':');
710 char *ep;
711
712 ctx->userFunc(ctx, 1);
713
714 if (c == NULL)
715 c = name;
716 else
717 c++;
718
719 ep = ctx->name + strlen(ctx->name) - strlen(c) - 1;
720 *ep = 0;
721 }
722
723 static void
724 xml_cdata(void *userData, const XML_Char *s, int len)
725 {
726 struct xml_ctx *ctx = (struct xml_ctx *)userData;
727 if (ctx->cdata)
728 free(ctx->cdata);
729 ctx->cdata = xmalloc(len + 1);
730 safe_strncpy(ctx->cdata, s, len + 1);
731 }
732
733 static int remote_ls(struct alt_base *repo, const char *path, int flags,
734 void (*userFunc)(struct remote_ls_ctx *ls),
735 void *userData);
736
737 static void handle_remote_ls_ctx(struct xml_ctx *ctx, int tag_closed)
738 {
739 struct remote_ls_ctx *ls = (struct remote_ls_ctx *)ctx->userData;
740
741 if (tag_closed) {
742 if (!strcmp(ctx->name, DAV_PROPFIND_RESP) && ls->dentry_name) {
743 if (ls->dentry_flags & IS_DIR) {
744 if (ls->flags & PROCESS_DIRS) {
745 ls->userFunc(ls);
746 }
747 if (strcmp(ls->dentry_name, ls->path) &&
748 ls->flags & RECURSIVE) {
749 ls->rc = remote_ls(ls->repo,
750 ls->dentry_name,
751 ls->flags,
752 ls->userFunc,
753 ls->userData);
754 }
755 } else if (ls->flags & PROCESS_FILES) {
756 ls->userFunc(ls);
757 }
758 } else if (!strcmp(ctx->name, DAV_PROPFIND_NAME) && ctx->cdata) {
759 ls->dentry_name = xmalloc(strlen(ctx->cdata) -
760 ls->repo->path_len + 1);
761 strcpy(ls->dentry_name, ctx->cdata + ls->repo->path_len);
762 } else if (!strcmp(ctx->name, DAV_PROPFIND_COLLECTION)) {
763 ls->dentry_flags |= IS_DIR;
764 }
765 } else if (!strcmp(ctx->name, DAV_PROPFIND_RESP)) {
766 if (ls->dentry_name) {
767 free(ls->dentry_name);
768 }
769 ls->dentry_name = NULL;
770 ls->dentry_flags = 0;
771 }
772 }
773
774 static int remote_ls(struct alt_base *repo, const char *path, int flags,
775 void (*userFunc)(struct remote_ls_ctx *ls),
776 void *userData)
777 {
778 char *url = xmalloc(strlen(repo->base) + strlen(path) + 1);
779 struct active_request_slot *slot;
780 struct slot_results results;
781 struct buffer in_buffer;
782 struct buffer out_buffer;
783 char *in_data;
784 char *out_data;
785 XML_Parser parser = XML_ParserCreate(NULL);
786 enum XML_Status result;
787 struct curl_slist *dav_headers = NULL;
788 struct xml_ctx ctx;
789 struct remote_ls_ctx ls;
790
791 ls.flags = flags;
792 ls.repo = repo;
793 ls.path = strdup(path);
794 ls.dentry_name = NULL;
795 ls.dentry_flags = 0;
796 ls.userData = userData;
797 ls.userFunc = userFunc;
798 ls.rc = 0;
799
800 sprintf(url, "%s%s", repo->base, path);
801
802 out_buffer.size = strlen(PROPFIND_ALL_REQUEST);
803 out_data = xmalloc(out_buffer.size + 1);
804 snprintf(out_data, out_buffer.size + 1, PROPFIND_ALL_REQUEST);
805 out_buffer.posn = 0;
806 out_buffer.buffer = out_data;
807
808 in_buffer.size = 4096;
809 in_data = xmalloc(in_buffer.size);
810 in_buffer.posn = 0;
811 in_buffer.buffer = in_data;
812
813 dav_headers = curl_slist_append(dav_headers, "Depth: 1");
814 dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml");
815
816 slot = get_active_slot();
817 slot->results = &results;
818 curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer);
819 curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.size);
820 curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
821 curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
822 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
823 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
824 curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
825 curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND);
826 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
827
828 if (start_active_slot(slot)) {
829 run_active_slot(slot);
830 if (results.curl_result == CURLE_OK) {
831 ctx.name = xcalloc(10, 1);
832 ctx.len = 0;
833 ctx.cdata = NULL;
834 ctx.userFunc = handle_remote_ls_ctx;
835 ctx.userData = &ls;
836 XML_SetUserData(parser, &ctx);
837 XML_SetElementHandler(parser, xml_start_tag,
838 xml_end_tag);
839 XML_SetCharacterDataHandler(parser, xml_cdata);
840 result = XML_Parse(parser, in_buffer.buffer,
841 in_buffer.posn, 1);
842 free(ctx.name);
843
844 if (result != XML_STATUS_OK) {
845 ls.rc = error("XML error: %s",
846 XML_ErrorString(
847 XML_GetErrorCode(parser)));
848 }
849 } else {
850 ls.rc = -1;
851 }
852 } else {
853 ls.rc = error("Unable to start PROPFIND request");
854 }
855
856 free(ls.path);
857 free(url);
858 free(out_data);
859 free(in_buffer.buffer);
860 curl_slist_free_all(dav_headers);
861
862 return ls.rc;
863 }
864
865 static void process_ls_pack(struct remote_ls_ctx *ls)
866 {
867 unsigned char sha1[20];
868
869 if (strlen(ls->dentry_name) == 63 &&
870 !strncmp(ls->dentry_name, "objects/pack/pack-", 18) &&
871 !strncmp(ls->dentry_name+58, ".pack", 5)) {
872 get_sha1_hex(ls->dentry_name + 18, sha1);
873 setup_index(ls->repo, sha1);
874 }
875 }
876 #endif
877
878 static int fetch_indices(struct alt_base *repo)
879 {
880 unsigned char sha1[20];
881 char *url;
882 struct buffer buffer;
883 char *data;
884 int i = 0;
885
886 struct active_request_slot *slot;
887 struct slot_results results;
888
889 if (repo->got_indices)
890 return 0;
891
892 data = xmalloc(4096);
893 buffer.size = 4096;
894 buffer.posn = 0;
895 buffer.buffer = data;
896
897 if (get_verbosely)
898 fprintf(stderr, "Getting pack list for %s\n", repo->base);
899
900 #ifndef NO_EXPAT
901 if (remote_ls(repo, "objects/pack/", PROCESS_FILES,
902 process_ls_pack, NULL) == 0)
903 return 0;
904 #endif
905
906 url = xmalloc(strlen(repo->base) + 21);
907 sprintf(url, "%s/objects/info/packs", repo->base);
908
909 slot = get_active_slot();
910 slot->results = &results;
911 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
912 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
913 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
914 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
915 if (start_active_slot(slot)) {
916 run_active_slot(slot);
917 if (results.curl_result != CURLE_OK) {
918 if (results.http_code == 404 ||
919 results.curl_result == CURLE_FILE_COULDNT_READ_FILE) {
920 repo->got_indices = 1;
921 free(buffer.buffer);
922 return 0;
923 } else {
924 repo->got_indices = 0;
925 free(buffer.buffer);
926 return error("%s", curl_errorstr);
927 }
928 }
929 } else {
930 repo->got_indices = 0;
931 free(buffer.buffer);
932 return error("Unable to start request");
933 }
934
935 data = buffer.buffer;
936 while (i < buffer.posn) {
937 switch (data[i]) {
938 case 'P':
939 i++;
940 if (i + 52 <= buffer.posn &&
941 !strncmp(data + i, " pack-", 6) &&
942 !strncmp(data + i + 46, ".pack\n", 6)) {
943 get_sha1_hex(data + i + 6, sha1);
944 setup_index(repo, sha1);
945 i += 51;
946 break;
947 }
948 default:
949 while (i < buffer.posn && data[i] != '\n')
950 i++;
951 }
952 i++;
953 }
954
955 free(buffer.buffer);
956 repo->got_indices = 1;
957 return 0;
958 }
959
960 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
961 {
962 char *url;
963 struct packed_git *target;
964 struct packed_git **lst;
965 FILE *packfile;
966 char *filename;
967 char tmpfile[PATH_MAX];
968 int ret;
969 long prev_posn = 0;
970 char range[RANGE_HEADER_SIZE];
971 struct curl_slist *range_header = NULL;
972
973 struct active_request_slot *slot;
974 struct slot_results results;
975
976 if (fetch_indices(repo))
977 return -1;
978 target = find_sha1_pack(sha1, repo->packs);
979 if (!target)
980 return -1;
981
982 if (get_verbosely) {
983 fprintf(stderr, "Getting pack %s\n",
984 sha1_to_hex(target->sha1));
985 fprintf(stderr, " which contains %s\n",
986 sha1_to_hex(sha1));
987 }
988
989 url = xmalloc(strlen(repo->base) + 65);
990 sprintf(url, "%s/objects/pack/pack-%s.pack",
991 repo->base, sha1_to_hex(target->sha1));
992
993 filename = sha1_pack_name(target->sha1);
994 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
995 packfile = fopen(tmpfile, "a");
996 if (!packfile)
997 return error("Unable to open local file %s for pack",
998 filename);
999
1000 slot = get_active_slot();
1001 slot->results = &results;
1002 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
1003 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1004 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1005 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1006 slot->local = packfile;
1007
1008 /* If there is data present from a previous transfer attempt,
1009 resume where it left off */
1010 prev_posn = ftell(packfile);
1011 if (prev_posn>0) {
1012 if (get_verbosely)
1013 fprintf(stderr,
1014 "Resuming fetch of pack %s at byte %ld\n",
1015 sha1_to_hex(target->sha1), prev_posn);
1016 sprintf(range, "Range: bytes=%ld-", prev_posn);
1017 range_header = curl_slist_append(range_header, range);
1018 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
1019 }
1020
1021 if (start_active_slot(slot)) {
1022 run_active_slot(slot);
1023 if (results.curl_result != CURLE_OK) {
1024 fclose(packfile);
1025 return error("Unable to get pack file %s\n%s", url,
1026 curl_errorstr);
1027 }
1028 } else {
1029 fclose(packfile);
1030 return error("Unable to start request");
1031 }
1032
1033 fclose(packfile);
1034
1035 ret = move_temp_to_file(tmpfile, filename);
1036 if (ret)
1037 return ret;
1038
1039 lst = &repo->packs;
1040 while (*lst != target)
1041 lst = &((*lst)->next);
1042 *lst = (*lst)->next;
1043
1044 if (verify_pack(target, 0))
1045 return -1;
1046 install_packed_git(target);
1047
1048 return 0;
1049 }
1050
1051 static void abort_object_request(struct object_request *obj_req)
1052 {
1053 if (obj_req->local >= 0) {
1054 close(obj_req->local);
1055 obj_req->local = -1;
1056 }
1057 unlink(obj_req->tmpfile);
1058 if (obj_req->slot) {
1059 release_active_slot(obj_req->slot);
1060 obj_req->slot = NULL;
1061 }
1062 release_object_request(obj_req);
1063 }
1064
1065 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1066 {
1067 char *hex = sha1_to_hex(sha1);
1068 int ret = 0;
1069 struct object_request *obj_req = object_queue_head;
1070
1071 while (obj_req != NULL && memcmp(obj_req->sha1, sha1, 20))
1072 obj_req = obj_req->next;
1073 if (obj_req == NULL)
1074 return error("Couldn't find request for %s in the queue", hex);
1075
1076 if (has_sha1_file(obj_req->sha1)) {
1077 abort_object_request(obj_req);
1078 return 0;
1079 }
1080
1081 #ifdef USE_CURL_MULTI
1082 while (obj_req->state == WAITING) {
1083 step_active_slots();
1084 }
1085 #else
1086 start_object_request(obj_req);
1087 #endif
1088
1089 while (obj_req->state == ACTIVE) {
1090 run_active_slot(obj_req->slot);
1091 }
1092 if (obj_req->local != -1) {
1093 close(obj_req->local); obj_req->local = -1;
1094 }
1095
1096 if (obj_req->state == ABORTED) {
1097 ret = error("Request for %s aborted", hex);
1098 } else if (obj_req->curl_result != CURLE_OK &&
1099 obj_req->http_code != 416) {
1100 if (obj_req->http_code == 404 ||
1101 obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE)
1102 ret = -1; /* Be silent, it is probably in a pack. */
1103 else
1104 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1105 obj_req->errorstr, obj_req->curl_result,
1106 obj_req->http_code, hex);
1107 } else if (obj_req->zret != Z_STREAM_END) {
1108 corrupt_object_found++;
1109 ret = error("File %s (%s) corrupt", hex, obj_req->url);
1110 } else if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
1111 ret = error("File %s has bad hash", hex);
1112 } else if (obj_req->rename < 0) {
1113 ret = error("unable to write sha1 filename %s",
1114 obj_req->filename);
1115 }
1116
1117 release_object_request(obj_req);
1118 return ret;
1119 }
1120
1121 int fetch(unsigned char *sha1)
1122 {
1123 struct alt_base *altbase = alt;
1124
1125 if (!fetch_object(altbase, sha1))
1126 return 0;
1127 while (altbase) {
1128 if (!fetch_pack(altbase, sha1))
1129 return 0;
1130 fetch_alternates(alt->base);
1131 altbase = altbase->next;
1132 }
1133 return error("Unable to find %s under %s", sha1_to_hex(sha1),
1134 alt->base);
1135 }
1136
1137 static inline int needs_quote(int ch)
1138 {
1139 if (((ch >= 'A') && (ch <= 'Z'))
1140 || ((ch >= 'a') && (ch <= 'z'))
1141 || ((ch >= '0') && (ch <= '9'))
1142 || (ch == '/')
1143 || (ch == '-')
1144 || (ch == '.'))
1145 return 0;
1146 return 1;
1147 }
1148
1149 static inline int hex(int v)
1150 {
1151 if (v < 10) return '0' + v;
1152 else return 'A' + v - 10;
1153 }
1154
1155 static char *quote_ref_url(const char *base, const char *ref)
1156 {
1157 const char *cp;
1158 char *dp, *qref;
1159 int len, baselen, ch;
1160
1161 baselen = strlen(base);
1162 len = baselen + 6; /* "refs/" + NUL */
1163 for (cp = ref; (ch = *cp) != 0; cp++, len++)
1164 if (needs_quote(ch))
1165 len += 2; /* extra two hex plus replacement % */
1166 qref = xmalloc(len);
1167 memcpy(qref, base, baselen);
1168 memcpy(qref + baselen, "refs/", 5);
1169 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1170 if (needs_quote(ch)) {
1171 *dp++ = '%';
1172 *dp++ = hex((ch >> 4) & 0xF);
1173 *dp++ = hex(ch & 0xF);
1174 }
1175 else
1176 *dp++ = ch;
1177 }
1178 *dp = 0;
1179
1180 return qref;
1181 }
1182
1183 int fetch_ref(char *ref, unsigned char *sha1)
1184 {
1185 char *url;
1186 char hex[42];
1187 struct buffer buffer;
1188 char *base = alt->base;
1189 struct active_request_slot *slot;
1190 struct slot_results results;
1191 buffer.size = 41;
1192 buffer.posn = 0;
1193 buffer.buffer = hex;
1194 hex[41] = '\0';
1195
1196 url = quote_ref_url(base, ref);
1197 slot = get_active_slot();
1198 slot->results = &results;
1199 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1200 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1201 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1202 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1203 if (start_active_slot(slot)) {
1204 run_active_slot(slot);
1205 if (results.curl_result != CURLE_OK)
1206 return error("Couldn't get %s for %s\n%s",
1207 url, ref, curl_errorstr);
1208 } else {
1209 return error("Unable to start request");
1210 }
1211
1212 hex[40] = '\0';
1213 get_sha1_hex(hex, sha1);
1214 return 0;
1215 }
1216
1217 int main(int argc, char **argv)
1218 {
1219 char *commit_id;
1220 char *url;
1221 char *path;
1222 int arg = 1;
1223 int rc = 0;
1224
1225 setup_git_directory();
1226 git_config(git_default_config);
1227
1228 while (arg < argc && argv[arg][0] == '-') {
1229 if (argv[arg][1] == 't') {
1230 get_tree = 1;
1231 } else if (argv[arg][1] == 'c') {
1232 get_history = 1;
1233 } else if (argv[arg][1] == 'a') {
1234 get_all = 1;
1235 get_tree = 1;
1236 get_history = 1;
1237 } else if (argv[arg][1] == 'v') {
1238 get_verbosely = 1;
1239 } else if (argv[arg][1] == 'w') {
1240 write_ref = argv[arg + 1];
1241 arg++;
1242 } else if (!strcmp(argv[arg], "--recover")) {
1243 get_recover = 1;
1244 }
1245 arg++;
1246 }
1247 if (argc < arg + 2) {
1248 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1249 return 1;
1250 }
1251 commit_id = argv[arg];
1252 url = argv[arg + 1];
1253 write_ref_log_details = url;
1254
1255 http_init();
1256
1257 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1258
1259 alt = xmalloc(sizeof(*alt));
1260 alt->base = url;
1261 alt->got_indices = 0;
1262 alt->packs = NULL;
1263 alt->next = NULL;
1264 path = strstr(url, "//");
1265 if (path) {
1266 path = strchr(path+2, '/');
1267 if (path)
1268 alt->path_len = strlen(path);
1269 }
1270
1271 if (pull(commit_id))
1272 rc = 1;
1273
1274 http_cleanup();
1275
1276 curl_slist_free_all(no_pragma_header);
1277
1278 if (corrupt_object_found) {
1279 fprintf(stderr,
1280 "Some loose object were found to be corrupt, but they might be just\n"
1281 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1282 "status code. Suggest running git fsck-objects.\n");
1283 }
1284 return rc;
1285 }