Merge branch 'maint'
[git/git.git] / http-fetch.c
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
5 #include "http.h"
6
7 #ifndef NO_EXPAT
8 #include <expat.h>
9
10 /* Definitions for DAV requests */
11 #define DAV_PROPFIND "PROPFIND"
12 #define DAV_PROPFIND_RESP ".multistatus.response"
13 #define DAV_PROPFIND_NAME ".multistatus.response.href"
14 #define DAV_PROPFIND_COLLECTION ".multistatus.response.propstat.prop.resourcetype.collection"
15 #define PROPFIND_ALL_REQUEST "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<D:propfind xmlns:D=\"DAV:\">\n<D:allprop/>\n</D:propfind>"
16
17 /* Definitions for processing XML DAV responses */
18 #ifndef XML_STATUS_OK
19 enum XML_Status {
20 XML_STATUS_OK = 1,
21 XML_STATUS_ERROR = 0
22 };
23 #define XML_STATUS_OK 1
24 #define XML_STATUS_ERROR 0
25 #endif
26
27 /* Flags that control remote_ls processing */
28 #define PROCESS_FILES (1u << 0)
29 #define PROCESS_DIRS (1u << 1)
30 #define RECURSIVE (1u << 2)
31
32 /* Flags that remote_ls passes to callback functions */
33 #define IS_DIR (1u << 0)
34 #endif
35
36 #define PREV_BUF_SIZE 4096
37 #define RANGE_HEADER_SIZE 30
38
39 static int commits_on_stdin;
40
41 static int got_alternates = -1;
42 static int corrupt_object_found;
43
44 static struct curl_slist *no_pragma_header;
45
46 struct alt_base
47 {
48 const char *base;
49 int path_len;
50 int got_indices;
51 struct packed_git *packs;
52 struct alt_base *next;
53 };
54
55 static struct alt_base *alt;
56
57 enum object_request_state {
58 WAITING,
59 ABORTED,
60 ACTIVE,
61 COMPLETE,
62 };
63
64 struct object_request
65 {
66 unsigned char sha1[20];
67 struct alt_base *repo;
68 char *url;
69 char filename[PATH_MAX];
70 char tmpfile[PATH_MAX];
71 int local;
72 enum object_request_state state;
73 CURLcode curl_result;
74 char errorstr[CURL_ERROR_SIZE];
75 long http_code;
76 unsigned char real_sha1[20];
77 SHA_CTX c;
78 z_stream stream;
79 int zret;
80 int rename;
81 struct active_request_slot *slot;
82 struct object_request *next;
83 };
84
85 struct alternates_request {
86 const char *base;
87 char *url;
88 struct buffer *buffer;
89 struct active_request_slot *slot;
90 int http_specific;
91 };
92
93 #ifndef NO_EXPAT
94 struct xml_ctx
95 {
96 char *name;
97 int len;
98 char *cdata;
99 void (*userFunc)(struct xml_ctx *ctx, int tag_closed);
100 void *userData;
101 };
102
103 struct remote_ls_ctx
104 {
105 struct alt_base *repo;
106 char *path;
107 void (*userFunc)(struct remote_ls_ctx *ls);
108 void *userData;
109 int flags;
110 char *dentry_name;
111 int dentry_flags;
112 int rc;
113 struct remote_ls_ctx *parent;
114 };
115 #endif
116
117 static struct object_request *object_queue_head;
118
119 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
120 void *data)
121 {
122 unsigned char expn[4096];
123 size_t size = eltsize * nmemb;
124 int posn = 0;
125 struct object_request *obj_req = (struct object_request *)data;
126 do {
127 ssize_t retval = write(obj_req->local,
128 (char *) ptr + posn, size - posn);
129 if (retval < 0)
130 return posn;
131 posn += retval;
132 } while (posn < size);
133
134 obj_req->stream.avail_in = size;
135 obj_req->stream.next_in = ptr;
136 do {
137 obj_req->stream.next_out = expn;
138 obj_req->stream.avail_out = sizeof(expn);
139 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
140 SHA1_Update(&obj_req->c, expn,
141 sizeof(expn) - obj_req->stream.avail_out);
142 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
143 data_received++;
144 return size;
145 }
146
147 static void fetch_alternates(const char *base);
148
149 static void process_object_response(void *callback_data);
150
151 static void start_object_request(struct object_request *obj_req)
152 {
153 char *hex = sha1_to_hex(obj_req->sha1);
154 char prevfile[PATH_MAX];
155 char *url;
156 char *posn;
157 int prevlocal;
158 unsigned char prev_buf[PREV_BUF_SIZE];
159 ssize_t prev_read = 0;
160 long prev_posn = 0;
161 char range[RANGE_HEADER_SIZE];
162 struct curl_slist *range_header = NULL;
163 struct active_request_slot *slot;
164
165 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
166 unlink(prevfile);
167 rename(obj_req->tmpfile, prevfile);
168 unlink(obj_req->tmpfile);
169
170 if (obj_req->local != -1)
171 error("fd leakage in start: %d", obj_req->local);
172 obj_req->local = open(obj_req->tmpfile,
173 O_WRONLY | O_CREAT | O_EXCL, 0666);
174 /* This could have failed due to the "lazy directory creation";
175 * try to mkdir the last path component.
176 */
177 if (obj_req->local < 0 && errno == ENOENT) {
178 char *dir = strrchr(obj_req->tmpfile, '/');
179 if (dir) {
180 *dir = 0;
181 mkdir(obj_req->tmpfile, 0777);
182 *dir = '/';
183 }
184 obj_req->local = open(obj_req->tmpfile,
185 O_WRONLY | O_CREAT | O_EXCL, 0666);
186 }
187
188 if (obj_req->local < 0) {
189 obj_req->state = ABORTED;
190 error("Couldn't create temporary file %s for %s: %s",
191 obj_req->tmpfile, obj_req->filename, strerror(errno));
192 return;
193 }
194
195 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
196
197 inflateInit(&obj_req->stream);
198
199 SHA1_Init(&obj_req->c);
200
201 url = xmalloc(strlen(obj_req->repo->base) + 50);
202 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
203 strcpy(url, obj_req->repo->base);
204 posn = url + strlen(obj_req->repo->base);
205 strcpy(posn, "objects/");
206 posn += 8;
207 memcpy(posn, hex, 2);
208 posn += 2;
209 *(posn++) = '/';
210 strcpy(posn, hex + 2);
211 strcpy(obj_req->url, url);
212
213 /* If a previous temp file is present, process what was already
214 fetched. */
215 prevlocal = open(prevfile, O_RDONLY);
216 if (prevlocal != -1) {
217 do {
218 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
219 if (prev_read>0) {
220 if (fwrite_sha1_file(prev_buf,
221 1,
222 prev_read,
223 obj_req) == prev_read) {
224 prev_posn += prev_read;
225 } else {
226 prev_read = -1;
227 }
228 }
229 } while (prev_read > 0);
230 close(prevlocal);
231 }
232 unlink(prevfile);
233
234 /* Reset inflate/SHA1 if there was an error reading the previous temp
235 file; also rewind to the beginning of the local file. */
236 if (prev_read == -1) {
237 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
238 inflateInit(&obj_req->stream);
239 SHA1_Init(&obj_req->c);
240 if (prev_posn>0) {
241 prev_posn = 0;
242 lseek(obj_req->local, SEEK_SET, 0);
243 ftruncate(obj_req->local, 0);
244 }
245 }
246
247 slot = get_active_slot();
248 slot->callback_func = process_object_response;
249 slot->callback_data = obj_req;
250 obj_req->slot = slot;
251
252 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
253 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
254 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
255 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
256 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
257
258 /* If we have successfully processed data from a previous fetch
259 attempt, only fetch the data we don't already have. */
260 if (prev_posn>0) {
261 if (get_verbosely)
262 fprintf(stderr,
263 "Resuming fetch of object %s at byte %ld\n",
264 hex, prev_posn);
265 sprintf(range, "Range: bytes=%ld-", prev_posn);
266 range_header = curl_slist_append(range_header, range);
267 curl_easy_setopt(slot->curl,
268 CURLOPT_HTTPHEADER, range_header);
269 }
270
271 /* Try to get the request started, abort the request on error */
272 obj_req->state = ACTIVE;
273 if (!start_active_slot(slot)) {
274 obj_req->state = ABORTED;
275 obj_req->slot = NULL;
276 close(obj_req->local); obj_req->local = -1;
277 free(obj_req->url);
278 return;
279 }
280 }
281
282 static void finish_object_request(struct object_request *obj_req)
283 {
284 struct stat st;
285
286 fchmod(obj_req->local, 0444);
287 close(obj_req->local); obj_req->local = -1;
288
289 if (obj_req->http_code == 416) {
290 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
291 } else if (obj_req->curl_result != CURLE_OK) {
292 if (stat(obj_req->tmpfile, &st) == 0)
293 if (st.st_size == 0)
294 unlink(obj_req->tmpfile);
295 return;
296 }
297
298 inflateEnd(&obj_req->stream);
299 SHA1_Final(obj_req->real_sha1, &obj_req->c);
300 if (obj_req->zret != Z_STREAM_END) {
301 unlink(obj_req->tmpfile);
302 return;
303 }
304 if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
305 unlink(obj_req->tmpfile);
306 return;
307 }
308 obj_req->rename =
309 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
310
311 if (obj_req->rename == 0)
312 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
313 }
314
315 static void process_object_response(void *callback_data)
316 {
317 struct object_request *obj_req =
318 (struct object_request *)callback_data;
319
320 obj_req->curl_result = obj_req->slot->curl_result;
321 obj_req->http_code = obj_req->slot->http_code;
322 obj_req->slot = NULL;
323 obj_req->state = COMPLETE;
324
325 /* Use alternates if necessary */
326 if (obj_req->http_code == 404 ||
327 obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE) {
328 fetch_alternates(alt->base);
329 if (obj_req->repo->next != NULL) {
330 obj_req->repo =
331 obj_req->repo->next;
332 close(obj_req->local);
333 obj_req->local = -1;
334 start_object_request(obj_req);
335 return;
336 }
337 }
338
339 finish_object_request(obj_req);
340 }
341
342 static void release_object_request(struct object_request *obj_req)
343 {
344 struct object_request *entry = object_queue_head;
345
346 if (obj_req->local != -1)
347 error("fd leakage in release: %d", obj_req->local);
348 if (obj_req == object_queue_head) {
349 object_queue_head = obj_req->next;
350 } else {
351 while (entry->next != NULL && entry->next != obj_req)
352 entry = entry->next;
353 if (entry->next == obj_req)
354 entry->next = entry->next->next;
355 }
356
357 free(obj_req->url);
358 free(obj_req);
359 }
360
361 #ifdef USE_CURL_MULTI
362 void fill_active_slots(void)
363 {
364 struct object_request *obj_req = object_queue_head;
365 struct active_request_slot *slot = active_queue_head;
366 int num_transfers;
367
368 while (active_requests < max_requests && obj_req != NULL) {
369 if (obj_req->state == WAITING) {
370 if (has_sha1_file(obj_req->sha1))
371 obj_req->state = COMPLETE;
372 else
373 start_object_request(obj_req);
374 curl_multi_perform(curlm, &num_transfers);
375 }
376 obj_req = obj_req->next;
377 }
378
379 while (slot != NULL) {
380 if (!slot->in_use && slot->curl != NULL) {
381 curl_easy_cleanup(slot->curl);
382 slot->curl = NULL;
383 }
384 slot = slot->next;
385 }
386 }
387 #endif
388
389 void prefetch(unsigned char *sha1)
390 {
391 struct object_request *newreq;
392 struct object_request *tail;
393 char *filename = sha1_file_name(sha1);
394
395 newreq = xmalloc(sizeof(*newreq));
396 memcpy(newreq->sha1, sha1, 20);
397 newreq->repo = alt;
398 newreq->url = NULL;
399 newreq->local = -1;
400 newreq->state = WAITING;
401 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
402 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
403 "%s.temp", filename);
404 newreq->slot = NULL;
405 newreq->next = NULL;
406
407 if (object_queue_head == NULL) {
408 object_queue_head = newreq;
409 } else {
410 tail = object_queue_head;
411 while (tail->next != NULL) {
412 tail = tail->next;
413 }
414 tail->next = newreq;
415 }
416
417 #ifdef USE_CURL_MULTI
418 fill_active_slots();
419 step_active_slots();
420 #endif
421 }
422
423 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
424 {
425 char *hex = sha1_to_hex(sha1);
426 char *filename;
427 char *url;
428 char tmpfile[PATH_MAX];
429 long prev_posn = 0;
430 char range[RANGE_HEADER_SIZE];
431 struct curl_slist *range_header = NULL;
432
433 FILE *indexfile;
434 struct active_request_slot *slot;
435 struct slot_results results;
436
437 if (has_pack_index(sha1))
438 return 0;
439
440 if (get_verbosely)
441 fprintf(stderr, "Getting index for pack %s\n", hex);
442
443 url = xmalloc(strlen(repo->base) + 64);
444 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
445
446 filename = sha1_pack_index_name(sha1);
447 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
448 indexfile = fopen(tmpfile, "a");
449 if (!indexfile)
450 return error("Unable to open local file %s for pack index",
451 filename);
452
453 slot = get_active_slot();
454 slot->results = &results;
455 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
456 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
457 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
458 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
459 slot->local = indexfile;
460
461 /* If there is data present from a previous transfer attempt,
462 resume where it left off */
463 prev_posn = ftell(indexfile);
464 if (prev_posn>0) {
465 if (get_verbosely)
466 fprintf(stderr,
467 "Resuming fetch of index for pack %s at byte %ld\n",
468 hex, prev_posn);
469 sprintf(range, "Range: bytes=%ld-", prev_posn);
470 range_header = curl_slist_append(range_header, range);
471 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
472 }
473
474 if (start_active_slot(slot)) {
475 run_active_slot(slot);
476 if (results.curl_result != CURLE_OK) {
477 fclose(indexfile);
478 return error("Unable to get pack index %s\n%s", url,
479 curl_errorstr);
480 }
481 } else {
482 fclose(indexfile);
483 return error("Unable to start request");
484 }
485
486 fclose(indexfile);
487
488 return move_temp_to_file(tmpfile, filename);
489 }
490
491 static int setup_index(struct alt_base *repo, unsigned char *sha1)
492 {
493 struct packed_git *new_pack;
494 if (has_pack_file(sha1))
495 return 0; /* don't list this as something we can get */
496
497 if (fetch_index(repo, sha1))
498 return -1;
499
500 new_pack = parse_pack_index(sha1);
501 new_pack->next = repo->packs;
502 repo->packs = new_pack;
503 return 0;
504 }
505
506 static void process_alternates_response(void *callback_data)
507 {
508 struct alternates_request *alt_req =
509 (struct alternates_request *)callback_data;
510 struct active_request_slot *slot = alt_req->slot;
511 struct alt_base *tail = alt;
512 const char *base = alt_req->base;
513 static const char null_byte = '\0';
514 char *data;
515 int i = 0;
516
517 if (alt_req->http_specific) {
518 if (slot->curl_result != CURLE_OK ||
519 !alt_req->buffer->posn) {
520
521 /* Try reusing the slot to get non-http alternates */
522 alt_req->http_specific = 0;
523 sprintf(alt_req->url, "%s/objects/info/alternates",
524 base);
525 curl_easy_setopt(slot->curl, CURLOPT_URL,
526 alt_req->url);
527 active_requests++;
528 slot->in_use = 1;
529 if (slot->finished != NULL)
530 (*slot->finished) = 0;
531 if (!start_active_slot(slot)) {
532 got_alternates = -1;
533 slot->in_use = 0;
534 if (slot->finished != NULL)
535 (*slot->finished) = 1;
536 }
537 return;
538 }
539 } else if (slot->curl_result != CURLE_OK) {
540 if (slot->http_code != 404 &&
541 slot->curl_result != CURLE_FILE_COULDNT_READ_FILE) {
542 got_alternates = -1;
543 return;
544 }
545 }
546
547 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
548 alt_req->buffer->posn--;
549 data = alt_req->buffer->buffer;
550
551 while (i < alt_req->buffer->posn) {
552 int posn = i;
553 while (posn < alt_req->buffer->posn && data[posn] != '\n')
554 posn++;
555 if (data[posn] == '\n') {
556 int okay = 0;
557 int serverlen = 0;
558 struct alt_base *newalt;
559 char *target = NULL;
560 char *path;
561 if (data[i] == '/') {
562 serverlen = strchr(base + 8, '/') - base;
563 okay = 1;
564 } else if (!memcmp(data + i, "../", 3)) {
565 i += 3;
566 serverlen = strlen(base);
567 while (i + 2 < posn &&
568 !memcmp(data + i, "../", 3)) {
569 do {
570 serverlen--;
571 } while (serverlen &&
572 base[serverlen - 1] != '/');
573 i += 3;
574 }
575 /* If the server got removed, give up. */
576 okay = strchr(base, ':') - base + 3 <
577 serverlen;
578 } else if (alt_req->http_specific) {
579 char *colon = strchr(data + i, ':');
580 char *slash = strchr(data + i, '/');
581 if (colon && slash && colon < data + posn &&
582 slash < data + posn && colon < slash) {
583 okay = 1;
584 }
585 }
586 /* skip 'objects' at end */
587 if (okay) {
588 target = xmalloc(serverlen + posn - i - 6);
589 strlcpy(target, base, serverlen);
590 strlcpy(target + serverlen, data + i, posn - i - 6);
591 if (get_verbosely)
592 fprintf(stderr,
593 "Also look at %s\n", target);
594 newalt = xmalloc(sizeof(*newalt));
595 newalt->next = NULL;
596 newalt->base = target;
597 newalt->got_indices = 0;
598 newalt->packs = NULL;
599 path = strstr(target, "//");
600 if (path) {
601 path = strchr(path+2, '/');
602 if (path)
603 newalt->path_len = strlen(path);
604 }
605
606 while (tail->next != NULL)
607 tail = tail->next;
608 tail->next = newalt;
609 }
610 }
611 i = posn + 1;
612 }
613
614 got_alternates = 1;
615 }
616
617 static void fetch_alternates(const char *base)
618 {
619 struct buffer buffer;
620 char *url;
621 char *data;
622 struct active_request_slot *slot;
623 struct alternates_request alt_req;
624
625 /* If another request has already started fetching alternates,
626 wait for them to arrive and return to processing this request's
627 curl message */
628 #ifdef USE_CURL_MULTI
629 while (got_alternates == 0) {
630 step_active_slots();
631 }
632 #endif
633
634 /* Nothing to do if they've already been fetched */
635 if (got_alternates == 1)
636 return;
637
638 /* Start the fetch */
639 got_alternates = 0;
640
641 data = xmalloc(4096);
642 buffer.size = 4096;
643 buffer.posn = 0;
644 buffer.buffer = data;
645
646 if (get_verbosely)
647 fprintf(stderr, "Getting alternates list for %s\n", base);
648
649 url = xmalloc(strlen(base) + 31);
650 sprintf(url, "%s/objects/info/http-alternates", base);
651
652 /* Use a callback to process the result, since another request
653 may fail and need to have alternates loaded before continuing */
654 slot = get_active_slot();
655 slot->callback_func = process_alternates_response;
656 slot->callback_data = &alt_req;
657
658 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
659 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
660 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
661
662 alt_req.base = base;
663 alt_req.url = url;
664 alt_req.buffer = &buffer;
665 alt_req.http_specific = 1;
666 alt_req.slot = slot;
667
668 if (start_active_slot(slot))
669 run_active_slot(slot);
670 else
671 got_alternates = -1;
672
673 free(data);
674 free(url);
675 }
676
677 #ifndef NO_EXPAT
678 static void
679 xml_start_tag(void *userData, const char *name, const char **atts)
680 {
681 struct xml_ctx *ctx = (struct xml_ctx *)userData;
682 const char *c = strchr(name, ':');
683 int new_len;
684
685 if (c == NULL)
686 c = name;
687 else
688 c++;
689
690 new_len = strlen(ctx->name) + strlen(c) + 2;
691
692 if (new_len > ctx->len) {
693 ctx->name = xrealloc(ctx->name, new_len);
694 ctx->len = new_len;
695 }
696 strcat(ctx->name, ".");
697 strcat(ctx->name, c);
698
699 if (ctx->cdata) {
700 free(ctx->cdata);
701 ctx->cdata = NULL;
702 }
703
704 ctx->userFunc(ctx, 0);
705 }
706
707 static void
708 xml_end_tag(void *userData, const char *name)
709 {
710 struct xml_ctx *ctx = (struct xml_ctx *)userData;
711 const char *c = strchr(name, ':');
712 char *ep;
713
714 ctx->userFunc(ctx, 1);
715
716 if (c == NULL)
717 c = name;
718 else
719 c++;
720
721 ep = ctx->name + strlen(ctx->name) - strlen(c) - 1;
722 *ep = 0;
723 }
724
725 static void
726 xml_cdata(void *userData, const XML_Char *s, int len)
727 {
728 struct xml_ctx *ctx = (struct xml_ctx *)userData;
729 if (ctx->cdata)
730 free(ctx->cdata);
731 ctx->cdata = xmalloc(len + 1);
732 strlcpy(ctx->cdata, s, len + 1);
733 }
734
735 static int remote_ls(struct alt_base *repo, const char *path, int flags,
736 void (*userFunc)(struct remote_ls_ctx *ls),
737 void *userData);
738
739 static void handle_remote_ls_ctx(struct xml_ctx *ctx, int tag_closed)
740 {
741 struct remote_ls_ctx *ls = (struct remote_ls_ctx *)ctx->userData;
742
743 if (tag_closed) {
744 if (!strcmp(ctx->name, DAV_PROPFIND_RESP) && ls->dentry_name) {
745 if (ls->dentry_flags & IS_DIR) {
746 if (ls->flags & PROCESS_DIRS) {
747 ls->userFunc(ls);
748 }
749 if (strcmp(ls->dentry_name, ls->path) &&
750 ls->flags & RECURSIVE) {
751 ls->rc = remote_ls(ls->repo,
752 ls->dentry_name,
753 ls->flags,
754 ls->userFunc,
755 ls->userData);
756 }
757 } else if (ls->flags & PROCESS_FILES) {
758 ls->userFunc(ls);
759 }
760 } else if (!strcmp(ctx->name, DAV_PROPFIND_NAME) && ctx->cdata) {
761 ls->dentry_name = xmalloc(strlen(ctx->cdata) -
762 ls->repo->path_len + 1);
763 strcpy(ls->dentry_name, ctx->cdata + ls->repo->path_len);
764 } else if (!strcmp(ctx->name, DAV_PROPFIND_COLLECTION)) {
765 ls->dentry_flags |= IS_DIR;
766 }
767 } else if (!strcmp(ctx->name, DAV_PROPFIND_RESP)) {
768 if (ls->dentry_name) {
769 free(ls->dentry_name);
770 }
771 ls->dentry_name = NULL;
772 ls->dentry_flags = 0;
773 }
774 }
775
776 static int remote_ls(struct alt_base *repo, const char *path, int flags,
777 void (*userFunc)(struct remote_ls_ctx *ls),
778 void *userData)
779 {
780 char *url = xmalloc(strlen(repo->base) + strlen(path) + 1);
781 struct active_request_slot *slot;
782 struct slot_results results;
783 struct buffer in_buffer;
784 struct buffer out_buffer;
785 char *in_data;
786 char *out_data;
787 XML_Parser parser = XML_ParserCreate(NULL);
788 enum XML_Status result;
789 struct curl_slist *dav_headers = NULL;
790 struct xml_ctx ctx;
791 struct remote_ls_ctx ls;
792
793 ls.flags = flags;
794 ls.repo = repo;
795 ls.path = strdup(path);
796 ls.dentry_name = NULL;
797 ls.dentry_flags = 0;
798 ls.userData = userData;
799 ls.userFunc = userFunc;
800 ls.rc = 0;
801
802 sprintf(url, "%s%s", repo->base, path);
803
804 out_buffer.size = strlen(PROPFIND_ALL_REQUEST);
805 out_data = xmalloc(out_buffer.size + 1);
806 snprintf(out_data, out_buffer.size + 1, PROPFIND_ALL_REQUEST);
807 out_buffer.posn = 0;
808 out_buffer.buffer = out_data;
809
810 in_buffer.size = 4096;
811 in_data = xmalloc(in_buffer.size);
812 in_buffer.posn = 0;
813 in_buffer.buffer = in_data;
814
815 dav_headers = curl_slist_append(dav_headers, "Depth: 1");
816 dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml");
817
818 slot = get_active_slot();
819 slot->results = &results;
820 curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer);
821 curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.size);
822 curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
823 curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
824 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
825 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
826 curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
827 curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND);
828 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
829
830 if (start_active_slot(slot)) {
831 run_active_slot(slot);
832 if (results.curl_result == CURLE_OK) {
833 ctx.name = xcalloc(10, 1);
834 ctx.len = 0;
835 ctx.cdata = NULL;
836 ctx.userFunc = handle_remote_ls_ctx;
837 ctx.userData = &ls;
838 XML_SetUserData(parser, &ctx);
839 XML_SetElementHandler(parser, xml_start_tag,
840 xml_end_tag);
841 XML_SetCharacterDataHandler(parser, xml_cdata);
842 result = XML_Parse(parser, in_buffer.buffer,
843 in_buffer.posn, 1);
844 free(ctx.name);
845
846 if (result != XML_STATUS_OK) {
847 ls.rc = error("XML error: %s",
848 XML_ErrorString(
849 XML_GetErrorCode(parser)));
850 }
851 } else {
852 ls.rc = -1;
853 }
854 } else {
855 ls.rc = error("Unable to start PROPFIND request");
856 }
857
858 free(ls.path);
859 free(url);
860 free(out_data);
861 free(in_buffer.buffer);
862 curl_slist_free_all(dav_headers);
863
864 return ls.rc;
865 }
866
867 static void process_ls_pack(struct remote_ls_ctx *ls)
868 {
869 unsigned char sha1[20];
870
871 if (strlen(ls->dentry_name) == 63 &&
872 !strncmp(ls->dentry_name, "objects/pack/pack-", 18) &&
873 has_extension(ls->dentry_name, ".pack")) {
874 get_sha1_hex(ls->dentry_name + 18, sha1);
875 setup_index(ls->repo, sha1);
876 }
877 }
878 #endif
879
880 static int fetch_indices(struct alt_base *repo)
881 {
882 unsigned char sha1[20];
883 char *url;
884 struct buffer buffer;
885 char *data;
886 int i = 0;
887
888 struct active_request_slot *slot;
889 struct slot_results results;
890
891 if (repo->got_indices)
892 return 0;
893
894 data = xmalloc(4096);
895 buffer.size = 4096;
896 buffer.posn = 0;
897 buffer.buffer = data;
898
899 if (get_verbosely)
900 fprintf(stderr, "Getting pack list for %s\n", repo->base);
901
902 #ifndef NO_EXPAT
903 if (remote_ls(repo, "objects/pack/", PROCESS_FILES,
904 process_ls_pack, NULL) == 0)
905 return 0;
906 #endif
907
908 url = xmalloc(strlen(repo->base) + 21);
909 sprintf(url, "%s/objects/info/packs", repo->base);
910
911 slot = get_active_slot();
912 slot->results = &results;
913 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
914 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
915 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
916 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
917 if (start_active_slot(slot)) {
918 run_active_slot(slot);
919 if (results.curl_result != CURLE_OK) {
920 if (results.http_code == 404 ||
921 results.curl_result == CURLE_FILE_COULDNT_READ_FILE) {
922 repo->got_indices = 1;
923 free(buffer.buffer);
924 return 0;
925 } else {
926 repo->got_indices = 0;
927 free(buffer.buffer);
928 return error("%s", curl_errorstr);
929 }
930 }
931 } else {
932 repo->got_indices = 0;
933 free(buffer.buffer);
934 return error("Unable to start request");
935 }
936
937 data = buffer.buffer;
938 while (i < buffer.posn) {
939 switch (data[i]) {
940 case 'P':
941 i++;
942 if (i + 52 <= buffer.posn &&
943 !strncmp(data + i, " pack-", 6) &&
944 !strncmp(data + i + 46, ".pack\n", 6)) {
945 get_sha1_hex(data + i + 6, sha1);
946 setup_index(repo, sha1);
947 i += 51;
948 break;
949 }
950 default:
951 while (i < buffer.posn && data[i] != '\n')
952 i++;
953 }
954 i++;
955 }
956
957 free(buffer.buffer);
958 repo->got_indices = 1;
959 return 0;
960 }
961
962 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
963 {
964 char *url;
965 struct packed_git *target;
966 struct packed_git **lst;
967 FILE *packfile;
968 char *filename;
969 char tmpfile[PATH_MAX];
970 int ret;
971 long prev_posn = 0;
972 char range[RANGE_HEADER_SIZE];
973 struct curl_slist *range_header = NULL;
974
975 struct active_request_slot *slot;
976 struct slot_results results;
977
978 if (fetch_indices(repo))
979 return -1;
980 target = find_sha1_pack(sha1, repo->packs);
981 if (!target)
982 return -1;
983
984 if (get_verbosely) {
985 fprintf(stderr, "Getting pack %s\n",
986 sha1_to_hex(target->sha1));
987 fprintf(stderr, " which contains %s\n",
988 sha1_to_hex(sha1));
989 }
990
991 url = xmalloc(strlen(repo->base) + 65);
992 sprintf(url, "%s/objects/pack/pack-%s.pack",
993 repo->base, sha1_to_hex(target->sha1));
994
995 filename = sha1_pack_name(target->sha1);
996 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
997 packfile = fopen(tmpfile, "a");
998 if (!packfile)
999 return error("Unable to open local file %s for pack",
1000 filename);
1001
1002 slot = get_active_slot();
1003 slot->results = &results;
1004 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
1005 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1006 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1007 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1008 slot->local = packfile;
1009
1010 /* If there is data present from a previous transfer attempt,
1011 resume where it left off */
1012 prev_posn = ftell(packfile);
1013 if (prev_posn>0) {
1014 if (get_verbosely)
1015 fprintf(stderr,
1016 "Resuming fetch of pack %s at byte %ld\n",
1017 sha1_to_hex(target->sha1), prev_posn);
1018 sprintf(range, "Range: bytes=%ld-", prev_posn);
1019 range_header = curl_slist_append(range_header, range);
1020 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
1021 }
1022
1023 if (start_active_slot(slot)) {
1024 run_active_slot(slot);
1025 if (results.curl_result != CURLE_OK) {
1026 fclose(packfile);
1027 return error("Unable to get pack file %s\n%s", url,
1028 curl_errorstr);
1029 }
1030 } else {
1031 fclose(packfile);
1032 return error("Unable to start request");
1033 }
1034
1035 fclose(packfile);
1036
1037 ret = move_temp_to_file(tmpfile, filename);
1038 if (ret)
1039 return ret;
1040
1041 lst = &repo->packs;
1042 while (*lst != target)
1043 lst = &((*lst)->next);
1044 *lst = (*lst)->next;
1045
1046 if (verify_pack(target, 0))
1047 return -1;
1048 install_packed_git(target);
1049
1050 return 0;
1051 }
1052
1053 static void abort_object_request(struct object_request *obj_req)
1054 {
1055 if (obj_req->local >= 0) {
1056 close(obj_req->local);
1057 obj_req->local = -1;
1058 }
1059 unlink(obj_req->tmpfile);
1060 if (obj_req->slot) {
1061 release_active_slot(obj_req->slot);
1062 obj_req->slot = NULL;
1063 }
1064 release_object_request(obj_req);
1065 }
1066
1067 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1068 {
1069 char *hex = sha1_to_hex(sha1);
1070 int ret = 0;
1071 struct object_request *obj_req = object_queue_head;
1072
1073 while (obj_req != NULL && memcmp(obj_req->sha1, sha1, 20))
1074 obj_req = obj_req->next;
1075 if (obj_req == NULL)
1076 return error("Couldn't find request for %s in the queue", hex);
1077
1078 if (has_sha1_file(obj_req->sha1)) {
1079 abort_object_request(obj_req);
1080 return 0;
1081 }
1082
1083 #ifdef USE_CURL_MULTI
1084 while (obj_req->state == WAITING) {
1085 step_active_slots();
1086 }
1087 #else
1088 start_object_request(obj_req);
1089 #endif
1090
1091 while (obj_req->state == ACTIVE) {
1092 run_active_slot(obj_req->slot);
1093 }
1094 if (obj_req->local != -1) {
1095 close(obj_req->local); obj_req->local = -1;
1096 }
1097
1098 if (obj_req->state == ABORTED) {
1099 ret = error("Request for %s aborted", hex);
1100 } else if (obj_req->curl_result != CURLE_OK &&
1101 obj_req->http_code != 416) {
1102 if (obj_req->http_code == 404 ||
1103 obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE)
1104 ret = -1; /* Be silent, it is probably in a pack. */
1105 else
1106 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1107 obj_req->errorstr, obj_req->curl_result,
1108 obj_req->http_code, hex);
1109 } else if (obj_req->zret != Z_STREAM_END) {
1110 corrupt_object_found++;
1111 ret = error("File %s (%s) corrupt", hex, obj_req->url);
1112 } else if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
1113 ret = error("File %s has bad hash", hex);
1114 } else if (obj_req->rename < 0) {
1115 ret = error("unable to write sha1 filename %s",
1116 obj_req->filename);
1117 }
1118
1119 release_object_request(obj_req);
1120 return ret;
1121 }
1122
1123 int fetch(unsigned char *sha1)
1124 {
1125 struct alt_base *altbase = alt;
1126
1127 if (!fetch_object(altbase, sha1))
1128 return 0;
1129 while (altbase) {
1130 if (!fetch_pack(altbase, sha1))
1131 return 0;
1132 fetch_alternates(alt->base);
1133 altbase = altbase->next;
1134 }
1135 return error("Unable to find %s under %s", sha1_to_hex(sha1),
1136 alt->base);
1137 }
1138
1139 static inline int needs_quote(int ch)
1140 {
1141 if (((ch >= 'A') && (ch <= 'Z'))
1142 || ((ch >= 'a') && (ch <= 'z'))
1143 || ((ch >= '0') && (ch <= '9'))
1144 || (ch == '/')
1145 || (ch == '-')
1146 || (ch == '.'))
1147 return 0;
1148 return 1;
1149 }
1150
1151 static inline int hex(int v)
1152 {
1153 if (v < 10) return '0' + v;
1154 else return 'A' + v - 10;
1155 }
1156
1157 static char *quote_ref_url(const char *base, const char *ref)
1158 {
1159 const char *cp;
1160 char *dp, *qref;
1161 int len, baselen, ch;
1162
1163 baselen = strlen(base);
1164 len = baselen + 6; /* "refs/" + NUL */
1165 for (cp = ref; (ch = *cp) != 0; cp++, len++)
1166 if (needs_quote(ch))
1167 len += 2; /* extra two hex plus replacement % */
1168 qref = xmalloc(len);
1169 memcpy(qref, base, baselen);
1170 memcpy(qref + baselen, "refs/", 5);
1171 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1172 if (needs_quote(ch)) {
1173 *dp++ = '%';
1174 *dp++ = hex((ch >> 4) & 0xF);
1175 *dp++ = hex(ch & 0xF);
1176 }
1177 else
1178 *dp++ = ch;
1179 }
1180 *dp = 0;
1181
1182 return qref;
1183 }
1184
1185 int fetch_ref(char *ref, unsigned char *sha1)
1186 {
1187 char *url;
1188 char hex[42];
1189 struct buffer buffer;
1190 const char *base = alt->base;
1191 struct active_request_slot *slot;
1192 struct slot_results results;
1193 buffer.size = 41;
1194 buffer.posn = 0;
1195 buffer.buffer = hex;
1196 hex[41] = '\0';
1197
1198 url = quote_ref_url(base, ref);
1199 slot = get_active_slot();
1200 slot->results = &results;
1201 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1202 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1203 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1204 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1205 if (start_active_slot(slot)) {
1206 run_active_slot(slot);
1207 if (results.curl_result != CURLE_OK)
1208 return error("Couldn't get %s for %s\n%s",
1209 url, ref, curl_errorstr);
1210 } else {
1211 return error("Unable to start request");
1212 }
1213
1214 hex[40] = '\0';
1215 get_sha1_hex(hex, sha1);
1216 return 0;
1217 }
1218
1219 int main(int argc, const char **argv)
1220 {
1221 int commits;
1222 const char **write_ref = NULL;
1223 char **commit_id;
1224 const char *url;
1225 char *path;
1226 int arg = 1;
1227 int rc = 0;
1228
1229 setup_ident();
1230 setup_git_directory();
1231 git_config(git_default_config);
1232
1233 while (arg < argc && argv[arg][0] == '-') {
1234 if (argv[arg][1] == 't') {
1235 get_tree = 1;
1236 } else if (argv[arg][1] == 'c') {
1237 get_history = 1;
1238 } else if (argv[arg][1] == 'a') {
1239 get_all = 1;
1240 get_tree = 1;
1241 get_history = 1;
1242 } else if (argv[arg][1] == 'v') {
1243 get_verbosely = 1;
1244 } else if (argv[arg][1] == 'w') {
1245 write_ref = &argv[arg + 1];
1246 arg++;
1247 } else if (!strcmp(argv[arg], "--recover")) {
1248 get_recover = 1;
1249 } else if (!strcmp(argv[arg], "--stdin")) {
1250 commits_on_stdin = 1;
1251 }
1252 arg++;
1253 }
1254 if (argc < arg + 2 - commits_on_stdin) {
1255 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1256 return 1;
1257 }
1258 if (commits_on_stdin) {
1259 commits = pull_targets_stdin(&commit_id, &write_ref);
1260 } else {
1261 commit_id = (char **) &argv[arg++];
1262 commits = 1;
1263 }
1264 url = argv[arg];
1265
1266 http_init();
1267
1268 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1269
1270 alt = xmalloc(sizeof(*alt));
1271 alt->base = url;
1272 alt->got_indices = 0;
1273 alt->packs = NULL;
1274 alt->next = NULL;
1275 path = strstr(url, "//");
1276 if (path) {
1277 path = strchr(path+2, '/');
1278 if (path)
1279 alt->path_len = strlen(path);
1280 }
1281
1282 if (pull(commits, commit_id, write_ref, url))
1283 rc = 1;
1284
1285 http_cleanup();
1286
1287 curl_slist_free_all(no_pragma_header);
1288
1289 if (commits_on_stdin)
1290 pull_targets_free(commits, commit_id, write_ref);
1291
1292 if (corrupt_object_found) {
1293 fprintf(stderr,
1294 "Some loose object were found to be corrupt, but they might be just\n"
1295 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1296 "status code. Suggest running git fsck-objects.\n");
1297 }
1298 return rc;
1299 }