Merge branch 'maint' to sync with 1.5.3.2
[git/git.git] / http-fetch.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
271421cd 3#include "pack.h"
215a7ad1 4#include "fetch.h"
29508e1e 5#include "http.h"
7baa3e86 6
49a0f240
NH
7#define PREV_BUF_SIZE 4096
8#define RANGE_HEADER_SIZE 30
9
96f1e58f 10static int commits_on_stdin;
8e29f6a0 11
acc075a8 12static int got_alternates = -1;
96f1e58f 13static int corrupt_object_found;
1d389ab6 14
1db69b57 15static struct curl_slist *no_pragma_header;
6eb7ed54 16
b3661567
DB
17struct alt_base
18{
2afea3bc 19 char *base;
b3661567
DB
20 int got_indices;
21 struct packed_git *packs;
22 struct alt_base *next;
23};
24
96f1e58f 25static struct alt_base *alt;
6eb7ed54 26
e388ab74 27enum object_request_state {
1d389ab6
NH
28 WAITING,
29 ABORTED,
30 ACTIVE,
31 COMPLETE,
32};
6eb7ed54 33
e388ab74 34struct object_request
1d389ab6
NH
35{
36 unsigned char sha1[20];
37 struct alt_base *repo;
38 char *url;
39 char filename[PATH_MAX];
40 char tmpfile[PATH_MAX];
41 int local;
e388ab74 42 enum object_request_state state;
1d389ab6
NH
43 CURLcode curl_result;
44 char errorstr[CURL_ERROR_SIZE];
45 long http_code;
46 unsigned char real_sha1[20];
47 SHA_CTX c;
48 z_stream stream;
49 int zret;
50 int rename;
51 struct active_request_slot *slot;
e388ab74 52 struct object_request *next;
1d389ab6
NH
53};
54
e388ab74 55struct alternates_request {
8e29f6a0 56 const char *base;
acc075a8
NH
57 char *url;
58 struct buffer *buffer;
59 struct active_request_slot *slot;
60 int http_specific;
61};
62
96f1e58f 63static struct object_request *object_queue_head;
bc8f2652 64
182005b9
DB
65static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
66 void *data)
67{
bf0f910d 68 unsigned char expn[4096];
6eb7ed54
DB
69 size_t size = eltsize * nmemb;
70 int posn = 0;
e388ab74 71 struct object_request *obj_req = (struct object_request *)data;
6eb7ed54 72 do {
93822c22 73 ssize_t retval = xwrite(obj_req->local,
1d7f171c 74 (char *) ptr + posn, size - posn);
6eb7ed54
DB
75 if (retval < 0)
76 return posn;
77 posn += retval;
78 } while (posn < size);
79
e388ab74
NH
80 obj_req->stream.avail_in = size;
81 obj_req->stream.next_in = ptr;
6eb7ed54 82 do {
e388ab74
NH
83 obj_req->stream.next_out = expn;
84 obj_req->stream.avail_out = sizeof(expn);
85 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
86 SHA1_Update(&obj_req->c, expn,
87 sizeof(expn) - obj_req->stream.avail_out);
88 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
1d389ab6 89 data_received++;
6eb7ed54
DB
90 return size;
91}
92
be4a015b
JH
93static int missing__target(int code, int result)
94{
95 return /* file:// URL -- do we ever use one??? */
96 (result == CURLE_FILE_COULDNT_READ_FILE) ||
97 /* http:// and https:// URL */
4adffc7b
JH
98 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
99 /* ftp:// URL */
100 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
be4a015b
JH
101 ;
102}
103
104#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
105
8e29f6a0 106static void fetch_alternates(const char *base);
1d389ab6 107
29508e1e 108static void process_object_response(void *callback_data);
1d389ab6 109
e388ab74 110static void start_object_request(struct object_request *obj_req)
1d389ab6 111{
e388ab74 112 char *hex = sha1_to_hex(obj_req->sha1);
1d389ab6
NH
113 char prevfile[PATH_MAX];
114 char *url;
115 char *posn;
116 int prevlocal;
117 unsigned char prev_buf[PREV_BUF_SIZE];
118 ssize_t prev_read = 0;
119 long prev_posn = 0;
120 char range[RANGE_HEADER_SIZE];
121 struct curl_slist *range_header = NULL;
122 struct active_request_slot *slot;
123
e388ab74 124 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
1d389ab6 125 unlink(prevfile);
e388ab74
NH
126 rename(obj_req->tmpfile, prevfile);
127 unlink(obj_req->tmpfile);
1d389ab6 128
e388ab74
NH
129 if (obj_req->local != -1)
130 error("fd leakage in start: %d", obj_req->local);
131 obj_req->local = open(obj_req->tmpfile,
1d389ab6 132 O_WRONLY | O_CREAT | O_EXCL, 0666);
b721e01f
JH
133 /* This could have failed due to the "lazy directory creation";
134 * try to mkdir the last path component.
135 */
e388ab74
NH
136 if (obj_req->local < 0 && errno == ENOENT) {
137 char *dir = strrchr(obj_req->tmpfile, '/');
b721e01f
JH
138 if (dir) {
139 *dir = 0;
e388ab74 140 mkdir(obj_req->tmpfile, 0777);
b721e01f
JH
141 *dir = '/';
142 }
e388ab74 143 obj_req->local = open(obj_req->tmpfile,
b721e01f
JH
144 O_WRONLY | O_CREAT | O_EXCL, 0666);
145 }
146
e388ab74
NH
147 if (obj_req->local < 0) {
148 obj_req->state = ABORTED;
bd2afde8 149 error("Couldn't create temporary file %s for %s: %s",
e388ab74 150 obj_req->tmpfile, obj_req->filename, strerror(errno));
1d389ab6
NH
151 return;
152 }
153
e388ab74 154 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
1d389ab6 155
e388ab74 156 inflateInit(&obj_req->stream);
1d389ab6 157
e388ab74 158 SHA1_Init(&obj_req->c);
1d389ab6 159
2afea3bc
GP
160 url = xmalloc(strlen(obj_req->repo->base) + 51);
161 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
e388ab74
NH
162 strcpy(url, obj_req->repo->base);
163 posn = url + strlen(obj_req->repo->base);
2afea3bc
GP
164 strcpy(posn, "/objects/");
165 posn += 9;
1d389ab6
NH
166 memcpy(posn, hex, 2);
167 posn += 2;
168 *(posn++) = '/';
169 strcpy(posn, hex + 2);
e388ab74 170 strcpy(obj_req->url, url);
1d389ab6
NH
171
172 /* If a previous temp file is present, process what was already
173 fetched. */
174 prevlocal = open(prevfile, O_RDONLY);
175 if (prevlocal != -1) {
176 do {
93d26e4c 177 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1d389ab6
NH
178 if (prev_read>0) {
179 if (fwrite_sha1_file(prev_buf,
180 1,
181 prev_read,
e388ab74 182 obj_req) == prev_read) {
1d389ab6
NH
183 prev_posn += prev_read;
184 } else {
185 prev_read = -1;
186 }
187 }
188 } while (prev_read > 0);
189 close(prevlocal);
190 }
191 unlink(prevfile);
192
193 /* Reset inflate/SHA1 if there was an error reading the previous temp
194 file; also rewind to the beginning of the local file. */
195 if (prev_read == -1) {
e388ab74
NH
196 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
197 inflateInit(&obj_req->stream);
198 SHA1_Init(&obj_req->c);
1d389ab6
NH
199 if (prev_posn>0) {
200 prev_posn = 0;
b5da2467 201 lseek(obj_req->local, 0, SEEK_SET);
e388ab74 202 ftruncate(obj_req->local, 0);
1d389ab6
NH
203 }
204 }
205
206 slot = get_active_slot();
29508e1e 207 slot->callback_func = process_object_response;
e388ab74
NH
208 slot->callback_data = obj_req;
209 obj_req->slot = slot;
29508e1e 210
e388ab74 211 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
1d389ab6 212 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
e388ab74 213 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
1d389ab6 214 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 215 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
216
217 /* If we have successfully processed data from a previous fetch
218 attempt, only fetch the data we don't already have. */
219 if (prev_posn>0) {
220 if (get_verbosely)
221 fprintf(stderr,
222 "Resuming fetch of object %s at byte %ld\n",
223 hex, prev_posn);
224 sprintf(range, "Range: bytes=%ld-", prev_posn);
225 range_header = curl_slist_append(range_header, range);
226 curl_easy_setopt(slot->curl,
227 CURLOPT_HTTPHEADER, range_header);
228 }
229
a7a8d378 230 /* Try to get the request started, abort the request on error */
e388ab74 231 obj_req->state = ACTIVE;
1d389ab6 232 if (!start_active_slot(slot)) {
e388ab74
NH
233 obj_req->state = ABORTED;
234 obj_req->slot = NULL;
235 close(obj_req->local); obj_req->local = -1;
236 free(obj_req->url);
237 return;
1d389ab6 238 }
1d389ab6
NH
239}
240
e388ab74 241static void finish_object_request(struct object_request *obj_req)
1d389ab6 242{
50496b21
NH
243 struct stat st;
244
e388ab74
NH
245 fchmod(obj_req->local, 0444);
246 close(obj_req->local); obj_req->local = -1;
1d389ab6 247
e388ab74 248 if (obj_req->http_code == 416) {
1d389ab6 249 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
e388ab74
NH
250 } else if (obj_req->curl_result != CURLE_OK) {
251 if (stat(obj_req->tmpfile, &st) == 0)
50496b21 252 if (st.st_size == 0)
e388ab74 253 unlink(obj_req->tmpfile);
1d389ab6
NH
254 return;
255 }
256
e388ab74
NH
257 inflateEnd(&obj_req->stream);
258 SHA1_Final(obj_req->real_sha1, &obj_req->c);
259 if (obj_req->zret != Z_STREAM_END) {
260 unlink(obj_req->tmpfile);
1d389ab6
NH
261 return;
262 }
a89fccd2 263 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
e388ab74 264 unlink(obj_req->tmpfile);
1d389ab6
NH
265 return;
266 }
e388ab74
NH
267 obj_req->rename =
268 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
1d389ab6 269
e388ab74
NH
270 if (obj_req->rename == 0)
271 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
1d389ab6
NH
272}
273
29508e1e
NH
274static void process_object_response(void *callback_data)
275{
e388ab74
NH
276 struct object_request *obj_req =
277 (struct object_request *)callback_data;
29508e1e 278
e388ab74
NH
279 obj_req->curl_result = obj_req->slot->curl_result;
280 obj_req->http_code = obj_req->slot->http_code;
281 obj_req->slot = NULL;
282 obj_req->state = COMPLETE;
29508e1e
NH
283
284 /* Use alternates if necessary */
be4a015b 285 if (missing_target(obj_req)) {
29508e1e 286 fetch_alternates(alt->base);
e388ab74
NH
287 if (obj_req->repo->next != NULL) {
288 obj_req->repo =
289 obj_req->repo->next;
290 close(obj_req->local);
291 obj_req->local = -1;
292 start_object_request(obj_req);
29508e1e
NH
293 return;
294 }
295 }
296
e388ab74 297 finish_object_request(obj_req);
29508e1e
NH
298}
299
e388ab74 300static void release_object_request(struct object_request *obj_req)
1d389ab6 301{
e388ab74 302 struct object_request *entry = object_queue_head;
1d389ab6 303
e388ab74
NH
304 if (obj_req->local != -1)
305 error("fd leakage in release: %d", obj_req->local);
306 if (obj_req == object_queue_head) {
307 object_queue_head = obj_req->next;
1d389ab6 308 } else {
e388ab74 309 while (entry->next != NULL && entry->next != obj_req)
1d389ab6 310 entry = entry->next;
e388ab74 311 if (entry->next == obj_req)
1d389ab6
NH
312 entry->next = entry->next->next;
313 }
314
e388ab74
NH
315 free(obj_req->url);
316 free(obj_req);
1d389ab6
NH
317}
318
a7a8d378 319#ifdef USE_CURL_MULTI
29508e1e 320void fill_active_slots(void)
1d389ab6 321{
e388ab74 322 struct object_request *obj_req = object_queue_head;
f1a906a3 323 struct active_request_slot *slot = active_queue_head;
1d389ab6
NH
324 int num_transfers;
325
e388ab74
NH
326 while (active_requests < max_requests && obj_req != NULL) {
327 if (obj_req->state == WAITING) {
328 if (has_sha1_file(obj_req->sha1))
09db444f 329 obj_req->state = COMPLETE;
11f0dafe 330 else
e388ab74 331 start_object_request(obj_req);
1d389ab6
NH
332 curl_multi_perform(curlm, &num_transfers);
333 }
e388ab74 334 obj_req = obj_req->next;
1d389ab6 335 }
f1a906a3
NH
336
337 while (slot != NULL) {
338 if (!slot->in_use && slot->curl != NULL) {
339 curl_easy_cleanup(slot->curl);
340 slot->curl = NULL;
341 }
342 slot = slot->next;
8fcf7f9a 343 }
1d389ab6 344}
a7a8d378 345#endif
1d389ab6
NH
346
347void prefetch(unsigned char *sha1)
348{
e388ab74
NH
349 struct object_request *newreq;
350 struct object_request *tail;
1d389ab6
NH
351 char *filename = sha1_file_name(sha1);
352
353 newreq = xmalloc(sizeof(*newreq));
e702496e 354 hashcpy(newreq->sha1, sha1);
1d389ab6
NH
355 newreq->repo = alt;
356 newreq->url = NULL;
357 newreq->local = -1;
358 newreq->state = WAITING;
359 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
360 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
361 "%s.temp", filename);
e8dff6ba 362 newreq->slot = NULL;
1d389ab6
NH
363 newreq->next = NULL;
364
e388ab74
NH
365 if (object_queue_head == NULL) {
366 object_queue_head = newreq;
1d389ab6 367 } else {
e388ab74 368 tail = object_queue_head;
1d389ab6
NH
369 while (tail->next != NULL) {
370 tail = tail->next;
371 }
372 tail->next = newreq;
373 }
29508e1e 374
a7a8d378 375#ifdef USE_CURL_MULTI
29508e1e
NH
376 fill_active_slots();
377 step_active_slots();
a7a8d378 378#endif
1d389ab6
NH
379}
380
b3661567 381static int fetch_index(struct alt_base *repo, unsigned char *sha1)
182005b9 382{
1d389ab6 383 char *hex = sha1_to_hex(sha1);
182005b9
DB
384 char *filename;
385 char *url;
49a0f240 386 char tmpfile[PATH_MAX];
49a0f240
NH
387 long prev_posn = 0;
388 char range[RANGE_HEADER_SIZE];
389 struct curl_slist *range_header = NULL;
182005b9
DB
390
391 FILE *indexfile;
1d389ab6 392 struct active_request_slot *slot;
cb754fdf 393 struct slot_results results;
182005b9
DB
394
395 if (has_pack_index(sha1))
396 return 0;
397
398 if (get_verbosely)
1d389ab6 399 fprintf(stderr, "Getting index for pack %s\n", hex);
8fcf7f9a 400
b3661567 401 url = xmalloc(strlen(repo->base) + 64);
1d389ab6 402 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
8fcf7f9a 403
182005b9 404 filename = sha1_pack_index_name(sha1);
49a0f240
NH
405 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
406 indexfile = fopen(tmpfile, "a");
182005b9
DB
407 if (!indexfile)
408 return error("Unable to open local file %s for pack index",
409 filename);
410
1d389ab6 411 slot = get_active_slot();
c8568e13 412 slot->results = &results;
1d389ab6
NH
413 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
414 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
415 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 416 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
417 slot->local = indexfile;
418
49a0f240
NH
419 /* If there is data present from a previous transfer attempt,
420 resume where it left off */
421 prev_posn = ftell(indexfile);
422 if (prev_posn>0) {
423 if (get_verbosely)
424 fprintf(stderr,
425 "Resuming fetch of index for pack %s at byte %ld\n",
1d389ab6 426 hex, prev_posn);
49a0f240
NH
427 sprintf(range, "Range: bytes=%ld-", prev_posn);
428 range_header = curl_slist_append(range_header, range);
1d389ab6 429 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
430 }
431
1d389ab6
NH
432 if (start_active_slot(slot)) {
433 run_active_slot(slot);
c8568e13 434 if (results.curl_result != CURLE_OK) {
1d389ab6
NH
435 fclose(indexfile);
436 return error("Unable to get pack index %s\n%s", url,
437 curl_errorstr);
438 }
439 } else {
313c4714 440 fclose(indexfile);
1d389ab6 441 return error("Unable to start request");
182005b9
DB
442 }
443
444 fclose(indexfile);
49a0f240 445
b721e01f 446 return move_temp_to_file(tmpfile, filename);
182005b9
DB
447}
448
b3661567 449static int setup_index(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
450{
451 struct packed_git *new_pack;
452 if (has_pack_file(sha1))
a9486b02 453 return 0; /* don't list this as something we can get */
182005b9 454
b3661567 455 if (fetch_index(repo, sha1))
182005b9
DB
456 return -1;
457
458 new_pack = parse_pack_index(sha1);
b3661567
DB
459 new_pack->next = repo->packs;
460 repo->packs = new_pack;
182005b9
DB
461 return 0;
462}
463
e388ab74 464static void process_alternates_response(void *callback_data)
b3661567 465{
e388ab74
NH
466 struct alternates_request *alt_req =
467 (struct alternates_request *)callback_data;
acc075a8 468 struct active_request_slot *slot = alt_req->slot;
1d389ab6 469 struct alt_base *tail = alt;
8e29f6a0 470 const char *base = alt_req->base;
bc8f2652 471 static const char null_byte = '\0';
acc075a8
NH
472 char *data;
473 int i = 0;
1d389ab6 474
acc075a8
NH
475 if (alt_req->http_specific) {
476 if (slot->curl_result != CURLE_OK ||
477 !alt_req->buffer->posn) {
478
479 /* Try reusing the slot to get non-http alternates */
480 alt_req->http_specific = 0;
481 sprintf(alt_req->url, "%s/objects/info/alternates",
482 base);
483 curl_easy_setopt(slot->curl, CURLOPT_URL,
484 alt_req->url);
485 active_requests++;
486 slot->in_use = 1;
c9826473
NH
487 if (slot->finished != NULL)
488 (*slot->finished) = 0;
a3f583cb 489 if (!start_active_slot(slot)) {
acc075a8 490 got_alternates = -1;
29508e1e 491 slot->in_use = 0;
c9826473
NH
492 if (slot->finished != NULL)
493 (*slot->finished) = 1;
1d389ab6 494 }
a3f583cb 495 return;
b3661567 496 }
acc075a8 497 } else if (slot->curl_result != CURLE_OK) {
be4a015b 498 if (!missing_target(slot)) {
acc075a8
NH
499 got_alternates = -1;
500 return;
501 }
b3661567
DB
502 }
503
29508e1e 504 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
acc075a8
NH
505 alt_req->buffer->posn--;
506 data = alt_req->buffer->buffer;
1b0c1e67 507
acc075a8 508 while (i < alt_req->buffer->posn) {
b3661567 509 int posn = i;
acc075a8 510 while (posn < alt_req->buffer->posn && data[posn] != '\n')
b3661567
DB
511 posn++;
512 if (data[posn] == '\n') {
1b0c1e67
DB
513 int okay = 0;
514 int serverlen = 0;
515 struct alt_base *newalt;
516 char *target = NULL;
b3661567 517 if (data[i] == '/') {
5df1e0d0
JH
518 /* This counts
519 * http://git.host/pub/scm/linux.git/
520 * -----------here^
521 * so memcpy(dst, base, serverlen) will
522 * copy up to "...git.host".
523 */
524 const char *colon_ss = strstr(base,"://");
525 if (colon_ss) {
526 serverlen = (strchr(colon_ss + 3, '/')
527 - base);
528 okay = 1;
529 }
1b0c1e67 530 } else if (!memcmp(data + i, "../", 3)) {
5df1e0d0
JH
531 /* Relative URL; chop the corresponding
532 * number of subpath from base (and ../
533 * from data), and concatenate the result.
534 *
535 * The code first drops ../ from data, and
536 * then drops one ../ from data and one path
537 * from base. IOW, one extra ../ is dropped
538 * from data than path is dropped from base.
539 *
540 * This is not wrong. The alternate in
541 * http://git.host/pub/scm/linux.git/
542 * to borrow from
543 * http://git.host/pub/scm/linus.git/
544 * is ../../linus.git/objects/. You need
545 * two ../../ to borrow from your direct
546 * neighbour.
547 */
1b0c1e67
DB
548 i += 3;
549 serverlen = strlen(base);
8fcf7f9a 550 while (i + 2 < posn &&
1b0c1e67
DB
551 !memcmp(data + i, "../", 3)) {
552 do {
553 serverlen--;
554 } while (serverlen &&
555 base[serverlen - 1] != '/');
556 i += 3;
557 }
a9486b02 558 /* If the server got removed, give up. */
8fcf7f9a 559 okay = strchr(base, ':') - base + 3 <
1b0c1e67 560 serverlen;
acc075a8 561 } else if (alt_req->http_specific) {
1b0c1e67
DB
562 char *colon = strchr(data + i, ':');
563 char *slash = strchr(data + i, '/');
564 if (colon && slash && colon < data + posn &&
565 slash < data + posn && colon < slash) {
566 okay = 1;
567 }
568 }
5df1e0d0 569 /* skip "objects\n" at end */
1b0c1e67
DB
570 if (okay) {
571 target = xmalloc(serverlen + posn - i - 6);
5df1e0d0
JH
572 memcpy(target, base, serverlen);
573 memcpy(target + serverlen, data + i,
574 posn - i - 7);
575 target[serverlen + posn - i - 7] = 0;
b3661567 576 if (get_verbosely)
8fcf7f9a 577 fprintf(stderr,
b3661567
DB
578 "Also look at %s\n", target);
579 newalt = xmalloc(sizeof(*newalt));
1d389ab6 580 newalt->next = NULL;
b3661567
DB
581 newalt->base = target;
582 newalt->got_indices = 0;
583 newalt->packs = NULL;
8d9fbe57 584
1d389ab6
NH
585 while (tail->next != NULL)
586 tail = tail->next;
587 tail->next = newalt;
b3661567
DB
588 }
589 }
590 i = posn + 1;
591 }
bc8f2652 592
f7eb290f 593 got_alternates = 1;
acc075a8
NH
594}
595
8e29f6a0 596static void fetch_alternates(const char *base)
acc075a8
NH
597{
598 struct buffer buffer;
599 char *url;
600 char *data;
601 struct active_request_slot *slot;
cb754fdf 602 struct alternates_request alt_req;
acc075a8
NH
603
604 /* If another request has already started fetching alternates,
605 wait for them to arrive and return to processing this request's
606 curl message */
29508e1e 607#ifdef USE_CURL_MULTI
acc075a8 608 while (got_alternates == 0) {
29508e1e 609 step_active_slots();
acc075a8 610 }
29508e1e 611#endif
acc075a8
NH
612
613 /* Nothing to do if they've already been fetched */
614 if (got_alternates == 1)
615 return;
616
617 /* Start the fetch */
618 got_alternates = 0;
619
620 data = xmalloc(4096);
621 buffer.size = 4096;
622 buffer.posn = 0;
623 buffer.buffer = data;
624
625 if (get_verbosely)
626 fprintf(stderr, "Getting alternates list for %s\n", base);
8fcf7f9a 627
acc075a8
NH
628 url = xmalloc(strlen(base) + 31);
629 sprintf(url, "%s/objects/info/http-alternates", base);
630
631 /* Use a callback to process the result, since another request
632 may fail and need to have alternates loaded before continuing */
633 slot = get_active_slot();
e388ab74 634 slot->callback_func = process_alternates_response;
acc075a8
NH
635 slot->callback_data = &alt_req;
636
637 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 638 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
acc075a8
NH
639 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
640
641 alt_req.base = base;
642 alt_req.url = url;
643 alt_req.buffer = &buffer;
644 alt_req.http_specific = 1;
645 alt_req.slot = slot;
646
647 if (start_active_slot(slot))
648 run_active_slot(slot);
649 else
650 got_alternates = -1;
651
652 free(data);
653 free(url);
b3661567
DB
654}
655
656static int fetch_indices(struct alt_base *repo)
182005b9
DB
657{
658 unsigned char sha1[20];
659 char *url;
660 struct buffer buffer;
661 char *data;
662 int i = 0;
663
1d389ab6 664 struct active_request_slot *slot;
cb754fdf 665 struct slot_results results;
1d389ab6 666
b3661567 667 if (repo->got_indices)
182005b9
DB
668 return 0;
669
670 data = xmalloc(4096);
671 buffer.size = 4096;
672 buffer.posn = 0;
673 buffer.buffer = data;
674
675 if (get_verbosely)
6fd72e39 676 fprintf(stderr, "Getting pack list for %s\n", repo->base);
8fcf7f9a 677
b3661567
DB
678 url = xmalloc(strlen(repo->base) + 21);
679 sprintf(url, "%s/objects/info/packs", repo->base);
182005b9 680
1d389ab6 681 slot = get_active_slot();
c8568e13 682 slot->results = &results;
1d389ab6 683 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 684 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1d389ab6
NH
685 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
686 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
687 if (start_active_slot(slot)) {
688 run_active_slot(slot);
c8568e13 689 if (results.curl_result != CURLE_OK) {
be4a015b 690 if (missing_target(&results)) {
5e3a7691
NH
691 repo->got_indices = 1;
692 free(buffer.buffer);
693 return 0;
694 } else {
695 repo->got_indices = 0;
696 free(buffer.buffer);
697 return error("%s", curl_errorstr);
698 }
bc8f2652 699 }
1d389ab6 700 } else {
5e3a7691 701 repo->got_indices = 0;
bc8f2652 702 free(buffer.buffer);
1d389ab6
NH
703 return error("Unable to start request");
704 }
182005b9 705
bc8f2652 706 data = buffer.buffer;
b3661567 707 while (i < buffer.posn) {
182005b9
DB
708 switch (data[i]) {
709 case 'P':
710 i++;
455c161c 711 if (i + 52 <= buffer.posn &&
cc44c765 712 !prefixcmp(data + i, " pack-") &&
1968d77d 713 !prefixcmp(data + i + 46, ".pack\n")) {
182005b9 714 get_sha1_hex(data + i + 6, sha1);
b3661567 715 setup_index(repo, sha1);
182005b9
DB
716 i += 51;
717 break;
718 }
719 default:
455c161c 720 while (i < buffer.posn && data[i] != '\n')
182005b9
DB
721 i++;
722 }
723 i++;
b3661567 724 }
182005b9 725
bc8f2652 726 free(buffer.buffer);
b3661567 727 repo->got_indices = 1;
182005b9
DB
728 return 0;
729}
730
b3661567 731static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
732{
733 char *url;
734 struct packed_git *target;
735 struct packed_git **lst;
736 FILE *packfile;
737 char *filename;
49a0f240
NH
738 char tmpfile[PATH_MAX];
739 int ret;
740 long prev_posn = 0;
741 char range[RANGE_HEADER_SIZE];
742 struct curl_slist *range_header = NULL;
1d389ab6
NH
743
744 struct active_request_slot *slot;
cb754fdf 745 struct slot_results results;
182005b9 746
b3661567 747 if (fetch_indices(repo))
182005b9 748 return -1;
b3661567 749 target = find_sha1_pack(sha1, repo->packs);
182005b9 750 if (!target)
b3661567 751 return -1;
182005b9
DB
752
753 if (get_verbosely) {
754 fprintf(stderr, "Getting pack %s\n",
755 sha1_to_hex(target->sha1));
756 fprintf(stderr, " which contains %s\n",
757 sha1_to_hex(sha1));
758 }
759
b3661567 760 url = xmalloc(strlen(repo->base) + 65);
182005b9 761 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 762 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
763
764 filename = sha1_pack_name(target->sha1);
49a0f240
NH
765 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
766 packfile = fopen(tmpfile, "a");
182005b9
DB
767 if (!packfile)
768 return error("Unable to open local file %s for pack",
769 filename);
770
1d389ab6 771 slot = get_active_slot();
c8568e13 772 slot->results = &results;
1d389ab6
NH
773 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
774 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
775 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 776 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 777 slot->local = packfile;
1ddea77e 778
49a0f240
NH
779 /* If there is data present from a previous transfer attempt,
780 resume where it left off */
781 prev_posn = ftell(packfile);
782 if (prev_posn>0) {
783 if (get_verbosely)
784 fprintf(stderr,
785 "Resuming fetch of pack %s at byte %ld\n",
786 sha1_to_hex(target->sha1), prev_posn);
787 sprintf(range, "Range: bytes=%ld-", prev_posn);
788 range_header = curl_slist_append(range_header, range);
1d389ab6 789 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
790 }
791
1d389ab6
NH
792 if (start_active_slot(slot)) {
793 run_active_slot(slot);
c8568e13 794 if (results.curl_result != CURLE_OK) {
1d389ab6
NH
795 fclose(packfile);
796 return error("Unable to get pack file %s\n%s", url,
797 curl_errorstr);
798 }
799 } else {
313c4714 800 fclose(packfile);
1d389ab6 801 return error("Unable to start request");
182005b9
DB
802 }
803
1c23d794 804 target->pack_size = ftell(packfile);
182005b9
DB
805 fclose(packfile);
806
b721e01f 807 ret = move_temp_to_file(tmpfile, filename);
49a0f240 808 if (ret)
b721e01f 809 return ret;
49a0f240 810
b3661567 811 lst = &repo->packs;
182005b9
DB
812 while (*lst != target)
813 lst = &((*lst)->next);
814 *lst = (*lst)->next;
815
271421cd
JH
816 if (verify_pack(target, 0))
817 return -1;
182005b9
DB
818 install_packed_git(target);
819
820 return 0;
821}
822
53f31389
MW
823static void abort_object_request(struct object_request *obj_req)
824{
825 if (obj_req->local >= 0) {
826 close(obj_req->local);
827 obj_req->local = -1;
828 }
829 unlink(obj_req->tmpfile);
830 if (obj_req->slot) {
a6080a0a 831 release_active_slot(obj_req->slot);
53f31389
MW
832 obj_req->slot = NULL;
833 }
834 release_object_request(obj_req);
835}
836
a7928f8e 837static int fetch_object(struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
838{
839 char *hex = sha1_to_hex(sha1);
29508e1e 840 int ret = 0;
e388ab74 841 struct object_request *obj_req = object_queue_head;
1d389ab6 842
a89fccd2 843 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
e388ab74
NH
844 obj_req = obj_req->next;
845 if (obj_req == NULL)
1d389ab6
NH
846 return error("Couldn't find request for %s in the queue", hex);
847
e388ab74 848 if (has_sha1_file(obj_req->sha1)) {
53f31389 849 abort_object_request(obj_req);
11f0dafe
NH
850 return 0;
851 }
852
a7a8d378 853#ifdef USE_CURL_MULTI
e388ab74 854 while (obj_req->state == WAITING) {
29508e1e 855 step_active_slots();
1d389ab6 856 }
a7a8d378 857#else
e388ab74 858 start_object_request(obj_req);
a7a8d378 859#endif
6eb7ed54 860
e388ab74
NH
861 while (obj_req->state == ACTIVE) {
862 run_active_slot(obj_req->slot);
a7a8d378 863 }
e388ab74
NH
864 if (obj_req->local != -1) {
865 close(obj_req->local); obj_req->local = -1;
313c4714 866 }
6eb7ed54 867
e388ab74 868 if (obj_req->state == ABORTED) {
29508e1e 869 ret = error("Request for %s aborted", hex);
e388ab74
NH
870 } else if (obj_req->curl_result != CURLE_OK &&
871 obj_req->http_code != 416) {
be4a015b 872 if (missing_target(obj_req))
e2029eb9
PB
873 ret = -1; /* Be silent, it is probably in a pack. */
874 else
875 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
e388ab74
NH
876 obj_req->errorstr, obj_req->curl_result,
877 obj_req->http_code, hex);
878 } else if (obj_req->zret != Z_STREAM_END) {
bb528079 879 corrupt_object_found++;
bd2afde8 880 ret = error("File %s (%s) corrupt", hex, obj_req->url);
a89fccd2 881 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
bd2afde8 882 ret = error("File %s has bad hash", hex);
e388ab74 883 } else if (obj_req->rename < 0) {
7b934ec0
MW
884 ret = error("unable to write sha1 filename %s",
885 obj_req->filename);
6eb7ed54 886 }
49a0f240 887
e388ab74 888 release_object_request(obj_req);
29508e1e 889 return ret;
6eb7ed54
DB
890}
891
b3661567
DB
892int fetch(unsigned char *sha1)
893{
894 struct alt_base *altbase = alt;
1d389ab6
NH
895
896 if (!fetch_object(altbase, sha1))
897 return 0;
b3661567 898 while (altbase) {
b3661567
DB
899 if (!fetch_pack(altbase, sha1))
900 return 0;
f7eb290f 901 fetch_alternates(alt->base);
b3661567
DB
902 altbase = altbase->next;
903 }
bd2afde8 904 return error("Unable to find %s under %s", sha1_to_hex(sha1),
1d389ab6 905 alt->base);
b3661567
DB
906}
907
94fa447a
JH
908static inline int needs_quote(int ch)
909{
cfd432e6
FF
910 if (((ch >= 'A') && (ch <= 'Z'))
911 || ((ch >= 'a') && (ch <= 'z'))
912 || ((ch >= '0') && (ch <= '9'))
913 || (ch == '/')
914 || (ch == '-')
915 || (ch == '.'))
94fa447a 916 return 0;
cfd432e6 917 return 1;
94fa447a
JH
918}
919
920static inline int hex(int v)
921{
922 if (v < 10) return '0' + v;
923 else return 'A' + v - 10;
924}
925
926static char *quote_ref_url(const char *base, const char *ref)
927{
928 const char *cp;
929 char *dp, *qref;
930 int len, baselen, ch;
931
932 baselen = strlen(base);
2afea3bc 933 len = baselen + 7; /* "/refs/" + NUL */
94fa447a
JH
934 for (cp = ref; (ch = *cp) != 0; cp++, len++)
935 if (needs_quote(ch))
936 len += 2; /* extra two hex plus replacement % */
937 qref = xmalloc(len);
938 memcpy(qref, base, baselen);
2afea3bc
GP
939 memcpy(qref + baselen, "/refs/", 6);
940 for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
94fa447a
JH
941 if (needs_quote(ch)) {
942 *dp++ = '%';
943 *dp++ = hex((ch >> 4) & 0xF);
944 *dp++ = hex(ch & 0xF);
945 }
946 else
947 *dp++ = ch;
948 }
949 *dp = 0;
950
951 return qref;
952}
953
cd541a68
DB
954int fetch_ref(char *ref, unsigned char *sha1)
955{
94fa447a 956 char *url;
fa3e0655
DB
957 char hex[42];
958 struct buffer buffer;
8e29f6a0 959 const char *base = alt->base;
1d389ab6 960 struct active_request_slot *slot;
cb754fdf 961 struct slot_results results;
fa3e0655
DB
962 buffer.size = 41;
963 buffer.posn = 0;
964 buffer.buffer = hex;
965 hex[41] = '\0';
8fcf7f9a 966
94fa447a 967 url = quote_ref_url(base, ref);
1d389ab6 968 slot = get_active_slot();
c8568e13 969 slot->results = &results;
1d389ab6
NH
970 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
971 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
972 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
973 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
974 if (start_active_slot(slot)) {
975 run_active_slot(slot);
c8568e13 976 if (results.curl_result != CURLE_OK)
1d389ab6
NH
977 return error("Couldn't get %s for %s\n%s",
978 url, ref, curl_errorstr);
979 } else {
980 return error("Unable to start request");
981 }
fa3e0655
DB
982
983 hex[40] = '\0';
984 get_sha1_hex(hex, sha1);
985 return 0;
cd541a68
DB
986}
987
8e29f6a0 988int main(int argc, const char **argv)
6eb7ed54 989{
8e29f6a0
PB
990 int commits;
991 const char **write_ref = NULL;
992 char **commit_id;
993 const char *url;
9c880b3e 994 char *s;
6eb7ed54 995 int arg = 1;
7b9ae53e 996 int rc = 0;
6eb7ed54 997
5a327713 998 setup_git_directory();
d0740d92 999 git_config(git_default_config);
5a327713 1000
6eb7ed54
DB
1001 while (arg < argc && argv[arg][0] == '-') {
1002 if (argv[arg][1] == 't') {
4250a5e5 1003 get_tree = 1;
6eb7ed54 1004 } else if (argv[arg][1] == 'c') {
4250a5e5 1005 get_history = 1;
6eb7ed54 1006 } else if (argv[arg][1] == 'a') {
4250a5e5
DB
1007 get_all = 1;
1008 get_tree = 1;
1009 get_history = 1;
e78d9772
JH
1010 } else if (argv[arg][1] == 'v') {
1011 get_verbosely = 1;
fa3e0655 1012 } else if (argv[arg][1] == 'w') {
8e29f6a0 1013 write_ref = &argv[arg + 1];
fa3e0655 1014 arg++;
820eca68
DB
1015 } else if (!strcmp(argv[arg], "--recover")) {
1016 get_recover = 1;
8e29f6a0
PB
1017 } else if (!strcmp(argv[arg], "--stdin")) {
1018 commits_on_stdin = 1;
6eb7ed54
DB
1019 }
1020 arg++;
1021 }
8e29f6a0
PB
1022 if (argc < arg + 2 - commits_on_stdin) {
1023 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
6eb7ed54
DB
1024 return 1;
1025 }
8e29f6a0
PB
1026 if (commits_on_stdin) {
1027 commits = pull_targets_stdin(&commit_id, &write_ref);
1028 } else {
1029 commit_id = (char **) &argv[arg++];
1030 commits = 1;
1031 }
1032 url = argv[arg];
6eb7ed54 1033
29508e1e 1034 http_init();
d402d556 1035
1db69b57 1036 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
3dcb90f5 1037
b3661567 1038 alt = xmalloc(sizeof(*alt));
2afea3bc
GP
1039 alt->base = xmalloc(strlen(url) + 1);
1040 strcpy(alt->base, url);
9c880b3e
GP
1041 for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s)
1042 *s = 0;
b3661567
DB
1043 alt->got_indices = 0;
1044 alt->packs = NULL;
1045 alt->next = NULL;
6eb7ed54 1046
8e29f6a0 1047 if (pull(commits, commit_id, write_ref, url))
7b9ae53e 1048 rc = 1;
6eb7ed54 1049
29508e1e
NH
1050 http_cleanup();
1051
07001f95
SE
1052 curl_slist_free_all(no_pragma_header);
1053
8e29f6a0
PB
1054 if (commits_on_stdin)
1055 pull_targets_free(commits, commit_id, write_ref);
1056
bb528079
JH
1057 if (corrupt_object_found) {
1058 fprintf(stderr,
1059"Some loose object were found to be corrupt, but they might be just\n"
1060"a false '404 Not Found' error message sent with incorrect HTTP\n"
df391b19 1061"status code. Suggest running git-fsck.\n");
bb528079 1062 }
7b9ae53e 1063 return rc;
6eb7ed54 1064}