Remove obsolete commit-walkers
[git/git.git] / http-fetch.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
271421cd 3#include "pack.h"
215a7ad1 4#include "fetch.h"
29508e1e 5#include "http.h"
7baa3e86 6
49a0f240
NH
7#define PREV_BUF_SIZE 4096
8#define RANGE_HEADER_SIZE 30
9
96f1e58f 10static int commits_on_stdin;
8e29f6a0 11
acc075a8 12static int got_alternates = -1;
96f1e58f 13static int corrupt_object_found;
1d389ab6 14
1db69b57 15static struct curl_slist *no_pragma_header;
6eb7ed54 16
b3661567
DB
17struct alt_base
18{
2afea3bc 19 char *base;
b3661567
DB
20 int got_indices;
21 struct packed_git *packs;
22 struct alt_base *next;
23};
24
96f1e58f 25static struct alt_base *alt;
6eb7ed54 26
e388ab74 27enum object_request_state {
1d389ab6
NH
28 WAITING,
29 ABORTED,
30 ACTIVE,
31 COMPLETE,
32};
6eb7ed54 33
e388ab74 34struct object_request
1d389ab6
NH
35{
36 unsigned char sha1[20];
37 struct alt_base *repo;
38 char *url;
39 char filename[PATH_MAX];
40 char tmpfile[PATH_MAX];
41 int local;
e388ab74 42 enum object_request_state state;
1d389ab6
NH
43 CURLcode curl_result;
44 char errorstr[CURL_ERROR_SIZE];
45 long http_code;
46 unsigned char real_sha1[20];
47 SHA_CTX c;
48 z_stream stream;
49 int zret;
50 int rename;
51 struct active_request_slot *slot;
e388ab74 52 struct object_request *next;
1d389ab6
NH
53};
54
e388ab74 55struct alternates_request {
8e29f6a0 56 const char *base;
acc075a8
NH
57 char *url;
58 struct buffer *buffer;
59 struct active_request_slot *slot;
60 int http_specific;
61};
62
96f1e58f 63static struct object_request *object_queue_head;
bc8f2652 64
182005b9
DB
65static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
66 void *data)
67{
bf0f910d 68 unsigned char expn[4096];
6eb7ed54
DB
69 size_t size = eltsize * nmemb;
70 int posn = 0;
e388ab74 71 struct object_request *obj_req = (struct object_request *)data;
6eb7ed54 72 do {
93822c22 73 ssize_t retval = xwrite(obj_req->local,
1d7f171c 74 (char *) ptr + posn, size - posn);
6eb7ed54
DB
75 if (retval < 0)
76 return posn;
77 posn += retval;
78 } while (posn < size);
79
e388ab74
NH
80 obj_req->stream.avail_in = size;
81 obj_req->stream.next_in = ptr;
6eb7ed54 82 do {
e388ab74
NH
83 obj_req->stream.next_out = expn;
84 obj_req->stream.avail_out = sizeof(expn);
85 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
86 SHA1_Update(&obj_req->c, expn,
87 sizeof(expn) - obj_req->stream.avail_out);
88 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
1d389ab6 89 data_received++;
6eb7ed54
DB
90 return size;
91}
92
be4a015b
JH
93static int missing__target(int code, int result)
94{
95 return /* file:// URL -- do we ever use one??? */
96 (result == CURLE_FILE_COULDNT_READ_FILE) ||
97 /* http:// and https:// URL */
4adffc7b
JH
98 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
99 /* ftp:// URL */
100 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
be4a015b
JH
101 ;
102}
103
104#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
105
8e29f6a0 106static void fetch_alternates(const char *base);
1d389ab6 107
29508e1e 108static void process_object_response(void *callback_data);
1d389ab6 109
e388ab74 110static void start_object_request(struct object_request *obj_req)
1d389ab6 111{
e388ab74 112 char *hex = sha1_to_hex(obj_req->sha1);
1d389ab6
NH
113 char prevfile[PATH_MAX];
114 char *url;
115 char *posn;
116 int prevlocal;
117 unsigned char prev_buf[PREV_BUF_SIZE];
118 ssize_t prev_read = 0;
119 long prev_posn = 0;
120 char range[RANGE_HEADER_SIZE];
121 struct curl_slist *range_header = NULL;
122 struct active_request_slot *slot;
123
e388ab74 124 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
1d389ab6 125 unlink(prevfile);
e388ab74
NH
126 rename(obj_req->tmpfile, prevfile);
127 unlink(obj_req->tmpfile);
1d389ab6 128
e388ab74
NH
129 if (obj_req->local != -1)
130 error("fd leakage in start: %d", obj_req->local);
131 obj_req->local = open(obj_req->tmpfile,
1d389ab6 132 O_WRONLY | O_CREAT | O_EXCL, 0666);
b721e01f
JH
133 /* This could have failed due to the "lazy directory creation";
134 * try to mkdir the last path component.
135 */
e388ab74
NH
136 if (obj_req->local < 0 && errno == ENOENT) {
137 char *dir = strrchr(obj_req->tmpfile, '/');
b721e01f
JH
138 if (dir) {
139 *dir = 0;
e388ab74 140 mkdir(obj_req->tmpfile, 0777);
b721e01f
JH
141 *dir = '/';
142 }
e388ab74 143 obj_req->local = open(obj_req->tmpfile,
b721e01f
JH
144 O_WRONLY | O_CREAT | O_EXCL, 0666);
145 }
146
e388ab74
NH
147 if (obj_req->local < 0) {
148 obj_req->state = ABORTED;
bd2afde8 149 error("Couldn't create temporary file %s for %s: %s",
e388ab74 150 obj_req->tmpfile, obj_req->filename, strerror(errno));
1d389ab6
NH
151 return;
152 }
153
e388ab74 154 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
1d389ab6 155
e388ab74 156 inflateInit(&obj_req->stream);
1d389ab6 157
e388ab74 158 SHA1_Init(&obj_req->c);
1d389ab6 159
2afea3bc
GP
160 url = xmalloc(strlen(obj_req->repo->base) + 51);
161 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
e388ab74
NH
162 strcpy(url, obj_req->repo->base);
163 posn = url + strlen(obj_req->repo->base);
2afea3bc
GP
164 strcpy(posn, "/objects/");
165 posn += 9;
1d389ab6
NH
166 memcpy(posn, hex, 2);
167 posn += 2;
168 *(posn++) = '/';
169 strcpy(posn, hex + 2);
e388ab74 170 strcpy(obj_req->url, url);
1d389ab6
NH
171
172 /* If a previous temp file is present, process what was already
173 fetched. */
174 prevlocal = open(prevfile, O_RDONLY);
175 if (prevlocal != -1) {
176 do {
93d26e4c 177 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1d389ab6
NH
178 if (prev_read>0) {
179 if (fwrite_sha1_file(prev_buf,
180 1,
181 prev_read,
e388ab74 182 obj_req) == prev_read) {
1d389ab6
NH
183 prev_posn += prev_read;
184 } else {
185 prev_read = -1;
186 }
187 }
188 } while (prev_read > 0);
189 close(prevlocal);
190 }
191 unlink(prevfile);
192
193 /* Reset inflate/SHA1 if there was an error reading the previous temp
194 file; also rewind to the beginning of the local file. */
195 if (prev_read == -1) {
e388ab74
NH
196 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
197 inflateInit(&obj_req->stream);
198 SHA1_Init(&obj_req->c);
1d389ab6
NH
199 if (prev_posn>0) {
200 prev_posn = 0;
b5da2467 201 lseek(obj_req->local, 0, SEEK_SET);
e388ab74 202 ftruncate(obj_req->local, 0);
1d389ab6
NH
203 }
204 }
205
206 slot = get_active_slot();
29508e1e 207 slot->callback_func = process_object_response;
e388ab74
NH
208 slot->callback_data = obj_req;
209 obj_req->slot = slot;
29508e1e 210
e388ab74 211 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
1d389ab6 212 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
e388ab74 213 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
1d389ab6 214 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 215 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
216
217 /* If we have successfully processed data from a previous fetch
218 attempt, only fetch the data we don't already have. */
219 if (prev_posn>0) {
220 if (get_verbosely)
221 fprintf(stderr,
222 "Resuming fetch of object %s at byte %ld\n",
223 hex, prev_posn);
224 sprintf(range, "Range: bytes=%ld-", prev_posn);
225 range_header = curl_slist_append(range_header, range);
226 curl_easy_setopt(slot->curl,
227 CURLOPT_HTTPHEADER, range_header);
228 }
229
a7a8d378 230 /* Try to get the request started, abort the request on error */
e388ab74 231 obj_req->state = ACTIVE;
1d389ab6 232 if (!start_active_slot(slot)) {
e388ab74
NH
233 obj_req->state = ABORTED;
234 obj_req->slot = NULL;
235 close(obj_req->local); obj_req->local = -1;
236 free(obj_req->url);
237 return;
1d389ab6 238 }
1d389ab6
NH
239}
240
e388ab74 241static void finish_object_request(struct object_request *obj_req)
1d389ab6 242{
50496b21
NH
243 struct stat st;
244
e388ab74
NH
245 fchmod(obj_req->local, 0444);
246 close(obj_req->local); obj_req->local = -1;
1d389ab6 247
e388ab74 248 if (obj_req->http_code == 416) {
1d389ab6 249 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
e388ab74
NH
250 } else if (obj_req->curl_result != CURLE_OK) {
251 if (stat(obj_req->tmpfile, &st) == 0)
50496b21 252 if (st.st_size == 0)
e388ab74 253 unlink(obj_req->tmpfile);
1d389ab6
NH
254 return;
255 }
256
e388ab74
NH
257 inflateEnd(&obj_req->stream);
258 SHA1_Final(obj_req->real_sha1, &obj_req->c);
259 if (obj_req->zret != Z_STREAM_END) {
260 unlink(obj_req->tmpfile);
1d389ab6
NH
261 return;
262 }
a89fccd2 263 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
e388ab74 264 unlink(obj_req->tmpfile);
1d389ab6
NH
265 return;
266 }
e388ab74
NH
267 obj_req->rename =
268 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
1d389ab6 269
e388ab74
NH
270 if (obj_req->rename == 0)
271 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
1d389ab6
NH
272}
273
29508e1e
NH
274static void process_object_response(void *callback_data)
275{
e388ab74
NH
276 struct object_request *obj_req =
277 (struct object_request *)callback_data;
29508e1e 278
e388ab74
NH
279 obj_req->curl_result = obj_req->slot->curl_result;
280 obj_req->http_code = obj_req->slot->http_code;
281 obj_req->slot = NULL;
282 obj_req->state = COMPLETE;
29508e1e
NH
283
284 /* Use alternates if necessary */
be4a015b 285 if (missing_target(obj_req)) {
29508e1e 286 fetch_alternates(alt->base);
e388ab74
NH
287 if (obj_req->repo->next != NULL) {
288 obj_req->repo =
289 obj_req->repo->next;
290 close(obj_req->local);
291 obj_req->local = -1;
292 start_object_request(obj_req);
29508e1e
NH
293 return;
294 }
295 }
296
e388ab74 297 finish_object_request(obj_req);
29508e1e
NH
298}
299
e388ab74 300static void release_object_request(struct object_request *obj_req)
1d389ab6 301{
e388ab74 302 struct object_request *entry = object_queue_head;
1d389ab6 303
e388ab74
NH
304 if (obj_req->local != -1)
305 error("fd leakage in release: %d", obj_req->local);
306 if (obj_req == object_queue_head) {
307 object_queue_head = obj_req->next;
1d389ab6 308 } else {
e388ab74 309 while (entry->next != NULL && entry->next != obj_req)
1d389ab6 310 entry = entry->next;
e388ab74 311 if (entry->next == obj_req)
1d389ab6
NH
312 entry->next = entry->next->next;
313 }
314
e388ab74
NH
315 free(obj_req->url);
316 free(obj_req);
1d389ab6
NH
317}
318
a7a8d378 319#ifdef USE_CURL_MULTI
fc57b6aa 320static int fill_active_slot(void *unused)
1d389ab6 321{
45c17412 322 struct object_request *obj_req;
1d389ab6 323
45c17412 324 for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
e388ab74
NH
325 if (obj_req->state == WAITING) {
326 if (has_sha1_file(obj_req->sha1))
09db444f 327 obj_req->state = COMPLETE;
45c17412 328 else {
e388ab74 329 start_object_request(obj_req);
45c17412
DB
330 return 1;
331 }
f1a906a3 332 }
8fcf7f9a 333 }
45c17412 334 return 0;
1d389ab6 335}
a7a8d378 336#endif
1d389ab6
NH
337
338void prefetch(unsigned char *sha1)
339{
e388ab74
NH
340 struct object_request *newreq;
341 struct object_request *tail;
1d389ab6
NH
342 char *filename = sha1_file_name(sha1);
343
344 newreq = xmalloc(sizeof(*newreq));
e702496e 345 hashcpy(newreq->sha1, sha1);
1d389ab6
NH
346 newreq->repo = alt;
347 newreq->url = NULL;
348 newreq->local = -1;
349 newreq->state = WAITING;
350 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
351 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
352 "%s.temp", filename);
e8dff6ba 353 newreq->slot = NULL;
1d389ab6
NH
354 newreq->next = NULL;
355
e388ab74
NH
356 if (object_queue_head == NULL) {
357 object_queue_head = newreq;
1d389ab6 358 } else {
e388ab74 359 tail = object_queue_head;
1d389ab6
NH
360 while (tail->next != NULL) {
361 tail = tail->next;
362 }
363 tail->next = newreq;
364 }
29508e1e 365
a7a8d378 366#ifdef USE_CURL_MULTI
29508e1e
NH
367 fill_active_slots();
368 step_active_slots();
a7a8d378 369#endif
1d389ab6
NH
370}
371
b3661567 372static int fetch_index(struct alt_base *repo, unsigned char *sha1)
182005b9 373{
1d389ab6 374 char *hex = sha1_to_hex(sha1);
182005b9
DB
375 char *filename;
376 char *url;
49a0f240 377 char tmpfile[PATH_MAX];
49a0f240
NH
378 long prev_posn = 0;
379 char range[RANGE_HEADER_SIZE];
380 struct curl_slist *range_header = NULL;
182005b9
DB
381
382 FILE *indexfile;
1d389ab6 383 struct active_request_slot *slot;
cb754fdf 384 struct slot_results results;
182005b9
DB
385
386 if (has_pack_index(sha1))
387 return 0;
388
389 if (get_verbosely)
1d389ab6 390 fprintf(stderr, "Getting index for pack %s\n", hex);
8fcf7f9a 391
b3661567 392 url = xmalloc(strlen(repo->base) + 64);
1d389ab6 393 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
8fcf7f9a 394
182005b9 395 filename = sha1_pack_index_name(sha1);
49a0f240
NH
396 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
397 indexfile = fopen(tmpfile, "a");
182005b9
DB
398 if (!indexfile)
399 return error("Unable to open local file %s for pack index",
400 filename);
401
1d389ab6 402 slot = get_active_slot();
c8568e13 403 slot->results = &results;
1d389ab6
NH
404 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
405 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
406 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 407 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
408 slot->local = indexfile;
409
49a0f240
NH
410 /* If there is data present from a previous transfer attempt,
411 resume where it left off */
412 prev_posn = ftell(indexfile);
413 if (prev_posn>0) {
414 if (get_verbosely)
415 fprintf(stderr,
416 "Resuming fetch of index for pack %s at byte %ld\n",
1d389ab6 417 hex, prev_posn);
49a0f240
NH
418 sprintf(range, "Range: bytes=%ld-", prev_posn);
419 range_header = curl_slist_append(range_header, range);
1d389ab6 420 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
421 }
422
1d389ab6
NH
423 if (start_active_slot(slot)) {
424 run_active_slot(slot);
c8568e13 425 if (results.curl_result != CURLE_OK) {
1d389ab6
NH
426 fclose(indexfile);
427 return error("Unable to get pack index %s\n%s", url,
428 curl_errorstr);
429 }
430 } else {
313c4714 431 fclose(indexfile);
1d389ab6 432 return error("Unable to start request");
182005b9
DB
433 }
434
435 fclose(indexfile);
49a0f240 436
b721e01f 437 return move_temp_to_file(tmpfile, filename);
182005b9
DB
438}
439
b3661567 440static int setup_index(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
441{
442 struct packed_git *new_pack;
443 if (has_pack_file(sha1))
a9486b02 444 return 0; /* don't list this as something we can get */
182005b9 445
b3661567 446 if (fetch_index(repo, sha1))
182005b9
DB
447 return -1;
448
449 new_pack = parse_pack_index(sha1);
b3661567
DB
450 new_pack->next = repo->packs;
451 repo->packs = new_pack;
182005b9
DB
452 return 0;
453}
454
e388ab74 455static void process_alternates_response(void *callback_data)
b3661567 456{
e388ab74
NH
457 struct alternates_request *alt_req =
458 (struct alternates_request *)callback_data;
acc075a8 459 struct active_request_slot *slot = alt_req->slot;
1d389ab6 460 struct alt_base *tail = alt;
8e29f6a0 461 const char *base = alt_req->base;
bc8f2652 462 static const char null_byte = '\0';
acc075a8
NH
463 char *data;
464 int i = 0;
1d389ab6 465
acc075a8
NH
466 if (alt_req->http_specific) {
467 if (slot->curl_result != CURLE_OK ||
468 !alt_req->buffer->posn) {
469
470 /* Try reusing the slot to get non-http alternates */
471 alt_req->http_specific = 0;
472 sprintf(alt_req->url, "%s/objects/info/alternates",
473 base);
474 curl_easy_setopt(slot->curl, CURLOPT_URL,
475 alt_req->url);
476 active_requests++;
477 slot->in_use = 1;
c9826473
NH
478 if (slot->finished != NULL)
479 (*slot->finished) = 0;
a3f583cb 480 if (!start_active_slot(slot)) {
acc075a8 481 got_alternates = -1;
29508e1e 482 slot->in_use = 0;
c9826473
NH
483 if (slot->finished != NULL)
484 (*slot->finished) = 1;
1d389ab6 485 }
a3f583cb 486 return;
b3661567 487 }
acc075a8 488 } else if (slot->curl_result != CURLE_OK) {
be4a015b 489 if (!missing_target(slot)) {
acc075a8
NH
490 got_alternates = -1;
491 return;
492 }
b3661567
DB
493 }
494
29508e1e 495 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
acc075a8
NH
496 alt_req->buffer->posn--;
497 data = alt_req->buffer->buffer;
1b0c1e67 498
acc075a8 499 while (i < alt_req->buffer->posn) {
b3661567 500 int posn = i;
acc075a8 501 while (posn < alt_req->buffer->posn && data[posn] != '\n')
b3661567
DB
502 posn++;
503 if (data[posn] == '\n') {
1b0c1e67
DB
504 int okay = 0;
505 int serverlen = 0;
506 struct alt_base *newalt;
507 char *target = NULL;
b3661567 508 if (data[i] == '/') {
5df1e0d0
JH
509 /* This counts
510 * http://git.host/pub/scm/linux.git/
511 * -----------here^
512 * so memcpy(dst, base, serverlen) will
513 * copy up to "...git.host".
514 */
515 const char *colon_ss = strstr(base,"://");
516 if (colon_ss) {
517 serverlen = (strchr(colon_ss + 3, '/')
518 - base);
519 okay = 1;
520 }
1b0c1e67 521 } else if (!memcmp(data + i, "../", 3)) {
5df1e0d0
JH
522 /* Relative URL; chop the corresponding
523 * number of subpath from base (and ../
524 * from data), and concatenate the result.
525 *
526 * The code first drops ../ from data, and
527 * then drops one ../ from data and one path
528 * from base. IOW, one extra ../ is dropped
529 * from data than path is dropped from base.
530 *
531 * This is not wrong. The alternate in
532 * http://git.host/pub/scm/linux.git/
533 * to borrow from
534 * http://git.host/pub/scm/linus.git/
535 * is ../../linus.git/objects/. You need
536 * two ../../ to borrow from your direct
537 * neighbour.
538 */
1b0c1e67
DB
539 i += 3;
540 serverlen = strlen(base);
8fcf7f9a 541 while (i + 2 < posn &&
1b0c1e67
DB
542 !memcmp(data + i, "../", 3)) {
543 do {
544 serverlen--;
545 } while (serverlen &&
546 base[serverlen - 1] != '/');
547 i += 3;
548 }
a9486b02 549 /* If the server got removed, give up. */
8fcf7f9a 550 okay = strchr(base, ':') - base + 3 <
1b0c1e67 551 serverlen;
acc075a8 552 } else if (alt_req->http_specific) {
1b0c1e67
DB
553 char *colon = strchr(data + i, ':');
554 char *slash = strchr(data + i, '/');
555 if (colon && slash && colon < data + posn &&
556 slash < data + posn && colon < slash) {
557 okay = 1;
558 }
559 }
5df1e0d0 560 /* skip "objects\n" at end */
1b0c1e67
DB
561 if (okay) {
562 target = xmalloc(serverlen + posn - i - 6);
5df1e0d0
JH
563 memcpy(target, base, serverlen);
564 memcpy(target + serverlen, data + i,
565 posn - i - 7);
566 target[serverlen + posn - i - 7] = 0;
b3661567 567 if (get_verbosely)
8fcf7f9a 568 fprintf(stderr,
b3661567
DB
569 "Also look at %s\n", target);
570 newalt = xmalloc(sizeof(*newalt));
1d389ab6 571 newalt->next = NULL;
b3661567
DB
572 newalt->base = target;
573 newalt->got_indices = 0;
574 newalt->packs = NULL;
8d9fbe57 575
1d389ab6
NH
576 while (tail->next != NULL)
577 tail = tail->next;
578 tail->next = newalt;
b3661567
DB
579 }
580 }
581 i = posn + 1;
582 }
bc8f2652 583
f7eb290f 584 got_alternates = 1;
acc075a8
NH
585}
586
8e29f6a0 587static void fetch_alternates(const char *base)
acc075a8
NH
588{
589 struct buffer buffer;
590 char *url;
591 char *data;
592 struct active_request_slot *slot;
cb754fdf 593 struct alternates_request alt_req;
acc075a8
NH
594
595 /* If another request has already started fetching alternates,
596 wait for them to arrive and return to processing this request's
597 curl message */
29508e1e 598#ifdef USE_CURL_MULTI
acc075a8 599 while (got_alternates == 0) {
29508e1e 600 step_active_slots();
acc075a8 601 }
29508e1e 602#endif
acc075a8
NH
603
604 /* Nothing to do if they've already been fetched */
605 if (got_alternates == 1)
606 return;
607
608 /* Start the fetch */
609 got_alternates = 0;
610
611 data = xmalloc(4096);
612 buffer.size = 4096;
613 buffer.posn = 0;
614 buffer.buffer = data;
615
616 if (get_verbosely)
617 fprintf(stderr, "Getting alternates list for %s\n", base);
8fcf7f9a 618
acc075a8
NH
619 url = xmalloc(strlen(base) + 31);
620 sprintf(url, "%s/objects/info/http-alternates", base);
621
622 /* Use a callback to process the result, since another request
623 may fail and need to have alternates loaded before continuing */
624 slot = get_active_slot();
e388ab74 625 slot->callback_func = process_alternates_response;
acc075a8
NH
626 slot->callback_data = &alt_req;
627
628 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 629 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
acc075a8
NH
630 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
631
632 alt_req.base = base;
633 alt_req.url = url;
634 alt_req.buffer = &buffer;
635 alt_req.http_specific = 1;
636 alt_req.slot = slot;
637
638 if (start_active_slot(slot))
639 run_active_slot(slot);
640 else
641 got_alternates = -1;
642
643 free(data);
644 free(url);
b3661567
DB
645}
646
647static int fetch_indices(struct alt_base *repo)
182005b9
DB
648{
649 unsigned char sha1[20];
650 char *url;
651 struct buffer buffer;
652 char *data;
653 int i = 0;
654
1d389ab6 655 struct active_request_slot *slot;
cb754fdf 656 struct slot_results results;
1d389ab6 657
b3661567 658 if (repo->got_indices)
182005b9
DB
659 return 0;
660
661 data = xmalloc(4096);
662 buffer.size = 4096;
663 buffer.posn = 0;
664 buffer.buffer = data;
665
666 if (get_verbosely)
6fd72e39 667 fprintf(stderr, "Getting pack list for %s\n", repo->base);
8fcf7f9a 668
b3661567
DB
669 url = xmalloc(strlen(repo->base) + 21);
670 sprintf(url, "%s/objects/info/packs", repo->base);
182005b9 671
1d389ab6 672 slot = get_active_slot();
c8568e13 673 slot->results = &results;
1d389ab6 674 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 675 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1d389ab6
NH
676 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
677 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
678 if (start_active_slot(slot)) {
679 run_active_slot(slot);
c8568e13 680 if (results.curl_result != CURLE_OK) {
be4a015b 681 if (missing_target(&results)) {
5e3a7691
NH
682 repo->got_indices = 1;
683 free(buffer.buffer);
684 return 0;
685 } else {
686 repo->got_indices = 0;
687 free(buffer.buffer);
688 return error("%s", curl_errorstr);
689 }
bc8f2652 690 }
1d389ab6 691 } else {
5e3a7691 692 repo->got_indices = 0;
bc8f2652 693 free(buffer.buffer);
1d389ab6
NH
694 return error("Unable to start request");
695 }
182005b9 696
bc8f2652 697 data = buffer.buffer;
b3661567 698 while (i < buffer.posn) {
182005b9
DB
699 switch (data[i]) {
700 case 'P':
701 i++;
455c161c 702 if (i + 52 <= buffer.posn &&
cc44c765 703 !prefixcmp(data + i, " pack-") &&
1968d77d 704 !prefixcmp(data + i + 46, ".pack\n")) {
182005b9 705 get_sha1_hex(data + i + 6, sha1);
b3661567 706 setup_index(repo, sha1);
182005b9
DB
707 i += 51;
708 break;
709 }
710 default:
455c161c 711 while (i < buffer.posn && data[i] != '\n')
182005b9
DB
712 i++;
713 }
714 i++;
b3661567 715 }
182005b9 716
bc8f2652 717 free(buffer.buffer);
b3661567 718 repo->got_indices = 1;
182005b9
DB
719 return 0;
720}
721
b3661567 722static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
723{
724 char *url;
725 struct packed_git *target;
726 struct packed_git **lst;
727 FILE *packfile;
728 char *filename;
49a0f240
NH
729 char tmpfile[PATH_MAX];
730 int ret;
731 long prev_posn = 0;
732 char range[RANGE_HEADER_SIZE];
733 struct curl_slist *range_header = NULL;
1d389ab6
NH
734
735 struct active_request_slot *slot;
cb754fdf 736 struct slot_results results;
182005b9 737
b3661567 738 if (fetch_indices(repo))
182005b9 739 return -1;
b3661567 740 target = find_sha1_pack(sha1, repo->packs);
182005b9 741 if (!target)
b3661567 742 return -1;
182005b9
DB
743
744 if (get_verbosely) {
745 fprintf(stderr, "Getting pack %s\n",
746 sha1_to_hex(target->sha1));
747 fprintf(stderr, " which contains %s\n",
748 sha1_to_hex(sha1));
749 }
750
b3661567 751 url = xmalloc(strlen(repo->base) + 65);
182005b9 752 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 753 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
754
755 filename = sha1_pack_name(target->sha1);
49a0f240
NH
756 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
757 packfile = fopen(tmpfile, "a");
182005b9
DB
758 if (!packfile)
759 return error("Unable to open local file %s for pack",
760 filename);
761
1d389ab6 762 slot = get_active_slot();
c8568e13 763 slot->results = &results;
1d389ab6
NH
764 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
765 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
766 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
03126006 767 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 768 slot->local = packfile;
1ddea77e 769
49a0f240
NH
770 /* If there is data present from a previous transfer attempt,
771 resume where it left off */
772 prev_posn = ftell(packfile);
773 if (prev_posn>0) {
774 if (get_verbosely)
775 fprintf(stderr,
776 "Resuming fetch of pack %s at byte %ld\n",
777 sha1_to_hex(target->sha1), prev_posn);
778 sprintf(range, "Range: bytes=%ld-", prev_posn);
779 range_header = curl_slist_append(range_header, range);
1d389ab6 780 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
781 }
782
1d389ab6
NH
783 if (start_active_slot(slot)) {
784 run_active_slot(slot);
c8568e13 785 if (results.curl_result != CURLE_OK) {
1d389ab6
NH
786 fclose(packfile);
787 return error("Unable to get pack file %s\n%s", url,
788 curl_errorstr);
789 }
790 } else {
313c4714 791 fclose(packfile);
1d389ab6 792 return error("Unable to start request");
182005b9
DB
793 }
794
1c23d794 795 target->pack_size = ftell(packfile);
182005b9
DB
796 fclose(packfile);
797
b721e01f 798 ret = move_temp_to_file(tmpfile, filename);
49a0f240 799 if (ret)
b721e01f 800 return ret;
49a0f240 801
b3661567 802 lst = &repo->packs;
182005b9
DB
803 while (*lst != target)
804 lst = &((*lst)->next);
805 *lst = (*lst)->next;
806
271421cd
JH
807 if (verify_pack(target, 0))
808 return -1;
182005b9
DB
809 install_packed_git(target);
810
811 return 0;
812}
813
53f31389
MW
814static void abort_object_request(struct object_request *obj_req)
815{
816 if (obj_req->local >= 0) {
817 close(obj_req->local);
818 obj_req->local = -1;
819 }
820 unlink(obj_req->tmpfile);
821 if (obj_req->slot) {
a6080a0a 822 release_active_slot(obj_req->slot);
53f31389
MW
823 obj_req->slot = NULL;
824 }
825 release_object_request(obj_req);
826}
827
a7928f8e 828static int fetch_object(struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
829{
830 char *hex = sha1_to_hex(sha1);
29508e1e 831 int ret = 0;
e388ab74 832 struct object_request *obj_req = object_queue_head;
1d389ab6 833
a89fccd2 834 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
e388ab74
NH
835 obj_req = obj_req->next;
836 if (obj_req == NULL)
1d389ab6
NH
837 return error("Couldn't find request for %s in the queue", hex);
838
e388ab74 839 if (has_sha1_file(obj_req->sha1)) {
53f31389 840 abort_object_request(obj_req);
11f0dafe
NH
841 return 0;
842 }
843
a7a8d378 844#ifdef USE_CURL_MULTI
e388ab74 845 while (obj_req->state == WAITING) {
29508e1e 846 step_active_slots();
1d389ab6 847 }
a7a8d378 848#else
e388ab74 849 start_object_request(obj_req);
a7a8d378 850#endif
6eb7ed54 851
e388ab74
NH
852 while (obj_req->state == ACTIVE) {
853 run_active_slot(obj_req->slot);
a7a8d378 854 }
e388ab74
NH
855 if (obj_req->local != -1) {
856 close(obj_req->local); obj_req->local = -1;
313c4714 857 }
6eb7ed54 858
e388ab74 859 if (obj_req->state == ABORTED) {
29508e1e 860 ret = error("Request for %s aborted", hex);
e388ab74
NH
861 } else if (obj_req->curl_result != CURLE_OK &&
862 obj_req->http_code != 416) {
be4a015b 863 if (missing_target(obj_req))
e2029eb9
PB
864 ret = -1; /* Be silent, it is probably in a pack. */
865 else
866 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
e388ab74
NH
867 obj_req->errorstr, obj_req->curl_result,
868 obj_req->http_code, hex);
869 } else if (obj_req->zret != Z_STREAM_END) {
bb528079 870 corrupt_object_found++;
bd2afde8 871 ret = error("File %s (%s) corrupt", hex, obj_req->url);
a89fccd2 872 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
bd2afde8 873 ret = error("File %s has bad hash", hex);
e388ab74 874 } else if (obj_req->rename < 0) {
7b934ec0
MW
875 ret = error("unable to write sha1 filename %s",
876 obj_req->filename);
6eb7ed54 877 }
49a0f240 878
e388ab74 879 release_object_request(obj_req);
29508e1e 880 return ret;
6eb7ed54
DB
881}
882
b3661567
DB
883int fetch(unsigned char *sha1)
884{
885 struct alt_base *altbase = alt;
1d389ab6
NH
886
887 if (!fetch_object(altbase, sha1))
888 return 0;
b3661567 889 while (altbase) {
b3661567
DB
890 if (!fetch_pack(altbase, sha1))
891 return 0;
f7eb290f 892 fetch_alternates(alt->base);
b3661567
DB
893 altbase = altbase->next;
894 }
bd2afde8 895 return error("Unable to find %s under %s", sha1_to_hex(sha1),
1d389ab6 896 alt->base);
b3661567
DB
897}
898
94fa447a
JH
899static inline int needs_quote(int ch)
900{
cfd432e6
FF
901 if (((ch >= 'A') && (ch <= 'Z'))
902 || ((ch >= 'a') && (ch <= 'z'))
903 || ((ch >= '0') && (ch <= '9'))
904 || (ch == '/')
905 || (ch == '-')
906 || (ch == '.'))
94fa447a 907 return 0;
cfd432e6 908 return 1;
94fa447a
JH
909}
910
911static inline int hex(int v)
912{
913 if (v < 10) return '0' + v;
914 else return 'A' + v - 10;
915}
916
917static char *quote_ref_url(const char *base, const char *ref)
918{
919 const char *cp;
920 char *dp, *qref;
921 int len, baselen, ch;
922
923 baselen = strlen(base);
2afea3bc 924 len = baselen + 7; /* "/refs/" + NUL */
94fa447a
JH
925 for (cp = ref; (ch = *cp) != 0; cp++, len++)
926 if (needs_quote(ch))
927 len += 2; /* extra two hex plus replacement % */
928 qref = xmalloc(len);
929 memcpy(qref, base, baselen);
2afea3bc
GP
930 memcpy(qref + baselen, "/refs/", 6);
931 for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
94fa447a
JH
932 if (needs_quote(ch)) {
933 *dp++ = '%';
934 *dp++ = hex((ch >> 4) & 0xF);
935 *dp++ = hex(ch & 0xF);
936 }
937 else
938 *dp++ = ch;
939 }
940 *dp = 0;
941
942 return qref;
943}
944
cd541a68
DB
945int fetch_ref(char *ref, unsigned char *sha1)
946{
94fa447a 947 char *url;
fa3e0655
DB
948 char hex[42];
949 struct buffer buffer;
8e29f6a0 950 const char *base = alt->base;
1d389ab6 951 struct active_request_slot *slot;
cb754fdf 952 struct slot_results results;
fa3e0655
DB
953 buffer.size = 41;
954 buffer.posn = 0;
955 buffer.buffer = hex;
956 hex[41] = '\0';
8fcf7f9a 957
94fa447a 958 url = quote_ref_url(base, ref);
1d389ab6 959 slot = get_active_slot();
c8568e13 960 slot->results = &results;
1d389ab6
NH
961 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
962 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
963 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
964 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
965 if (start_active_slot(slot)) {
966 run_active_slot(slot);
c8568e13 967 if (results.curl_result != CURLE_OK)
1d389ab6
NH
968 return error("Couldn't get %s for %s\n%s",
969 url, ref, curl_errorstr);
970 } else {
971 return error("Unable to start request");
972 }
fa3e0655
DB
973
974 hex[40] = '\0';
975 get_sha1_hex(hex, sha1);
976 return 0;
cd541a68
DB
977}
978
8e29f6a0 979int main(int argc, const char **argv)
6eb7ed54 980{
8e29f6a0
PB
981 int commits;
982 const char **write_ref = NULL;
983 char **commit_id;
984 const char *url;
9c880b3e 985 char *s;
6eb7ed54 986 int arg = 1;
7b9ae53e 987 int rc = 0;
6eb7ed54 988
5a327713 989 setup_git_directory();
d0740d92 990 git_config(git_default_config);
5a327713 991
6eb7ed54
DB
992 while (arg < argc && argv[arg][0] == '-') {
993 if (argv[arg][1] == 't') {
4250a5e5 994 get_tree = 1;
6eb7ed54 995 } else if (argv[arg][1] == 'c') {
4250a5e5 996 get_history = 1;
6eb7ed54 997 } else if (argv[arg][1] == 'a') {
4250a5e5
DB
998 get_all = 1;
999 get_tree = 1;
1000 get_history = 1;
e78d9772
JH
1001 } else if (argv[arg][1] == 'v') {
1002 get_verbosely = 1;
fa3e0655 1003 } else if (argv[arg][1] == 'w') {
8e29f6a0 1004 write_ref = &argv[arg + 1];
fa3e0655 1005 arg++;
820eca68
DB
1006 } else if (!strcmp(argv[arg], "--recover")) {
1007 get_recover = 1;
8e29f6a0
PB
1008 } else if (!strcmp(argv[arg], "--stdin")) {
1009 commits_on_stdin = 1;
6eb7ed54
DB
1010 }
1011 arg++;
1012 }
8e29f6a0
PB
1013 if (argc < arg + 2 - commits_on_stdin) {
1014 usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
6eb7ed54
DB
1015 return 1;
1016 }
8e29f6a0
PB
1017 if (commits_on_stdin) {
1018 commits = pull_targets_stdin(&commit_id, &write_ref);
1019 } else {
1020 commit_id = (char **) &argv[arg++];
1021 commits = 1;
1022 }
1023 url = argv[arg];
6eb7ed54 1024
29508e1e 1025 http_init();
d402d556 1026
1db69b57 1027 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
3dcb90f5 1028
b3661567 1029 alt = xmalloc(sizeof(*alt));
2afea3bc
GP
1030 alt->base = xmalloc(strlen(url) + 1);
1031 strcpy(alt->base, url);
9c880b3e
GP
1032 for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s)
1033 *s = 0;
b3661567
DB
1034 alt->got_indices = 0;
1035 alt->packs = NULL;
1036 alt->next = NULL;
6eb7ed54 1037
fc57b6aa
DB
1038#ifdef USE_CURL_MULTI
1039 add_fill_function(NULL, fill_active_slot);
1040#endif
1041
8e29f6a0 1042 if (pull(commits, commit_id, write_ref, url))
7b9ae53e 1043 rc = 1;
6eb7ed54 1044
29508e1e
NH
1045 http_cleanup();
1046
07001f95
SE
1047 curl_slist_free_all(no_pragma_header);
1048
8e29f6a0
PB
1049 if (commits_on_stdin)
1050 pull_targets_free(commits, commit_id, write_ref);
1051
bb528079
JH
1052 if (corrupt_object_found) {
1053 fprintf(stderr,
1054"Some loose object were found to be corrupt, but they might be just\n"
1055"a false '404 Not Found' error message sent with incorrect HTTP\n"
df391b19 1056"status code. Suggest running git-fsck.\n");
bb528079 1057 }
7b9ae53e 1058 return rc;
6eb7ed54 1059}