http: use new http API in fetch_index()
[git/git.git] / http-walker.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
271421cd 3#include "pack.h"
30ae764b 4#include "walker.h"
29508e1e 5#include "http.h"
7baa3e86 6
49a0f240 7#define PREV_BUF_SIZE 4096
49a0f240 8
b3661567
DB
9struct alt_base
10{
2afea3bc 11 char *base;
b3661567
DB
12 int got_indices;
13 struct packed_git *packs;
14 struct alt_base *next;
15};
16
e388ab74 17enum object_request_state {
1d389ab6
NH
18 WAITING,
19 ABORTED,
20 ACTIVE,
21 COMPLETE,
22};
6eb7ed54 23
e388ab74 24struct object_request
1d389ab6 25{
30ae764b 26 struct walker *walker;
1d389ab6
NH
27 unsigned char sha1[20];
28 struct alt_base *repo;
29 char *url;
30 char filename[PATH_MAX];
31 char tmpfile[PATH_MAX];
32 int local;
e388ab74 33 enum object_request_state state;
1d389ab6
NH
34 CURLcode curl_result;
35 char errorstr[CURL_ERROR_SIZE];
36 long http_code;
37 unsigned char real_sha1[20];
9126f009 38 git_SHA_CTX c;
1d389ab6
NH
39 z_stream stream;
40 int zret;
41 int rename;
42 struct active_request_slot *slot;
e388ab74 43 struct object_request *next;
1d389ab6
NH
44};
45
e388ab74 46struct alternates_request {
30ae764b 47 struct walker *walker;
8e29f6a0 48 const char *base;
acc075a8 49 char *url;
028c2976 50 struct strbuf *buffer;
acc075a8
NH
51 struct active_request_slot *slot;
52 int http_specific;
53};
54
30ae764b
DB
55struct walker_data {
56 const char *url;
57 int got_alternates;
58 struct alt_base *alt;
30ae764b
DB
59};
60
96f1e58f 61static struct object_request *object_queue_head;
bc8f2652 62
182005b9
DB
63static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
64 void *data)
65{
bf0f910d 66 unsigned char expn[4096];
6eb7ed54
DB
67 size_t size = eltsize * nmemb;
68 int posn = 0;
e388ab74 69 struct object_request *obj_req = (struct object_request *)data;
6eb7ed54 70 do {
93822c22 71 ssize_t retval = xwrite(obj_req->local,
4c42aa1a 72 (char *) ptr + posn, size - posn);
6eb7ed54
DB
73 if (retval < 0)
74 return posn;
75 posn += retval;
76 } while (posn < size);
77
e388ab74
NH
78 obj_req->stream.avail_in = size;
79 obj_req->stream.next_in = ptr;
6eb7ed54 80 do {
e388ab74
NH
81 obj_req->stream.next_out = expn;
82 obj_req->stream.avail_out = sizeof(expn);
39c68542 83 obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);
9126f009 84 git_SHA1_Update(&obj_req->c, expn,
4c42aa1a 85 sizeof(expn) - obj_req->stream.avail_out);
e388ab74 86 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
1d389ab6 87 data_received++;
6eb7ed54
DB
88 return size;
89}
90
30ae764b 91static void fetch_alternates(struct walker *walker, const char *base);
1d389ab6 92
29508e1e 93static void process_object_response(void *callback_data);
1d389ab6 94
30ae764b
DB
95static void start_object_request(struct walker *walker,
96 struct object_request *obj_req)
1d389ab6 97{
e388ab74 98 char *hex = sha1_to_hex(obj_req->sha1);
1d389ab6
NH
99 char prevfile[PATH_MAX];
100 char *url;
101 char *posn;
102 int prevlocal;
103 unsigned char prev_buf[PREV_BUF_SIZE];
104 ssize_t prev_read = 0;
105 long prev_posn = 0;
106 char range[RANGE_HEADER_SIZE];
107 struct curl_slist *range_header = NULL;
108 struct active_request_slot *slot;
109
e388ab74 110 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
691f1a28 111 unlink_or_warn(prevfile);
e388ab74 112 rename(obj_req->tmpfile, prevfile);
691f1a28 113 unlink_or_warn(obj_req->tmpfile);
1d389ab6 114
e388ab74
NH
115 if (obj_req->local != -1)
116 error("fd leakage in start: %d", obj_req->local);
117 obj_req->local = open(obj_req->tmpfile,
1d389ab6 118 O_WRONLY | O_CREAT | O_EXCL, 0666);
4c42aa1a
TRC
119 /*
120 * This could have failed due to the "lazy directory creation";
b721e01f
JH
121 * try to mkdir the last path component.
122 */
e388ab74
NH
123 if (obj_req->local < 0 && errno == ENOENT) {
124 char *dir = strrchr(obj_req->tmpfile, '/');
b721e01f
JH
125 if (dir) {
126 *dir = 0;
e388ab74 127 mkdir(obj_req->tmpfile, 0777);
b721e01f
JH
128 *dir = '/';
129 }
e388ab74 130 obj_req->local = open(obj_req->tmpfile,
b721e01f
JH
131 O_WRONLY | O_CREAT | O_EXCL, 0666);
132 }
133
e388ab74
NH
134 if (obj_req->local < 0) {
135 obj_req->state = ABORTED;
bd2afde8 136 error("Couldn't create temporary file %s for %s: %s",
e388ab74 137 obj_req->tmpfile, obj_req->filename, strerror(errno));
1d389ab6
NH
138 return;
139 }
140
e388ab74 141 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
1d389ab6 142
39c68542 143 git_inflate_init(&obj_req->stream);
1d389ab6 144
9126f009 145 git_SHA1_Init(&obj_req->c);
1d389ab6 146
2afea3bc
GP
147 url = xmalloc(strlen(obj_req->repo->base) + 51);
148 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
e388ab74
NH
149 strcpy(url, obj_req->repo->base);
150 posn = url + strlen(obj_req->repo->base);
2afea3bc
GP
151 strcpy(posn, "/objects/");
152 posn += 9;
1d389ab6
NH
153 memcpy(posn, hex, 2);
154 posn += 2;
155 *(posn++) = '/';
156 strcpy(posn, hex + 2);
e388ab74 157 strcpy(obj_req->url, url);
1d389ab6 158
4c42aa1a
TRC
159 /*
160 * If a previous temp file is present, process what was already
161 * fetched.
162 */
1d389ab6
NH
163 prevlocal = open(prevfile, O_RDONLY);
164 if (prevlocal != -1) {
165 do {
93d26e4c 166 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1d389ab6
NH
167 if (prev_read>0) {
168 if (fwrite_sha1_file(prev_buf,
169 1,
170 prev_read,
4c42aa1a 171 obj_req) == prev_read)
1d389ab6 172 prev_posn += prev_read;
4c42aa1a 173 else
1d389ab6 174 prev_read = -1;
1d389ab6
NH
175 }
176 } while (prev_read > 0);
177 close(prevlocal);
178 }
691f1a28 179 unlink_or_warn(prevfile);
1d389ab6 180
4c42aa1a
TRC
181 /*
182 * Reset inflate/SHA1 if there was an error reading the previous temp
183 * file; also rewind to the beginning of the local file.
184 */
1d389ab6 185 if (prev_read == -1) {
e388ab74 186 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
39c68542 187 git_inflate_init(&obj_req->stream);
9126f009 188 git_SHA1_Init(&obj_req->c);
1d389ab6
NH
189 if (prev_posn>0) {
190 prev_posn = 0;
b5da2467 191 lseek(obj_req->local, 0, SEEK_SET);
e388ab74 192 ftruncate(obj_req->local, 0);
1d389ab6
NH
193 }
194 }
195
196 slot = get_active_slot();
29508e1e 197 slot->callback_func = process_object_response;
e388ab74
NH
198 slot->callback_data = obj_req;
199 obj_req->slot = slot;
29508e1e 200
e388ab74 201 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
1d389ab6 202 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
e388ab74 203 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
1d389ab6 204 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
e9176745 205 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 206
4c42aa1a
TRC
207 /*
208 * If we have successfully processed data from a previous fetch
209 * attempt, only fetch the data we don't already have.
210 */
1d389ab6 211 if (prev_posn>0) {
30ae764b 212 if (walker->get_verbosely)
1d389ab6
NH
213 fprintf(stderr,
214 "Resuming fetch of object %s at byte %ld\n",
215 hex, prev_posn);
216 sprintf(range, "Range: bytes=%ld-", prev_posn);
217 range_header = curl_slist_append(range_header, range);
218 curl_easy_setopt(slot->curl,
219 CURLOPT_HTTPHEADER, range_header);
220 }
221
a7a8d378 222 /* Try to get the request started, abort the request on error */
e388ab74 223 obj_req->state = ACTIVE;
1d389ab6 224 if (!start_active_slot(slot)) {
e388ab74
NH
225 obj_req->state = ABORTED;
226 obj_req->slot = NULL;
4c42aa1a
TRC
227 close(obj_req->local);
228 obj_req->local = -1;
e388ab74
NH
229 free(obj_req->url);
230 return;
1d389ab6 231 }
1d389ab6
NH
232}
233
e388ab74 234static void finish_object_request(struct object_request *obj_req)
1d389ab6 235{
50496b21
NH
236 struct stat st;
237
4c42aa1a
TRC
238 close(obj_req->local);
239 obj_req->local = -1;
1d389ab6 240
e388ab74 241 if (obj_req->http_code == 416) {
1d389ab6 242 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
e388ab74
NH
243 } else if (obj_req->curl_result != CURLE_OK) {
244 if (stat(obj_req->tmpfile, &st) == 0)
50496b21 245 if (st.st_size == 0)
691f1a28 246 unlink_or_warn(obj_req->tmpfile);
1d389ab6
NH
247 return;
248 }
249
39c68542 250 git_inflate_end(&obj_req->stream);
9126f009 251 git_SHA1_Final(obj_req->real_sha1, &obj_req->c);
e388ab74 252 if (obj_req->zret != Z_STREAM_END) {
691f1a28 253 unlink_or_warn(obj_req->tmpfile);
1d389ab6
NH
254 return;
255 }
a89fccd2 256 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
691f1a28 257 unlink_or_warn(obj_req->tmpfile);
1d389ab6
NH
258 return;
259 }
e388ab74
NH
260 obj_req->rename =
261 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
1d389ab6 262
e388ab74 263 if (obj_req->rename == 0)
30ae764b 264 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
1d389ab6
NH
265}
266
29508e1e
NH
267static void process_object_response(void *callback_data)
268{
e388ab74
NH
269 struct object_request *obj_req =
270 (struct object_request *)callback_data;
30ae764b
DB
271 struct walker *walker = obj_req->walker;
272 struct walker_data *data = walker->data;
273 struct alt_base *alt = data->alt;
29508e1e 274
e388ab74
NH
275 obj_req->curl_result = obj_req->slot->curl_result;
276 obj_req->http_code = obj_req->slot->http_code;
277 obj_req->slot = NULL;
278 obj_req->state = COMPLETE;
29508e1e
NH
279
280 /* Use alternates if necessary */
be4a015b 281 if (missing_target(obj_req)) {
30ae764b 282 fetch_alternates(walker, alt->base);
e388ab74
NH
283 if (obj_req->repo->next != NULL) {
284 obj_req->repo =
285 obj_req->repo->next;
286 close(obj_req->local);
287 obj_req->local = -1;
30ae764b 288 start_object_request(walker, obj_req);
29508e1e
NH
289 return;
290 }
291 }
292
e388ab74 293 finish_object_request(obj_req);
29508e1e
NH
294}
295
e388ab74 296static void release_object_request(struct object_request *obj_req)
1d389ab6 297{
e388ab74 298 struct object_request *entry = object_queue_head;
1d389ab6 299
e388ab74
NH
300 if (obj_req->local != -1)
301 error("fd leakage in release: %d", obj_req->local);
302 if (obj_req == object_queue_head) {
303 object_queue_head = obj_req->next;
1d389ab6 304 } else {
e388ab74 305 while (entry->next != NULL && entry->next != obj_req)
1d389ab6 306 entry = entry->next;
e388ab74 307 if (entry->next == obj_req)
1d389ab6
NH
308 entry->next = entry->next->next;
309 }
310
e388ab74
NH
311 free(obj_req->url);
312 free(obj_req);
1d389ab6
NH
313}
314
a7a8d378 315#ifdef USE_CURL_MULTI
30ae764b 316static int fill_active_slot(struct walker *walker)
1d389ab6 317{
45c17412 318 struct object_request *obj_req;
1d389ab6 319
45c17412 320 for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
e388ab74
NH
321 if (obj_req->state == WAITING) {
322 if (has_sha1_file(obj_req->sha1))
09db444f 323 obj_req->state = COMPLETE;
45c17412 324 else {
30ae764b 325 start_object_request(walker, obj_req);
45c17412
DB
326 return 1;
327 }
f1a906a3 328 }
8fcf7f9a 329 }
45c17412 330 return 0;
1d389ab6 331}
a7a8d378 332#endif
1d389ab6 333
30ae764b 334static void prefetch(struct walker *walker, unsigned char *sha1)
1d389ab6 335{
e388ab74
NH
336 struct object_request *newreq;
337 struct object_request *tail;
30ae764b 338 struct walker_data *data = walker->data;
1d389ab6
NH
339 char *filename = sha1_file_name(sha1);
340
341 newreq = xmalloc(sizeof(*newreq));
30ae764b 342 newreq->walker = walker;
e702496e 343 hashcpy(newreq->sha1, sha1);
30ae764b 344 newreq->repo = data->alt;
1d389ab6
NH
345 newreq->url = NULL;
346 newreq->local = -1;
347 newreq->state = WAITING;
348 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
349 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
350 "%s.temp", filename);
e8dff6ba 351 newreq->slot = NULL;
1d389ab6
NH
352 newreq->next = NULL;
353
e9176745
TRC
354 http_is_verbose = walker->get_verbosely;
355
e388ab74
NH
356 if (object_queue_head == NULL) {
357 object_queue_head = newreq;
1d389ab6 358 } else {
e388ab74 359 tail = object_queue_head;
4c42aa1a 360 while (tail->next != NULL)
1d389ab6 361 tail = tail->next;
1d389ab6
NH
362 tail->next = newreq;
363 }
29508e1e 364
a7a8d378 365#ifdef USE_CURL_MULTI
29508e1e
NH
366 fill_active_slots();
367 step_active_slots();
a7a8d378 368#endif
1d389ab6
NH
369}
370
e388ab74 371static void process_alternates_response(void *callback_data)
b3661567 372{
e388ab74
NH
373 struct alternates_request *alt_req =
374 (struct alternates_request *)callback_data;
30ae764b
DB
375 struct walker *walker = alt_req->walker;
376 struct walker_data *cdata = walker->data;
acc075a8 377 struct active_request_slot *slot = alt_req->slot;
30ae764b 378 struct alt_base *tail = cdata->alt;
8e29f6a0 379 const char *base = alt_req->base;
bc8f2652 380 static const char null_byte = '\0';
acc075a8
NH
381 char *data;
382 int i = 0;
1d389ab6 383
acc075a8
NH
384 if (alt_req->http_specific) {
385 if (slot->curl_result != CURLE_OK ||
028c2976 386 !alt_req->buffer->len) {
acc075a8
NH
387
388 /* Try reusing the slot to get non-http alternates */
389 alt_req->http_specific = 0;
390 sprintf(alt_req->url, "%s/objects/info/alternates",
391 base);
392 curl_easy_setopt(slot->curl, CURLOPT_URL,
393 alt_req->url);
394 active_requests++;
395 slot->in_use = 1;
c9826473
NH
396 if (slot->finished != NULL)
397 (*slot->finished) = 0;
a3f583cb 398 if (!start_active_slot(slot)) {
30ae764b 399 cdata->got_alternates = -1;
29508e1e 400 slot->in_use = 0;
c9826473
NH
401 if (slot->finished != NULL)
402 (*slot->finished) = 1;
1d389ab6 403 }
a3f583cb 404 return;
b3661567 405 }
acc075a8 406 } else if (slot->curl_result != CURLE_OK) {
be4a015b 407 if (!missing_target(slot)) {
30ae764b 408 cdata->got_alternates = -1;
acc075a8
NH
409 return;
410 }
b3661567
DB
411 }
412
29508e1e 413 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
028c2976
MH
414 alt_req->buffer->len--;
415 data = alt_req->buffer->buf;
1b0c1e67 416
028c2976 417 while (i < alt_req->buffer->len) {
b3661567 418 int posn = i;
028c2976 419 while (posn < alt_req->buffer->len && data[posn] != '\n')
b3661567
DB
420 posn++;
421 if (data[posn] == '\n') {
1b0c1e67
DB
422 int okay = 0;
423 int serverlen = 0;
424 struct alt_base *newalt;
425 char *target = NULL;
b3661567 426 if (data[i] == '/') {
4c42aa1a
TRC
427 /*
428 * This counts
5df1e0d0
JH
429 * http://git.host/pub/scm/linux.git/
430 * -----------here^
431 * so memcpy(dst, base, serverlen) will
432 * copy up to "...git.host".
433 */
434 const char *colon_ss = strstr(base,"://");
435 if (colon_ss) {
436 serverlen = (strchr(colon_ss + 3, '/')
437 - base);
438 okay = 1;
439 }
1b0c1e67 440 } else if (!memcmp(data + i, "../", 3)) {
4c42aa1a
TRC
441 /*
442 * Relative URL; chop the corresponding
5df1e0d0
JH
443 * number of subpath from base (and ../
444 * from data), and concatenate the result.
445 *
446 * The code first drops ../ from data, and
447 * then drops one ../ from data and one path
448 * from base. IOW, one extra ../ is dropped
449 * from data than path is dropped from base.
450 *
451 * This is not wrong. The alternate in
452 * http://git.host/pub/scm/linux.git/
453 * to borrow from
454 * http://git.host/pub/scm/linus.git/
455 * is ../../linus.git/objects/. You need
456 * two ../../ to borrow from your direct
457 * neighbour.
458 */
1b0c1e67
DB
459 i += 3;
460 serverlen = strlen(base);
8fcf7f9a 461 while (i + 2 < posn &&
1b0c1e67
DB
462 !memcmp(data + i, "../", 3)) {
463 do {
464 serverlen--;
465 } while (serverlen &&
466 base[serverlen - 1] != '/');
467 i += 3;
468 }
a9486b02 469 /* If the server got removed, give up. */
8fcf7f9a 470 okay = strchr(base, ':') - base + 3 <
4c42aa1a 471 serverlen;
acc075a8 472 } else if (alt_req->http_specific) {
1b0c1e67
DB
473 char *colon = strchr(data + i, ':');
474 char *slash = strchr(data + i, '/');
475 if (colon && slash && colon < data + posn &&
476 slash < data + posn && colon < slash) {
477 okay = 1;
478 }
479 }
5df1e0d0 480 /* skip "objects\n" at end */
1b0c1e67
DB
481 if (okay) {
482 target = xmalloc(serverlen + posn - i - 6);
5df1e0d0
JH
483 memcpy(target, base, serverlen);
484 memcpy(target + serverlen, data + i,
485 posn - i - 7);
486 target[serverlen + posn - i - 7] = 0;
30ae764b 487 if (walker->get_verbosely)
8fcf7f9a 488 fprintf(stderr,
b3661567
DB
489 "Also look at %s\n", target);
490 newalt = xmalloc(sizeof(*newalt));
1d389ab6 491 newalt->next = NULL;
b3661567
DB
492 newalt->base = target;
493 newalt->got_indices = 0;
494 newalt->packs = NULL;
8d9fbe57 495
1d389ab6
NH
496 while (tail->next != NULL)
497 tail = tail->next;
498 tail->next = newalt;
b3661567
DB
499 }
500 }
501 i = posn + 1;
502 }
bc8f2652 503
30ae764b 504 cdata->got_alternates = 1;
acc075a8
NH
505}
506
30ae764b 507static void fetch_alternates(struct walker *walker, const char *base)
acc075a8 508{
028c2976 509 struct strbuf buffer = STRBUF_INIT;
acc075a8 510 char *url;
acc075a8 511 struct active_request_slot *slot;
cb754fdf 512 struct alternates_request alt_req;
30ae764b 513 struct walker_data *cdata = walker->data;
acc075a8 514
4c42aa1a
TRC
515 /*
516 * If another request has already started fetching alternates,
517 * wait for them to arrive and return to processing this request's
518 * curl message
519 */
29508e1e 520#ifdef USE_CURL_MULTI
30ae764b 521 while (cdata->got_alternates == 0) {
29508e1e 522 step_active_slots();
acc075a8 523 }
29508e1e 524#endif
acc075a8
NH
525
526 /* Nothing to do if they've already been fetched */
30ae764b 527 if (cdata->got_alternates == 1)
acc075a8
NH
528 return;
529
530 /* Start the fetch */
30ae764b 531 cdata->got_alternates = 0;
acc075a8 532
30ae764b 533 if (walker->get_verbosely)
acc075a8 534 fprintf(stderr, "Getting alternates list for %s\n", base);
8fcf7f9a 535
acc075a8
NH
536 url = xmalloc(strlen(base) + 31);
537 sprintf(url, "%s/objects/info/http-alternates", base);
538
4c42aa1a
TRC
539 /*
540 * Use a callback to process the result, since another request
541 * may fail and need to have alternates loaded before continuing
542 */
acc075a8 543 slot = get_active_slot();
e388ab74 544 slot->callback_func = process_alternates_response;
30ae764b 545 alt_req.walker = walker;
acc075a8
NH
546 slot->callback_data = &alt_req;
547
548 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 549 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
acc075a8
NH
550 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
551
552 alt_req.base = base;
553 alt_req.url = url;
554 alt_req.buffer = &buffer;
555 alt_req.http_specific = 1;
556 alt_req.slot = slot;
557
558 if (start_active_slot(slot))
559 run_active_slot(slot);
560 else
30ae764b 561 cdata->got_alternates = -1;
acc075a8 562
028c2976 563 strbuf_release(&buffer);
acc075a8 564 free(url);
b3661567
DB
565}
566
30ae764b 567static int fetch_indices(struct walker *walker, struct alt_base *repo)
182005b9 568{
b8caac2b 569 int ret;
1d389ab6 570
b3661567 571 if (repo->got_indices)
182005b9
DB
572 return 0;
573
30ae764b 574 if (walker->get_verbosely)
6fd72e39 575 fprintf(stderr, "Getting pack list for %s\n", repo->base);
8fcf7f9a 576
b8caac2b
TRC
577 switch (http_get_info_packs(repo->base, &repo->packs)) {
578 case HTTP_OK:
579 case HTTP_MISSING_TARGET:
580 repo->got_indices = 1;
581 ret = 0;
582 break;
583 default:
5e3a7691 584 repo->got_indices = 0;
b8caac2b 585 ret = -1;
b3661567 586 }
182005b9 587
3a462bc9 588 return ret;
182005b9
DB
589}
590
30ae764b 591static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
182005b9
DB
592{
593 char *url;
594 struct packed_git *target;
595 struct packed_git **lst;
596 FILE *packfile;
597 char *filename;
49a0f240
NH
598 char tmpfile[PATH_MAX];
599 int ret;
600 long prev_posn = 0;
601 char range[RANGE_HEADER_SIZE];
602 struct curl_slist *range_header = NULL;
1d389ab6
NH
603
604 struct active_request_slot *slot;
cb754fdf 605 struct slot_results results;
182005b9 606
30ae764b 607 if (fetch_indices(walker, repo))
182005b9 608 return -1;
b3661567 609 target = find_sha1_pack(sha1, repo->packs);
182005b9 610 if (!target)
b3661567 611 return -1;
182005b9 612
30ae764b 613 if (walker->get_verbosely) {
182005b9
DB
614 fprintf(stderr, "Getting pack %s\n",
615 sha1_to_hex(target->sha1));
616 fprintf(stderr, " which contains %s\n",
617 sha1_to_hex(sha1));
618 }
619
b3661567 620 url = xmalloc(strlen(repo->base) + 65);
182005b9 621 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 622 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
623
624 filename = sha1_pack_name(target->sha1);
49a0f240
NH
625 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
626 packfile = fopen(tmpfile, "a");
182005b9
DB
627 if (!packfile)
628 return error("Unable to open local file %s for pack",
84ef0338 629 tmpfile);
182005b9 630
1d389ab6 631 slot = get_active_slot();
c8568e13 632 slot->results = &results;
1d389ab6
NH
633 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
634 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
635 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
e9176745 636 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 637 slot->local = packfile;
1ddea77e 638
4c42aa1a
TRC
639 /*
640 * If there is data present from a previous transfer attempt,
641 * resume where it left off
642 */
49a0f240
NH
643 prev_posn = ftell(packfile);
644 if (prev_posn>0) {
30ae764b 645 if (walker->get_verbosely)
49a0f240
NH
646 fprintf(stderr,
647 "Resuming fetch of pack %s at byte %ld\n",
648 sha1_to_hex(target->sha1), prev_posn);
649 sprintf(range, "Range: bytes=%ld-", prev_posn);
650 range_header = curl_slist_append(range_header, range);
1d389ab6 651 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
652 }
653
1d389ab6
NH
654 if (start_active_slot(slot)) {
655 run_active_slot(slot);
c8568e13 656 if (results.curl_result != CURLE_OK) {
1d389ab6 657 fclose(packfile);
16493eb0 658 slot->local = NULL;
1d389ab6
NH
659 return error("Unable to get pack file %s\n%s", url,
660 curl_errorstr);
661 }
662 } else {
313c4714 663 fclose(packfile);
16493eb0 664 slot->local = NULL;
1d389ab6 665 return error("Unable to start request");
182005b9
DB
666 }
667
1c23d794 668 target->pack_size = ftell(packfile);
182005b9 669 fclose(packfile);
16493eb0 670 slot->local = NULL;
182005b9 671
b721e01f 672 ret = move_temp_to_file(tmpfile, filename);
49a0f240 673 if (ret)
b721e01f 674 return ret;
49a0f240 675
b3661567 676 lst = &repo->packs;
182005b9
DB
677 while (*lst != target)
678 lst = &((*lst)->next);
679 *lst = (*lst)->next;
680
77d3ecee 681 if (verify_pack(target))
271421cd 682 return -1;
182005b9
DB
683 install_packed_git(target);
684
685 return 0;
686}
687
53f31389
MW
688static void abort_object_request(struct object_request *obj_req)
689{
690 if (obj_req->local >= 0) {
691 close(obj_req->local);
692 obj_req->local = -1;
693 }
691f1a28 694 unlink_or_warn(obj_req->tmpfile);
53f31389 695 if (obj_req->slot) {
a6080a0a 696 release_active_slot(obj_req->slot);
53f31389
MW
697 obj_req->slot = NULL;
698 }
699 release_object_request(obj_req);
700}
701
30ae764b 702static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
703{
704 char *hex = sha1_to_hex(sha1);
29508e1e 705 int ret = 0;
e388ab74 706 struct object_request *obj_req = object_queue_head;
1d389ab6 707
a89fccd2 708 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
e388ab74
NH
709 obj_req = obj_req->next;
710 if (obj_req == NULL)
1d389ab6
NH
711 return error("Couldn't find request for %s in the queue", hex);
712
e388ab74 713 if (has_sha1_file(obj_req->sha1)) {
53f31389 714 abort_object_request(obj_req);
11f0dafe
NH
715 return 0;
716 }
717
a7a8d378 718#ifdef USE_CURL_MULTI
4c42aa1a 719 while (obj_req->state == WAITING)
29508e1e 720 step_active_slots();
a7a8d378 721#else
30ae764b 722 start_object_request(walker, obj_req);
a7a8d378 723#endif
6eb7ed54 724
4c42aa1a 725 while (obj_req->state == ACTIVE)
e388ab74 726 run_active_slot(obj_req->slot);
4c42aa1a 727
e388ab74 728 if (obj_req->local != -1) {
4c42aa1a
TRC
729 close(obj_req->local);
730 obj_req->local = -1;
313c4714 731 }
6eb7ed54 732
e388ab74 733 if (obj_req->state == ABORTED) {
29508e1e 734 ret = error("Request for %s aborted", hex);
e388ab74
NH
735 } else if (obj_req->curl_result != CURLE_OK &&
736 obj_req->http_code != 416) {
be4a015b 737 if (missing_target(obj_req))
e2029eb9
PB
738 ret = -1; /* Be silent, it is probably in a pack. */
739 else
740 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
e388ab74
NH
741 obj_req->errorstr, obj_req->curl_result,
742 obj_req->http_code, hex);
743 } else if (obj_req->zret != Z_STREAM_END) {
30ae764b 744 walker->corrupt_object_found++;
bd2afde8 745 ret = error("File %s (%s) corrupt", hex, obj_req->url);
a89fccd2 746 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
bd2afde8 747 ret = error("File %s has bad hash", hex);
e388ab74 748 } else if (obj_req->rename < 0) {
7b934ec0
MW
749 ret = error("unable to write sha1 filename %s",
750 obj_req->filename);
6eb7ed54 751 }
49a0f240 752
e388ab74 753 release_object_request(obj_req);
29508e1e 754 return ret;
6eb7ed54
DB
755}
756
30ae764b 757static int fetch(struct walker *walker, unsigned char *sha1)
b3661567 758{
30ae764b
DB
759 struct walker_data *data = walker->data;
760 struct alt_base *altbase = data->alt;
1d389ab6 761
30ae764b 762 if (!fetch_object(walker, altbase, sha1))
1d389ab6 763 return 0;
b3661567 764 while (altbase) {
30ae764b 765 if (!fetch_pack(walker, altbase, sha1))
b3661567 766 return 0;
30ae764b 767 fetch_alternates(walker, data->alt->base);
b3661567
DB
768 altbase = altbase->next;
769 }
bd2afde8 770 return error("Unable to find %s under %s", sha1_to_hex(sha1),
30ae764b 771 data->alt->base);
b3661567
DB
772}
773
c13b2633 774static int fetch_ref(struct walker *walker, struct ref *ref)
cd541a68 775{
30ae764b 776 struct walker_data *data = walker->data;
c13b2633 777 return http_fetch_ref(data->alt->base, ref);
cd541a68
DB
778}
779
30ae764b
DB
780static void cleanup(struct walker *walker)
781{
30ae764b 782 http_cleanup();
30ae764b
DB
783}
784
9fc6440d 785struct walker *get_http_walker(const char *url, struct remote *remote)
6eb7ed54 786{
9c880b3e 787 char *s;
30ae764b
DB
788 struct walker_data *data = xmalloc(sizeof(struct walker_data));
789 struct walker *walker = xmalloc(sizeof(struct walker));
6eb7ed54 790
9fc6440d 791 http_init(remote);
d402d556 792
30ae764b
DB
793 data->alt = xmalloc(sizeof(*data->alt));
794 data->alt->base = xmalloc(strlen(url) + 1);
795 strcpy(data->alt->base, url);
796 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
9c880b3e 797 *s = 0;
6eb7ed54 798
30ae764b
DB
799 data->alt->got_indices = 0;
800 data->alt->packs = NULL;
801 data->alt->next = NULL;
802 data->got_alternates = -1;
fc57b6aa 803
30ae764b
DB
804 walker->corrupt_object_found = 0;
805 walker->fetch = fetch;
806 walker->fetch_ref = fetch_ref;
807 walker->prefetch = prefetch;
808 walker->cleanup = cleanup;
809 walker->data = data;
6eb7ed54 810
30ae764b
DB
811#ifdef USE_CURL_MULTI
812 add_fill_function(walker, (int (*)(void *)) fill_active_slot);
813#endif
8e29f6a0 814
30ae764b 815 return walker;
6eb7ed54 816}