http-push.c::fetch_symref(): use the new http API
[git/git.git] / http-walker.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
271421cd 3#include "pack.h"
30ae764b 4#include "walker.h"
29508e1e 5#include "http.h"
7baa3e86 6
49a0f240 7#define PREV_BUF_SIZE 4096
49a0f240 8
b3661567
DB
9struct alt_base
10{
2afea3bc 11 char *base;
b3661567
DB
12 int got_indices;
13 struct packed_git *packs;
14 struct alt_base *next;
15};
16
e388ab74 17enum object_request_state {
1d389ab6
NH
18 WAITING,
19 ABORTED,
20 ACTIVE,
21 COMPLETE,
22};
6eb7ed54 23
e388ab74 24struct object_request
1d389ab6 25{
30ae764b 26 struct walker *walker;
1d389ab6
NH
27 unsigned char sha1[20];
28 struct alt_base *repo;
29 char *url;
30 char filename[PATH_MAX];
31 char tmpfile[PATH_MAX];
32 int local;
e388ab74 33 enum object_request_state state;
1d389ab6
NH
34 CURLcode curl_result;
35 char errorstr[CURL_ERROR_SIZE];
36 long http_code;
37 unsigned char real_sha1[20];
9126f009 38 git_SHA_CTX c;
1d389ab6
NH
39 z_stream stream;
40 int zret;
41 int rename;
42 struct active_request_slot *slot;
e388ab74 43 struct object_request *next;
1d389ab6
NH
44};
45
e388ab74 46struct alternates_request {
30ae764b 47 struct walker *walker;
8e29f6a0 48 const char *base;
acc075a8 49 char *url;
028c2976 50 struct strbuf *buffer;
acc075a8
NH
51 struct active_request_slot *slot;
52 int http_specific;
53};
54
30ae764b
DB
55struct walker_data {
56 const char *url;
57 int got_alternates;
58 struct alt_base *alt;
30ae764b
DB
59};
60
96f1e58f 61static struct object_request *object_queue_head;
bc8f2652 62
182005b9
DB
63static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
64 void *data)
65{
bf0f910d 66 unsigned char expn[4096];
6eb7ed54
DB
67 size_t size = eltsize * nmemb;
68 int posn = 0;
e388ab74 69 struct object_request *obj_req = (struct object_request *)data;
6eb7ed54 70 do {
93822c22 71 ssize_t retval = xwrite(obj_req->local,
4c42aa1a 72 (char *) ptr + posn, size - posn);
6eb7ed54
DB
73 if (retval < 0)
74 return posn;
75 posn += retval;
76 } while (posn < size);
77
e388ab74
NH
78 obj_req->stream.avail_in = size;
79 obj_req->stream.next_in = ptr;
6eb7ed54 80 do {
e388ab74
NH
81 obj_req->stream.next_out = expn;
82 obj_req->stream.avail_out = sizeof(expn);
39c68542 83 obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);
9126f009 84 git_SHA1_Update(&obj_req->c, expn,
4c42aa1a 85 sizeof(expn) - obj_req->stream.avail_out);
e388ab74 86 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
1d389ab6 87 data_received++;
6eb7ed54
DB
88 return size;
89}
90
30ae764b 91static void fetch_alternates(struct walker *walker, const char *base);
1d389ab6 92
29508e1e 93static void process_object_response(void *callback_data);
1d389ab6 94
30ae764b
DB
95static void start_object_request(struct walker *walker,
96 struct object_request *obj_req)
1d389ab6 97{
e388ab74 98 char *hex = sha1_to_hex(obj_req->sha1);
1d389ab6
NH
99 char prevfile[PATH_MAX];
100 char *url;
101 char *posn;
102 int prevlocal;
103 unsigned char prev_buf[PREV_BUF_SIZE];
104 ssize_t prev_read = 0;
105 long prev_posn = 0;
106 char range[RANGE_HEADER_SIZE];
107 struct curl_slist *range_header = NULL;
108 struct active_request_slot *slot;
109
e388ab74 110 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
691f1a28 111 unlink_or_warn(prevfile);
e388ab74 112 rename(obj_req->tmpfile, prevfile);
691f1a28 113 unlink_or_warn(obj_req->tmpfile);
1d389ab6 114
e388ab74
NH
115 if (obj_req->local != -1)
116 error("fd leakage in start: %d", obj_req->local);
117 obj_req->local = open(obj_req->tmpfile,
1d389ab6 118 O_WRONLY | O_CREAT | O_EXCL, 0666);
4c42aa1a
TRC
119 /*
120 * This could have failed due to the "lazy directory creation";
b721e01f
JH
121 * try to mkdir the last path component.
122 */
e388ab74
NH
123 if (obj_req->local < 0 && errno == ENOENT) {
124 char *dir = strrchr(obj_req->tmpfile, '/');
b721e01f
JH
125 if (dir) {
126 *dir = 0;
e388ab74 127 mkdir(obj_req->tmpfile, 0777);
b721e01f
JH
128 *dir = '/';
129 }
e388ab74 130 obj_req->local = open(obj_req->tmpfile,
b721e01f
JH
131 O_WRONLY | O_CREAT | O_EXCL, 0666);
132 }
133
e388ab74
NH
134 if (obj_req->local < 0) {
135 obj_req->state = ABORTED;
bd2afde8 136 error("Couldn't create temporary file %s for %s: %s",
e388ab74 137 obj_req->tmpfile, obj_req->filename, strerror(errno));
1d389ab6
NH
138 return;
139 }
140
e388ab74 141 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
1d389ab6 142
39c68542 143 git_inflate_init(&obj_req->stream);
1d389ab6 144
9126f009 145 git_SHA1_Init(&obj_req->c);
1d389ab6 146
2afea3bc
GP
147 url = xmalloc(strlen(obj_req->repo->base) + 51);
148 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
e388ab74
NH
149 strcpy(url, obj_req->repo->base);
150 posn = url + strlen(obj_req->repo->base);
2afea3bc
GP
151 strcpy(posn, "/objects/");
152 posn += 9;
1d389ab6
NH
153 memcpy(posn, hex, 2);
154 posn += 2;
155 *(posn++) = '/';
156 strcpy(posn, hex + 2);
e388ab74 157 strcpy(obj_req->url, url);
1d389ab6 158
4c42aa1a
TRC
159 /*
160 * If a previous temp file is present, process what was already
161 * fetched.
162 */
1d389ab6
NH
163 prevlocal = open(prevfile, O_RDONLY);
164 if (prevlocal != -1) {
165 do {
93d26e4c 166 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1d389ab6
NH
167 if (prev_read>0) {
168 if (fwrite_sha1_file(prev_buf,
169 1,
170 prev_read,
4c42aa1a 171 obj_req) == prev_read)
1d389ab6 172 prev_posn += prev_read;
4c42aa1a 173 else
1d389ab6 174 prev_read = -1;
1d389ab6
NH
175 }
176 } while (prev_read > 0);
177 close(prevlocal);
178 }
691f1a28 179 unlink_or_warn(prevfile);
1d389ab6 180
4c42aa1a
TRC
181 /*
182 * Reset inflate/SHA1 if there was an error reading the previous temp
183 * file; also rewind to the beginning of the local file.
184 */
1d389ab6 185 if (prev_read == -1) {
e388ab74 186 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
39c68542 187 git_inflate_init(&obj_req->stream);
9126f009 188 git_SHA1_Init(&obj_req->c);
1d389ab6
NH
189 if (prev_posn>0) {
190 prev_posn = 0;
b5da2467 191 lseek(obj_req->local, 0, SEEK_SET);
e388ab74 192 ftruncate(obj_req->local, 0);
1d389ab6
NH
193 }
194 }
195
196 slot = get_active_slot();
29508e1e 197 slot->callback_func = process_object_response;
e388ab74
NH
198 slot->callback_data = obj_req;
199 obj_req->slot = slot;
29508e1e 200
e388ab74 201 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
1d389ab6 202 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
e388ab74 203 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
1d389ab6 204 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
e9176745 205 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 206
4c42aa1a
TRC
207 /*
208 * If we have successfully processed data from a previous fetch
209 * attempt, only fetch the data we don't already have.
210 */
1d389ab6 211 if (prev_posn>0) {
30ae764b 212 if (walker->get_verbosely)
1d389ab6
NH
213 fprintf(stderr,
214 "Resuming fetch of object %s at byte %ld\n",
215 hex, prev_posn);
216 sprintf(range, "Range: bytes=%ld-", prev_posn);
217 range_header = curl_slist_append(range_header, range);
218 curl_easy_setopt(slot->curl,
219 CURLOPT_HTTPHEADER, range_header);
220 }
221
a7a8d378 222 /* Try to get the request started, abort the request on error */
e388ab74 223 obj_req->state = ACTIVE;
1d389ab6 224 if (!start_active_slot(slot)) {
e388ab74
NH
225 obj_req->state = ABORTED;
226 obj_req->slot = NULL;
4c42aa1a
TRC
227 close(obj_req->local);
228 obj_req->local = -1;
e388ab74
NH
229 free(obj_req->url);
230 return;
1d389ab6 231 }
1d389ab6
NH
232}
233
e388ab74 234static void finish_object_request(struct object_request *obj_req)
1d389ab6 235{
50496b21
NH
236 struct stat st;
237
4c42aa1a
TRC
238 close(obj_req->local);
239 obj_req->local = -1;
1d389ab6 240
e388ab74 241 if (obj_req->http_code == 416) {
1d389ab6 242 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
e388ab74
NH
243 } else if (obj_req->curl_result != CURLE_OK) {
244 if (stat(obj_req->tmpfile, &st) == 0)
50496b21 245 if (st.st_size == 0)
691f1a28 246 unlink_or_warn(obj_req->tmpfile);
1d389ab6
NH
247 return;
248 }
249
39c68542 250 git_inflate_end(&obj_req->stream);
9126f009 251 git_SHA1_Final(obj_req->real_sha1, &obj_req->c);
e388ab74 252 if (obj_req->zret != Z_STREAM_END) {
691f1a28 253 unlink_or_warn(obj_req->tmpfile);
1d389ab6
NH
254 return;
255 }
a89fccd2 256 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
691f1a28 257 unlink_or_warn(obj_req->tmpfile);
1d389ab6
NH
258 return;
259 }
e388ab74
NH
260 obj_req->rename =
261 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
1d389ab6 262
e388ab74 263 if (obj_req->rename == 0)
30ae764b 264 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
1d389ab6
NH
265}
266
29508e1e
NH
267static void process_object_response(void *callback_data)
268{
e388ab74
NH
269 struct object_request *obj_req =
270 (struct object_request *)callback_data;
30ae764b
DB
271 struct walker *walker = obj_req->walker;
272 struct walker_data *data = walker->data;
273 struct alt_base *alt = data->alt;
29508e1e 274
e388ab74
NH
275 obj_req->curl_result = obj_req->slot->curl_result;
276 obj_req->http_code = obj_req->slot->http_code;
277 obj_req->slot = NULL;
278 obj_req->state = COMPLETE;
29508e1e
NH
279
280 /* Use alternates if necessary */
be4a015b 281 if (missing_target(obj_req)) {
30ae764b 282 fetch_alternates(walker, alt->base);
e388ab74
NH
283 if (obj_req->repo->next != NULL) {
284 obj_req->repo =
285 obj_req->repo->next;
286 close(obj_req->local);
287 obj_req->local = -1;
30ae764b 288 start_object_request(walker, obj_req);
29508e1e
NH
289 return;
290 }
291 }
292
e388ab74 293 finish_object_request(obj_req);
29508e1e
NH
294}
295
e388ab74 296static void release_object_request(struct object_request *obj_req)
1d389ab6 297{
e388ab74 298 struct object_request *entry = object_queue_head;
1d389ab6 299
e388ab74
NH
300 if (obj_req->local != -1)
301 error("fd leakage in release: %d", obj_req->local);
302 if (obj_req == object_queue_head) {
303 object_queue_head = obj_req->next;
1d389ab6 304 } else {
e388ab74 305 while (entry->next != NULL && entry->next != obj_req)
1d389ab6 306 entry = entry->next;
e388ab74 307 if (entry->next == obj_req)
1d389ab6
NH
308 entry->next = entry->next->next;
309 }
310
e388ab74
NH
311 free(obj_req->url);
312 free(obj_req);
1d389ab6
NH
313}
314
a7a8d378 315#ifdef USE_CURL_MULTI
30ae764b 316static int fill_active_slot(struct walker *walker)
1d389ab6 317{
45c17412 318 struct object_request *obj_req;
1d389ab6 319
45c17412 320 for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
e388ab74
NH
321 if (obj_req->state == WAITING) {
322 if (has_sha1_file(obj_req->sha1))
09db444f 323 obj_req->state = COMPLETE;
45c17412 324 else {
30ae764b 325 start_object_request(walker, obj_req);
45c17412
DB
326 return 1;
327 }
f1a906a3 328 }
8fcf7f9a 329 }
45c17412 330 return 0;
1d389ab6 331}
a7a8d378 332#endif
1d389ab6 333
30ae764b 334static void prefetch(struct walker *walker, unsigned char *sha1)
1d389ab6 335{
e388ab74
NH
336 struct object_request *newreq;
337 struct object_request *tail;
30ae764b 338 struct walker_data *data = walker->data;
1d389ab6
NH
339 char *filename = sha1_file_name(sha1);
340
341 newreq = xmalloc(sizeof(*newreq));
30ae764b 342 newreq->walker = walker;
e702496e 343 hashcpy(newreq->sha1, sha1);
30ae764b 344 newreq->repo = data->alt;
1d389ab6
NH
345 newreq->url = NULL;
346 newreq->local = -1;
347 newreq->state = WAITING;
348 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
349 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
350 "%s.temp", filename);
e8dff6ba 351 newreq->slot = NULL;
1d389ab6
NH
352 newreq->next = NULL;
353
e9176745
TRC
354 http_is_verbose = walker->get_verbosely;
355
e388ab74
NH
356 if (object_queue_head == NULL) {
357 object_queue_head = newreq;
1d389ab6 358 } else {
e388ab74 359 tail = object_queue_head;
4c42aa1a 360 while (tail->next != NULL)
1d389ab6 361 tail = tail->next;
1d389ab6
NH
362 tail->next = newreq;
363 }
29508e1e 364
a7a8d378 365#ifdef USE_CURL_MULTI
29508e1e
NH
366 fill_active_slots();
367 step_active_slots();
a7a8d378 368#endif
1d389ab6
NH
369}
370
30ae764b 371static int fetch_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
182005b9 372{
20cfb3aa
TRC
373 int ret = 0;
374 char *hex = xstrdup(sha1_to_hex(sha1));
182005b9
DB
375 char *filename;
376 char *url;
49a0f240 377 char tmpfile[PATH_MAX];
49a0f240
NH
378 long prev_posn = 0;
379 char range[RANGE_HEADER_SIZE];
380 struct curl_slist *range_header = NULL;
182005b9
DB
381
382 FILE *indexfile;
1d389ab6 383 struct active_request_slot *slot;
cb754fdf 384 struct slot_results results;
182005b9 385
48188c25
TRC
386 /* Don't use the index if the pack isn't there */
387 url = xmalloc(strlen(repo->base) + 64);
388 sprintf(url, "%s/objects/pack/pack-%s.pack", repo->base, hex);
389 slot = get_active_slot();
390 slot->results = &results;
391 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
392 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
393 if (start_active_slot(slot)) {
394 run_active_slot(slot);
395 if (results.curl_result != CURLE_OK) {
20cfb3aa 396 ret = error("Unable to verify pack %s is available",
48188c25 397 hex);
20cfb3aa 398 goto cleanup_pack;
48188c25
TRC
399 }
400 } else {
20cfb3aa
TRC
401 ret = error("Unable to start request");
402 goto cleanup_pack;
48188c25
TRC
403 }
404
405 if (has_pack_index(sha1)) {
20cfb3aa
TRC
406 ret = 0;
407 goto cleanup_pack;
48188c25 408 }
182005b9 409
30ae764b 410 if (walker->get_verbosely)
1d389ab6 411 fprintf(stderr, "Getting index for pack %s\n", hex);
8fcf7f9a 412
1d389ab6 413 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
8fcf7f9a 414
182005b9 415 filename = sha1_pack_index_name(sha1);
49a0f240
NH
416 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
417 indexfile = fopen(tmpfile, "a");
48188c25 418 if (!indexfile) {
20cfb3aa
TRC
419 ret = error("Unable to open local file %s for pack index",
420 tmpfile);
421 goto cleanup_pack;
48188c25 422 }
182005b9 423
1d389ab6 424 slot = get_active_slot();
c8568e13 425 slot->results = &results;
48188c25
TRC
426 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
427 curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1d389ab6
NH
428 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
429 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
430 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
e9176745 431 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6
NH
432 slot->local = indexfile;
433
4c42aa1a
TRC
434 /*
435 * If there is data present from a previous transfer attempt,
436 * resume where it left off
437 */
49a0f240
NH
438 prev_posn = ftell(indexfile);
439 if (prev_posn>0) {
30ae764b 440 if (walker->get_verbosely)
49a0f240
NH
441 fprintf(stderr,
442 "Resuming fetch of index for pack %s at byte %ld\n",
1d389ab6 443 hex, prev_posn);
49a0f240
NH
444 sprintf(range, "Range: bytes=%ld-", prev_posn);
445 range_header = curl_slist_append(range_header, range);
1d389ab6 446 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
447 }
448
1d389ab6
NH
449 if (start_active_slot(slot)) {
450 run_active_slot(slot);
c8568e13 451 if (results.curl_result != CURLE_OK) {
20cfb3aa
TRC
452 ret = error("Unable to get pack index %s\n%s", url,
453 curl_errorstr);
454 goto cleanup_index;
1d389ab6
NH
455 }
456 } else {
20cfb3aa
TRC
457 ret = error("Unable to start request");
458 goto cleanup_index;
182005b9
DB
459 }
460
20cfb3aa
TRC
461 ret = move_temp_to_file(tmpfile, filename);
462
463cleanup_index:
182005b9 464 fclose(indexfile);
16493eb0 465 slot->local = NULL;
20cfb3aa
TRC
466cleanup_pack:
467 free(url);
468 free(hex);
469 return ret;
182005b9
DB
470}
471
30ae764b 472static int setup_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
182005b9
DB
473{
474 struct packed_git *new_pack;
475 if (has_pack_file(sha1))
a9486b02 476 return 0; /* don't list this as something we can get */
182005b9 477
30ae764b 478 if (fetch_index(walker, repo, sha1))
182005b9
DB
479 return -1;
480
481 new_pack = parse_pack_index(sha1);
a70c232e
JH
482 if (!new_pack)
483 return -1; /* parse_pack_index() already issued error message */
b3661567
DB
484 new_pack->next = repo->packs;
485 repo->packs = new_pack;
182005b9
DB
486 return 0;
487}
488
e388ab74 489static void process_alternates_response(void *callback_data)
b3661567 490{
e388ab74
NH
491 struct alternates_request *alt_req =
492 (struct alternates_request *)callback_data;
30ae764b
DB
493 struct walker *walker = alt_req->walker;
494 struct walker_data *cdata = walker->data;
acc075a8 495 struct active_request_slot *slot = alt_req->slot;
30ae764b 496 struct alt_base *tail = cdata->alt;
8e29f6a0 497 const char *base = alt_req->base;
bc8f2652 498 static const char null_byte = '\0';
acc075a8
NH
499 char *data;
500 int i = 0;
1d389ab6 501
acc075a8
NH
502 if (alt_req->http_specific) {
503 if (slot->curl_result != CURLE_OK ||
028c2976 504 !alt_req->buffer->len) {
acc075a8
NH
505
506 /* Try reusing the slot to get non-http alternates */
507 alt_req->http_specific = 0;
508 sprintf(alt_req->url, "%s/objects/info/alternates",
509 base);
510 curl_easy_setopt(slot->curl, CURLOPT_URL,
511 alt_req->url);
512 active_requests++;
513 slot->in_use = 1;
c9826473
NH
514 if (slot->finished != NULL)
515 (*slot->finished) = 0;
a3f583cb 516 if (!start_active_slot(slot)) {
30ae764b 517 cdata->got_alternates = -1;
29508e1e 518 slot->in_use = 0;
c9826473
NH
519 if (slot->finished != NULL)
520 (*slot->finished) = 1;
1d389ab6 521 }
a3f583cb 522 return;
b3661567 523 }
acc075a8 524 } else if (slot->curl_result != CURLE_OK) {
be4a015b 525 if (!missing_target(slot)) {
30ae764b 526 cdata->got_alternates = -1;
acc075a8
NH
527 return;
528 }
b3661567
DB
529 }
530
29508e1e 531 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
028c2976
MH
532 alt_req->buffer->len--;
533 data = alt_req->buffer->buf;
1b0c1e67 534
028c2976 535 while (i < alt_req->buffer->len) {
b3661567 536 int posn = i;
028c2976 537 while (posn < alt_req->buffer->len && data[posn] != '\n')
b3661567
DB
538 posn++;
539 if (data[posn] == '\n') {
1b0c1e67
DB
540 int okay = 0;
541 int serverlen = 0;
542 struct alt_base *newalt;
543 char *target = NULL;
b3661567 544 if (data[i] == '/') {
4c42aa1a
TRC
545 /*
546 * This counts
5df1e0d0
JH
547 * http://git.host/pub/scm/linux.git/
548 * -----------here^
549 * so memcpy(dst, base, serverlen) will
550 * copy up to "...git.host".
551 */
552 const char *colon_ss = strstr(base,"://");
553 if (colon_ss) {
554 serverlen = (strchr(colon_ss + 3, '/')
555 - base);
556 okay = 1;
557 }
1b0c1e67 558 } else if (!memcmp(data + i, "../", 3)) {
4c42aa1a
TRC
559 /*
560 * Relative URL; chop the corresponding
5df1e0d0
JH
561 * number of subpath from base (and ../
562 * from data), and concatenate the result.
563 *
564 * The code first drops ../ from data, and
565 * then drops one ../ from data and one path
566 * from base. IOW, one extra ../ is dropped
567 * from data than path is dropped from base.
568 *
569 * This is not wrong. The alternate in
570 * http://git.host/pub/scm/linux.git/
571 * to borrow from
572 * http://git.host/pub/scm/linus.git/
573 * is ../../linus.git/objects/. You need
574 * two ../../ to borrow from your direct
575 * neighbour.
576 */
1b0c1e67
DB
577 i += 3;
578 serverlen = strlen(base);
8fcf7f9a 579 while (i + 2 < posn &&
1b0c1e67
DB
580 !memcmp(data + i, "../", 3)) {
581 do {
582 serverlen--;
583 } while (serverlen &&
584 base[serverlen - 1] != '/');
585 i += 3;
586 }
a9486b02 587 /* If the server got removed, give up. */
8fcf7f9a 588 okay = strchr(base, ':') - base + 3 <
4c42aa1a 589 serverlen;
acc075a8 590 } else if (alt_req->http_specific) {
1b0c1e67
DB
591 char *colon = strchr(data + i, ':');
592 char *slash = strchr(data + i, '/');
593 if (colon && slash && colon < data + posn &&
594 slash < data + posn && colon < slash) {
595 okay = 1;
596 }
597 }
5df1e0d0 598 /* skip "objects\n" at end */
1b0c1e67
DB
599 if (okay) {
600 target = xmalloc(serverlen + posn - i - 6);
5df1e0d0
JH
601 memcpy(target, base, serverlen);
602 memcpy(target + serverlen, data + i,
603 posn - i - 7);
604 target[serverlen + posn - i - 7] = 0;
30ae764b 605 if (walker->get_verbosely)
8fcf7f9a 606 fprintf(stderr,
b3661567
DB
607 "Also look at %s\n", target);
608 newalt = xmalloc(sizeof(*newalt));
1d389ab6 609 newalt->next = NULL;
b3661567
DB
610 newalt->base = target;
611 newalt->got_indices = 0;
612 newalt->packs = NULL;
8d9fbe57 613
1d389ab6
NH
614 while (tail->next != NULL)
615 tail = tail->next;
616 tail->next = newalt;
b3661567
DB
617 }
618 }
619 i = posn + 1;
620 }
bc8f2652 621
30ae764b 622 cdata->got_alternates = 1;
acc075a8
NH
623}
624
30ae764b 625static void fetch_alternates(struct walker *walker, const char *base)
acc075a8 626{
028c2976 627 struct strbuf buffer = STRBUF_INIT;
acc075a8 628 char *url;
acc075a8 629 struct active_request_slot *slot;
cb754fdf 630 struct alternates_request alt_req;
30ae764b 631 struct walker_data *cdata = walker->data;
acc075a8 632
4c42aa1a
TRC
633 /*
634 * If another request has already started fetching alternates,
635 * wait for them to arrive and return to processing this request's
636 * curl message
637 */
29508e1e 638#ifdef USE_CURL_MULTI
30ae764b 639 while (cdata->got_alternates == 0) {
29508e1e 640 step_active_slots();
acc075a8 641 }
29508e1e 642#endif
acc075a8
NH
643
644 /* Nothing to do if they've already been fetched */
30ae764b 645 if (cdata->got_alternates == 1)
acc075a8
NH
646 return;
647
648 /* Start the fetch */
30ae764b 649 cdata->got_alternates = 0;
acc075a8 650
30ae764b 651 if (walker->get_verbosely)
acc075a8 652 fprintf(stderr, "Getting alternates list for %s\n", base);
8fcf7f9a 653
acc075a8
NH
654 url = xmalloc(strlen(base) + 31);
655 sprintf(url, "%s/objects/info/http-alternates", base);
656
4c42aa1a
TRC
657 /*
658 * Use a callback to process the result, since another request
659 * may fail and need to have alternates loaded before continuing
660 */
acc075a8 661 slot = get_active_slot();
e388ab74 662 slot->callback_func = process_alternates_response;
30ae764b 663 alt_req.walker = walker;
acc075a8
NH
664 slot->callback_data = &alt_req;
665
666 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 667 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
acc075a8
NH
668 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
669
670 alt_req.base = base;
671 alt_req.url = url;
672 alt_req.buffer = &buffer;
673 alt_req.http_specific = 1;
674 alt_req.slot = slot;
675
676 if (start_active_slot(slot))
677 run_active_slot(slot);
678 else
30ae764b 679 cdata->got_alternates = -1;
acc075a8 680
028c2976 681 strbuf_release(&buffer);
acc075a8 682 free(url);
b3661567
DB
683}
684
30ae764b 685static int fetch_indices(struct walker *walker, struct alt_base *repo)
182005b9
DB
686{
687 unsigned char sha1[20];
688 char *url;
028c2976 689 struct strbuf buffer = STRBUF_INIT;
182005b9
DB
690 char *data;
691 int i = 0;
3a462bc9 692 int ret = 0;
182005b9 693
1d389ab6 694 struct active_request_slot *slot;
cb754fdf 695 struct slot_results results;
1d389ab6 696
b3661567 697 if (repo->got_indices)
182005b9
DB
698 return 0;
699
30ae764b 700 if (walker->get_verbosely)
6fd72e39 701 fprintf(stderr, "Getting pack list for %s\n", repo->base);
8fcf7f9a 702
b3661567
DB
703 url = xmalloc(strlen(repo->base) + 21);
704 sprintf(url, "%s/objects/info/packs", repo->base);
182005b9 705
1d389ab6 706 slot = get_active_slot();
c8568e13 707 slot->results = &results;
1d389ab6 708 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e 709 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1d389ab6
NH
710 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
711 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
712 if (start_active_slot(slot)) {
713 run_active_slot(slot);
c8568e13 714 if (results.curl_result != CURLE_OK) {
be4a015b 715 if (missing_target(&results)) {
5e3a7691 716 repo->got_indices = 1;
3a462bc9 717 goto cleanup;
5e3a7691
NH
718 } else {
719 repo->got_indices = 0;
3a462bc9
MH
720 ret = error("%s", curl_errorstr);
721 goto cleanup;
5e3a7691 722 }
bc8f2652 723 }
1d389ab6 724 } else {
5e3a7691 725 repo->got_indices = 0;
3a462bc9
MH
726 ret = error("Unable to start request");
727 goto cleanup;
1d389ab6 728 }
182005b9 729
028c2976
MH
730 data = buffer.buf;
731 while (i < buffer.len) {
182005b9
DB
732 switch (data[i]) {
733 case 'P':
734 i++;
028c2976 735 if (i + 52 <= buffer.len &&
cc44c765 736 !prefixcmp(data + i, " pack-") &&
1968d77d 737 !prefixcmp(data + i + 46, ".pack\n")) {
182005b9 738 get_sha1_hex(data + i + 6, sha1);
30ae764b 739 setup_index(walker, repo, sha1);
182005b9
DB
740 i += 51;
741 break;
742 }
743 default:
028c2976 744 while (i < buffer.len && data[i] != '\n')
182005b9
DB
745 i++;
746 }
747 i++;
b3661567 748 }
182005b9 749
b3661567 750 repo->got_indices = 1;
3a462bc9
MH
751cleanup:
752 strbuf_release(&buffer);
753 free(url);
754 return ret;
182005b9
DB
755}
756
30ae764b 757static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
182005b9
DB
758{
759 char *url;
760 struct packed_git *target;
761 struct packed_git **lst;
762 FILE *packfile;
763 char *filename;
49a0f240
NH
764 char tmpfile[PATH_MAX];
765 int ret;
766 long prev_posn = 0;
767 char range[RANGE_HEADER_SIZE];
768 struct curl_slist *range_header = NULL;
1d389ab6
NH
769
770 struct active_request_slot *slot;
cb754fdf 771 struct slot_results results;
182005b9 772
30ae764b 773 if (fetch_indices(walker, repo))
182005b9 774 return -1;
b3661567 775 target = find_sha1_pack(sha1, repo->packs);
182005b9 776 if (!target)
b3661567 777 return -1;
182005b9 778
30ae764b 779 if (walker->get_verbosely) {
182005b9
DB
780 fprintf(stderr, "Getting pack %s\n",
781 sha1_to_hex(target->sha1));
782 fprintf(stderr, " which contains %s\n",
783 sha1_to_hex(sha1));
784 }
785
b3661567 786 url = xmalloc(strlen(repo->base) + 65);
182005b9 787 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 788 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
789
790 filename = sha1_pack_name(target->sha1);
49a0f240
NH
791 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
792 packfile = fopen(tmpfile, "a");
182005b9
DB
793 if (!packfile)
794 return error("Unable to open local file %s for pack",
84ef0338 795 tmpfile);
182005b9 796
1d389ab6 797 slot = get_active_slot();
c8568e13 798 slot->results = &results;
1d389ab6
NH
799 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
800 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
801 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
e9176745 802 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1d389ab6 803 slot->local = packfile;
1ddea77e 804
4c42aa1a
TRC
805 /*
806 * If there is data present from a previous transfer attempt,
807 * resume where it left off
808 */
49a0f240
NH
809 prev_posn = ftell(packfile);
810 if (prev_posn>0) {
30ae764b 811 if (walker->get_verbosely)
49a0f240
NH
812 fprintf(stderr,
813 "Resuming fetch of pack %s at byte %ld\n",
814 sha1_to_hex(target->sha1), prev_posn);
815 sprintf(range, "Range: bytes=%ld-", prev_posn);
816 range_header = curl_slist_append(range_header, range);
1d389ab6 817 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
49a0f240
NH
818 }
819
1d389ab6
NH
820 if (start_active_slot(slot)) {
821 run_active_slot(slot);
c8568e13 822 if (results.curl_result != CURLE_OK) {
1d389ab6 823 fclose(packfile);
16493eb0 824 slot->local = NULL;
1d389ab6
NH
825 return error("Unable to get pack file %s\n%s", url,
826 curl_errorstr);
827 }
828 } else {
313c4714 829 fclose(packfile);
16493eb0 830 slot->local = NULL;
1d389ab6 831 return error("Unable to start request");
182005b9
DB
832 }
833
1c23d794 834 target->pack_size = ftell(packfile);
182005b9 835 fclose(packfile);
16493eb0 836 slot->local = NULL;
182005b9 837
b721e01f 838 ret = move_temp_to_file(tmpfile, filename);
49a0f240 839 if (ret)
b721e01f 840 return ret;
49a0f240 841
b3661567 842 lst = &repo->packs;
182005b9
DB
843 while (*lst != target)
844 lst = &((*lst)->next);
845 *lst = (*lst)->next;
846
77d3ecee 847 if (verify_pack(target))
271421cd 848 return -1;
182005b9
DB
849 install_packed_git(target);
850
851 return 0;
852}
853
53f31389
MW
854static void abort_object_request(struct object_request *obj_req)
855{
856 if (obj_req->local >= 0) {
857 close(obj_req->local);
858 obj_req->local = -1;
859 }
691f1a28 860 unlink_or_warn(obj_req->tmpfile);
53f31389 861 if (obj_req->slot) {
a6080a0a 862 release_active_slot(obj_req->slot);
53f31389
MW
863 obj_req->slot = NULL;
864 }
865 release_object_request(obj_req);
866}
867
30ae764b 868static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
869{
870 char *hex = sha1_to_hex(sha1);
29508e1e 871 int ret = 0;
e388ab74 872 struct object_request *obj_req = object_queue_head;
1d389ab6 873
a89fccd2 874 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
e388ab74
NH
875 obj_req = obj_req->next;
876 if (obj_req == NULL)
1d389ab6
NH
877 return error("Couldn't find request for %s in the queue", hex);
878
e388ab74 879 if (has_sha1_file(obj_req->sha1)) {
53f31389 880 abort_object_request(obj_req);
11f0dafe
NH
881 return 0;
882 }
883
a7a8d378 884#ifdef USE_CURL_MULTI
4c42aa1a 885 while (obj_req->state == WAITING)
29508e1e 886 step_active_slots();
a7a8d378 887#else
30ae764b 888 start_object_request(walker, obj_req);
a7a8d378 889#endif
6eb7ed54 890
4c42aa1a 891 while (obj_req->state == ACTIVE)
e388ab74 892 run_active_slot(obj_req->slot);
4c42aa1a 893
e388ab74 894 if (obj_req->local != -1) {
4c42aa1a
TRC
895 close(obj_req->local);
896 obj_req->local = -1;
313c4714 897 }
6eb7ed54 898
e388ab74 899 if (obj_req->state == ABORTED) {
29508e1e 900 ret = error("Request for %s aborted", hex);
e388ab74
NH
901 } else if (obj_req->curl_result != CURLE_OK &&
902 obj_req->http_code != 416) {
be4a015b 903 if (missing_target(obj_req))
e2029eb9
PB
904 ret = -1; /* Be silent, it is probably in a pack. */
905 else
906 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
e388ab74
NH
907 obj_req->errorstr, obj_req->curl_result,
908 obj_req->http_code, hex);
909 } else if (obj_req->zret != Z_STREAM_END) {
30ae764b 910 walker->corrupt_object_found++;
bd2afde8 911 ret = error("File %s (%s) corrupt", hex, obj_req->url);
a89fccd2 912 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
bd2afde8 913 ret = error("File %s has bad hash", hex);
e388ab74 914 } else if (obj_req->rename < 0) {
7b934ec0
MW
915 ret = error("unable to write sha1 filename %s",
916 obj_req->filename);
6eb7ed54 917 }
49a0f240 918
e388ab74 919 release_object_request(obj_req);
29508e1e 920 return ret;
6eb7ed54
DB
921}
922
30ae764b 923static int fetch(struct walker *walker, unsigned char *sha1)
b3661567 924{
30ae764b
DB
925 struct walker_data *data = walker->data;
926 struct alt_base *altbase = data->alt;
1d389ab6 927
30ae764b 928 if (!fetch_object(walker, altbase, sha1))
1d389ab6 929 return 0;
b3661567 930 while (altbase) {
30ae764b 931 if (!fetch_pack(walker, altbase, sha1))
b3661567 932 return 0;
30ae764b 933 fetch_alternates(walker, data->alt->base);
b3661567
DB
934 altbase = altbase->next;
935 }
bd2afde8 936 return error("Unable to find %s under %s", sha1_to_hex(sha1),
30ae764b 937 data->alt->base);
b3661567
DB
938}
939
c13b2633 940static int fetch_ref(struct walker *walker, struct ref *ref)
cd541a68 941{
30ae764b 942 struct walker_data *data = walker->data;
c13b2633 943 return http_fetch_ref(data->alt->base, ref);
cd541a68
DB
944}
945
30ae764b
DB
946static void cleanup(struct walker *walker)
947{
30ae764b 948 http_cleanup();
30ae764b
DB
949}
950
9fc6440d 951struct walker *get_http_walker(const char *url, struct remote *remote)
6eb7ed54 952{
9c880b3e 953 char *s;
30ae764b
DB
954 struct walker_data *data = xmalloc(sizeof(struct walker_data));
955 struct walker *walker = xmalloc(sizeof(struct walker));
6eb7ed54 956
9fc6440d 957 http_init(remote);
d402d556 958
30ae764b
DB
959 data->alt = xmalloc(sizeof(*data->alt));
960 data->alt->base = xmalloc(strlen(url) + 1);
961 strcpy(data->alt->base, url);
962 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
9c880b3e 963 *s = 0;
6eb7ed54 964
30ae764b
DB
965 data->alt->got_indices = 0;
966 data->alt->packs = NULL;
967 data->alt->next = NULL;
968 data->got_alternates = -1;
fc57b6aa 969
30ae764b
DB
970 walker->corrupt_object_found = 0;
971 walker->fetch = fetch;
972 walker->fetch_ref = fetch_ref;
973 walker->prefetch = prefetch;
974 walker->cleanup = cleanup;
975 walker->data = data;
6eb7ed54 976
30ae764b
DB
977#ifdef USE_CURL_MULTI
978 add_fill_function(walker, (int (*)(void *)) fill_active_slot);
979#endif
8e29f6a0 980
30ae764b 981 return walker;
6eb7ed54 982}