object-store: move packed_git and packed_git_mru to object store
[git/git.git] / http-walker.c
1 #include "cache.h"
2 #include "commit.h"
3 #include "walker.h"
4 #include "http.h"
5 #include "list.h"
6 #include "transport.h"
7 #include "packfile.h"
8 #include "object-store.h"
9
10 struct alt_base {
11 char *base;
12 int got_indices;
13 struct packed_git *packs;
14 struct alt_base *next;
15 };
16
17 enum object_request_state {
18 WAITING,
19 ABORTED,
20 ACTIVE,
21 COMPLETE
22 };
23
24 struct object_request {
25 struct walker *walker;
26 unsigned char sha1[20];
27 struct alt_base *repo;
28 enum object_request_state state;
29 struct http_object_request *req;
30 struct list_head node;
31 };
32
33 struct alternates_request {
34 struct walker *walker;
35 const char *base;
36 struct strbuf *url;
37 struct strbuf *buffer;
38 struct active_request_slot *slot;
39 int http_specific;
40 };
41
42 struct walker_data {
43 const char *url;
44 int got_alternates;
45 struct alt_base *alt;
46 };
47
48 static LIST_HEAD(object_queue_head);
49
50 static void fetch_alternates(struct walker *walker, const char *base);
51
52 static void process_object_response(void *callback_data);
53
54 static void start_object_request(struct walker *walker,
55 struct object_request *obj_req)
56 {
57 struct active_request_slot *slot;
58 struct http_object_request *req;
59
60 req = new_http_object_request(obj_req->repo->base, obj_req->sha1);
61 if (req == NULL) {
62 obj_req->state = ABORTED;
63 return;
64 }
65 obj_req->req = req;
66
67 slot = req->slot;
68 slot->callback_func = process_object_response;
69 slot->callback_data = obj_req;
70
71 /* Try to get the request started, abort the request on error */
72 obj_req->state = ACTIVE;
73 if (!start_active_slot(slot)) {
74 obj_req->state = ABORTED;
75 release_http_object_request(req);
76 return;
77 }
78 }
79
80 static void finish_object_request(struct object_request *obj_req)
81 {
82 if (finish_http_object_request(obj_req->req))
83 return;
84
85 if (obj_req->req->rename == 0)
86 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
87 }
88
89 static void process_object_response(void *callback_data)
90 {
91 struct object_request *obj_req =
92 (struct object_request *)callback_data;
93 struct walker *walker = obj_req->walker;
94 struct walker_data *data = walker->data;
95 struct alt_base *alt = data->alt;
96
97 process_http_object_request(obj_req->req);
98 obj_req->state = COMPLETE;
99
100 /* Use alternates if necessary */
101 if (missing_target(obj_req->req)) {
102 fetch_alternates(walker, alt->base);
103 if (obj_req->repo->next != NULL) {
104 obj_req->repo =
105 obj_req->repo->next;
106 release_http_object_request(obj_req->req);
107 start_object_request(walker, obj_req);
108 return;
109 }
110 }
111
112 finish_object_request(obj_req);
113 }
114
115 static void release_object_request(struct object_request *obj_req)
116 {
117 if (obj_req->req !=NULL && obj_req->req->localfile != -1)
118 error("fd leakage in release: %d", obj_req->req->localfile);
119
120 list_del(&obj_req->node);
121 free(obj_req);
122 }
123
124 #ifdef USE_CURL_MULTI
125 static int fill_active_slot(struct walker *walker)
126 {
127 struct object_request *obj_req;
128 struct list_head *pos, *tmp, *head = &object_queue_head;
129
130 list_for_each_safe(pos, tmp, head) {
131 obj_req = list_entry(pos, struct object_request, node);
132 if (obj_req->state == WAITING) {
133 if (has_sha1_file(obj_req->sha1))
134 obj_req->state = COMPLETE;
135 else {
136 start_object_request(walker, obj_req);
137 return 1;
138 }
139 }
140 }
141 return 0;
142 }
143 #endif
144
145 static void prefetch(struct walker *walker, unsigned char *sha1)
146 {
147 struct object_request *newreq;
148 struct walker_data *data = walker->data;
149
150 newreq = xmalloc(sizeof(*newreq));
151 newreq->walker = walker;
152 hashcpy(newreq->sha1, sha1);
153 newreq->repo = data->alt;
154 newreq->state = WAITING;
155 newreq->req = NULL;
156
157 http_is_verbose = walker->get_verbosely;
158 list_add_tail(&newreq->node, &object_queue_head);
159
160 #ifdef USE_CURL_MULTI
161 fill_active_slots();
162 step_active_slots();
163 #endif
164 }
165
166 static int is_alternate_allowed(const char *url)
167 {
168 const char *protocols[] = {
169 "http", "https", "ftp", "ftps"
170 };
171 int i;
172
173 if (http_follow_config != HTTP_FOLLOW_ALWAYS) {
174 warning("alternate disabled by http.followRedirects: %s", url);
175 return 0;
176 }
177
178 for (i = 0; i < ARRAY_SIZE(protocols); i++) {
179 const char *end;
180 if (skip_prefix(url, protocols[i], &end) &&
181 starts_with(end, "://"))
182 break;
183 }
184
185 if (i >= ARRAY_SIZE(protocols)) {
186 warning("ignoring alternate with unknown protocol: %s", url);
187 return 0;
188 }
189 if (!is_transport_allowed(protocols[i], 0)) {
190 warning("ignoring alternate with restricted protocol: %s", url);
191 return 0;
192 }
193
194 return 1;
195 }
196
197 static void process_alternates_response(void *callback_data)
198 {
199 struct alternates_request *alt_req =
200 (struct alternates_request *)callback_data;
201 struct walker *walker = alt_req->walker;
202 struct walker_data *cdata = walker->data;
203 struct active_request_slot *slot = alt_req->slot;
204 struct alt_base *tail = cdata->alt;
205 const char *base = alt_req->base;
206 const char null_byte = '\0';
207 char *data;
208 int i = 0;
209
210 if (alt_req->http_specific) {
211 if (slot->curl_result != CURLE_OK ||
212 !alt_req->buffer->len) {
213
214 /* Try reusing the slot to get non-http alternates */
215 alt_req->http_specific = 0;
216 strbuf_reset(alt_req->url);
217 strbuf_addf(alt_req->url, "%s/objects/info/alternates",
218 base);
219 curl_easy_setopt(slot->curl, CURLOPT_URL,
220 alt_req->url->buf);
221 active_requests++;
222 slot->in_use = 1;
223 if (slot->finished != NULL)
224 (*slot->finished) = 0;
225 if (!start_active_slot(slot)) {
226 cdata->got_alternates = -1;
227 slot->in_use = 0;
228 if (slot->finished != NULL)
229 (*slot->finished) = 1;
230 }
231 return;
232 }
233 } else if (slot->curl_result != CURLE_OK) {
234 if (!missing_target(slot)) {
235 cdata->got_alternates = -1;
236 return;
237 }
238 }
239
240 fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
241 alt_req->buffer->len--;
242 data = alt_req->buffer->buf;
243
244 while (i < alt_req->buffer->len) {
245 int posn = i;
246 while (posn < alt_req->buffer->len && data[posn] != '\n')
247 posn++;
248 if (data[posn] == '\n') {
249 int okay = 0;
250 int serverlen = 0;
251 struct alt_base *newalt;
252 if (data[i] == '/') {
253 /*
254 * This counts
255 * http://git.host/pub/scm/linux.git/
256 * -----------here^
257 * so memcpy(dst, base, serverlen) will
258 * copy up to "...git.host".
259 */
260 const char *colon_ss = strstr(base,"://");
261 if (colon_ss) {
262 serverlen = (strchr(colon_ss + 3, '/')
263 - base);
264 okay = 1;
265 }
266 } else if (!memcmp(data + i, "../", 3)) {
267 /*
268 * Relative URL; chop the corresponding
269 * number of subpath from base (and ../
270 * from data), and concatenate the result.
271 *
272 * The code first drops ../ from data, and
273 * then drops one ../ from data and one path
274 * from base. IOW, one extra ../ is dropped
275 * from data than path is dropped from base.
276 *
277 * This is not wrong. The alternate in
278 * http://git.host/pub/scm/linux.git/
279 * to borrow from
280 * http://git.host/pub/scm/linus.git/
281 * is ../../linus.git/objects/. You need
282 * two ../../ to borrow from your direct
283 * neighbour.
284 */
285 i += 3;
286 serverlen = strlen(base);
287 while (i + 2 < posn &&
288 !memcmp(data + i, "../", 3)) {
289 do {
290 serverlen--;
291 } while (serverlen &&
292 base[serverlen - 1] != '/');
293 i += 3;
294 }
295 /* If the server got removed, give up. */
296 okay = strchr(base, ':') - base + 3 <
297 serverlen;
298 } else if (alt_req->http_specific) {
299 char *colon = strchr(data + i, ':');
300 char *slash = strchr(data + i, '/');
301 if (colon && slash && colon < data + posn &&
302 slash < data + posn && colon < slash) {
303 okay = 1;
304 }
305 }
306 if (okay) {
307 struct strbuf target = STRBUF_INIT;
308 strbuf_add(&target, base, serverlen);
309 strbuf_add(&target, data + i, posn - i);
310 if (!strbuf_strip_suffix(&target, "objects")) {
311 warning("ignoring alternate that does"
312 " not end in 'objects': %s",
313 target.buf);
314 strbuf_release(&target);
315 } else if (is_alternate_allowed(target.buf)) {
316 warning("adding alternate object store: %s",
317 target.buf);
318 newalt = xmalloc(sizeof(*newalt));
319 newalt->next = NULL;
320 newalt->base = strbuf_detach(&target, NULL);
321 newalt->got_indices = 0;
322 newalt->packs = NULL;
323
324 while (tail->next != NULL)
325 tail = tail->next;
326 tail->next = newalt;
327 } else {
328 strbuf_release(&target);
329 }
330 }
331 }
332 i = posn + 1;
333 }
334
335 cdata->got_alternates = 1;
336 }
337
338 static void fetch_alternates(struct walker *walker, const char *base)
339 {
340 struct strbuf buffer = STRBUF_INIT;
341 struct strbuf url = STRBUF_INIT;
342 struct active_request_slot *slot;
343 struct alternates_request alt_req;
344 struct walker_data *cdata = walker->data;
345
346 /*
347 * If another request has already started fetching alternates,
348 * wait for them to arrive and return to processing this request's
349 * curl message
350 */
351 #ifdef USE_CURL_MULTI
352 while (cdata->got_alternates == 0) {
353 step_active_slots();
354 }
355 #endif
356
357 /* Nothing to do if they've already been fetched */
358 if (cdata->got_alternates == 1)
359 return;
360
361 /* Start the fetch */
362 cdata->got_alternates = 0;
363
364 if (walker->get_verbosely)
365 fprintf(stderr, "Getting alternates list for %s\n", base);
366
367 strbuf_addf(&url, "%s/objects/info/http-alternates", base);
368
369 /*
370 * Use a callback to process the result, since another request
371 * may fail and need to have alternates loaded before continuing
372 */
373 slot = get_active_slot();
374 slot->callback_func = process_alternates_response;
375 alt_req.walker = walker;
376 slot->callback_data = &alt_req;
377
378 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
379 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
380 curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
381
382 alt_req.base = base;
383 alt_req.url = &url;
384 alt_req.buffer = &buffer;
385 alt_req.http_specific = 1;
386 alt_req.slot = slot;
387
388 if (start_active_slot(slot))
389 run_active_slot(slot);
390 else
391 cdata->got_alternates = -1;
392
393 strbuf_release(&buffer);
394 strbuf_release(&url);
395 }
396
397 static int fetch_indices(struct walker *walker, struct alt_base *repo)
398 {
399 int ret;
400
401 if (repo->got_indices)
402 return 0;
403
404 if (walker->get_verbosely)
405 fprintf(stderr, "Getting pack list for %s\n", repo->base);
406
407 switch (http_get_info_packs(repo->base, &repo->packs)) {
408 case HTTP_OK:
409 case HTTP_MISSING_TARGET:
410 repo->got_indices = 1;
411 ret = 0;
412 break;
413 default:
414 repo->got_indices = 0;
415 ret = -1;
416 }
417
418 return ret;
419 }
420
421 static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
422 {
423 struct packed_git *target;
424 int ret;
425 struct slot_results results;
426 struct http_pack_request *preq;
427
428 if (fetch_indices(walker, repo))
429 return -1;
430 target = find_sha1_pack(sha1, repo->packs);
431 if (!target)
432 return -1;
433
434 if (walker->get_verbosely) {
435 fprintf(stderr, "Getting pack %s\n",
436 sha1_to_hex(target->sha1));
437 fprintf(stderr, " which contains %s\n",
438 sha1_to_hex(sha1));
439 }
440
441 preq = new_http_pack_request(target, repo->base);
442 if (preq == NULL)
443 goto abort;
444 preq->lst = &repo->packs;
445 preq->slot->results = &results;
446
447 if (start_active_slot(preq->slot)) {
448 run_active_slot(preq->slot);
449 if (results.curl_result != CURLE_OK) {
450 error("Unable to get pack file %s\n%s", preq->url,
451 curl_errorstr);
452 goto abort;
453 }
454 } else {
455 error("Unable to start request");
456 goto abort;
457 }
458
459 ret = finish_http_pack_request(preq);
460 release_http_pack_request(preq);
461 if (ret)
462 return ret;
463
464 return 0;
465
466 abort:
467 return -1;
468 }
469
470 static void abort_object_request(struct object_request *obj_req)
471 {
472 release_object_request(obj_req);
473 }
474
475 static int fetch_object(struct walker *walker, unsigned char *sha1)
476 {
477 char *hex = sha1_to_hex(sha1);
478 int ret = 0;
479 struct object_request *obj_req = NULL;
480 struct http_object_request *req;
481 struct list_head *pos, *head = &object_queue_head;
482
483 list_for_each(pos, head) {
484 obj_req = list_entry(pos, struct object_request, node);
485 if (!hashcmp(obj_req->sha1, sha1))
486 break;
487 }
488 if (obj_req == NULL)
489 return error("Couldn't find request for %s in the queue", hex);
490
491 if (has_sha1_file(obj_req->sha1)) {
492 if (obj_req->req != NULL)
493 abort_http_object_request(obj_req->req);
494 abort_object_request(obj_req);
495 return 0;
496 }
497
498 #ifdef USE_CURL_MULTI
499 while (obj_req->state == WAITING)
500 step_active_slots();
501 #else
502 start_object_request(walker, obj_req);
503 #endif
504
505 /*
506 * obj_req->req might change when fetching alternates in the callback
507 * process_object_response; therefore, the "shortcut" variable, req,
508 * is used only after we're done with slots.
509 */
510 while (obj_req->state == ACTIVE)
511 run_active_slot(obj_req->req->slot);
512
513 req = obj_req->req;
514
515 if (req->localfile != -1) {
516 close(req->localfile);
517 req->localfile = -1;
518 }
519
520 /*
521 * we turned off CURLOPT_FAILONERROR to avoid losing a
522 * persistent connection and got CURLE_OK.
523 */
524 if (req->http_code >= 300 && req->curl_result == CURLE_OK &&
525 (starts_with(req->url, "http://") ||
526 starts_with(req->url, "https://"))) {
527 req->curl_result = CURLE_HTTP_RETURNED_ERROR;
528 xsnprintf(req->errorstr, sizeof(req->errorstr),
529 "HTTP request failed");
530 }
531
532 if (obj_req->state == ABORTED) {
533 ret = error("Request for %s aborted", hex);
534 } else if (req->curl_result != CURLE_OK &&
535 req->http_code != 416) {
536 if (missing_target(req))
537 ret = -1; /* Be silent, it is probably in a pack. */
538 else
539 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
540 req->errorstr, req->curl_result,
541 req->http_code, hex);
542 } else if (req->zret != Z_STREAM_END) {
543 walker->corrupt_object_found++;
544 ret = error("File %s (%s) corrupt", hex, req->url);
545 } else if (hashcmp(obj_req->sha1, req->real_sha1)) {
546 ret = error("File %s has bad hash", hex);
547 } else if (req->rename < 0) {
548 struct strbuf buf = STRBUF_INIT;
549 sha1_file_name(&buf, req->sha1);
550 ret = error("unable to write sha1 filename %s", buf.buf);
551 strbuf_release(&buf);
552 }
553
554 release_http_object_request(req);
555 release_object_request(obj_req);
556 return ret;
557 }
558
559 static int fetch(struct walker *walker, unsigned char *sha1)
560 {
561 struct walker_data *data = walker->data;
562 struct alt_base *altbase = data->alt;
563
564 if (!fetch_object(walker, sha1))
565 return 0;
566 while (altbase) {
567 if (!http_fetch_pack(walker, altbase, sha1))
568 return 0;
569 fetch_alternates(walker, data->alt->base);
570 altbase = altbase->next;
571 }
572 return error("Unable to find %s under %s", sha1_to_hex(sha1),
573 data->alt->base);
574 }
575
576 static int fetch_ref(struct walker *walker, struct ref *ref)
577 {
578 struct walker_data *data = walker->data;
579 return http_fetch_ref(data->alt->base, ref);
580 }
581
582 static void cleanup(struct walker *walker)
583 {
584 struct walker_data *data = walker->data;
585 struct alt_base *alt, *alt_next;
586
587 if (data) {
588 alt = data->alt;
589 while (alt) {
590 alt_next = alt->next;
591
592 free(alt->base);
593 free(alt);
594
595 alt = alt_next;
596 }
597 free(data);
598 walker->data = NULL;
599 }
600 }
601
602 struct walker *get_http_walker(const char *url)
603 {
604 char *s;
605 struct walker_data *data = xmalloc(sizeof(struct walker_data));
606 struct walker *walker = xmalloc(sizeof(struct walker));
607
608 data->alt = xmalloc(sizeof(*data->alt));
609 data->alt->base = xstrdup(url);
610 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
611 *s = 0;
612
613 data->alt->got_indices = 0;
614 data->alt->packs = NULL;
615 data->alt->next = NULL;
616 data->got_alternates = -1;
617
618 walker->corrupt_object_found = 0;
619 walker->fetch = fetch;
620 walker->fetch_ref = fetch_ref;
621 walker->prefetch = prefetch;
622 walker->cleanup = cleanup;
623 walker->data = data;
624
625 #ifdef USE_CURL_MULTI
626 add_fill_function(walker, (int (*)(void *)) fill_active_slot);
627 #endif
628
629 return walker;
630 }