Git 2.4.10
[git/git.git] / http.c
CommitLineData
1c4b6604 1#include "git-compat-util.h"
29508e1e 2#include "http.h"
2264dfa5 3#include "pack.h"
de1a2fdd 4#include "sideband.h"
fe72d420 5#include "run-command.h"
f39f72d8 6#include "url.h"
6a56993b 7#include "urlmatch.h"
148bb6a7 8#include "credential.h"
745c7c8e 9#include "version.h"
047ec602 10#include "pkt-line.h"
93f7d910 11#include "gettext.h"
f4113cac 12#include "transport.h"
29508e1e 13
4251ccbd 14int active_requests;
e9176745 15int http_is_verbose;
de1a2fdd 16size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
29508e1e 17
b8ac9230
MS
18#if LIBCURL_VERSION_NUM >= 0x070a06
19#define LIBCURL_CAN_HANDLE_AUTH_ANY
20#endif
21
ad75ebe5
TRC
22static int min_curl_sessions = 1;
23static int curl_session_count;
29508e1e 24#ifdef USE_CURL_MULTI
cc3530e8
MH
25static int max_requests = -1;
26static CURLM *curlm;
29508e1e
NH
27#endif
28#ifndef NO_CURL_EASY_DUPHANDLE
cc3530e8 29static CURL *curl_default;
29508e1e 30#endif
5424bc55
TRC
31
32#define PREV_BUF_SIZE 4096
33#define RANGE_HEADER_SIZE 30
34
29508e1e
NH
35char curl_errorstr[CURL_ERROR_SIZE];
36
cc3530e8 37static int curl_ssl_verify = -1;
4bc444eb 38static int curl_ssl_try;
4251ccbd 39static const char *ssl_cert;
ef52aafa 40#if LIBCURL_VERSION_NUM >= 0x070903
4251ccbd 41static const char *ssl_key;
29508e1e
NH
42#endif
43#if LIBCURL_VERSION_NUM >= 0x070908
4251ccbd 44static const char *ssl_capath;
29508e1e 45#endif
4251ccbd 46static const char *ssl_cainfo;
cc3530e8
MH
47static long curl_low_speed_limit = -1;
48static long curl_low_speed_time = -1;
4251ccbd
JH
49static int curl_ftp_no_epsv;
50static const char *curl_http_proxy;
bcfb95dd 51static const char *curl_cookie_file;
912b2acf 52static int curl_save_cookies;
2501aff8 53struct credential http_auth = CREDENTIAL_INIT;
a4ddbc33 54static int http_proactive_auth;
b1d1058c 55static const char *user_agent;
29508e1e 56
30dd9163
ML
57#if LIBCURL_VERSION_NUM >= 0x071700
58/* Use CURLOPT_KEYPASSWD as is */
59#elif LIBCURL_VERSION_NUM >= 0x070903
60#define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
61#else
62#define CURLOPT_KEYPASSWD CURLOPT_SSLCERTPASSWD
63#endif
64
148bb6a7 65static struct credential cert_auth = CREDENTIAL_INIT;
30dd9163 66static int ssl_cert_password_required;
4dbe6646 67#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
68static unsigned long http_auth_methods = CURLAUTH_ANY;
69#endif
30dd9163 70
cc3530e8 71static struct curl_slist *pragma_header;
5424bc55 72static struct curl_slist *no_pragma_header;
e9176745 73
4251ccbd 74static struct active_request_slot *active_queue_head;
29508e1e 75
f18604bb
YE
76static char *cached_accept_language;
77
a04ff3ec 78size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
29508e1e
NH
79{
80 size_t size = eltsize * nmemb;
f444e528
JH
81 struct buffer *buffer = buffer_;
82
028c2976
MH
83 if (size > buffer->buf.len - buffer->posn)
84 size = buffer->buf.len - buffer->posn;
85 memcpy(ptr, buffer->buf.buf + buffer->posn, size);
29508e1e 86 buffer->posn += size;
028c2976 87
29508e1e
NH
88 return size;
89}
90
3944ba0c
MS
91#ifndef NO_CURL_IOCTL
92curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
93{
94 struct buffer *buffer = clientp;
95
96 switch (cmd) {
97 case CURLIOCMD_NOP:
98 return CURLIOE_OK;
99
100 case CURLIOCMD_RESTARTREAD:
101 buffer->posn = 0;
102 return CURLIOE_OK;
103
104 default:
105 return CURLIOE_UNKNOWNCMD;
106 }
107}
108#endif
109
a04ff3ec 110size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
29508e1e
NH
111{
112 size_t size = eltsize * nmemb;
f444e528
JH
113 struct strbuf *buffer = buffer_;
114
028c2976 115 strbuf_add(buffer, ptr, size);
29508e1e
NH
116 return size;
117}
118
a04ff3ec 119size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)
29508e1e 120{
29508e1e
NH
121 return eltsize * nmemb;
122}
123
b90a3d7b
JH
124static void closedown_active_slot(struct active_request_slot *slot)
125{
126 active_requests--;
127 slot->in_use = 0;
128}
129
130static void finish_active_slot(struct active_request_slot *slot)
131{
132 closedown_active_slot(slot);
133 curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
134
135 if (slot->finished != NULL)
136 (*slot->finished) = 1;
137
138 /* Store slot results so they can be read after the slot is reused */
139 if (slot->results != NULL) {
140 slot->results->curl_result = slot->curl_result;
141 slot->results->http_code = slot->http_code;
142#if LIBCURL_VERSION_NUM >= 0x070a08
143 curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
144 &slot->results->auth_avail);
145#else
146 slot->results->auth_avail = 0;
147#endif
148 }
149
150 /* Run callback if appropriate */
151 if (slot->callback_func != NULL)
152 slot->callback_func(slot->callback_data);
153}
154
29508e1e
NH
155#ifdef USE_CURL_MULTI
156static void process_curl_messages(void)
157{
158 int num_messages;
159 struct active_request_slot *slot;
160 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
161
162 while (curl_message != NULL) {
163 if (curl_message->msg == CURLMSG_DONE) {
164 int curl_result = curl_message->data.result;
165 slot = active_queue_head;
166 while (slot != NULL &&
167 slot->curl != curl_message->easy_handle)
168 slot = slot->next;
169 if (slot != NULL) {
170 curl_multi_remove_handle(curlm, slot->curl);
171 slot->curl_result = curl_result;
172 finish_active_slot(slot);
173 } else {
174 fprintf(stderr, "Received DONE message for unknown request!\n");
175 }
176 } else {
177 fprintf(stderr, "Unknown CURL message received: %d\n",
178 (int)curl_message->msg);
179 }
180 curl_message = curl_multi_info_read(curlm, &num_messages);
181 }
182}
183#endif
184
ef90d6d4 185static int http_options(const char *var, const char *value, void *cb)
29508e1e
NH
186{
187 if (!strcmp("http.sslverify", var)) {
7059cd99 188 curl_ssl_verify = git_config_bool(var, value);
29508e1e
NH
189 return 0;
190 }
7059cd99
JH
191 if (!strcmp("http.sslcert", var))
192 return git_config_string(&ssl_cert, var, value);
ef52aafa 193#if LIBCURL_VERSION_NUM >= 0x070903
7059cd99
JH
194 if (!strcmp("http.sslkey", var))
195 return git_config_string(&ssl_key, var, value);
29508e1e
NH
196#endif
197#if LIBCURL_VERSION_NUM >= 0x070908
7059cd99
JH
198 if (!strcmp("http.sslcapath", var))
199 return git_config_string(&ssl_capath, var, value);
29508e1e 200#endif
7059cd99
JH
201 if (!strcmp("http.sslcainfo", var))
202 return git_config_string(&ssl_cainfo, var, value);
754ae192 203 if (!strcmp("http.sslcertpasswordprotected", var)) {
3f4ccd2b 204 ssl_cert_password_required = git_config_bool(var, value);
754ae192
ML
205 return 0;
206 }
4bc444eb
MV
207 if (!strcmp("http.ssltry", var)) {
208 curl_ssl_try = git_config_bool(var, value);
209 return 0;
210 }
ad75ebe5
TRC
211 if (!strcmp("http.minsessions", var)) {
212 min_curl_sessions = git_config_int(var, value);
213#ifndef USE_CURL_MULTI
214 if (min_curl_sessions > 1)
215 min_curl_sessions = 1;
216#endif
217 return 0;
218 }
a6080a0a 219#ifdef USE_CURL_MULTI
29508e1e 220 if (!strcmp("http.maxrequests", var)) {
7059cd99 221 max_requests = git_config_int(var, value);
29508e1e
NH
222 return 0;
223 }
224#endif
29508e1e 225 if (!strcmp("http.lowspeedlimit", var)) {
7059cd99 226 curl_low_speed_limit = (long)git_config_int(var, value);
29508e1e
NH
227 return 0;
228 }
229 if (!strcmp("http.lowspeedtime", var)) {
7059cd99 230 curl_low_speed_time = (long)git_config_int(var, value);
29508e1e
NH
231 return 0;
232 }
233
3ea099d4
SK
234 if (!strcmp("http.noepsv", var)) {
235 curl_ftp_no_epsv = git_config_bool(var, value);
236 return 0;
237 }
7059cd99
JH
238 if (!strcmp("http.proxy", var))
239 return git_config_string(&curl_http_proxy, var, value);
3ea099d4 240
bcfb95dd
DB
241 if (!strcmp("http.cookiefile", var))
242 return git_config_string(&curl_cookie_file, var, value);
912b2acf
DB
243 if (!strcmp("http.savecookies", var)) {
244 curl_save_cookies = git_config_bool(var, value);
245 return 0;
246 }
bcfb95dd 247
de1a2fdd
SP
248 if (!strcmp("http.postbuffer", var)) {
249 http_post_buffer = git_config_int(var, value);
250 if (http_post_buffer < LARGE_PACKET_MAX)
251 http_post_buffer = LARGE_PACKET_MAX;
252 return 0;
253 }
254
b1d1058c
SO
255 if (!strcmp("http.useragent", var))
256 return git_config_string(&user_agent, var, value);
257
29508e1e 258 /* Fall back on the default ones */
ef90d6d4 259 return git_default_config(var, value, cb);
29508e1e
NH
260}
261
c33976cb
JH
262static void init_curl_http_auth(CURL *result)
263{
6f4c347c
JK
264 if (!http_auth.username)
265 return;
266
267 credential_fill(&http_auth);
268
269#if LIBCURL_VERSION_NUM >= 0x071301
270 curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
271 curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
272#else
273 {
aa0834a0 274 static struct strbuf up = STRBUF_INIT;
a94cf2cb
BC
275 /*
276 * Note that we assume we only ever have a single set of
277 * credentials in a given program run, so we do not have
278 * to worry about updating this buffer, only setting its
279 * initial value.
280 */
281 if (!up.len)
282 strbuf_addf(&up, "%s:%s",
283 http_auth.username, http_auth.password);
aa0834a0 284 curl_easy_setopt(result, CURLOPT_USERPWD, up.buf);
c33976cb 285 }
6f4c347c 286#endif
c33976cb
JH
287}
288
30dd9163
ML
289static int has_cert_password(void)
290{
30dd9163
ML
291 if (ssl_cert == NULL || ssl_cert_password_required != 1)
292 return 0;
148bb6a7
JK
293 if (!cert_auth.password) {
294 cert_auth.protocol = xstrdup("cert");
75e9a405 295 cert_auth.username = xstrdup("");
148bb6a7
JK
296 cert_auth.path = xstrdup(ssl_cert);
297 credential_fill(&cert_auth);
298 }
299 return 1;
30dd9163
ML
300}
301
47ce1153
JK
302#if LIBCURL_VERSION_NUM >= 0x071900
303static void set_curl_keepalive(CURL *c)
304{
305 curl_easy_setopt(c, CURLOPT_TCP_KEEPALIVE, 1);
306}
307
308#elif LIBCURL_VERSION_NUM >= 0x071000
a15d069a
EW
309static int sockopt_callback(void *client, curl_socket_t fd, curlsocktype type)
310{
311 int ka = 1;
312 int rc;
313 socklen_t len = (socklen_t)sizeof(ka);
314
315 if (type != CURLSOCKTYPE_IPCXN)
316 return 0;
317
318 rc = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&ka, len);
319 if (rc < 0)
320 warning("unable to set SO_KEEPALIVE on socket %s",
321 strerror(errno));
322
323 return 0; /* CURL_SOCKOPT_OK only exists since curl 7.21.5 */
324}
325
47ce1153
JK
326static void set_curl_keepalive(CURL *c)
327{
328 curl_easy_setopt(c, CURLOPT_SOCKOPTFUNCTION, sockopt_callback);
329}
330
331#else
332static void set_curl_keepalive(CURL *c)
333{
334 /* not supported on older curl versions */
335}
336#endif
337
4251ccbd 338static CURL *get_curl_handle(void)
11979b98 339{
4251ccbd 340 CURL *result = curl_easy_init();
f4113cac 341 long allowed_protocols = 0;
11979b98 342
faa3807c
BR
343 if (!result)
344 die("curl_easy_init failed");
345
a5ccc597
JH
346 if (!curl_ssl_verify) {
347 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
348 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
349 } else {
350 /* Verify authenticity of the peer's certificate */
351 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
352 /* The name in the cert must match whom we tried to connect */
353 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
354 }
355
11979b98
JH
356#if LIBCURL_VERSION_NUM >= 0x070907
357 curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
358#endif
b8ac9230 359#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
525ecd26 360 curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
b8ac9230 361#endif
11979b98 362
a4ddbc33
JK
363 if (http_proactive_auth)
364 init_curl_http_auth(result);
365
11979b98
JH
366 if (ssl_cert != NULL)
367 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
30dd9163 368 if (has_cert_password())
148bb6a7 369 curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
ef52aafa 370#if LIBCURL_VERSION_NUM >= 0x070903
11979b98
JH
371 if (ssl_key != NULL)
372 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
373#endif
374#if LIBCURL_VERSION_NUM >= 0x070908
375 if (ssl_capath != NULL)
376 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
377#endif
378 if (ssl_cainfo != NULL)
379 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
11979b98
JH
380
381 if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
382 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
383 curl_low_speed_limit);
384 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
385 curl_low_speed_time);
386 }
387
388 curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
b2581164 389 curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
311e2ea0
TRC
390#if LIBCURL_VERSION_NUM >= 0x071301
391 curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
392#elif LIBCURL_VERSION_NUM >= 0x071101
393 curl_easy_setopt(result, CURLOPT_POST301, 1);
394#endif
f4113cac
BB
395#if LIBCURL_VERSION_NUM >= 0x071304
396 if (is_transport_allowed("http"))
397 allowed_protocols |= CURLPROTO_HTTP;
398 if (is_transport_allowed("https"))
399 allowed_protocols |= CURLPROTO_HTTPS;
400 if (is_transport_allowed("ftp"))
401 allowed_protocols |= CURLPROTO_FTP;
402 if (is_transport_allowed("ftps"))
403 allowed_protocols |= CURLPROTO_FTPS;
404 curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS, allowed_protocols);
405#else
406 if (transport_restrict_protocols())
407 warning("protocol restrictions not applied to curl redirects because\n"
408 "your curl version is too old (>= 7.19.4)");
409#endif
11979b98 410
7982d74e
MW
411 if (getenv("GIT_CURL_VERBOSE"))
412 curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
413
b1d1058c 414 curl_easy_setopt(result, CURLOPT_USERAGENT,
745c7c8e 415 user_agent ? user_agent : git_user_agent());
20fc9bc5 416
3ea099d4
SK
417 if (curl_ftp_no_epsv)
418 curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
419
4bc444eb
MV
420#ifdef CURLOPT_USE_SSL
421 if (curl_ssl_try)
422 curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
423#endif
424
dd613997 425 if (curl_http_proxy) {
9c5665aa 426 curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
5841520b 427 }
1c2dbf20 428#if LIBCURL_VERSION_NUM >= 0x070a07
5841520b 429 curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
1c2dbf20 430#endif
9c5665aa 431
47ce1153 432 set_curl_keepalive(result);
a15d069a 433
11979b98
JH
434 return result;
435}
436
7059cd99
JH
437static void set_from_env(const char **var, const char *envname)
438{
439 const char *val = getenv(envname);
440 if (val)
441 *var = val;
442}
443
a4ddbc33 444void http_init(struct remote *remote, const char *url, int proactive_auth)
29508e1e
NH
445{
446 char *low_speed_limit;
447 char *low_speed_time;
6a56993b
KM
448 char *normalized_url;
449 struct urlmatch_config config = { STRING_LIST_INIT_DUP };
450
451 config.section = "http";
452 config.key = NULL;
453 config.collect_fn = http_options;
454 config.cascade_fn = git_default_config;
455 config.cb = NULL;
29508e1e 456
e9176745 457 http_is_verbose = 0;
6a56993b 458 normalized_url = url_normalize(url, &config.url);
e9176745 459
6a56993b
KM
460 git_config(urlmatch_config_entry, &config);
461 free(normalized_url);
7059cd99 462
faa3807c
BR
463 if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
464 die("curl_global_init failed");
29508e1e 465
a4ddbc33
JK
466 http_proactive_auth = proactive_auth;
467
9fc6440d
MH
468 if (remote && remote->http_proxy)
469 curl_http_proxy = xstrdup(remote->http_proxy);
470
29508e1e 471 pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
e9176745 472 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
29508e1e
NH
473
474#ifdef USE_CURL_MULTI
475 {
476 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
477 if (http_max_requests != NULL)
478 max_requests = atoi(http_max_requests);
479 }
480
481 curlm = curl_multi_init();
8837eb47
JK
482 if (!curlm)
483 die("curl_multi_init failed");
29508e1e
NH
484#endif
485
486 if (getenv("GIT_SSL_NO_VERIFY"))
487 curl_ssl_verify = 0;
488
7059cd99 489 set_from_env(&ssl_cert, "GIT_SSL_CERT");
ef52aafa 490#if LIBCURL_VERSION_NUM >= 0x070903
7059cd99 491 set_from_env(&ssl_key, "GIT_SSL_KEY");
29508e1e
NH
492#endif
493#if LIBCURL_VERSION_NUM >= 0x070908
7059cd99 494 set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
29508e1e 495#endif
7059cd99 496 set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
29508e1e 497
b1d1058c
SO
498 set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
499
29508e1e
NH
500 low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
501 if (low_speed_limit != NULL)
502 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
503 low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
504 if (low_speed_time != NULL)
505 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
506
29508e1e
NH
507 if (curl_ssl_verify == -1)
508 curl_ssl_verify = 1;
509
ad75ebe5 510 curl_session_count = 0;
29508e1e
NH
511#ifdef USE_CURL_MULTI
512 if (max_requests < 1)
513 max_requests = DEFAULT_MAX_REQUESTS;
514#endif
515
3ea099d4
SK
516 if (getenv("GIT_CURL_FTP_NO_EPSV"))
517 curl_ftp_no_epsv = 1;
518
deba4937 519 if (url) {
148bb6a7 520 credential_from_url(&http_auth, url);
754ae192
ML
521 if (!ssl_cert_password_required &&
522 getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
59556548 523 starts_with(url, "https://"))
30dd9163
ML
524 ssl_cert_password_required = 1;
525 }
c33976cb 526
29508e1e
NH
527#ifndef NO_CURL_EASY_DUPHANDLE
528 curl_default = get_curl_handle();
529#endif
530}
531
532void http_cleanup(void)
533{
534 struct active_request_slot *slot = active_queue_head;
29508e1e
NH
535
536 while (slot != NULL) {
3278cd0a 537 struct active_request_slot *next = slot->next;
f23d1f76 538 if (slot->curl != NULL) {
29508e1e 539#ifdef USE_CURL_MULTI
f23d1f76 540 curl_multi_remove_handle(curlm, slot->curl);
29508e1e 541#endif
29508e1e 542 curl_easy_cleanup(slot->curl);
f23d1f76 543 }
3278cd0a
SP
544 free(slot);
545 slot = next;
29508e1e 546 }
3278cd0a 547 active_queue_head = NULL;
29508e1e
NH
548
549#ifndef NO_CURL_EASY_DUPHANDLE
550 curl_easy_cleanup(curl_default);
551#endif
552
553#ifdef USE_CURL_MULTI
554 curl_multi_cleanup(curlm);
555#endif
556 curl_global_cleanup();
b3ca4e4e
NH
557
558 curl_slist_free_all(pragma_header);
3278cd0a 559 pragma_header = NULL;
9fc6440d 560
e9176745
TRC
561 curl_slist_free_all(no_pragma_header);
562 no_pragma_header = NULL;
563
9fc6440d 564 if (curl_http_proxy) {
e4a80ecf 565 free((void *)curl_http_proxy);
9fc6440d
MH
566 curl_http_proxy = NULL;
567 }
30dd9163 568
148bb6a7
JK
569 if (cert_auth.password != NULL) {
570 memset(cert_auth.password, 0, strlen(cert_auth.password));
571 free(cert_auth.password);
572 cert_auth.password = NULL;
30dd9163
ML
573 }
574 ssl_cert_password_required = 0;
f18604bb
YE
575
576 free(cached_accept_language);
577 cached_accept_language = NULL;
29508e1e
NH
578}
579
29508e1e
NH
580struct active_request_slot *get_active_slot(void)
581{
582 struct active_request_slot *slot = active_queue_head;
583 struct active_request_slot *newslot;
584
585#ifdef USE_CURL_MULTI
586 int num_transfers;
587
588 /* Wait for a slot to open up if the queue is full */
589 while (active_requests >= max_requests) {
590 curl_multi_perform(curlm, &num_transfers);
4251ccbd 591 if (num_transfers < active_requests)
29508e1e 592 process_curl_messages();
29508e1e
NH
593 }
594#endif
595
4251ccbd 596 while (slot != NULL && slot->in_use)
29508e1e 597 slot = slot->next;
4251ccbd 598
29508e1e
NH
599 if (slot == NULL) {
600 newslot = xmalloc(sizeof(*newslot));
601 newslot->curl = NULL;
602 newslot->in_use = 0;
603 newslot->next = NULL;
604
605 slot = active_queue_head;
606 if (slot == NULL) {
607 active_queue_head = newslot;
608 } else {
4251ccbd 609 while (slot->next != NULL)
29508e1e 610 slot = slot->next;
29508e1e
NH
611 slot->next = newslot;
612 }
613 slot = newslot;
614 }
615
616 if (slot->curl == NULL) {
617#ifdef NO_CURL_EASY_DUPHANDLE
618 slot->curl = get_curl_handle();
619#else
620 slot->curl = curl_easy_duphandle(curl_default);
621#endif
ad75ebe5 622 curl_session_count++;
29508e1e
NH
623 }
624
625 active_requests++;
626 slot->in_use = 1;
c8568e13 627 slot->results = NULL;
baa7b67d 628 slot->finished = NULL;
29508e1e
NH
629 slot->callback_data = NULL;
630 slot->callback_func = NULL;
bcfb95dd 631 curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
912b2acf
DB
632 if (curl_save_cookies)
633 curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
29508e1e 634 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
29508e1e 635 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
9094950d
NH
636 curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
637 curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
638 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
1e41827d 639 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
9094950d
NH
640 curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
641 curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
b793acf1 642 curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1);
4dbe6646 643#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
644 curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
645#endif
dfa1725a
JK
646 if (http_auth.password)
647 init_curl_http_auth(slot->curl);
29508e1e
NH
648
649 return slot;
650}
651
652int start_active_slot(struct active_request_slot *slot)
653{
654#ifdef USE_CURL_MULTI
655 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
45c17412 656 int num_transfers;
29508e1e
NH
657
658 if (curlm_result != CURLM_OK &&
659 curlm_result != CURLM_CALL_MULTI_PERFORM) {
660 active_requests--;
661 slot->in_use = 0;
662 return 0;
663 }
45c17412
DB
664
665 /*
666 * We know there must be something to do, since we just added
667 * something.
668 */
669 curl_multi_perform(curlm, &num_transfers);
29508e1e
NH
670#endif
671 return 1;
672}
673
674#ifdef USE_CURL_MULTI
fc57b6aa
DB
675struct fill_chain {
676 void *data;
677 int (*fill)(void *);
678 struct fill_chain *next;
679};
680
4251ccbd 681static struct fill_chain *fill_cfg;
fc57b6aa
DB
682
683void add_fill_function(void *data, int (*fill)(void *))
684{
e8eec71d 685 struct fill_chain *new = xmalloc(sizeof(*new));
fc57b6aa
DB
686 struct fill_chain **linkp = &fill_cfg;
687 new->data = data;
688 new->fill = fill;
689 new->next = NULL;
690 while (*linkp)
691 linkp = &(*linkp)->next;
692 *linkp = new;
693}
694
45c17412
DB
695void fill_active_slots(void)
696{
697 struct active_request_slot *slot = active_queue_head;
698
fc57b6aa
DB
699 while (active_requests < max_requests) {
700 struct fill_chain *fill;
701 for (fill = fill_cfg; fill; fill = fill->next)
702 if (fill->fill(fill->data))
703 break;
704
705 if (!fill)
45c17412 706 break;
fc57b6aa 707 }
45c17412
DB
708
709 while (slot != NULL) {
ad75ebe5
TRC
710 if (!slot->in_use && slot->curl != NULL
711 && curl_session_count > min_curl_sessions) {
45c17412
DB
712 curl_easy_cleanup(slot->curl);
713 slot->curl = NULL;
ad75ebe5 714 curl_session_count--;
45c17412
DB
715 }
716 slot = slot->next;
717 }
718}
719
29508e1e
NH
720void step_active_slots(void)
721{
722 int num_transfers;
723 CURLMcode curlm_result;
724
725 do {
726 curlm_result = curl_multi_perform(curlm, &num_transfers);
727 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
728 if (num_transfers < active_requests) {
729 process_curl_messages();
730 fill_active_slots();
731 }
732}
733#endif
734
735void run_active_slot(struct active_request_slot *slot)
736{
737#ifdef USE_CURL_MULTI
29508e1e
NH
738 fd_set readfds;
739 fd_set writefds;
740 fd_set excfds;
741 int max_fd;
742 struct timeval select_timeout;
baa7b67d 743 int finished = 0;
29508e1e 744
baa7b67d
NH
745 slot->finished = &finished;
746 while (!finished) {
29508e1e
NH
747 step_active_slots();
748
df26c471 749 if (slot->in_use) {
eb56c821
MF
750#if LIBCURL_VERSION_NUM >= 0x070f04
751 long curl_timeout;
752 curl_multi_timeout(curlm, &curl_timeout);
753 if (curl_timeout == 0) {
754 continue;
755 } else if (curl_timeout == -1) {
756 select_timeout.tv_sec = 0;
757 select_timeout.tv_usec = 50000;
758 } else {
759 select_timeout.tv_sec = curl_timeout / 1000;
760 select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
761 }
762#else
763 select_timeout.tv_sec = 0;
764 select_timeout.tv_usec = 50000;
765#endif
29508e1e 766
6f9dd67f 767 max_fd = -1;
29508e1e
NH
768 FD_ZERO(&readfds);
769 FD_ZERO(&writefds);
770 FD_ZERO(&excfds);
6f9dd67f 771 curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
eb56c821 772
7202b81f
SZ
773 /*
774 * It can happen that curl_multi_timeout returns a pathologically
775 * long timeout when curl_multi_fdset returns no file descriptors
776 * to read. See commit message for more details.
777 */
778 if (max_fd < 0 &&
779 (select_timeout.tv_sec > 0 ||
780 select_timeout.tv_usec > 50000)) {
781 select_timeout.tv_sec = 0;
782 select_timeout.tv_usec = 50000;
783 }
784
6f9dd67f 785 select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
29508e1e
NH
786 }
787 }
788#else
789 while (slot->in_use) {
790 slot->curl_result = curl_easy_perform(slot->curl);
791 finish_active_slot(slot);
792 }
793#endif
794}
795
83e41e2e 796static void release_active_slot(struct active_request_slot *slot)
53f31389
MW
797{
798 closedown_active_slot(slot);
ad75ebe5 799 if (slot->curl && curl_session_count > min_curl_sessions) {
b3ca4e4e 800#ifdef USE_CURL_MULTI
53f31389 801 curl_multi_remove_handle(curlm, slot->curl);
b3ca4e4e 802#endif
53f31389
MW
803 curl_easy_cleanup(slot->curl);
804 slot->curl = NULL;
ad75ebe5 805 curl_session_count--;
53f31389 806 }
b3ca4e4e 807#ifdef USE_CURL_MULTI
53f31389 808 fill_active_slots();
b3ca4e4e 809#endif
53f31389
MW
810}
811
29508e1e
NH
812void finish_all_active_slots(void)
813{
814 struct active_request_slot *slot = active_queue_head;
815
816 while (slot != NULL)
817 if (slot->in_use) {
818 run_active_slot(slot);
819 slot = active_queue_head;
820 } else {
821 slot = slot->next;
822 }
823}
d7e92806 824
5ace994f 825/* Helpers for modifying and creating URLs */
d7e92806
MH
826static inline int needs_quote(int ch)
827{
828 if (((ch >= 'A') && (ch <= 'Z'))
829 || ((ch >= 'a') && (ch <= 'z'))
830 || ((ch >= '0') && (ch <= '9'))
831 || (ch == '/')
832 || (ch == '-')
833 || (ch == '.'))
834 return 0;
835 return 1;
836}
837
d7e92806
MH
838static char *quote_ref_url(const char *base, const char *ref)
839{
113106e0 840 struct strbuf buf = STRBUF_INIT;
d7e92806 841 const char *cp;
113106e0 842 int ch;
d7e92806 843
5ace994f 844 end_url_with_slash(&buf, base);
113106e0
TRC
845
846 for (cp = ref; (ch = *cp) != 0; cp++)
d7e92806 847 if (needs_quote(ch))
113106e0 848 strbuf_addf(&buf, "%%%02x", ch);
d7e92806 849 else
113106e0 850 strbuf_addch(&buf, *cp);
d7e92806 851
113106e0 852 return strbuf_detach(&buf, NULL);
d7e92806
MH
853}
854
5424bc55
TRC
855void append_remote_object_url(struct strbuf *buf, const char *url,
856 const char *hex,
857 int only_two_digit_prefix)
858{
800324c3
TRC
859 end_url_with_slash(buf, url);
860
861 strbuf_addf(buf, "objects/%.*s/", 2, hex);
5424bc55
TRC
862 if (!only_two_digit_prefix)
863 strbuf_addf(buf, "%s", hex+2);
864}
865
866char *get_remote_object_url(const char *url, const char *hex,
867 int only_two_digit_prefix)
868{
869 struct strbuf buf = STRBUF_INIT;
870 append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
871 return strbuf_detach(&buf, NULL);
872}
873
b90a3d7b 874static int handle_curl_result(struct slot_results *results)
88097030 875{
6d052d78
JK
876 /*
877 * If we see a failing http code with CURLE_OK, we have turned off
878 * FAILONERROR (to keep the server's custom error response), and should
879 * translate the code into failure here.
880 */
881 if (results->curl_result == CURLE_OK &&
882 results->http_code >= 400) {
883 results->curl_result = CURLE_HTTP_RETURNED_ERROR;
884 /*
885 * Normally curl will already have put the "reason phrase"
886 * from the server into curl_errorstr; unfortunately without
887 * FAILONERROR it is lost, so we can give only the numeric
888 * status code.
889 */
890 snprintf(curl_errorstr, sizeof(curl_errorstr),
891 "The requested URL returned error: %ld",
892 results->http_code);
893 }
894
88097030
JK
895 if (results->curl_result == CURLE_OK) {
896 credential_approve(&http_auth);
897 return HTTP_OK;
898 } else if (missing_target(results))
899 return HTTP_MISSING_TARGET;
900 else if (results->http_code == 401) {
901 if (http_auth.username && http_auth.password) {
902 credential_reject(&http_auth);
903 return HTTP_NOAUTH;
904 } else {
4dbe6646 905#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
906 http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
907#endif
88097030
JK
908 return HTTP_REAUTH;
909 }
910 } else {
3503e9ab 911#if LIBCURL_VERSION_NUM >= 0x070c00
88097030
JK
912 if (!curl_errorstr[0])
913 strlcpy(curl_errorstr,
914 curl_easy_strerror(results->curl_result),
915 sizeof(curl_errorstr));
3503e9ab 916#endif
88097030
JK
917 return HTTP_ERROR;
918 }
919}
920
beed336c
JK
921int run_one_slot(struct active_request_slot *slot,
922 struct slot_results *results)
923{
924 slot->results = results;
925 if (!start_active_slot(slot)) {
926 snprintf(curl_errorstr, sizeof(curl_errorstr),
927 "failed to start HTTP request");
928 return HTTP_START_FAILED;
929 }
930
931 run_active_slot(slot);
932 return handle_curl_result(results);
933}
934
132b70a2
JK
935static CURLcode curlinfo_strbuf(CURL *curl, CURLINFO info, struct strbuf *buf)
936{
937 char *ptr;
938 CURLcode ret;
939
940 strbuf_reset(buf);
941 ret = curl_easy_getinfo(curl, info, &ptr);
942 if (!ret && ptr)
943 strbuf_addstr(buf, ptr);
944 return ret;
945}
946
e3131626
JK
947/*
948 * Check for and extract a content-type parameter. "raw"
949 * should be positioned at the start of the potential
950 * parameter, with any whitespace already removed.
951 *
952 * "name" is the name of the parameter. The value is appended
953 * to "out".
954 */
955static int extract_param(const char *raw, const char *name,
956 struct strbuf *out)
957{
958 size_t len = strlen(name);
959
960 if (strncasecmp(raw, name, len))
961 return -1;
962 raw += len;
963
964 if (*raw != '=')
965 return -1;
966 raw++;
967
f34a655d 968 while (*raw && !isspace(*raw) && *raw != ';')
e3131626
JK
969 strbuf_addch(out, *raw++);
970 return 0;
971}
972
bf197fd7
JK
973/*
974 * Extract a normalized version of the content type, with any
975 * spaces suppressed, all letters lowercased, and no trailing ";"
976 * or parameters.
977 *
978 * Note that we will silently remove even invalid whitespace. For
979 * example, "text / plain" is specifically forbidden by RFC 2616,
980 * but "text/plain" is the only reasonable output, and this keeps
981 * our code simple.
982 *
e3131626
JK
983 * If the "charset" argument is not NULL, store the value of any
984 * charset parameter there.
985 *
bf197fd7 986 * Example:
e3131626 987 * "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
bf197fd7
JK
988 * "text / plain" -> "text/plain"
989 */
e3131626
JK
990static void extract_content_type(struct strbuf *raw, struct strbuf *type,
991 struct strbuf *charset)
bf197fd7
JK
992{
993 const char *p;
994
995 strbuf_reset(type);
996 strbuf_grow(type, raw->len);
997 for (p = raw->buf; *p; p++) {
998 if (isspace(*p))
999 continue;
e3131626
JK
1000 if (*p == ';') {
1001 p++;
bf197fd7 1002 break;
e3131626 1003 }
bf197fd7
JK
1004 strbuf_addch(type, tolower(*p));
1005 }
e3131626
JK
1006
1007 if (!charset)
1008 return;
1009
1010 strbuf_reset(charset);
1011 while (*p) {
f34a655d 1012 while (isspace(*p) || *p == ';')
e3131626
JK
1013 p++;
1014 if (!extract_param(p, "charset", charset))
1015 return;
1016 while (*p && !isspace(*p))
1017 p++;
1018 }
c553fd1c
JK
1019
1020 if (!charset->len && starts_with(type->buf, "text/"))
1021 strbuf_addstr(charset, "ISO-8859-1");
bf197fd7
JK
1022}
1023
f18604bb
YE
1024static void write_accept_language(struct strbuf *buf)
1025{
1026 /*
1027 * MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
1028 * that, q-value will be smaller than 0.001, the minimum q-value the
1029 * HTTP specification allows. See
1030 * http://tools.ietf.org/html/rfc7231#section-5.3.1 for q-value.
1031 */
1032 const int MAX_DECIMAL_PLACES = 3;
1033 const int MAX_LANGUAGE_TAGS = 1000;
1034 const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
1035 char **language_tags = NULL;
1036 int num_langs = 0;
1037 const char *s = get_preferred_languages();
1038 int i;
1039 struct strbuf tag = STRBUF_INIT;
1040
1041 /* Don't add Accept-Language header if no language is preferred. */
1042 if (!s)
1043 return;
1044
1045 /*
1046 * Split the colon-separated string of preferred languages into
1047 * language_tags array.
1048 */
1049 do {
1050 /* collect language tag */
1051 for (; *s && (isalnum(*s) || *s == '_'); s++)
1052 strbuf_addch(&tag, *s == '_' ? '-' : *s);
1053
1054 /* skip .codeset, @modifier and any other unnecessary parts */
1055 while (*s && *s != ':')
1056 s++;
1057
1058 if (tag.len) {
1059 num_langs++;
1060 REALLOC_ARRAY(language_tags, num_langs);
1061 language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
1062 if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '*' */
1063 break;
1064 }
1065 } while (*s++);
1066
1067 /* write Accept-Language header into buf */
1068 if (num_langs) {
1069 int last_buf_len = 0;
1070 int max_q;
1071 int decimal_places;
1072 char q_format[32];
1073
1074 /* add '*' */
1075 REALLOC_ARRAY(language_tags, num_langs + 1);
1076 language_tags[num_langs++] = "*"; /* it's OK; this won't be freed */
1077
1078 /* compute decimal_places */
1079 for (max_q = 1, decimal_places = 0;
1080 max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
1081 decimal_places++, max_q *= 10)
1082 ;
1083
1084 sprintf(q_format, ";q=0.%%0%dd", decimal_places);
1085
1086 strbuf_addstr(buf, "Accept-Language: ");
1087
1088 for (i = 0; i < num_langs; i++) {
1089 if (i > 0)
1090 strbuf_addstr(buf, ", ");
1091
1092 strbuf_addstr(buf, language_tags[i]);
1093
1094 if (i > 0)
1095 strbuf_addf(buf, q_format, max_q - i);
1096
1097 if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
1098 strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
1099 break;
1100 }
1101
1102 last_buf_len = buf->len;
1103 }
1104 }
1105
1106 /* free language tags -- last one is a static '*' */
1107 for (i = 0; i < num_langs - 1; i++)
1108 free(language_tags[i]);
1109 free(language_tags);
1110}
1111
1112/*
1113 * Get an Accept-Language header which indicates user's preferred languages.
1114 *
1115 * Examples:
1116 * LANGUAGE= -> ""
1117 * LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
1118 * LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
1119 * LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
1120 * LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
1121 * LANGUAGE= LANG=C -> ""
1122 */
1123static const char *get_accept_language(void)
1124{
1125 if (!cached_accept_language) {
1126 struct strbuf buf = STRBUF_INIT;
1127 write_accept_language(&buf);
1128 if (buf.len > 0)
1129 cached_accept_language = strbuf_detach(&buf, NULL);
1130 }
1131
1132 return cached_accept_language;
1133}
1134
e929cd20
MH
1135/* http_request() targets */
1136#define HTTP_REQUEST_STRBUF 0
1137#define HTTP_REQUEST_FILE 1
1138
1bbcc224
JK
1139static int http_request(const char *url,
1140 void *result, int target,
1141 const struct http_get_options *options)
e929cd20
MH
1142{
1143 struct active_request_slot *slot;
1144 struct slot_results results;
1145 struct curl_slist *headers = NULL;
1146 struct strbuf buf = STRBUF_INIT;
f18604bb 1147 const char *accept_language;
e929cd20
MH
1148 int ret;
1149
1150 slot = get_active_slot();
e929cd20
MH
1151 curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1152
1153 if (result == NULL) {
1154 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
1155 } else {
1156 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
1157 curl_easy_setopt(slot->curl, CURLOPT_FILE, result);
1158
1159 if (target == HTTP_REQUEST_FILE) {
1160 long posn = ftell(result);
1161 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1162 fwrite);
1163 if (posn > 0) {
1164 strbuf_addf(&buf, "Range: bytes=%ld-", posn);
1165 headers = curl_slist_append(headers, buf.buf);
1166 strbuf_reset(&buf);
1167 }
e929cd20
MH
1168 } else
1169 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1170 fwrite_buffer);
1171 }
1172
f18604bb
YE
1173 accept_language = get_accept_language();
1174
1175 if (accept_language)
1176 headers = curl_slist_append(headers, accept_language);
1177
e929cd20 1178 strbuf_addstr(&buf, "Pragma:");
1bbcc224 1179 if (options && options->no_cache)
e929cd20 1180 strbuf_addstr(&buf, " no-cache");
1bbcc224 1181 if (options && options->keep_error)
6d052d78 1182 curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0);
e929cd20
MH
1183
1184 headers = curl_slist_append(headers, buf.buf);
1185
1186 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1187 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
aa90b969 1188 curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip");
e929cd20 1189
beed336c 1190 ret = run_one_slot(slot, &results);
e929cd20 1191
bf197fd7
JK
1192 if (options && options->content_type) {
1193 struct strbuf raw = STRBUF_INIT;
1194 curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
e3131626
JK
1195 extract_content_type(&raw, options->content_type,
1196 options->charset);
bf197fd7
JK
1197 strbuf_release(&raw);
1198 }
4656bf47 1199
78868962
JK
1200 if (options && options->effective_url)
1201 curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
1202 options->effective_url);
4656bf47 1203
e929cd20
MH
1204 curl_slist_free_all(headers);
1205 strbuf_release(&buf);
1206
1207 return ret;
1208}
1209
c93c92f3
JK
1210/*
1211 * Update the "base" url to a more appropriate value, as deduced by
1212 * redirects seen when requesting a URL starting with "url".
1213 *
1214 * The "asked" parameter is a URL that we asked curl to access, and must begin
1215 * with "base".
1216 *
1217 * The "got" parameter is the URL that curl reported to us as where we ended
1218 * up.
1219 *
1220 * Returns 1 if we updated the base url, 0 otherwise.
1221 *
1222 * Our basic strategy is to compare "base" and "asked" to find the bits
1223 * specific to our request. We then strip those bits off of "got" to yield the
1224 * new base. So for example, if our base is "http://example.com/foo.git",
1225 * and we ask for "http://example.com/foo.git/info/refs", we might end up
1226 * with "https://other.example.com/foo.git/info/refs". We would want the
1227 * new URL to become "https://other.example.com/foo.git".
1228 *
1229 * Note that this assumes a sane redirect scheme. It's entirely possible
1230 * in the example above to end up at a URL that does not even end in
1231 * "info/refs". In such a case we simply punt, as there is not much we can
1232 * do (and such a scheme is unlikely to represent a real git repository,
1233 * which means we are likely about to abort anyway).
1234 */
1235static int update_url_from_redirect(struct strbuf *base,
1236 const char *asked,
1237 const struct strbuf *got)
1238{
1239 const char *tail;
1240 size_t tail_len;
1241
1242 if (!strcmp(asked, got->buf))
1243 return 0;
1244
de8118e1 1245 if (!skip_prefix(asked, base->buf, &tail))
c93c92f3
JK
1246 die("BUG: update_url_from_redirect: %s is not a superset of %s",
1247 asked, base->buf);
1248
c93c92f3
JK
1249 tail_len = strlen(tail);
1250
1251 if (got->len < tail_len ||
1252 strcmp(tail, got->buf + got->len - tail_len))
1253 return 0; /* insane redirect scheme */
1254
1255 strbuf_reset(base);
1256 strbuf_add(base, got->buf, got->len - tail_len);
1257 return 1;
1258}
1259
4656bf47 1260static int http_request_reauth(const char *url,
4656bf47 1261 void *result, int target,
1bbcc224 1262 struct http_get_options *options)
8d677edc 1263{
1bbcc224 1264 int ret = http_request(url, result, target, options);
c93c92f3
JK
1265
1266 if (options && options->effective_url && options->base_url) {
1267 if (update_url_from_redirect(options->base_url,
1268 url, options->effective_url)) {
1269 credential_from_url(&http_auth, options->base_url->buf);
1270 url = options->effective_url->buf;
1271 }
1272 }
1273
8d677edc
JK
1274 if (ret != HTTP_REAUTH)
1275 return ret;
6d052d78
JK
1276
1277 /*
1278 * If we are using KEEP_ERROR, the previous request may have
1279 * put cruft into our output stream; we should clear it out before
1280 * making our next request. We only know how to do this for
1281 * the strbuf case, but that is enough to satisfy current callers.
1282 */
1bbcc224 1283 if (options && options->keep_error) {
6d052d78
JK
1284 switch (target) {
1285 case HTTP_REQUEST_STRBUF:
1286 strbuf_reset(result);
1287 break;
1288 default:
1289 die("BUG: HTTP_KEEP_ERROR is only supported with strbufs");
1290 }
1291 }
2501aff8
JK
1292
1293 credential_fill(&http_auth);
1294
1bbcc224 1295 return http_request(url, result, target, options);
8d677edc
JK
1296}
1297
4656bf47 1298int http_get_strbuf(const char *url,
1bbcc224
JK
1299 struct strbuf *result,
1300 struct http_get_options *options)
e929cd20 1301{
1bbcc224 1302 return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options);
e929cd20
MH
1303}
1304
83e41e2e 1305/*
a7793a74 1306 * Downloads a URL and stores the result in the given file.
83e41e2e
JH
1307 *
1308 * If a previous interrupted download is detected (i.e. a previous temporary
1309 * file is still around) the download is resumed.
1310 */
1bbcc224
JK
1311static int http_get_file(const char *url, const char *filename,
1312 struct http_get_options *options)
e929cd20
MH
1313{
1314 int ret;
1315 struct strbuf tmpfile = STRBUF_INIT;
1316 FILE *result;
1317
1318 strbuf_addf(&tmpfile, "%s.temp", filename);
1319 result = fopen(tmpfile.buf, "a");
3d1fb769 1320 if (!result) {
e929cd20
MH
1321 error("Unable to open local file %s", tmpfile.buf);
1322 ret = HTTP_ERROR;
1323 goto cleanup;
1324 }
1325
1bbcc224 1326 ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);
e929cd20
MH
1327 fclose(result);
1328
3d1fb769 1329 if (ret == HTTP_OK && move_temp_to_file(tmpfile.buf, filename))
e929cd20
MH
1330 ret = HTTP_ERROR;
1331cleanup:
1332 strbuf_release(&tmpfile);
1333 return ret;
1334}
1335
c13b2633 1336int http_fetch_ref(const char *base, struct ref *ref)
d7e92806 1337{
1bbcc224 1338 struct http_get_options options = {0};
d7e92806
MH
1339 char *url;
1340 struct strbuf buffer = STRBUF_INIT;
0d5896e1 1341 int ret = -1;
d7e92806 1342
1bbcc224
JK
1343 options.no_cache = 1;
1344
c13b2633 1345 url = quote_ref_url(base, ref->name);
1bbcc224 1346 if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
0d5896e1
MH
1347 strbuf_rtrim(&buffer);
1348 if (buffer.len == 40)
1349 ret = get_sha1_hex(buffer.buf, ref->old_sha1);
59556548 1350 else if (starts_with(buffer.buf, "ref: ")) {
0d5896e1
MH
1351 ref->symref = xstrdup(buffer.buf + 5);
1352 ret = 0;
d7e92806 1353 }
d7e92806
MH
1354 }
1355
1356 strbuf_release(&buffer);
1357 free(url);
1358 return ret;
1359}
b8caac2b
TRC
1360
1361/* Helpers for fetching packs */
750ef425 1362static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
b8caac2b 1363{
750ef425 1364 char *url, *tmp;
b8caac2b 1365 struct strbuf buf = STRBUF_INIT;
b8caac2b 1366
b8caac2b 1367 if (http_is_verbose)
162eb5f8 1368 fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
b8caac2b
TRC
1369
1370 end_url_with_slash(&buf, base_url);
162eb5f8 1371 strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
b8caac2b
TRC
1372 url = strbuf_detach(&buf, NULL);
1373
750ef425
SP
1374 strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
1375 tmp = strbuf_detach(&buf, NULL);
1376
70900eda 1377 if (http_get_file(url, tmp, NULL) != HTTP_OK) {
82247e9b 1378 error("Unable to get pack index %s", url);
750ef425
SP
1379 free(tmp);
1380 tmp = NULL;
1381 }
b8caac2b 1382
b8caac2b 1383 free(url);
750ef425 1384 return tmp;
b8caac2b
TRC
1385}
1386
1387static int fetch_and_setup_pack_index(struct packed_git **packs_head,
1388 unsigned char *sha1, const char *base_url)
1389{
1390 struct packed_git *new_pack;
750ef425
SP
1391 char *tmp_idx = NULL;
1392 int ret;
b8caac2b 1393
750ef425 1394 if (has_pack_index(sha1)) {
8b9c2dd4 1395 new_pack = parse_pack_index(sha1, sha1_pack_index_name(sha1));
750ef425
SP
1396 if (!new_pack)
1397 return -1; /* parse_pack_index() already issued error message */
1398 goto add_pack;
1399 }
1400
1401 tmp_idx = fetch_pack_index(sha1, base_url);
1402 if (!tmp_idx)
b8caac2b
TRC
1403 return -1;
1404
750ef425
SP
1405 new_pack = parse_pack_index(sha1, tmp_idx);
1406 if (!new_pack) {
1407 unlink(tmp_idx);
1408 free(tmp_idx);
1409
b8caac2b 1410 return -1; /* parse_pack_index() already issued error message */
750ef425
SP
1411 }
1412
1413 ret = verify_pack_index(new_pack);
1414 if (!ret) {
1415 close_pack_index(new_pack);
1416 ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
1417 }
1418 free(tmp_idx);
1419 if (ret)
1420 return -1;
1421
1422add_pack:
b8caac2b
TRC
1423 new_pack->next = *packs_head;
1424 *packs_head = new_pack;
1425 return 0;
1426}
1427
1428int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
1429{
1bbcc224 1430 struct http_get_options options = {0};
b8caac2b
TRC
1431 int ret = 0, i = 0;
1432 char *url, *data;
1433 struct strbuf buf = STRBUF_INIT;
1434 unsigned char sha1[20];
1435
1436 end_url_with_slash(&buf, base_url);
1437 strbuf_addstr(&buf, "objects/info/packs");
1438 url = strbuf_detach(&buf, NULL);
1439
1bbcc224
JK
1440 options.no_cache = 1;
1441 ret = http_get_strbuf(url, &buf, &options);
b8caac2b
TRC
1442 if (ret != HTTP_OK)
1443 goto cleanup;
1444
1445 data = buf.buf;
1446 while (i < buf.len) {
1447 switch (data[i]) {
1448 case 'P':
1449 i++;
1450 if (i + 52 <= buf.len &&
59556548
CC
1451 starts_with(data + i, " pack-") &&
1452 starts_with(data + i + 46, ".pack\n")) {
b8caac2b
TRC
1453 get_sha1_hex(data + i + 6, sha1);
1454 fetch_and_setup_pack_index(packs_head, sha1,
1455 base_url);
1456 i += 51;
1457 break;
1458 }
1459 default:
1460 while (i < buf.len && data[i] != '\n')
1461 i++;
1462 }
1463 i++;
1464 }
1465
1466cleanup:
1467 free(url);
1468 return ret;
1469}
2264dfa5
TRC
1470
1471void release_http_pack_request(struct http_pack_request *preq)
1472{
1473 if (preq->packfile != NULL) {
1474 fclose(preq->packfile);
1475 preq->packfile = NULL;
2264dfa5
TRC
1476 }
1477 if (preq->range_header != NULL) {
1478 curl_slist_free_all(preq->range_header);
1479 preq->range_header = NULL;
1480 }
1481 preq->slot = NULL;
1482 free(preq->url);
826aed50 1483 free(preq);
2264dfa5
TRC
1484}
1485
1486int finish_http_pack_request(struct http_pack_request *preq)
1487{
2264dfa5 1488 struct packed_git **lst;
021ab6f0 1489 struct packed_git *p = preq->target;
fe72d420 1490 char *tmp_idx;
d3180279 1491 struct child_process ip = CHILD_PROCESS_INIT;
fe72d420 1492 const char *ip_argv[8];
2264dfa5 1493
fe72d420 1494 close_pack_index(p);
2264dfa5 1495
3065274c
SP
1496 fclose(preq->packfile);
1497 preq->packfile = NULL;
2264dfa5
TRC
1498
1499 lst = preq->lst;
021ab6f0 1500 while (*lst != p)
2264dfa5
TRC
1501 lst = &((*lst)->next);
1502 *lst = (*lst)->next;
1503
fe72d420
SP
1504 tmp_idx = xstrdup(preq->tmpfile);
1505 strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"),
1506 ".idx.temp");
1507
1508 ip_argv[0] = "index-pack";
1509 ip_argv[1] = "-o";
1510 ip_argv[2] = tmp_idx;
1511 ip_argv[3] = preq->tmpfile;
1512 ip_argv[4] = NULL;
1513
fe72d420
SP
1514 ip.argv = ip_argv;
1515 ip.git_cmd = 1;
1516 ip.no_stdin = 1;
1517 ip.no_stdout = 1;
1518
1519 if (run_command(&ip)) {
1520 unlink(preq->tmpfile);
1521 unlink(tmp_idx);
1522 free(tmp_idx);
2264dfa5 1523 return -1;
fe72d420
SP
1524 }
1525
1526 unlink(sha1_pack_index_name(p->sha1));
2264dfa5 1527
fe72d420
SP
1528 if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
1529 || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
1530 free(tmp_idx);
2264dfa5 1531 return -1;
fe72d420 1532 }
2264dfa5 1533
fe72d420
SP
1534 install_packed_git(p);
1535 free(tmp_idx);
2264dfa5
TRC
1536 return 0;
1537}
1538
1539struct http_pack_request *new_http_pack_request(
1540 struct packed_git *target, const char *base_url)
1541{
2264dfa5
TRC
1542 long prev_posn = 0;
1543 char range[RANGE_HEADER_SIZE];
1544 struct strbuf buf = STRBUF_INIT;
1545 struct http_pack_request *preq;
1546
ec99c9a8 1547 preq = xcalloc(1, sizeof(*preq));
2264dfa5 1548 preq->target = target;
2264dfa5
TRC
1549
1550 end_url_with_slash(&buf, base_url);
1551 strbuf_addf(&buf, "objects/pack/pack-%s.pack",
1552 sha1_to_hex(target->sha1));
bb99190e 1553 preq->url = strbuf_detach(&buf, NULL);
2264dfa5 1554
90d05713
TRC
1555 snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
1556 sha1_pack_name(target->sha1));
2264dfa5
TRC
1557 preq->packfile = fopen(preq->tmpfile, "a");
1558 if (!preq->packfile) {
1559 error("Unable to open local file %s for pack",
1560 preq->tmpfile);
1561 goto abort;
1562 }
1563
1564 preq->slot = get_active_slot();
2264dfa5
TRC
1565 curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
1566 curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
bb99190e 1567 curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2264dfa5
TRC
1568 curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1569 no_pragma_header);
1570
1571 /*
1572 * If there is data present from a previous transfer attempt,
1573 * resume where it left off
1574 */
1575 prev_posn = ftell(preq->packfile);
1576 if (prev_posn>0) {
1577 if (http_is_verbose)
1578 fprintf(stderr,
1579 "Resuming fetch of pack %s at byte %ld\n",
1580 sha1_to_hex(target->sha1), prev_posn);
1581 sprintf(range, "Range: bytes=%ld-", prev_posn);
1582 preq->range_header = curl_slist_append(NULL, range);
1583 curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1584 preq->range_header);
1585 }
1586
1587 return preq;
1588
1589abort:
bb99190e 1590 free(preq->url);
5ae9ebfd 1591 free(preq);
2264dfa5
TRC
1592 return NULL;
1593}
5424bc55
TRC
1594
1595/* Helpers for fetching objects (loose) */
a04ff3ec 1596static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
5424bc55
TRC
1597 void *data)
1598{
1599 unsigned char expn[4096];
1600 size_t size = eltsize * nmemb;
1601 int posn = 0;
1602 struct http_object_request *freq =
1603 (struct http_object_request *)data;
1604 do {
1605 ssize_t retval = xwrite(freq->localfile,
1606 (char *) ptr + posn, size - posn);
1607 if (retval < 0)
1608 return posn;
1609 posn += retval;
1610 } while (posn < size);
1611
1612 freq->stream.avail_in = size;
a04ff3ec 1613 freq->stream.next_in = (void *)ptr;
5424bc55
TRC
1614 do {
1615 freq->stream.next_out = expn;
1616 freq->stream.avail_out = sizeof(expn);
1617 freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
1618 git_SHA1_Update(&freq->c, expn,
1619 sizeof(expn) - freq->stream.avail_out);
1620 } while (freq->stream.avail_in && freq->zret == Z_OK);
5424bc55
TRC
1621 return size;
1622}
1623
1624struct http_object_request *new_http_object_request(const char *base_url,
1625 unsigned char *sha1)
1626{
1627 char *hex = sha1_to_hex(sha1);
30d6c6ea 1628 const char *filename;
5424bc55 1629 char prevfile[PATH_MAX];
5424bc55 1630 int prevlocal;
a04ff3ec 1631 char prev_buf[PREV_BUF_SIZE];
5424bc55
TRC
1632 ssize_t prev_read = 0;
1633 long prev_posn = 0;
1634 char range[RANGE_HEADER_SIZE];
1635 struct curl_slist *range_header = NULL;
1636 struct http_object_request *freq;
1637
ec99c9a8 1638 freq = xcalloc(1, sizeof(*freq));
5424bc55
TRC
1639 hashcpy(freq->sha1, sha1);
1640 freq->localfile = -1;
1641
1642 filename = sha1_file_name(sha1);
5424bc55
TRC
1643 snprintf(freq->tmpfile, sizeof(freq->tmpfile),
1644 "%s.temp", filename);
1645
1646 snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
1647 unlink_or_warn(prevfile);
1648 rename(freq->tmpfile, prevfile);
1649 unlink_or_warn(freq->tmpfile);
1650
1651 if (freq->localfile != -1)
1652 error("fd leakage in start: %d", freq->localfile);
1653 freq->localfile = open(freq->tmpfile,
1654 O_WRONLY | O_CREAT | O_EXCL, 0666);
1655 /*
1656 * This could have failed due to the "lazy directory creation";
1657 * try to mkdir the last path component.
1658 */
1659 if (freq->localfile < 0 && errno == ENOENT) {
1660 char *dir = strrchr(freq->tmpfile, '/');
1661 if (dir) {
1662 *dir = 0;
1663 mkdir(freq->tmpfile, 0777);
1664 *dir = '/';
1665 }
1666 freq->localfile = open(freq->tmpfile,
1667 O_WRONLY | O_CREAT | O_EXCL, 0666);
1668 }
1669
1670 if (freq->localfile < 0) {
0da8b2e7
SP
1671 error("Couldn't create temporary file %s: %s",
1672 freq->tmpfile, strerror(errno));
5424bc55
TRC
1673 goto abort;
1674 }
1675
5424bc55
TRC
1676 git_inflate_init(&freq->stream);
1677
1678 git_SHA1_Init(&freq->c);
1679
bb99190e 1680 freq->url = get_remote_object_url(base_url, hex, 0);
5424bc55
TRC
1681
1682 /*
1683 * If a previous temp file is present, process what was already
1684 * fetched.
1685 */
1686 prevlocal = open(prevfile, O_RDONLY);
1687 if (prevlocal != -1) {
1688 do {
1689 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1690 if (prev_read>0) {
1691 if (fwrite_sha1_file(prev_buf,
1692 1,
1693 prev_read,
1694 freq) == prev_read) {
1695 prev_posn += prev_read;
1696 } else {
1697 prev_read = -1;
1698 }
1699 }
1700 } while (prev_read > 0);
1701 close(prevlocal);
1702 }
1703 unlink_or_warn(prevfile);
1704
1705 /*
1706 * Reset inflate/SHA1 if there was an error reading the previous temp
1707 * file; also rewind to the beginning of the local file.
1708 */
1709 if (prev_read == -1) {
1710 memset(&freq->stream, 0, sizeof(freq->stream));
1711 git_inflate_init(&freq->stream);
1712 git_SHA1_Init(&freq->c);
1713 if (prev_posn>0) {
1714 prev_posn = 0;
1715 lseek(freq->localfile, 0, SEEK_SET);
0c4f21e4 1716 if (ftruncate(freq->localfile, 0) < 0) {
0da8b2e7
SP
1717 error("Couldn't truncate temporary file %s: %s",
1718 freq->tmpfile, strerror(errno));
0c4f21e4
JL
1719 goto abort;
1720 }
5424bc55
TRC
1721 }
1722 }
1723
1724 freq->slot = get_active_slot();
1725
1726 curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
1727 curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
1728 curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
bb99190e 1729 curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
5424bc55
TRC
1730 curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1731
1732 /*
1733 * If we have successfully processed data from a previous fetch
1734 * attempt, only fetch the data we don't already have.
1735 */
1736 if (prev_posn>0) {
1737 if (http_is_verbose)
1738 fprintf(stderr,
1739 "Resuming fetch of object %s at byte %ld\n",
1740 hex, prev_posn);
1741 sprintf(range, "Range: bytes=%ld-", prev_posn);
1742 range_header = curl_slist_append(range_header, range);
1743 curl_easy_setopt(freq->slot->curl,
1744 CURLOPT_HTTPHEADER, range_header);
1745 }
1746
1747 return freq;
1748
5424bc55 1749abort:
bb99190e 1750 free(freq->url);
5424bc55
TRC
1751 free(freq);
1752 return NULL;
1753}
1754
1755void process_http_object_request(struct http_object_request *freq)
1756{
1757 if (freq->slot == NULL)
1758 return;
1759 freq->curl_result = freq->slot->curl_result;
1760 freq->http_code = freq->slot->http_code;
1761 freq->slot = NULL;
1762}
1763
1764int finish_http_object_request(struct http_object_request *freq)
1765{
1766 struct stat st;
1767
1768 close(freq->localfile);
1769 freq->localfile = -1;
1770
1771 process_http_object_request(freq);
1772
1773 if (freq->http_code == 416) {
bd757c18 1774 warning("requested range invalid; we may already have all the data.");
5424bc55
TRC
1775 } else if (freq->curl_result != CURLE_OK) {
1776 if (stat(freq->tmpfile, &st) == 0)
1777 if (st.st_size == 0)
1778 unlink_or_warn(freq->tmpfile);
1779 return -1;
1780 }
1781
1782 git_inflate_end(&freq->stream);
1783 git_SHA1_Final(freq->real_sha1, &freq->c);
1784 if (freq->zret != Z_STREAM_END) {
1785 unlink_or_warn(freq->tmpfile);
1786 return -1;
1787 }
1788 if (hashcmp(freq->sha1, freq->real_sha1)) {
1789 unlink_or_warn(freq->tmpfile);
1790 return -1;
1791 }
1792 freq->rename =
0da8b2e7 1793 move_temp_to_file(freq->tmpfile, sha1_file_name(freq->sha1));
5424bc55
TRC
1794
1795 return freq->rename;
1796}
1797
1798void abort_http_object_request(struct http_object_request *freq)
1799{
1800 unlink_or_warn(freq->tmpfile);
1801
1802 release_http_object_request(freq);
1803}
1804
1805void release_http_object_request(struct http_object_request *freq)
1806{
1807 if (freq->localfile != -1) {
1808 close(freq->localfile);
1809 freq->localfile = -1;
1810 }
1811 if (freq->url != NULL) {
1812 free(freq->url);
1813 freq->url = NULL;
1814 }
4b9fa0e3
TRC
1815 if (freq->slot != NULL) {
1816 freq->slot->callback_func = NULL;
1817 freq->slot->callback_data = NULL;
1818 release_active_slot(freq->slot);
1819 freq->slot = NULL;
1820 }
5424bc55 1821}