Document extended SHA1 used by git-rev-parse.
[git/git.git] / http-fetch.c
CommitLineData
6eb7ed54
DB
1#include "cache.h"
2#include "commit.h"
6eb7ed54 3
215a7ad1 4#include "fetch.h"
4250a5e5 5
6eb7ed54
DB
6#include <curl/curl.h>
7#include <curl/easy.h>
8
4a30976e
JS
9#if LIBCURL_VERSION_NUM < 0x070704
10#define curl_global_cleanup() do { /* nothing */ } while(0)
11#endif
12#if LIBCURL_VERSION_NUM < 0x070800
13#define curl_global_init(a) do { /* nothing */ } while(0)
14#endif
4a30976e 15
6eb7ed54 16static CURL *curl;
1db69b57 17static struct curl_slist *no_pragma_header;
6eb7ed54 18
b3661567
DB
19static char *initial_base;
20
21struct alt_base
22{
23 char *base;
24 int got_indices;
25 struct packed_git *packs;
26 struct alt_base *next;
27};
28
29struct alt_base *alt = NULL;
6eb7ed54 30
6eb7ed54
DB
31static SHA_CTX c;
32static z_stream stream;
33
34static int local;
35static int zret;
36
3dcb90f5
DT
37static int curl_ssl_verify;
38
fa3e0655
DB
39struct buffer
40{
41 size_t posn;
42 size_t size;
43 void *buffer;
44};
45
46static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
182005b9
DB
47 struct buffer *buffer)
48{
fa3e0655
DB
49 size_t size = eltsize * nmemb;
50 if (size > buffer->size - buffer->posn)
51 size = buffer->size - buffer->posn;
52 memcpy(buffer->buffer + buffer->posn, ptr, size);
53 buffer->posn += size;
54 return size;
55}
56
182005b9
DB
57static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
58 void *data)
59{
bf0f910d 60 unsigned char expn[4096];
6eb7ed54
DB
61 size_t size = eltsize * nmemb;
62 int posn = 0;
63 do {
64 ssize_t retval = write(local, ptr + posn, size - posn);
65 if (retval < 0)
66 return posn;
67 posn += retval;
68 } while (posn < size);
69
70 stream.avail_in = size;
71 stream.next_in = ptr;
72 do {
73 stream.next_out = expn;
74 stream.avail_out = sizeof(expn);
75 zret = inflate(&stream, Z_SYNC_FLUSH);
76 SHA1_Update(&c, expn, sizeof(expn) - stream.avail_out);
77 } while (stream.avail_in && zret == Z_OK);
78 return size;
79}
80
1e8be59d
DB
81void prefetch(unsigned char *sha1)
82{
83}
84
b3661567 85static int got_alternates = 0;
182005b9 86
b3661567 87static int fetch_index(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
88{
89 char *filename;
90 char *url;
91
92 FILE *indexfile;
93
94 if (has_pack_index(sha1))
95 return 0;
96
97 if (get_verbosely)
98 fprintf(stderr, "Getting index for pack %s\n",
99 sha1_to_hex(sha1));
100
b3661567 101 url = xmalloc(strlen(repo->base) + 64);
182005b9 102 sprintf(url, "%s/objects/pack/pack-%s.idx",
b3661567 103 repo->base, sha1_to_hex(sha1));
182005b9
DB
104
105 filename = sha1_pack_index_name(sha1);
106 indexfile = fopen(filename, "w");
107 if (!indexfile)
108 return error("Unable to open local file %s for pack index",
109 filename);
110
111 curl_easy_setopt(curl, CURLOPT_FILE, indexfile);
112 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
113 curl_easy_setopt(curl, CURLOPT_URL, url);
1db69b57 114 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
182005b9
DB
115
116 if (curl_easy_perform(curl)) {
117 fclose(indexfile);
118 return error("Unable to get pack index %s", url);
119 }
120
121 fclose(indexfile);
122 return 0;
123}
124
b3661567 125static int setup_index(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
126{
127 struct packed_git *new_pack;
128 if (has_pack_file(sha1))
129 return 0; // don't list this as something we can get
130
b3661567 131 if (fetch_index(repo, sha1))
182005b9
DB
132 return -1;
133
134 new_pack = parse_pack_index(sha1);
b3661567
DB
135 new_pack->next = repo->packs;
136 repo->packs = new_pack;
182005b9
DB
137 return 0;
138}
139
b3661567
DB
140static int fetch_alternates(char *base)
141{
142 int ret = 0;
143 struct buffer buffer;
144 char *url;
145 char *data;
146 int i = 0;
147 if (got_alternates)
148 return 0;
149 data = xmalloc(4096);
150 buffer.size = 4096;
151 buffer.posn = 0;
152 buffer.buffer = data;
153
154 if (get_verbosely)
155 fprintf(stderr, "Getting alternates list\n");
156
157 url = xmalloc(strlen(base) + 31);
158 sprintf(url, "%s/objects/info/http-alternates", base);
159
160 curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
161 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
162 curl_easy_setopt(curl, CURLOPT_URL, url);
163
164 if (curl_easy_perform(curl) || !buffer.posn) {
165 sprintf(url, "%s/objects/info/alternates", base);
166
167 curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
168 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
169 curl_easy_setopt(curl, CURLOPT_URL, url);
170
171 if (curl_easy_perform(curl)) {
172 return 0;
173 }
174 }
175
176 while (i < buffer.posn) {
177 int posn = i;
178 while (posn < buffer.posn && data[posn] != '\n')
179 posn++;
180 if (data[posn] == '\n') {
181 if (data[i] == '/') {
182 int serverlen = strchr(base + 8, '/') - base;
183 // skip 'objects' at end
184 char *target =
185 xmalloc(serverlen + posn - i - 6);
186 struct alt_base *newalt;
187 strncpy(target, base, serverlen);
188 strncpy(target + serverlen, data + i,
189 posn - i - 7);
190 target[serverlen + posn - i - 7] = '\0';
191 if (get_verbosely)
192 fprintf(stderr,
193 "Also look at %s\n", target);
194 newalt = xmalloc(sizeof(*newalt));
195 newalt->next = alt;
196 newalt->base = target;
197 newalt->got_indices = 0;
198 newalt->packs = NULL;
199 alt = newalt;
200 ret++;
201 }
202 }
203 i = posn + 1;
204 }
205 got_alternates = 1;
206
207 return ret;
208}
209
210static int fetch_indices(struct alt_base *repo)
182005b9
DB
211{
212 unsigned char sha1[20];
213 char *url;
214 struct buffer buffer;
215 char *data;
216 int i = 0;
217
b3661567 218 if (repo->got_indices)
182005b9
DB
219 return 0;
220
221 data = xmalloc(4096);
222 buffer.size = 4096;
223 buffer.posn = 0;
224 buffer.buffer = data;
225
226 if (get_verbosely)
227 fprintf(stderr, "Getting pack list\n");
228
b3661567
DB
229 url = xmalloc(strlen(repo->base) + 21);
230 sprintf(url, "%s/objects/info/packs", repo->base);
182005b9
DB
231
232 curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
233 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
234 curl_easy_setopt(curl, CURLOPT_URL, url);
1db69b57 235 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
182005b9
DB
236
237 if (curl_easy_perform(curl)) {
238 return error("Unable to get pack index %s", url);
239 }
240
b3661567 241 while (i < buffer.posn) {
182005b9
DB
242 switch (data[i]) {
243 case 'P':
244 i++;
245 if (i + 52 < buffer.posn &&
246 !strncmp(data + i, " pack-", 6) &&
247 !strncmp(data + i + 46, ".pack\n", 6)) {
248 get_sha1_hex(data + i + 6, sha1);
b3661567 249 setup_index(repo, sha1);
182005b9
DB
250 i += 51;
251 break;
252 }
253 default:
254 while (data[i] != '\n')
255 i++;
256 }
257 i++;
b3661567 258 }
182005b9 259
b3661567 260 repo->got_indices = 1;
182005b9
DB
261 return 0;
262}
263
b3661567 264static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
182005b9
DB
265{
266 char *url;
267 struct packed_git *target;
268 struct packed_git **lst;
269 FILE *packfile;
270 char *filename;
271
b3661567 272 if (fetch_indices(repo))
182005b9 273 return -1;
b3661567 274 target = find_sha1_pack(sha1, repo->packs);
182005b9 275 if (!target)
b3661567 276 return -1;
182005b9
DB
277
278 if (get_verbosely) {
279 fprintf(stderr, "Getting pack %s\n",
280 sha1_to_hex(target->sha1));
281 fprintf(stderr, " which contains %s\n",
282 sha1_to_hex(sha1));
283 }
284
b3661567 285 url = xmalloc(strlen(repo->base) + 65);
182005b9 286 sprintf(url, "%s/objects/pack/pack-%s.pack",
b3661567 287 repo->base, sha1_to_hex(target->sha1));
182005b9
DB
288
289 filename = sha1_pack_name(target->sha1);
290 packfile = fopen(filename, "w");
291 if (!packfile)
292 return error("Unable to open local file %s for pack",
293 filename);
294
295 curl_easy_setopt(curl, CURLOPT_FILE, packfile);
296 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
297 curl_easy_setopt(curl, CURLOPT_URL, url);
1db69b57 298 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
182005b9
DB
299
300 if (curl_easy_perform(curl)) {
301 fclose(packfile);
302 return error("Unable to get pack file %s", url);
303 }
304
305 fclose(packfile);
306
b3661567 307 lst = &repo->packs;
182005b9
DB
308 while (*lst != target)
309 lst = &((*lst)->next);
310 *lst = (*lst)->next;
311
312 install_packed_git(target);
313
314 return 0;
315}
316
b3661567 317int fetch_object(struct alt_base *repo, unsigned char *sha1)
6eb7ed54
DB
318{
319 char *hex = sha1_to_hex(sha1);
320 char *filename = sha1_file_name(sha1);
bf0f910d 321 unsigned char real_sha1[20];
6eb7ed54
DB
322 char *url;
323 char *posn;
324
6eb7ed54
DB
325 local = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0666);
326
327 if (local < 0)
182005b9 328 return error("Couldn't open local object %s\n", filename);
6eb7ed54
DB
329
330 memset(&stream, 0, sizeof(stream));
331
332 inflateInit(&stream);
333
334 SHA1_Init(&c);
335
182005b9 336 curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
6eb7ed54
DB
337 curl_easy_setopt(curl, CURLOPT_FILE, NULL);
338 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
1db69b57 339 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
6eb7ed54 340
b3661567
DB
341 url = xmalloc(strlen(repo->base) + 50);
342 strcpy(url, repo->base);
343 posn = url + strlen(repo->base);
6eb7ed54
DB
344 strcpy(posn, "objects/");
345 posn += 8;
346 memcpy(posn, hex, 2);
347 posn += 2;
348 *(posn++) = '/';
349 strcpy(posn, hex + 2);
350
351 curl_easy_setopt(curl, CURLOPT_URL, url);
352
182005b9
DB
353 if (curl_easy_perform(curl)) {
354 unlink(filename);
b3661567 355 return -1;
182005b9 356 }
6eb7ed54
DB
357
358 close(local);
359 inflateEnd(&stream);
360 SHA1_Final(real_sha1, &c);
361 if (zret != Z_STREAM_END) {
362 unlink(filename);
363 return error("File %s (%s) corrupt\n", hex, url);
364 }
365 if (memcmp(sha1, real_sha1, 20)) {
366 unlink(filename);
367 return error("File %s has bad hash\n", hex);
368 }
369
e78d9772 370 pull_say("got %s\n", hex);
6eb7ed54
DB
371 return 0;
372}
373
b3661567
DB
374int fetch(unsigned char *sha1)
375{
376 struct alt_base *altbase = alt;
377 while (altbase) {
378 if (!fetch_object(altbase, sha1))
379 return 0;
380 if (!fetch_pack(altbase, sha1))
381 return 0;
382 if (fetch_alternates(altbase->base) > 0) {
383 altbase = alt;
384 continue;
385 }
386 altbase = altbase->next;
387 }
388 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
389 initial_base);
390}
391
cd541a68
DB
392int fetch_ref(char *ref, unsigned char *sha1)
393{
fa3e0655
DB
394 char *url, *posn;
395 char hex[42];
396 struct buffer buffer;
b3661567 397 char *base = initial_base;
fa3e0655
DB
398 buffer.size = 41;
399 buffer.posn = 0;
400 buffer.buffer = hex;
401 hex[41] = '\0';
402
403 curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
404 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1db69b57 405 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
fa3e0655
DB
406
407 url = xmalloc(strlen(base) + 6 + strlen(ref));
408 strcpy(url, base);
409 posn = url + strlen(base);
410 strcpy(posn, "refs/");
411 posn += 5;
412 strcpy(posn, ref);
413
414 curl_easy_setopt(curl, CURLOPT_URL, url);
415
416 if (curl_easy_perform(curl))
417 return error("Couldn't get %s for %s\n", url, ref);
418
419 hex[40] = '\0';
420 get_sha1_hex(hex, sha1);
421 return 0;
cd541a68
DB
422}
423
6eb7ed54
DB
424int main(int argc, char **argv)
425{
426 char *commit_id;
427 char *url;
428 int arg = 1;
6eb7ed54
DB
429
430 while (arg < argc && argv[arg][0] == '-') {
431 if (argv[arg][1] == 't') {
4250a5e5 432 get_tree = 1;
6eb7ed54 433 } else if (argv[arg][1] == 'c') {
4250a5e5 434 get_history = 1;
6eb7ed54 435 } else if (argv[arg][1] == 'a') {
4250a5e5
DB
436 get_all = 1;
437 get_tree = 1;
438 get_history = 1;
e78d9772
JH
439 } else if (argv[arg][1] == 'v') {
440 get_verbosely = 1;
fa3e0655
DB
441 } else if (argv[arg][1] == 'w') {
442 write_ref = argv[arg + 1];
443 arg++;
6eb7ed54
DB
444 }
445 arg++;
446 }
447 if (argc < arg + 2) {
215a7ad1 448 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
6eb7ed54
DB
449 return 1;
450 }
451 commit_id = argv[arg];
452 url = argv[arg + 1];
453
6eb7ed54
DB
454 curl_global_init(CURL_GLOBAL_ALL);
455
456 curl = curl_easy_init();
1db69b57 457 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
6eb7ed54 458
a9ab586a 459 curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1;
3dcb90f5 460 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
9f6cf65e 461#if LIBCURL_VERSION_NUM >= 0x070907
3dcb90f5 462 curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
9f6cf65e 463#endif
3dcb90f5 464
b3661567
DB
465 alt = xmalloc(sizeof(*alt));
466 alt->base = url;
467 alt->got_indices = 0;
468 alt->packs = NULL;
469 alt->next = NULL;
470 initial_base = url;
6eb7ed54 471
4250a5e5 472 if (pull(commit_id))
6eb7ed54
DB
473 return 1;
474
1db69b57 475 curl_slist_free_all(no_pragma_header);
6eb7ed54
DB
476 curl_global_cleanup();
477 return 0;
478}