Merge branch 'master' of git://repo.or.cz/git/fastimport into maint
[git/git.git] / convert.c
CommitLineData
6c510bee 1#include "cache.h"
35ebfd6a 2#include "attr.h"
3fed15f5 3#include "run-command.h"
35ebfd6a 4
6c510bee
LT
5/*
6 * convert.c - convert a file when checking it out and checking it in.
7 *
8 * This should use the pathname to decide on whether it wants to do some
9 * more interesting conversions (automatic gzip/unzip, general format
10 * conversions etc etc), but by default it just does automatic CRLF<->LF
11 * translation when the "auto_crlf" option is set.
12 */
13
163b9591
JH
14#define CRLF_GUESS (-1)
15#define CRLF_BINARY 0
16#define CRLF_TEXT 1
17#define CRLF_INPUT 2
18
6c510bee
LT
19struct text_stat {
20 /* CR, LF and CRLF counts */
21 unsigned cr, lf, crlf;
22
23 /* These are just approximations! */
24 unsigned printable, nonprintable;
25};
26
27static void gather_stats(const char *buf, unsigned long size, struct text_stat *stats)
28{
29 unsigned long i;
30
31 memset(stats, 0, sizeof(*stats));
32
33 for (i = 0; i < size; i++) {
34 unsigned char c = buf[i];
35 if (c == '\r') {
36 stats->cr++;
37 if (i+1 < size && buf[i+1] == '\n')
38 stats->crlf++;
39 continue;
40 }
41 if (c == '\n') {
42 stats->lf++;
43 continue;
44 }
45 if (c == 127)
46 /* DEL */
47 stats->nonprintable++;
48 else if (c < 32) {
49 switch (c) {
50 /* BS, HT, ESC and FF */
51 case '\b': case '\t': case '\033': case '\014':
52 stats->printable++;
53 break;
54 default:
55 stats->nonprintable++;
56 }
57 }
58 else
59 stats->printable++;
60 }
61}
62
63/*
64 * The same heuristics as diff.c::mmfile_is_binary()
65 */
66static int is_binary(unsigned long size, struct text_stat *stats)
67{
68
69 if ((stats->printable >> 7) < stats->nonprintable)
70 return 1;
71 /*
72 * Other heuristics? Average line length might be relevant,
73 * as might LF vs CR vs CRLF counts..
74 *
75 * NOTE! It might be normal to have a low ratio of CRLF to LF
76 * (somebody starts with a LF-only file and edits it with an editor
77 * that adds CRLF only to lines that are added..). But do we
78 * want to support CR-only? Probably not.
79 */
80 return 0;
81}
82
ac78e548 83static char *crlf_to_git(const char *path, const char *src, unsigned long *sizep, int action)
6c510bee 84{
ac78e548 85 char *buffer, *dst;
6c510bee
LT
86 unsigned long size, nsize;
87 struct text_stat stats;
88
760f0c62 89 if ((action == CRLF_BINARY) || !auto_crlf)
ac78e548 90 return NULL;
6c510bee
LT
91
92 size = *sizep;
93 if (!size)
ac78e548 94 return NULL;
6c510bee 95
ac78e548 96 gather_stats(src, size, &stats);
6c510bee
LT
97
98 /* No CR? Nothing to convert, regardless. */
99 if (!stats.cr)
ac78e548 100 return NULL;
6c510bee 101
163b9591 102 if (action == CRLF_GUESS) {
201ac8ef
JH
103 /*
104 * We're currently not going to even try to convert stuff
105 * that has bare CR characters. Does anybody do that crazy
106 * stuff?
107 */
108 if (stats.cr != stats.crlf)
ac78e548 109 return NULL;
201ac8ef
JH
110
111 /*
112 * And add some heuristics for binary vs text, of course...
113 */
114 if (is_binary(size, &stats))
ac78e548 115 return NULL;
201ac8ef 116 }
6c510bee
LT
117
118 /*
67e22ed5
AR
119 * Ok, allocate a new buffer, fill it in, and return it
120 * to let the caller know that we switched buffers.
6c510bee
LT
121 */
122 nsize = size - stats.crlf;
ac78e548 123 buffer = xmalloc(nsize);
6c510bee 124 *sizep = nsize;
201ac8ef 125
ac78e548 126 dst = buffer;
163b9591
JH
127 if (action == CRLF_GUESS) {
128 /*
129 * If we guessed, we already know we rejected a file with
130 * lone CR, and we can strip a CR without looking at what
131 * follow it.
132 */
201ac8ef 133 do {
ac78e548 134 unsigned char c = *src++;
201ac8ef 135 if (c != '\r')
ac78e548 136 *dst++ = c;
201ac8ef
JH
137 } while (--size);
138 } else {
139 do {
ac78e548 140 unsigned char c = *src++;
67e22ed5 141 if (! (c == '\r' && (1 < size && *src == '\n')))
ac78e548 142 *dst++ = c;
201ac8ef
JH
143 } while (--size);
144 }
6c510bee 145
ac78e548 146 return buffer;
6c510bee
LT
147}
148
ac78e548 149static char *crlf_to_worktree(const char *path, const char *src, unsigned long *sizep, int action)
6c510bee 150{
ac78e548 151 char *buffer, *dst;
6c510bee
LT
152 unsigned long size, nsize;
153 struct text_stat stats;
154 unsigned char last;
155
163b9591 156 if ((action == CRLF_BINARY) || (action == CRLF_INPUT) ||
760f0c62 157 auto_crlf <= 0)
ac78e548 158 return NULL;
6c510bee
LT
159
160 size = *sizep;
161 if (!size)
ac78e548 162 return NULL;
6c510bee 163
ac78e548 164 gather_stats(src, size, &stats);
6c510bee
LT
165
166 /* No LF? Nothing to convert, regardless. */
167 if (!stats.lf)
ac78e548 168 return NULL;
6c510bee
LT
169
170 /* Was it already in CRLF format? */
171 if (stats.lf == stats.crlf)
ac78e548 172 return NULL;
6c510bee 173
163b9591 174 if (action == CRLF_GUESS) {
201ac8ef
JH
175 /* If we have any bare CR characters, we're not going to touch it */
176 if (stats.cr != stats.crlf)
ac78e548 177 return NULL;
6c510bee 178
201ac8ef 179 if (is_binary(size, &stats))
ac78e548 180 return NULL;
201ac8ef 181 }
6c510bee
LT
182
183 /*
67e22ed5
AR
184 * Ok, allocate a new buffer, fill it in, and return it
185 * to let the caller know that we switched buffers.
6c510bee
LT
186 */
187 nsize = size + stats.lf - stats.crlf;
ac78e548 188 buffer = xmalloc(nsize);
6c510bee
LT
189 *sizep = nsize;
190 last = 0;
ac78e548
AR
191
192 dst = buffer;
6c510bee 193 do {
ac78e548 194 unsigned char c = *src++;
6c510bee 195 if (c == '\n' && last != '\r')
ac78e548
AR
196 *dst++ = '\r';
197 *dst++ = c;
6c510bee
LT
198 last = c;
199 } while (--size);
200
ac78e548 201 return buffer;
6c510bee 202}
35ebfd6a 203
aa4ed402
JH
204static int filter_buffer(const char *path, const char *src,
205 unsigned long size, const char *cmd)
206{
207 /*
208 * Spawn cmd and feed the buffer contents through its stdin.
209 */
210 struct child_process child_process;
211 int pipe_feed[2];
212 int write_err, status;
213
214 memset(&child_process, 0, sizeof(child_process));
215
216 if (pipe(pipe_feed) < 0) {
217 error("cannot create pipe to run external filter %s", cmd);
218 return 1;
219 }
220
221 child_process.pid = fork();
222 if (child_process.pid < 0) {
223 error("cannot fork to run external filter %s", cmd);
224 close(pipe_feed[0]);
225 close(pipe_feed[1]);
226 return 1;
227 }
228 if (!child_process.pid) {
229 dup2(pipe_feed[0], 0);
230 close(pipe_feed[0]);
231 close(pipe_feed[1]);
232 execlp("sh", "sh", "-c", cmd, NULL);
233 return 1;
234 }
235 close(pipe_feed[0]);
236
237 write_err = (write_in_full(pipe_feed[1], src, size) < 0);
238 if (close(pipe_feed[1]))
239 write_err = 1;
240 if (write_err)
241 error("cannot feed the input to external filter %s", cmd);
242
243 status = finish_command(&child_process);
244 if (status)
245 error("external filter %s failed %d", cmd, -status);
246 return (write_err || status);
247}
248
249static char *apply_filter(const char *path, const char *src,
250 unsigned long *sizep, const char *cmd)
251{
252 /*
253 * Create a pipeline to have the command filter the buffer's
254 * contents.
255 *
256 * (child --> cmd) --> us
257 */
258 const int SLOP = 4096;
259 int pipe_feed[2];
260 int status;
261 char *dst;
262 unsigned long dstsize, dstalloc;
263 struct child_process child_process;
264
265 if (!cmd)
266 return NULL;
267
268 memset(&child_process, 0, sizeof(child_process));
269
270 if (pipe(pipe_feed) < 0) {
271 error("cannot create pipe to run external filter %s", cmd);
272 return NULL;
273 }
274
275 fflush(NULL);
276 child_process.pid = fork();
277 if (child_process.pid < 0) {
278 error("cannot fork to run external filter %s", cmd);
279 close(pipe_feed[0]);
280 close(pipe_feed[1]);
281 return NULL;
282 }
283 if (!child_process.pid) {
284 dup2(pipe_feed[1], 1);
285 close(pipe_feed[0]);
286 close(pipe_feed[1]);
287 exit(filter_buffer(path, src, *sizep, cmd));
288 }
289 close(pipe_feed[1]);
290
291 dstalloc = *sizep;
292 dst = xmalloc(dstalloc);
293 dstsize = 0;
294
295 while (1) {
296 ssize_t numread = xread(pipe_feed[0], dst + dstsize,
297 dstalloc - dstsize);
298
299 if (numread <= 0) {
300 if (!numread)
301 break;
302 error("read from external filter %s failed", cmd);
303 free(dst);
304 dst = NULL;
305 break;
306 }
307 dstsize += numread;
308 if (dstalloc <= dstsize + SLOP) {
309 dstalloc = dstsize + SLOP;
310 dst = xrealloc(dst, dstalloc);
311 }
312 }
313 if (close(pipe_feed[0])) {
314 error("read from external filter %s failed", cmd);
315 free(dst);
316 dst = NULL;
317 }
318
319 status = finish_command(&child_process);
320 if (status) {
321 error("external filter %s failed %d", cmd, -status);
322 free(dst);
323 dst = NULL;
324 }
325
326 if (dst)
327 *sizep = dstsize;
328 return dst;
329}
330
331static struct convert_driver {
332 const char *name;
333 struct convert_driver *next;
334 char *smudge;
335 char *clean;
336} *user_convert, **user_convert_tail;
337
338static int read_convert_config(const char *var, const char *value)
339{
340 const char *ep, *name;
341 int namelen;
342 struct convert_driver *drv;
343
344 /*
345 * External conversion drivers are configured using
346 * "filter.<name>.variable".
347 */
348 if (prefixcmp(var, "filter.") || (ep = strrchr(var, '.')) == var + 6)
349 return 0;
350 name = var + 7;
351 namelen = ep - name;
352 for (drv = user_convert; drv; drv = drv->next)
353 if (!strncmp(drv->name, name, namelen) && !drv->name[namelen])
354 break;
355 if (!drv) {
356 char *namebuf;
357 drv = xcalloc(1, sizeof(struct convert_driver));
358 namebuf = xmalloc(namelen + 1);
359 memcpy(namebuf, name, namelen);
360 namebuf[namelen] = 0;
361 drv->name = namebuf;
362 drv->next = NULL;
363 *user_convert_tail = drv;
364 user_convert_tail = &(drv->next);
365 }
366
367 ep++;
368
369 /*
370 * filter.<name>.smudge and filter.<name>.clean specifies
371 * the command line:
372 *
373 * command-line
374 *
375 * The command-line will not be interpolated in any way.
376 */
377
378 if (!strcmp("smudge", ep)) {
379 if (!value)
380 return error("%s: lacks value", var);
381 drv->smudge = strdup(value);
382 return 0;
383 }
384
385 if (!strcmp("clean", ep)) {
386 if (!value)
387 return error("%s: lacks value", var);
388 drv->clean = strdup(value);
389 return 0;
390 }
391 return 0;
392}
393
6073ee85 394static void setup_convert_check(struct git_attr_check *check)
35ebfd6a
JH
395{
396 static struct git_attr *attr_crlf;
3fed15f5 397 static struct git_attr *attr_ident;
aa4ed402 398 static struct git_attr *attr_filter;
35ebfd6a 399
3fed15f5 400 if (!attr_crlf) {
35ebfd6a 401 attr_crlf = git_attr("crlf", 4);
3fed15f5 402 attr_ident = git_attr("ident", 5);
aa4ed402
JH
403 attr_filter = git_attr("filter", 6);
404 user_convert_tail = &user_convert;
405 git_config(read_convert_config);
3fed15f5
JH
406 }
407 check[0].attr = attr_crlf;
408 check[1].attr = attr_ident;
aa4ed402 409 check[2].attr = attr_filter;
3fed15f5
JH
410}
411
412static int count_ident(const char *cp, unsigned long size)
413{
414 /*
af9b54bb 415 * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$"
3fed15f5
JH
416 */
417 int cnt = 0;
418 char ch;
419
420 while (size) {
421 ch = *cp++;
422 size--;
423 if (ch != '$')
424 continue;
af9b54bb 425 if (size < 3)
3fed15f5 426 break;
af9b54bb 427 if (memcmp("Id", cp, 2))
3fed15f5 428 continue;
af9b54bb
AP
429 ch = cp[2];
430 cp += 3;
431 size -= 3;
3fed15f5 432 if (ch == '$')
af9b54bb 433 cnt++; /* $Id$ */
3fed15f5
JH
434 if (ch != ':')
435 continue;
436
437 /*
af9b54bb 438 * "$Id: ... "; scan up to the closing dollar sign and discard.
3fed15f5
JH
439 */
440 while (size) {
441 ch = *cp++;
442 size--;
443 if (ch == '$') {
444 cnt++;
445 break;
446 }
447 }
448 }
449 return cnt;
450}
451
452static char *ident_to_git(const char *path, const char *src, unsigned long *sizep, int ident)
453{
454 int cnt;
455 unsigned long size;
456 char *dst, *buf;
457
458 if (!ident)
459 return NULL;
460 size = *sizep;
461 cnt = count_ident(src, size);
462 if (!cnt)
463 return NULL;
464 buf = xmalloc(size);
465
466 for (dst = buf; size; size--) {
467 char ch = *src++;
468 *dst++ = ch;
af9b54bb
AP
469 if ((ch == '$') && (3 <= size) &&
470 !memcmp("Id:", src, 3)) {
471 unsigned long rem = size - 3;
472 const char *cp = src + 3;
3fed15f5
JH
473 do {
474 ch = *cp++;
475 if (ch == '$')
476 break;
477 rem--;
478 } while (rem);
479 if (!rem)
480 continue;
af9b54bb
AP
481 memcpy(dst, "Id$", 3);
482 dst += 3;
3fed15f5
JH
483 size -= (cp - src);
484 src = cp;
485 }
486 }
487
488 *sizep = dst - buf;
489 return buf;
490}
491
492static char *ident_to_worktree(const char *path, const char *src, unsigned long *sizep, int ident)
493{
494 int cnt;
495 unsigned long size;
496 char *dst, *buf;
497 unsigned char sha1[20];
498
499 if (!ident)
500 return NULL;
501
502 size = *sizep;
503 cnt = count_ident(src, size);
504 if (!cnt)
505 return NULL;
506
507 hash_sha1_file(src, size, "blob", sha1);
508 buf = xmalloc(size + cnt * 43);
509
510 for (dst = buf; size; size--) {
511 const char *cp;
512 char ch = *src++;
513 *dst++ = ch;
af9b54bb 514 if ((ch != '$') || (size < 3) || memcmp("Id", src, 2))
3fed15f5
JH
515 continue;
516
af9b54bb 517 if (src[2] == ':') {
3fed15f5 518 /* discard up to but not including the closing $ */
af9b54bb
AP
519 unsigned long rem = size - 3;
520 cp = src + 3;
3fed15f5
JH
521 do {
522 ch = *cp++;
523 if (ch == '$')
524 break;
525 rem--;
526 } while (rem);
527 if (!rem)
528 continue;
529 size -= (cp - src);
af9b54bb
AP
530 } else if (src[2] == '$')
531 cp = src + 2;
3fed15f5
JH
532 else
533 continue;
534
af9b54bb
AP
535 memcpy(dst, "Id: ", 4);
536 dst += 4;
3fed15f5
JH
537 memcpy(dst, sha1_to_hex(sha1), 40);
538 dst += 40;
539 *dst++ = ' ';
540 size -= (cp - src);
541 src = cp;
542 *dst++ = *src++;
543 size--;
544 }
545
546 *sizep = dst - buf;
547 return buf;
35ebfd6a
JH
548}
549
6073ee85 550static int git_path_check_crlf(const char *path, struct git_attr_check *check)
35ebfd6a 551{
6073ee85
JH
552 const char *value = check->value;
553
554 if (ATTR_TRUE(value))
555 return CRLF_TEXT;
556 else if (ATTR_FALSE(value))
557 return CRLF_BINARY;
558 else if (ATTR_UNSET(value))
559 ;
560 else if (!strcmp(value, "input"))
561 return CRLF_INPUT;
163b9591 562 return CRLF_GUESS;
35ebfd6a
JH
563}
564
aa4ed402
JH
565static struct convert_driver *git_path_check_convert(const char *path,
566 struct git_attr_check *check)
567{
568 const char *value = check->value;
569 struct convert_driver *drv;
570
571 if (ATTR_TRUE(value) || ATTR_FALSE(value) || ATTR_UNSET(value))
572 return NULL;
573 for (drv = user_convert; drv; drv = drv->next)
574 if (!strcmp(value, drv->name))
575 return drv;
576 return NULL;
577}
578
3fed15f5
JH
579static int git_path_check_ident(const char *path, struct git_attr_check *check)
580{
581 const char *value = check->value;
582
583 return !!ATTR_TRUE(value);
584}
585
ac78e548 586char *convert_to_git(const char *path, const char *src, unsigned long *sizep)
35ebfd6a 587{
aa4ed402 588 struct git_attr_check check[3];
6073ee85 589 int crlf = CRLF_GUESS;
3fed15f5 590 int ident = 0;
aa4ed402 591 char *filter = NULL;
3fed15f5 592 char *buf, *buf2;
6073ee85
JH
593
594 setup_convert_check(check);
3fed15f5 595 if (!git_checkattr(path, ARRAY_SIZE(check), check)) {
aa4ed402 596 struct convert_driver *drv;
3fed15f5
JH
597 crlf = git_path_check_crlf(path, check + 0);
598 ident = git_path_check_ident(path, check + 1);
aa4ed402
JH
599 drv = git_path_check_convert(path, check + 2);
600 if (drv && drv->clean)
601 filter = drv->clean;
3fed15f5
JH
602 }
603
aa4ed402
JH
604 buf = apply_filter(path, src, sizep, filter);
605
606 buf2 = crlf_to_git(path, buf ? buf : src, sizep, crlf);
607 if (buf2) {
608 free(buf);
609 buf = buf2;
610 }
3fed15f5
JH
611
612 buf2 = ident_to_git(path, buf ? buf : src, sizep, ident);
613 if (buf2) {
614 free(buf);
615 buf = buf2;
6073ee85 616 }
3fed15f5
JH
617
618 return buf;
35ebfd6a
JH
619}
620
ac78e548 621char *convert_to_working_tree(const char *path, const char *src, unsigned long *sizep)
35ebfd6a 622{
aa4ed402 623 struct git_attr_check check[3];
6073ee85 624 int crlf = CRLF_GUESS;
3fed15f5 625 int ident = 0;
aa4ed402 626 char *filter = NULL;
3fed15f5 627 char *buf, *buf2;
6073ee85
JH
628
629 setup_convert_check(check);
3fed15f5 630 if (!git_checkattr(path, ARRAY_SIZE(check), check)) {
aa4ed402 631 struct convert_driver *drv;
3fed15f5
JH
632 crlf = git_path_check_crlf(path, check + 0);
633 ident = git_path_check_ident(path, check + 1);
aa4ed402
JH
634 drv = git_path_check_convert(path, check + 2);
635 if (drv && drv->smudge)
636 filter = drv->smudge;
6073ee85 637 }
3fed15f5
JH
638
639 buf = ident_to_worktree(path, src, sizep, ident);
640
641 buf2 = crlf_to_worktree(path, buf ? buf : src, sizep, crlf);
642 if (buf2) {
643 free(buf);
644 buf = buf2;
645 }
646
aa4ed402
JH
647 buf2 = apply_filter(path, buf ? buf : src, sizep, filter);
648 if (buf2) {
649 free(buf);
650 buf = buf2;
651 }
652
3fed15f5 653 return buf;
35ebfd6a 654}
5e6cfc80
RS
655
656void *convert_sha1_file(const char *path, const unsigned char *sha1,
657 unsigned int mode, enum object_type *type,
658 unsigned long *size)
659{
660 void *buffer = read_sha1_file(sha1, type, size);
661 if (S_ISREG(mode) && buffer) {
662 void *converted = convert_to_working_tree(path, buffer, size);
663 if (converted) {
664 free(buffer);
665 buffer = converted;
666 }
667 }
668 return buffer;
669}