git-p4: auto-size the block
[git/git.git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # The block size is reduced automatically if required
51 defaultBlockSize = 1<<20
52
53 p4_access_checked = False
54
55 def p4_build_cmd(cmd):
56 """Build a suitable p4 command line.
57
58 This consolidates building and returning a p4 command line into one
59 location. It means that hooking into the environment, or other configuration
60 can be done more easily.
61 """
62 real_cmd = ["p4"]
63
64 user = gitConfig("git-p4.user")
65 if len(user) > 0:
66 real_cmd += ["-u",user]
67
68 password = gitConfig("git-p4.password")
69 if len(password) > 0:
70 real_cmd += ["-P", password]
71
72 port = gitConfig("git-p4.port")
73 if len(port) > 0:
74 real_cmd += ["-p", port]
75
76 host = gitConfig("git-p4.host")
77 if len(host) > 0:
78 real_cmd += ["-H", host]
79
80 client = gitConfig("git-p4.client")
81 if len(client) > 0:
82 real_cmd += ["-c", client]
83
84 retries = gitConfigInt("git-p4.retries")
85 if retries is None:
86 # Perform 3 retries by default
87 retries = 3
88 if retries > 0:
89 # Provide a way to not pass this option by setting git-p4.retries to 0
90 real_cmd += ["-r", str(retries)]
91
92 if isinstance(cmd,basestring):
93 real_cmd = ' '.join(real_cmd) + ' ' + cmd
94 else:
95 real_cmd += cmd
96
97 # now check that we can actually talk to the server
98 global p4_access_checked
99 if not p4_access_checked:
100 p4_access_checked = True # suppress access checks in p4_check_access itself
101 p4_check_access()
102
103 return real_cmd
104
105 def git_dir(path):
106 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
107 This won't automatically add ".git" to a directory.
108 """
109 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
110 if not d or len(d) == 0:
111 return None
112 else:
113 return d
114
115 def chdir(path, is_client_path=False):
116 """Do chdir to the given path, and set the PWD environment
117 variable for use by P4. It does not look at getcwd() output.
118 Since we're not using the shell, it is necessary to set the
119 PWD environment variable explicitly.
120
121 Normally, expand the path to force it to be absolute. This
122 addresses the use of relative path names inside P4 settings,
123 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
124 as given; it looks for .p4config using PWD.
125
126 If is_client_path, the path was handed to us directly by p4,
127 and may be a symbolic link. Do not call os.getcwd() in this
128 case, because it will cause p4 to think that PWD is not inside
129 the client path.
130 """
131
132 os.chdir(path)
133 if not is_client_path:
134 path = os.getcwd()
135 os.environ['PWD'] = path
136
137 def calcDiskFree():
138 """Return free space in bytes on the disk of the given dirname."""
139 if platform.system() == 'Windows':
140 free_bytes = ctypes.c_ulonglong(0)
141 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
142 return free_bytes.value
143 else:
144 st = os.statvfs(os.getcwd())
145 return st.f_bavail * st.f_frsize
146
147 def die(msg):
148 if verbose:
149 raise Exception(msg)
150 else:
151 sys.stderr.write(msg + "\n")
152 sys.exit(1)
153
154 def write_pipe(c, stdin):
155 if verbose:
156 sys.stderr.write('Writing pipe: %s\n' % str(c))
157
158 expand = isinstance(c,basestring)
159 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
160 pipe = p.stdin
161 val = pipe.write(stdin)
162 pipe.close()
163 if p.wait():
164 die('Command failed: %s' % str(c))
165
166 return val
167
168 def p4_write_pipe(c, stdin):
169 real_cmd = p4_build_cmd(c)
170 return write_pipe(real_cmd, stdin)
171
172 def read_pipe_full(c):
173 """ Read output from command. Returns a tuple
174 of the return status, stdout text and stderr
175 text.
176 """
177 if verbose:
178 sys.stderr.write('Reading pipe: %s\n' % str(c))
179
180 expand = isinstance(c,basestring)
181 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
182 (out, err) = p.communicate()
183 return (p.returncode, out, err)
184
185 def read_pipe(c, ignore_error=False):
186 """ Read output from command. Returns the output text on
187 success. On failure, terminates execution, unless
188 ignore_error is True, when it returns an empty string.
189 """
190 (retcode, out, err) = read_pipe_full(c)
191 if retcode != 0:
192 if ignore_error:
193 out = ""
194 else:
195 die('Command failed: %s\nError: %s' % (str(c), err))
196 return out
197
198 def read_pipe_text(c):
199 """ Read output from a command with trailing whitespace stripped.
200 On error, returns None.
201 """
202 (retcode, out, err) = read_pipe_full(c)
203 if retcode != 0:
204 return None
205 else:
206 return out.rstrip()
207
208 def p4_read_pipe(c, ignore_error=False):
209 real_cmd = p4_build_cmd(c)
210 return read_pipe(real_cmd, ignore_error)
211
212 def read_pipe_lines(c):
213 if verbose:
214 sys.stderr.write('Reading pipe: %s\n' % str(c))
215
216 expand = isinstance(c, basestring)
217 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
218 pipe = p.stdout
219 val = pipe.readlines()
220 if pipe.close() or p.wait():
221 die('Command failed: %s' % str(c))
222
223 return val
224
225 def p4_read_pipe_lines(c):
226 """Specifically invoke p4 on the command supplied. """
227 real_cmd = p4_build_cmd(c)
228 return read_pipe_lines(real_cmd)
229
230 def p4_has_command(cmd):
231 """Ask p4 for help on this command. If it returns an error, the
232 command does not exist in this version of p4."""
233 real_cmd = p4_build_cmd(["help", cmd])
234 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
235 stderr=subprocess.PIPE)
236 p.communicate()
237 return p.returncode == 0
238
239 def p4_has_move_command():
240 """See if the move command exists, that it supports -k, and that
241 it has not been administratively disabled. The arguments
242 must be correct, but the filenames do not have to exist. Use
243 ones with wildcards so even if they exist, it will fail."""
244
245 if not p4_has_command("move"):
246 return False
247 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
248 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
249 (out, err) = p.communicate()
250 # return code will be 1 in either case
251 if err.find("Invalid option") >= 0:
252 return False
253 if err.find("disabled") >= 0:
254 return False
255 # assume it failed because @... was invalid changelist
256 return True
257
258 def system(cmd, ignore_error=False):
259 expand = isinstance(cmd,basestring)
260 if verbose:
261 sys.stderr.write("executing %s\n" % str(cmd))
262 retcode = subprocess.call(cmd, shell=expand)
263 if retcode and not ignore_error:
264 raise CalledProcessError(retcode, cmd)
265
266 return retcode
267
268 def p4_system(cmd):
269 """Specifically invoke p4 as the system command. """
270 real_cmd = p4_build_cmd(cmd)
271 expand = isinstance(real_cmd, basestring)
272 retcode = subprocess.call(real_cmd, shell=expand)
273 if retcode:
274 raise CalledProcessError(retcode, real_cmd)
275
276 def die_bad_access(s):
277 die("failure accessing depot: {0}".format(s.rstrip()))
278
279 def p4_check_access(min_expiration=1):
280 """ Check if we can access Perforce - account still logged in
281 """
282 results = p4CmdList(["login", "-s"])
283
284 if len(results) == 0:
285 # should never get here: always get either some results, or a p4ExitCode
286 assert("could not parse response from perforce")
287
288 result = results[0]
289
290 if 'p4ExitCode' in result:
291 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
292 die_bad_access("could not run p4")
293
294 code = result.get("code")
295 if not code:
296 # we get here if we couldn't connect and there was nothing to unmarshal
297 die_bad_access("could not connect")
298
299 elif code == "stat":
300 expiry = result.get("TicketExpiration")
301 if expiry:
302 expiry = int(expiry)
303 if expiry > min_expiration:
304 # ok to carry on
305 return
306 else:
307 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
308
309 else:
310 # account without a timeout - all ok
311 return
312
313 elif code == "error":
314 data = result.get("data")
315 if data:
316 die_bad_access("p4 error: {0}".format(data))
317 else:
318 die_bad_access("unknown error")
319 else:
320 die_bad_access("unknown error code {0}".format(code))
321
322 _p4_version_string = None
323 def p4_version_string():
324 """Read the version string, showing just the last line, which
325 hopefully is the interesting version bit.
326
327 $ p4 -V
328 Perforce - The Fast Software Configuration Management System.
329 Copyright 1995-2011 Perforce Software. All rights reserved.
330 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
331 """
332 global _p4_version_string
333 if not _p4_version_string:
334 a = p4_read_pipe_lines(["-V"])
335 _p4_version_string = a[-1].rstrip()
336 return _p4_version_string
337
338 def p4_integrate(src, dest):
339 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
340
341 def p4_sync(f, *options):
342 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
343
344 def p4_add(f):
345 # forcibly add file names with wildcards
346 if wildcard_present(f):
347 p4_system(["add", "-f", f])
348 else:
349 p4_system(["add", f])
350
351 def p4_delete(f):
352 p4_system(["delete", wildcard_encode(f)])
353
354 def p4_edit(f, *options):
355 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
356
357 def p4_revert(f):
358 p4_system(["revert", wildcard_encode(f)])
359
360 def p4_reopen(type, f):
361 p4_system(["reopen", "-t", type, wildcard_encode(f)])
362
363 def p4_reopen_in_change(changelist, files):
364 cmd = ["reopen", "-c", str(changelist)] + files
365 p4_system(cmd)
366
367 def p4_move(src, dest):
368 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
369
370 def p4_last_change():
371 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
372 return int(results[0]['change'])
373
374 def p4_describe(change):
375 """Make sure it returns a valid result by checking for
376 the presence of field "time". Return a dict of the
377 results."""
378
379 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
380 if len(ds) != 1:
381 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
382
383 d = ds[0]
384
385 if "p4ExitCode" in d:
386 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
387 str(d)))
388 if "code" in d:
389 if d["code"] == "error":
390 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
391
392 if "time" not in d:
393 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
394
395 return d
396
397 #
398 # Canonicalize the p4 type and return a tuple of the
399 # base type, plus any modifiers. See "p4 help filetypes"
400 # for a list and explanation.
401 #
402 def split_p4_type(p4type):
403
404 p4_filetypes_historical = {
405 "ctempobj": "binary+Sw",
406 "ctext": "text+C",
407 "cxtext": "text+Cx",
408 "ktext": "text+k",
409 "kxtext": "text+kx",
410 "ltext": "text+F",
411 "tempobj": "binary+FSw",
412 "ubinary": "binary+F",
413 "uresource": "resource+F",
414 "uxbinary": "binary+Fx",
415 "xbinary": "binary+x",
416 "xltext": "text+Fx",
417 "xtempobj": "binary+Swx",
418 "xtext": "text+x",
419 "xunicode": "unicode+x",
420 "xutf16": "utf16+x",
421 }
422 if p4type in p4_filetypes_historical:
423 p4type = p4_filetypes_historical[p4type]
424 mods = ""
425 s = p4type.split("+")
426 base = s[0]
427 mods = ""
428 if len(s) > 1:
429 mods = s[1]
430 return (base, mods)
431
432 #
433 # return the raw p4 type of a file (text, text+ko, etc)
434 #
435 def p4_type(f):
436 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
437 return results[0]['headType']
438
439 #
440 # Given a type base and modifier, return a regexp matching
441 # the keywords that can be expanded in the file
442 #
443 def p4_keywords_regexp_for_type(base, type_mods):
444 if base in ("text", "unicode", "binary"):
445 kwords = None
446 if "ko" in type_mods:
447 kwords = 'Id|Header'
448 elif "k" in type_mods:
449 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
450 else:
451 return None
452 pattern = r"""
453 \$ # Starts with a dollar, followed by...
454 (%s) # one of the keywords, followed by...
455 (:[^$\n]+)? # possibly an old expansion, followed by...
456 \$ # another dollar
457 """ % kwords
458 return pattern
459 else:
460 return None
461
462 #
463 # Given a file, return a regexp matching the possible
464 # RCS keywords that will be expanded, or None for files
465 # with kw expansion turned off.
466 #
467 def p4_keywords_regexp_for_file(file):
468 if not os.path.exists(file):
469 return None
470 else:
471 (type_base, type_mods) = split_p4_type(p4_type(file))
472 return p4_keywords_regexp_for_type(type_base, type_mods)
473
474 def setP4ExecBit(file, mode):
475 # Reopens an already open file and changes the execute bit to match
476 # the execute bit setting in the passed in mode.
477
478 p4Type = "+x"
479
480 if not isModeExec(mode):
481 p4Type = getP4OpenedType(file)
482 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
483 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
484 if p4Type[-1] == "+":
485 p4Type = p4Type[0:-1]
486
487 p4_reopen(p4Type, file)
488
489 def getP4OpenedType(file):
490 # Returns the perforce file type for the given file.
491
492 result = p4_read_pipe(["opened", wildcard_encode(file)])
493 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
494 if match:
495 return match.group(1)
496 else:
497 die("Could not determine file type for %s (result: '%s')" % (file, result))
498
499 # Return the set of all p4 labels
500 def getP4Labels(depotPaths):
501 labels = set()
502 if isinstance(depotPaths,basestring):
503 depotPaths = [depotPaths]
504
505 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
506 label = l['label']
507 labels.add(label)
508
509 return labels
510
511 # Return the set of all git tags
512 def getGitTags():
513 gitTags = set()
514 for line in read_pipe_lines(["git", "tag"]):
515 tag = line.strip()
516 gitTags.add(tag)
517 return gitTags
518
519 def diffTreePattern():
520 # This is a simple generator for the diff tree regex pattern. This could be
521 # a class variable if this and parseDiffTreeEntry were a part of a class.
522 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
523 while True:
524 yield pattern
525
526 def parseDiffTreeEntry(entry):
527 """Parses a single diff tree entry into its component elements.
528
529 See git-diff-tree(1) manpage for details about the format of the diff
530 output. This method returns a dictionary with the following elements:
531
532 src_mode - The mode of the source file
533 dst_mode - The mode of the destination file
534 src_sha1 - The sha1 for the source file
535 dst_sha1 - The sha1 fr the destination file
536 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
537 status_score - The score for the status (applicable for 'C' and 'R'
538 statuses). This is None if there is no score.
539 src - The path for the source file.
540 dst - The path for the destination file. This is only present for
541 copy or renames. If it is not present, this is None.
542
543 If the pattern is not matched, None is returned."""
544
545 match = diffTreePattern().next().match(entry)
546 if match:
547 return {
548 'src_mode': match.group(1),
549 'dst_mode': match.group(2),
550 'src_sha1': match.group(3),
551 'dst_sha1': match.group(4),
552 'status': match.group(5),
553 'status_score': match.group(6),
554 'src': match.group(7),
555 'dst': match.group(10)
556 }
557 return None
558
559 def isModeExec(mode):
560 # Returns True if the given git mode represents an executable file,
561 # otherwise False.
562 return mode[-3:] == "755"
563
564 class P4Exception(Exception):
565 """ Base class for exceptions from the p4 client """
566 def __init__(self, exit_code):
567 self.p4ExitCode = exit_code
568
569 class P4ServerException(P4Exception):
570 """ Base class for exceptions where we get some kind of marshalled up result from the server """
571 def __init__(self, exit_code, p4_result):
572 super(P4ServerException, self).__init__(exit_code)
573 self.p4_result = p4_result
574 self.code = p4_result[0]['code']
575 self.data = p4_result[0]['data']
576
577 class P4RequestSizeException(P4ServerException):
578 """ One of the maxresults or maxscanrows errors """
579 def __init__(self, exit_code, p4_result, limit):
580 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
581 self.limit = limit
582
583 def isModeExecChanged(src_mode, dst_mode):
584 return isModeExec(src_mode) != isModeExec(dst_mode)
585
586 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
587 errors_as_exceptions=False):
588
589 if isinstance(cmd,basestring):
590 cmd = "-G " + cmd
591 expand = True
592 else:
593 cmd = ["-G"] + cmd
594 expand = False
595
596 cmd = p4_build_cmd(cmd)
597 if verbose:
598 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
599
600 # Use a temporary file to avoid deadlocks without
601 # subprocess.communicate(), which would put another copy
602 # of stdout into memory.
603 stdin_file = None
604 if stdin is not None:
605 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
606 if isinstance(stdin,basestring):
607 stdin_file.write(stdin)
608 else:
609 for i in stdin:
610 stdin_file.write(i + '\n')
611 stdin_file.flush()
612 stdin_file.seek(0)
613
614 p4 = subprocess.Popen(cmd,
615 shell=expand,
616 stdin=stdin_file,
617 stdout=subprocess.PIPE)
618
619 result = []
620 try:
621 while True:
622 entry = marshal.load(p4.stdout)
623 if skip_info:
624 if 'code' in entry and entry['code'] == 'info':
625 continue
626 if cb is not None:
627 cb(entry)
628 else:
629 result.append(entry)
630 except EOFError:
631 pass
632 exitCode = p4.wait()
633 if exitCode != 0:
634 if errors_as_exceptions:
635 if len(result) > 0:
636 data = result[0].get('data')
637 if data:
638 m = re.search('Too many rows scanned \(over (\d+)\)', data)
639 if not m:
640 m = re.search('Request too large \(over (\d+)\)', data)
641
642 if m:
643 limit = int(m.group(1))
644 raise P4RequestSizeException(exitCode, result, limit)
645
646 raise P4ServerException(exitCode, result)
647 else:
648 raise P4Exception(exitCode)
649 else:
650 entry = {}
651 entry["p4ExitCode"] = exitCode
652 result.append(entry)
653
654 return result
655
656 def p4Cmd(cmd):
657 list = p4CmdList(cmd)
658 result = {}
659 for entry in list:
660 result.update(entry)
661 return result;
662
663 def p4Where(depotPath):
664 if not depotPath.endswith("/"):
665 depotPath += "/"
666 depotPathLong = depotPath + "..."
667 outputList = p4CmdList(["where", depotPathLong])
668 output = None
669 for entry in outputList:
670 if "depotFile" in entry:
671 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
672 # The base path always ends with "/...".
673 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
674 output = entry
675 break
676 elif "data" in entry:
677 data = entry.get("data")
678 space = data.find(" ")
679 if data[:space] == depotPath:
680 output = entry
681 break
682 if output == None:
683 return ""
684 if output["code"] == "error":
685 return ""
686 clientPath = ""
687 if "path" in output:
688 clientPath = output.get("path")
689 elif "data" in output:
690 data = output.get("data")
691 lastSpace = data.rfind(" ")
692 clientPath = data[lastSpace + 1:]
693
694 if clientPath.endswith("..."):
695 clientPath = clientPath[:-3]
696 return clientPath
697
698 def currentGitBranch():
699 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
700
701 def isValidGitDir(path):
702 return git_dir(path) != None
703
704 def parseRevision(ref):
705 return read_pipe("git rev-parse %s" % ref).strip()
706
707 def branchExists(ref):
708 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
709 ignore_error=True)
710 return len(rev) > 0
711
712 def extractLogMessageFromGitCommit(commit):
713 logMessage = ""
714
715 ## fixme: title is first line of commit, not 1st paragraph.
716 foundTitle = False
717 for log in read_pipe_lines("git cat-file commit %s" % commit):
718 if not foundTitle:
719 if len(log) == 1:
720 foundTitle = True
721 continue
722
723 logMessage += log
724 return logMessage
725
726 def extractSettingsGitLog(log):
727 values = {}
728 for line in log.split("\n"):
729 line = line.strip()
730 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
731 if not m:
732 continue
733
734 assignments = m.group(1).split (':')
735 for a in assignments:
736 vals = a.split ('=')
737 key = vals[0].strip()
738 val = ('='.join (vals[1:])).strip()
739 if val.endswith ('\"') and val.startswith('"'):
740 val = val[1:-1]
741
742 values[key] = val
743
744 paths = values.get("depot-paths")
745 if not paths:
746 paths = values.get("depot-path")
747 if paths:
748 values['depot-paths'] = paths.split(',')
749 return values
750
751 def gitBranchExists(branch):
752 proc = subprocess.Popen(["git", "rev-parse", branch],
753 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
754 return proc.wait() == 0;
755
756 _gitConfig = {}
757
758 def gitConfig(key, typeSpecifier=None):
759 if not _gitConfig.has_key(key):
760 cmd = [ "git", "config" ]
761 if typeSpecifier:
762 cmd += [ typeSpecifier ]
763 cmd += [ key ]
764 s = read_pipe(cmd, ignore_error=True)
765 _gitConfig[key] = s.strip()
766 return _gitConfig[key]
767
768 def gitConfigBool(key):
769 """Return a bool, using git config --bool. It is True only if the
770 variable is set to true, and False if set to false or not present
771 in the config."""
772
773 if not _gitConfig.has_key(key):
774 _gitConfig[key] = gitConfig(key, '--bool') == "true"
775 return _gitConfig[key]
776
777 def gitConfigInt(key):
778 if not _gitConfig.has_key(key):
779 cmd = [ "git", "config", "--int", key ]
780 s = read_pipe(cmd, ignore_error=True)
781 v = s.strip()
782 try:
783 _gitConfig[key] = int(gitConfig(key, '--int'))
784 except ValueError:
785 _gitConfig[key] = None
786 return _gitConfig[key]
787
788 def gitConfigList(key):
789 if not _gitConfig.has_key(key):
790 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
791 _gitConfig[key] = s.strip().splitlines()
792 if _gitConfig[key] == ['']:
793 _gitConfig[key] = []
794 return _gitConfig[key]
795
796 def p4BranchesInGit(branchesAreInRemotes=True):
797 """Find all the branches whose names start with "p4/", looking
798 in remotes or heads as specified by the argument. Return
799 a dictionary of { branch: revision } for each one found.
800 The branch names are the short names, without any
801 "p4/" prefix."""
802
803 branches = {}
804
805 cmdline = "git rev-parse --symbolic "
806 if branchesAreInRemotes:
807 cmdline += "--remotes"
808 else:
809 cmdline += "--branches"
810
811 for line in read_pipe_lines(cmdline):
812 line = line.strip()
813
814 # only import to p4/
815 if not line.startswith('p4/'):
816 continue
817 # special symbolic ref to p4/master
818 if line == "p4/HEAD":
819 continue
820
821 # strip off p4/ prefix
822 branch = line[len("p4/"):]
823
824 branches[branch] = parseRevision(line)
825
826 return branches
827
828 def branch_exists(branch):
829 """Make sure that the given ref name really exists."""
830
831 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
832 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
833 out, _ = p.communicate()
834 if p.returncode:
835 return False
836 # expect exactly one line of output: the branch name
837 return out.rstrip() == branch
838
839 def findUpstreamBranchPoint(head = "HEAD"):
840 branches = p4BranchesInGit()
841 # map from depot-path to branch name
842 branchByDepotPath = {}
843 for branch in branches.keys():
844 tip = branches[branch]
845 log = extractLogMessageFromGitCommit(tip)
846 settings = extractSettingsGitLog(log)
847 if settings.has_key("depot-paths"):
848 paths = ",".join(settings["depot-paths"])
849 branchByDepotPath[paths] = "remotes/p4/" + branch
850
851 settings = None
852 parent = 0
853 while parent < 65535:
854 commit = head + "~%s" % parent
855 log = extractLogMessageFromGitCommit(commit)
856 settings = extractSettingsGitLog(log)
857 if settings.has_key("depot-paths"):
858 paths = ",".join(settings["depot-paths"])
859 if branchByDepotPath.has_key(paths):
860 return [branchByDepotPath[paths], settings]
861
862 parent = parent + 1
863
864 return ["", settings]
865
866 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
867 if not silent:
868 print ("Creating/updating branch(es) in %s based on origin branch(es)"
869 % localRefPrefix)
870
871 originPrefix = "origin/p4/"
872
873 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
874 line = line.strip()
875 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
876 continue
877
878 headName = line[len(originPrefix):]
879 remoteHead = localRefPrefix + headName
880 originHead = line
881
882 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
883 if (not original.has_key('depot-paths')
884 or not original.has_key('change')):
885 continue
886
887 update = False
888 if not gitBranchExists(remoteHead):
889 if verbose:
890 print "creating %s" % remoteHead
891 update = True
892 else:
893 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
894 if settings.has_key('change') > 0:
895 if settings['depot-paths'] == original['depot-paths']:
896 originP4Change = int(original['change'])
897 p4Change = int(settings['change'])
898 if originP4Change > p4Change:
899 print ("%s (%s) is newer than %s (%s). "
900 "Updating p4 branch from origin."
901 % (originHead, originP4Change,
902 remoteHead, p4Change))
903 update = True
904 else:
905 print ("Ignoring: %s was imported from %s while "
906 "%s was imported from %s"
907 % (originHead, ','.join(original['depot-paths']),
908 remoteHead, ','.join(settings['depot-paths'])))
909
910 if update:
911 system("git update-ref %s %s" % (remoteHead, originHead))
912
913 def originP4BranchesExist():
914 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
915
916
917 def p4ParseNumericChangeRange(parts):
918 changeStart = int(parts[0][1:])
919 if parts[1] == '#head':
920 changeEnd = p4_last_change()
921 else:
922 changeEnd = int(parts[1])
923
924 return (changeStart, changeEnd)
925
926 def chooseBlockSize(blockSize):
927 if blockSize:
928 return blockSize
929 else:
930 return defaultBlockSize
931
932 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
933 assert depotPaths
934
935 # Parse the change range into start and end. Try to find integer
936 # revision ranges as these can be broken up into blocks to avoid
937 # hitting server-side limits (maxrows, maxscanresults). But if
938 # that doesn't work, fall back to using the raw revision specifier
939 # strings, without using block mode.
940
941 if changeRange is None or changeRange == '':
942 changeStart = 1
943 changeEnd = p4_last_change()
944 block_size = chooseBlockSize(requestedBlockSize)
945 else:
946 parts = changeRange.split(',')
947 assert len(parts) == 2
948 try:
949 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
950 block_size = chooseBlockSize(requestedBlockSize)
951 except ValueError:
952 changeStart = parts[0][1:]
953 changeEnd = parts[1]
954 if requestedBlockSize:
955 die("cannot use --changes-block-size with non-numeric revisions")
956 block_size = None
957
958 changes = set()
959
960 # Retrieve changes a block at a time, to prevent running
961 # into a MaxResults/MaxScanRows error from the server. If
962 # we _do_ hit one of those errors, turn down the block size
963
964 while True:
965 cmd = ['changes']
966
967 if block_size:
968 end = min(changeEnd, changeStart + block_size)
969 revisionRange = "%d,%d" % (changeStart, end)
970 else:
971 revisionRange = "%s,%s" % (changeStart, changeEnd)
972
973 for p in depotPaths:
974 cmd += ["%s...@%s" % (p, revisionRange)]
975
976 # fetch the changes
977 try:
978 result = p4CmdList(cmd, errors_as_exceptions=True)
979 except P4RequestSizeException as e:
980 if not block_size:
981 block_size = e.limit
982 elif block_size > e.limit:
983 block_size = e.limit
984 else:
985 block_size = max(2, block_size // 2)
986
987 if verbose: print("block size error, retrying with block size {0}".format(block_size))
988 continue
989 except P4Exception as e:
990 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
991
992 # Insert changes in chronological order
993 for entry in reversed(result):
994 if not entry.has_key('change'):
995 continue
996 changes.add(int(entry['change']))
997
998 if not block_size:
999 break
1000
1001 if end >= changeEnd:
1002 break
1003
1004 changeStart = end + 1
1005
1006 changes = sorted(changes)
1007 return changes
1008
1009 def p4PathStartsWith(path, prefix):
1010 # This method tries to remedy a potential mixed-case issue:
1011 #
1012 # If UserA adds //depot/DirA/file1
1013 # and UserB adds //depot/dira/file2
1014 #
1015 # we may or may not have a problem. If you have core.ignorecase=true,
1016 # we treat DirA and dira as the same directory
1017 if gitConfigBool("core.ignorecase"):
1018 return path.lower().startswith(prefix.lower())
1019 return path.startswith(prefix)
1020
1021 def getClientSpec():
1022 """Look at the p4 client spec, create a View() object that contains
1023 all the mappings, and return it."""
1024
1025 specList = p4CmdList("client -o")
1026 if len(specList) != 1:
1027 die('Output from "client -o" is %d lines, expecting 1' %
1028 len(specList))
1029
1030 # dictionary of all client parameters
1031 entry = specList[0]
1032
1033 # the //client/ name
1034 client_name = entry["Client"]
1035
1036 # just the keys that start with "View"
1037 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1038
1039 # hold this new View
1040 view = View(client_name)
1041
1042 # append the lines, in order, to the view
1043 for view_num in range(len(view_keys)):
1044 k = "View%d" % view_num
1045 if k not in view_keys:
1046 die("Expected view key %s missing" % k)
1047 view.append(entry[k])
1048
1049 return view
1050
1051 def getClientRoot():
1052 """Grab the client directory."""
1053
1054 output = p4CmdList("client -o")
1055 if len(output) != 1:
1056 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1057
1058 entry = output[0]
1059 if "Root" not in entry:
1060 die('Client has no "Root"')
1061
1062 return entry["Root"]
1063
1064 #
1065 # P4 wildcards are not allowed in filenames. P4 complains
1066 # if you simply add them, but you can force it with "-f", in
1067 # which case it translates them into %xx encoding internally.
1068 #
1069 def wildcard_decode(path):
1070 # Search for and fix just these four characters. Do % last so
1071 # that fixing it does not inadvertently create new %-escapes.
1072 # Cannot have * in a filename in windows; untested as to
1073 # what p4 would do in such a case.
1074 if not platform.system() == "Windows":
1075 path = path.replace("%2A", "*")
1076 path = path.replace("%23", "#") \
1077 .replace("%40", "@") \
1078 .replace("%25", "%")
1079 return path
1080
1081 def wildcard_encode(path):
1082 # do % first to avoid double-encoding the %s introduced here
1083 path = path.replace("%", "%25") \
1084 .replace("*", "%2A") \
1085 .replace("#", "%23") \
1086 .replace("@", "%40")
1087 return path
1088
1089 def wildcard_present(path):
1090 m = re.search("[*#@%]", path)
1091 return m is not None
1092
1093 class LargeFileSystem(object):
1094 """Base class for large file system support."""
1095
1096 def __init__(self, writeToGitStream):
1097 self.largeFiles = set()
1098 self.writeToGitStream = writeToGitStream
1099
1100 def generatePointer(self, cloneDestination, contentFile):
1101 """Return the content of a pointer file that is stored in Git instead of
1102 the actual content."""
1103 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1104
1105 def pushFile(self, localLargeFile):
1106 """Push the actual content which is not stored in the Git repository to
1107 a server."""
1108 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1109
1110 def hasLargeFileExtension(self, relPath):
1111 return reduce(
1112 lambda a, b: a or b,
1113 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1114 False
1115 )
1116
1117 def generateTempFile(self, contents):
1118 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1119 for d in contents:
1120 contentFile.write(d)
1121 contentFile.close()
1122 return contentFile.name
1123
1124 def exceedsLargeFileThreshold(self, relPath, contents):
1125 if gitConfigInt('git-p4.largeFileThreshold'):
1126 contentsSize = sum(len(d) for d in contents)
1127 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1128 return True
1129 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1130 contentsSize = sum(len(d) for d in contents)
1131 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1132 return False
1133 contentTempFile = self.generateTempFile(contents)
1134 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1135 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1136 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1137 zf.close()
1138 compressedContentsSize = zf.infolist()[0].compress_size
1139 os.remove(contentTempFile)
1140 os.remove(compressedContentFile.name)
1141 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1142 return True
1143 return False
1144
1145 def addLargeFile(self, relPath):
1146 self.largeFiles.add(relPath)
1147
1148 def removeLargeFile(self, relPath):
1149 self.largeFiles.remove(relPath)
1150
1151 def isLargeFile(self, relPath):
1152 return relPath in self.largeFiles
1153
1154 def processContent(self, git_mode, relPath, contents):
1155 """Processes the content of git fast import. This method decides if a
1156 file is stored in the large file system and handles all necessary
1157 steps."""
1158 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1159 contentTempFile = self.generateTempFile(contents)
1160 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1161 if pointer_git_mode:
1162 git_mode = pointer_git_mode
1163 if localLargeFile:
1164 # Move temp file to final location in large file system
1165 largeFileDir = os.path.dirname(localLargeFile)
1166 if not os.path.isdir(largeFileDir):
1167 os.makedirs(largeFileDir)
1168 shutil.move(contentTempFile, localLargeFile)
1169 self.addLargeFile(relPath)
1170 if gitConfigBool('git-p4.largeFilePush'):
1171 self.pushFile(localLargeFile)
1172 if verbose:
1173 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1174 return (git_mode, contents)
1175
1176 class MockLFS(LargeFileSystem):
1177 """Mock large file system for testing."""
1178
1179 def generatePointer(self, contentFile):
1180 """The pointer content is the original content prefixed with "pointer-".
1181 The local filename of the large file storage is derived from the file content.
1182 """
1183 with open(contentFile, 'r') as f:
1184 content = next(f)
1185 gitMode = '100644'
1186 pointerContents = 'pointer-' + content
1187 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1188 return (gitMode, pointerContents, localLargeFile)
1189
1190 def pushFile(self, localLargeFile):
1191 """The remote filename of the large file storage is the same as the local
1192 one but in a different directory.
1193 """
1194 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1195 if not os.path.exists(remotePath):
1196 os.makedirs(remotePath)
1197 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1198
1199 class GitLFS(LargeFileSystem):
1200 """Git LFS as backend for the git-p4 large file system.
1201 See https://git-lfs.github.com/ for details."""
1202
1203 def __init__(self, *args):
1204 LargeFileSystem.__init__(self, *args)
1205 self.baseGitAttributes = []
1206
1207 def generatePointer(self, contentFile):
1208 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1209 mode and content which is stored in the Git repository instead of
1210 the actual content. Return also the new location of the actual
1211 content.
1212 """
1213 if os.path.getsize(contentFile) == 0:
1214 return (None, '', None)
1215
1216 pointerProcess = subprocess.Popen(
1217 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1218 stdout=subprocess.PIPE
1219 )
1220 pointerFile = pointerProcess.stdout.read()
1221 if pointerProcess.wait():
1222 os.remove(contentFile)
1223 die('git-lfs pointer command failed. Did you install the extension?')
1224
1225 # Git LFS removed the preamble in the output of the 'pointer' command
1226 # starting from version 1.2.0. Check for the preamble here to support
1227 # earlier versions.
1228 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1229 if pointerFile.startswith('Git LFS pointer for'):
1230 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1231
1232 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1233 localLargeFile = os.path.join(
1234 os.getcwd(),
1235 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1236 oid,
1237 )
1238 # LFS Spec states that pointer files should not have the executable bit set.
1239 gitMode = '100644'
1240 return (gitMode, pointerFile, localLargeFile)
1241
1242 def pushFile(self, localLargeFile):
1243 uploadProcess = subprocess.Popen(
1244 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1245 )
1246 if uploadProcess.wait():
1247 die('git-lfs push command failed. Did you define a remote?')
1248
1249 def generateGitAttributes(self):
1250 return (
1251 self.baseGitAttributes +
1252 [
1253 '\n',
1254 '#\n',
1255 '# Git LFS (see https://git-lfs.github.com/)\n',
1256 '#\n',
1257 ] +
1258 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1259 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1260 ] +
1261 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1262 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1263 ]
1264 )
1265
1266 def addLargeFile(self, relPath):
1267 LargeFileSystem.addLargeFile(self, relPath)
1268 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1269
1270 def removeLargeFile(self, relPath):
1271 LargeFileSystem.removeLargeFile(self, relPath)
1272 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1273
1274 def processContent(self, git_mode, relPath, contents):
1275 if relPath == '.gitattributes':
1276 self.baseGitAttributes = contents
1277 return (git_mode, self.generateGitAttributes())
1278 else:
1279 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1280
1281 class Command:
1282 def __init__(self):
1283 self.usage = "usage: %prog [options]"
1284 self.needsGit = True
1285 self.verbose = False
1286
1287 # This is required for the "append" cloneExclude action
1288 def ensure_value(self, attr, value):
1289 if not hasattr(self, attr) or getattr(self, attr) is None:
1290 setattr(self, attr, value)
1291 return getattr(self, attr)
1292
1293 class P4UserMap:
1294 def __init__(self):
1295 self.userMapFromPerforceServer = False
1296 self.myP4UserId = None
1297
1298 def p4UserId(self):
1299 if self.myP4UserId:
1300 return self.myP4UserId
1301
1302 results = p4CmdList("user -o")
1303 for r in results:
1304 if r.has_key('User'):
1305 self.myP4UserId = r['User']
1306 return r['User']
1307 die("Could not find your p4 user id")
1308
1309 def p4UserIsMe(self, p4User):
1310 # return True if the given p4 user is actually me
1311 me = self.p4UserId()
1312 if not p4User or p4User != me:
1313 return False
1314 else:
1315 return True
1316
1317 def getUserCacheFilename(self):
1318 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1319 return home + "/.gitp4-usercache.txt"
1320
1321 def getUserMapFromPerforceServer(self):
1322 if self.userMapFromPerforceServer:
1323 return
1324 self.users = {}
1325 self.emails = {}
1326
1327 for output in p4CmdList("users"):
1328 if not output.has_key("User"):
1329 continue
1330 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1331 self.emails[output["Email"]] = output["User"]
1332
1333 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1334 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1335 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1336 if mapUser and len(mapUser[0]) == 3:
1337 user = mapUser[0][0]
1338 fullname = mapUser[0][1]
1339 email = mapUser[0][2]
1340 self.users[user] = fullname + " <" + email + ">"
1341 self.emails[email] = user
1342
1343 s = ''
1344 for (key, val) in self.users.items():
1345 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1346
1347 open(self.getUserCacheFilename(), "wb").write(s)
1348 self.userMapFromPerforceServer = True
1349
1350 def loadUserMapFromCache(self):
1351 self.users = {}
1352 self.userMapFromPerforceServer = False
1353 try:
1354 cache = open(self.getUserCacheFilename(), "rb")
1355 lines = cache.readlines()
1356 cache.close()
1357 for line in lines:
1358 entry = line.strip().split("\t")
1359 self.users[entry[0]] = entry[1]
1360 except IOError:
1361 self.getUserMapFromPerforceServer()
1362
1363 class P4Debug(Command):
1364 def __init__(self):
1365 Command.__init__(self)
1366 self.options = []
1367 self.description = "A tool to debug the output of p4 -G."
1368 self.needsGit = False
1369
1370 def run(self, args):
1371 j = 0
1372 for output in p4CmdList(args):
1373 print 'Element: %d' % j
1374 j += 1
1375 print output
1376 return True
1377
1378 class P4RollBack(Command):
1379 def __init__(self):
1380 Command.__init__(self)
1381 self.options = [
1382 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1383 ]
1384 self.description = "A tool to debug the multi-branch import. Don't use :)"
1385 self.rollbackLocalBranches = False
1386
1387 def run(self, args):
1388 if len(args) != 1:
1389 return False
1390 maxChange = int(args[0])
1391
1392 if "p4ExitCode" in p4Cmd("changes -m 1"):
1393 die("Problems executing p4");
1394
1395 if self.rollbackLocalBranches:
1396 refPrefix = "refs/heads/"
1397 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1398 else:
1399 refPrefix = "refs/remotes/"
1400 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1401
1402 for line in lines:
1403 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1404 line = line.strip()
1405 ref = refPrefix + line
1406 log = extractLogMessageFromGitCommit(ref)
1407 settings = extractSettingsGitLog(log)
1408
1409 depotPaths = settings['depot-paths']
1410 change = settings['change']
1411
1412 changed = False
1413
1414 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1415 for p in depotPaths]))) == 0:
1416 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1417 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1418 continue
1419
1420 while change and int(change) > maxChange:
1421 changed = True
1422 if self.verbose:
1423 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1424 system("git update-ref %s \"%s^\"" % (ref, ref))
1425 log = extractLogMessageFromGitCommit(ref)
1426 settings = extractSettingsGitLog(log)
1427
1428
1429 depotPaths = settings['depot-paths']
1430 change = settings['change']
1431
1432 if changed:
1433 print "%s rewound to %s" % (ref, change)
1434
1435 return True
1436
1437 class P4Submit(Command, P4UserMap):
1438
1439 conflict_behavior_choices = ("ask", "skip", "quit")
1440
1441 def __init__(self):
1442 Command.__init__(self)
1443 P4UserMap.__init__(self)
1444 self.options = [
1445 optparse.make_option("--origin", dest="origin"),
1446 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1447 # preserve the user, requires relevant p4 permissions
1448 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1449 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1450 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1451 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1452 optparse.make_option("--conflict", dest="conflict_behavior",
1453 choices=self.conflict_behavior_choices),
1454 optparse.make_option("--branch", dest="branch"),
1455 optparse.make_option("--shelve", dest="shelve", action="store_true",
1456 help="Shelve instead of submit. Shelved files are reverted, "
1457 "restoring the workspace to the state before the shelve"),
1458 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1459 metavar="CHANGELIST",
1460 help="update an existing shelved changelist, implies --shelve, "
1461 "repeat in-order for multiple shelved changelists"),
1462 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1463 help="submit only the specified commit(s), one commit or xxx..xxx"),
1464 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1465 help="Disable rebase after submit is completed. Can be useful if you "
1466 "work from a local git branch that is not master"),
1467 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1468 help="Skip Perforce sync of p4/master after submit or shelve"),
1469 ]
1470 self.description = "Submit changes from git to the perforce depot."
1471 self.usage += " [name of git branch to submit into perforce depot]"
1472 self.origin = ""
1473 self.detectRenames = False
1474 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1475 self.dry_run = False
1476 self.shelve = False
1477 self.update_shelve = list()
1478 self.commit = ""
1479 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1480 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1481 self.prepare_p4_only = False
1482 self.conflict_behavior = None
1483 self.isWindows = (platform.system() == "Windows")
1484 self.exportLabels = False
1485 self.p4HasMoveCommand = p4_has_move_command()
1486 self.branch = None
1487
1488 if gitConfig('git-p4.largeFileSystem'):
1489 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1490
1491 def check(self):
1492 if len(p4CmdList("opened ...")) > 0:
1493 die("You have files opened with perforce! Close them before starting the sync.")
1494
1495 def separate_jobs_from_description(self, message):
1496 """Extract and return a possible Jobs field in the commit
1497 message. It goes into a separate section in the p4 change
1498 specification.
1499
1500 A jobs line starts with "Jobs:" and looks like a new field
1501 in a form. Values are white-space separated on the same
1502 line or on following lines that start with a tab.
1503
1504 This does not parse and extract the full git commit message
1505 like a p4 form. It just sees the Jobs: line as a marker
1506 to pass everything from then on directly into the p4 form,
1507 but outside the description section.
1508
1509 Return a tuple (stripped log message, jobs string)."""
1510
1511 m = re.search(r'^Jobs:', message, re.MULTILINE)
1512 if m is None:
1513 return (message, None)
1514
1515 jobtext = message[m.start():]
1516 stripped_message = message[:m.start()].rstrip()
1517 return (stripped_message, jobtext)
1518
1519 def prepareLogMessage(self, template, message, jobs):
1520 """Edits the template returned from "p4 change -o" to insert
1521 the message in the Description field, and the jobs text in
1522 the Jobs field."""
1523 result = ""
1524
1525 inDescriptionSection = False
1526
1527 for line in template.split("\n"):
1528 if line.startswith("#"):
1529 result += line + "\n"
1530 continue
1531
1532 if inDescriptionSection:
1533 if line.startswith("Files:") or line.startswith("Jobs:"):
1534 inDescriptionSection = False
1535 # insert Jobs section
1536 if jobs:
1537 result += jobs + "\n"
1538 else:
1539 continue
1540 else:
1541 if line.startswith("Description:"):
1542 inDescriptionSection = True
1543 line += "\n"
1544 for messageLine in message.split("\n"):
1545 line += "\t" + messageLine + "\n"
1546
1547 result += line + "\n"
1548
1549 return result
1550
1551 def patchRCSKeywords(self, file, pattern):
1552 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1553 (handle, outFileName) = tempfile.mkstemp(dir='.')
1554 try:
1555 outFile = os.fdopen(handle, "w+")
1556 inFile = open(file, "r")
1557 regexp = re.compile(pattern, re.VERBOSE)
1558 for line in inFile.readlines():
1559 line = regexp.sub(r'$\1$', line)
1560 outFile.write(line)
1561 inFile.close()
1562 outFile.close()
1563 # Forcibly overwrite the original file
1564 os.unlink(file)
1565 shutil.move(outFileName, file)
1566 except:
1567 # cleanup our temporary file
1568 os.unlink(outFileName)
1569 print "Failed to strip RCS keywords in %s" % file
1570 raise
1571
1572 print "Patched up RCS keywords in %s" % file
1573
1574 def p4UserForCommit(self,id):
1575 # Return the tuple (perforce user,git email) for a given git commit id
1576 self.getUserMapFromPerforceServer()
1577 gitEmail = read_pipe(["git", "log", "--max-count=1",
1578 "--format=%ae", id])
1579 gitEmail = gitEmail.strip()
1580 if not self.emails.has_key(gitEmail):
1581 return (None,gitEmail)
1582 else:
1583 return (self.emails[gitEmail],gitEmail)
1584
1585 def checkValidP4Users(self,commits):
1586 # check if any git authors cannot be mapped to p4 users
1587 for id in commits:
1588 (user,email) = self.p4UserForCommit(id)
1589 if not user:
1590 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1591 if gitConfigBool("git-p4.allowMissingP4Users"):
1592 print "%s" % msg
1593 else:
1594 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1595
1596 def lastP4Changelist(self):
1597 # Get back the last changelist number submitted in this client spec. This
1598 # then gets used to patch up the username in the change. If the same
1599 # client spec is being used by multiple processes then this might go
1600 # wrong.
1601 results = p4CmdList("client -o") # find the current client
1602 client = None
1603 for r in results:
1604 if r.has_key('Client'):
1605 client = r['Client']
1606 break
1607 if not client:
1608 die("could not get client spec")
1609 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1610 for r in results:
1611 if r.has_key('change'):
1612 return r['change']
1613 die("Could not get changelist number for last submit - cannot patch up user details")
1614
1615 def modifyChangelistUser(self, changelist, newUser):
1616 # fixup the user field of a changelist after it has been submitted.
1617 changes = p4CmdList("change -o %s" % changelist)
1618 if len(changes) != 1:
1619 die("Bad output from p4 change modifying %s to user %s" %
1620 (changelist, newUser))
1621
1622 c = changes[0]
1623 if c['User'] == newUser: return # nothing to do
1624 c['User'] = newUser
1625 input = marshal.dumps(c)
1626
1627 result = p4CmdList("change -f -i", stdin=input)
1628 for r in result:
1629 if r.has_key('code'):
1630 if r['code'] == 'error':
1631 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1632 if r.has_key('data'):
1633 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1634 return
1635 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1636
1637 def canChangeChangelists(self):
1638 # check to see if we have p4 admin or super-user permissions, either of
1639 # which are required to modify changelists.
1640 results = p4CmdList(["protects", self.depotPath])
1641 for r in results:
1642 if r.has_key('perm'):
1643 if r['perm'] == 'admin':
1644 return 1
1645 if r['perm'] == 'super':
1646 return 1
1647 return 0
1648
1649 def prepareSubmitTemplate(self, changelist=None):
1650 """Run "p4 change -o" to grab a change specification template.
1651 This does not use "p4 -G", as it is nice to keep the submission
1652 template in original order, since a human might edit it.
1653
1654 Remove lines in the Files section that show changes to files
1655 outside the depot path we're committing into."""
1656
1657 [upstream, settings] = findUpstreamBranchPoint()
1658
1659 template = """\
1660 # A Perforce Change Specification.
1661 #
1662 # Change: The change number. 'new' on a new changelist.
1663 # Date: The date this specification was last modified.
1664 # Client: The client on which the changelist was created. Read-only.
1665 # User: The user who created the changelist.
1666 # Status: Either 'pending' or 'submitted'. Read-only.
1667 # Type: Either 'public' or 'restricted'. Default is 'public'.
1668 # Description: Comments about the changelist. Required.
1669 # Jobs: What opened jobs are to be closed by this changelist.
1670 # You may delete jobs from this list. (New changelists only.)
1671 # Files: What opened files from the default changelist are to be added
1672 # to this changelist. You may delete files from this list.
1673 # (New changelists only.)
1674 """
1675 files_list = []
1676 inFilesSection = False
1677 change_entry = None
1678 args = ['change', '-o']
1679 if changelist:
1680 args.append(str(changelist))
1681 for entry in p4CmdList(args):
1682 if not entry.has_key('code'):
1683 continue
1684 if entry['code'] == 'stat':
1685 change_entry = entry
1686 break
1687 if not change_entry:
1688 die('Failed to decode output of p4 change -o')
1689 for key, value in change_entry.iteritems():
1690 if key.startswith('File'):
1691 if settings.has_key('depot-paths'):
1692 if not [p for p in settings['depot-paths']
1693 if p4PathStartsWith(value, p)]:
1694 continue
1695 else:
1696 if not p4PathStartsWith(value, self.depotPath):
1697 continue
1698 files_list.append(value)
1699 continue
1700 # Output in the order expected by prepareLogMessage
1701 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1702 if not change_entry.has_key(key):
1703 continue
1704 template += '\n'
1705 template += key + ':'
1706 if key == 'Description':
1707 template += '\n'
1708 for field_line in change_entry[key].splitlines():
1709 template += '\t'+field_line+'\n'
1710 if len(files_list) > 0:
1711 template += '\n'
1712 template += 'Files:\n'
1713 for path in files_list:
1714 template += '\t'+path+'\n'
1715 return template
1716
1717 def edit_template(self, template_file):
1718 """Invoke the editor to let the user change the submission
1719 message. Return true if okay to continue with the submit."""
1720
1721 # if configured to skip the editing part, just submit
1722 if gitConfigBool("git-p4.skipSubmitEdit"):
1723 return True
1724
1725 # look at the modification time, to check later if the user saved
1726 # the file
1727 mtime = os.stat(template_file).st_mtime
1728
1729 # invoke the editor
1730 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1731 editor = os.environ.get("P4EDITOR")
1732 else:
1733 editor = read_pipe("git var GIT_EDITOR").strip()
1734 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1735
1736 # If the file was not saved, prompt to see if this patch should
1737 # be skipped. But skip this verification step if configured so.
1738 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1739 return True
1740
1741 # modification time updated means user saved the file
1742 if os.stat(template_file).st_mtime > mtime:
1743 return True
1744
1745 while True:
1746 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1747 if response == 'y':
1748 return True
1749 if response == 'n':
1750 return False
1751
1752 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1753 # diff
1754 if os.environ.has_key("P4DIFF"):
1755 del(os.environ["P4DIFF"])
1756 diff = ""
1757 for editedFile in editedFiles:
1758 diff += p4_read_pipe(['diff', '-du',
1759 wildcard_encode(editedFile)])
1760
1761 # new file diff
1762 newdiff = ""
1763 for newFile in filesToAdd:
1764 newdiff += "==== new file ====\n"
1765 newdiff += "--- /dev/null\n"
1766 newdiff += "+++ %s\n" % newFile
1767
1768 is_link = os.path.islink(newFile)
1769 expect_link = newFile in symlinks
1770
1771 if is_link and expect_link:
1772 newdiff += "+%s\n" % os.readlink(newFile)
1773 else:
1774 f = open(newFile, "r")
1775 for line in f.readlines():
1776 newdiff += "+" + line
1777 f.close()
1778
1779 return (diff + newdiff).replace('\r\n', '\n')
1780
1781 def applyCommit(self, id):
1782 """Apply one commit, return True if it succeeded."""
1783
1784 print "Applying", read_pipe(["git", "show", "-s",
1785 "--format=format:%h %s", id])
1786
1787 (p4User, gitEmail) = self.p4UserForCommit(id)
1788
1789 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1790 filesToAdd = set()
1791 filesToChangeType = set()
1792 filesToDelete = set()
1793 editedFiles = set()
1794 pureRenameCopy = set()
1795 symlinks = set()
1796 filesToChangeExecBit = {}
1797 all_files = list()
1798
1799 for line in diff:
1800 diff = parseDiffTreeEntry(line)
1801 modifier = diff['status']
1802 path = diff['src']
1803 all_files.append(path)
1804
1805 if modifier == "M":
1806 p4_edit(path)
1807 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1808 filesToChangeExecBit[path] = diff['dst_mode']
1809 editedFiles.add(path)
1810 elif modifier == "A":
1811 filesToAdd.add(path)
1812 filesToChangeExecBit[path] = diff['dst_mode']
1813 if path in filesToDelete:
1814 filesToDelete.remove(path)
1815
1816 dst_mode = int(diff['dst_mode'], 8)
1817 if dst_mode == 0120000:
1818 symlinks.add(path)
1819
1820 elif modifier == "D":
1821 filesToDelete.add(path)
1822 if path in filesToAdd:
1823 filesToAdd.remove(path)
1824 elif modifier == "C":
1825 src, dest = diff['src'], diff['dst']
1826 p4_integrate(src, dest)
1827 pureRenameCopy.add(dest)
1828 if diff['src_sha1'] != diff['dst_sha1']:
1829 p4_edit(dest)
1830 pureRenameCopy.discard(dest)
1831 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1832 p4_edit(dest)
1833 pureRenameCopy.discard(dest)
1834 filesToChangeExecBit[dest] = diff['dst_mode']
1835 if self.isWindows:
1836 # turn off read-only attribute
1837 os.chmod(dest, stat.S_IWRITE)
1838 os.unlink(dest)
1839 editedFiles.add(dest)
1840 elif modifier == "R":
1841 src, dest = diff['src'], diff['dst']
1842 if self.p4HasMoveCommand:
1843 p4_edit(src) # src must be open before move
1844 p4_move(src, dest) # opens for (move/delete, move/add)
1845 else:
1846 p4_integrate(src, dest)
1847 if diff['src_sha1'] != diff['dst_sha1']:
1848 p4_edit(dest)
1849 else:
1850 pureRenameCopy.add(dest)
1851 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1852 if not self.p4HasMoveCommand:
1853 p4_edit(dest) # with move: already open, writable
1854 filesToChangeExecBit[dest] = diff['dst_mode']
1855 if not self.p4HasMoveCommand:
1856 if self.isWindows:
1857 os.chmod(dest, stat.S_IWRITE)
1858 os.unlink(dest)
1859 filesToDelete.add(src)
1860 editedFiles.add(dest)
1861 elif modifier == "T":
1862 filesToChangeType.add(path)
1863 else:
1864 die("unknown modifier %s for %s" % (modifier, path))
1865
1866 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1867 patchcmd = diffcmd + " | git apply "
1868 tryPatchCmd = patchcmd + "--check -"
1869 applyPatchCmd = patchcmd + "--check --apply -"
1870 patch_succeeded = True
1871
1872 if os.system(tryPatchCmd) != 0:
1873 fixed_rcs_keywords = False
1874 patch_succeeded = False
1875 print "Unfortunately applying the change failed!"
1876
1877 # Patch failed, maybe it's just RCS keyword woes. Look through
1878 # the patch to see if that's possible.
1879 if gitConfigBool("git-p4.attemptRCSCleanup"):
1880 file = None
1881 pattern = None
1882 kwfiles = {}
1883 for file in editedFiles | filesToDelete:
1884 # did this file's delta contain RCS keywords?
1885 pattern = p4_keywords_regexp_for_file(file)
1886
1887 if pattern:
1888 # this file is a possibility...look for RCS keywords.
1889 regexp = re.compile(pattern, re.VERBOSE)
1890 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1891 if regexp.search(line):
1892 if verbose:
1893 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1894 kwfiles[file] = pattern
1895 break
1896
1897 for file in kwfiles:
1898 if verbose:
1899 print "zapping %s with %s" % (line,pattern)
1900 # File is being deleted, so not open in p4. Must
1901 # disable the read-only bit on windows.
1902 if self.isWindows and file not in editedFiles:
1903 os.chmod(file, stat.S_IWRITE)
1904 self.patchRCSKeywords(file, kwfiles[file])
1905 fixed_rcs_keywords = True
1906
1907 if fixed_rcs_keywords:
1908 print "Retrying the patch with RCS keywords cleaned up"
1909 if os.system(tryPatchCmd) == 0:
1910 patch_succeeded = True
1911
1912 if not patch_succeeded:
1913 for f in editedFiles:
1914 p4_revert(f)
1915 return False
1916
1917 #
1918 # Apply the patch for real, and do add/delete/+x handling.
1919 #
1920 system(applyPatchCmd)
1921
1922 for f in filesToChangeType:
1923 p4_edit(f, "-t", "auto")
1924 for f in filesToAdd:
1925 p4_add(f)
1926 for f in filesToDelete:
1927 p4_revert(f)
1928 p4_delete(f)
1929
1930 # Set/clear executable bits
1931 for f in filesToChangeExecBit.keys():
1932 mode = filesToChangeExecBit[f]
1933 setP4ExecBit(f, mode)
1934
1935 update_shelve = 0
1936 if len(self.update_shelve) > 0:
1937 update_shelve = self.update_shelve.pop(0)
1938 p4_reopen_in_change(update_shelve, all_files)
1939
1940 #
1941 # Build p4 change description, starting with the contents
1942 # of the git commit message.
1943 #
1944 logMessage = extractLogMessageFromGitCommit(id)
1945 logMessage = logMessage.strip()
1946 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1947
1948 template = self.prepareSubmitTemplate(update_shelve)
1949 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1950
1951 if self.preserveUser:
1952 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1953
1954 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1955 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1956 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1957 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1958
1959 separatorLine = "######## everything below this line is just the diff #######\n"
1960 if not self.prepare_p4_only:
1961 submitTemplate += separatorLine
1962 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1963
1964 (handle, fileName) = tempfile.mkstemp()
1965 tmpFile = os.fdopen(handle, "w+b")
1966 if self.isWindows:
1967 submitTemplate = submitTemplate.replace("\n", "\r\n")
1968 tmpFile.write(submitTemplate)
1969 tmpFile.close()
1970
1971 if self.prepare_p4_only:
1972 #
1973 # Leave the p4 tree prepared, and the submit template around
1974 # and let the user decide what to do next
1975 #
1976 print
1977 print "P4 workspace prepared for submission."
1978 print "To submit or revert, go to client workspace"
1979 print " " + self.clientPath
1980 print
1981 print "To submit, use \"p4 submit\" to write a new description,"
1982 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1983 " \"git p4\"." % fileName
1984 print "You can delete the file \"%s\" when finished." % fileName
1985
1986 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1987 print "To preserve change ownership by user %s, you must\n" \
1988 "do \"p4 change -f <change>\" after submitting and\n" \
1989 "edit the User field."
1990 if pureRenameCopy:
1991 print "After submitting, renamed files must be re-synced."
1992 print "Invoke \"p4 sync -f\" on each of these files:"
1993 for f in pureRenameCopy:
1994 print " " + f
1995
1996 print
1997 print "To revert the changes, use \"p4 revert ...\", and delete"
1998 print "the submit template file \"%s\"" % fileName
1999 if filesToAdd:
2000 print "Since the commit adds new files, they must be deleted:"
2001 for f in filesToAdd:
2002 print " " + f
2003 print
2004 return True
2005
2006 #
2007 # Let the user edit the change description, then submit it.
2008 #
2009 submitted = False
2010
2011 try:
2012 if self.edit_template(fileName):
2013 # read the edited message and submit
2014 tmpFile = open(fileName, "rb")
2015 message = tmpFile.read()
2016 tmpFile.close()
2017 if self.isWindows:
2018 message = message.replace("\r\n", "\n")
2019 submitTemplate = message[:message.index(separatorLine)]
2020
2021 if update_shelve:
2022 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2023 elif self.shelve:
2024 p4_write_pipe(['shelve', '-i'], submitTemplate)
2025 else:
2026 p4_write_pipe(['submit', '-i'], submitTemplate)
2027 # The rename/copy happened by applying a patch that created a
2028 # new file. This leaves it writable, which confuses p4.
2029 for f in pureRenameCopy:
2030 p4_sync(f, "-f")
2031
2032 if self.preserveUser:
2033 if p4User:
2034 # Get last changelist number. Cannot easily get it from
2035 # the submit command output as the output is
2036 # unmarshalled.
2037 changelist = self.lastP4Changelist()
2038 self.modifyChangelistUser(changelist, p4User)
2039
2040 submitted = True
2041
2042 finally:
2043 # skip this patch
2044 if not submitted or self.shelve:
2045 if self.shelve:
2046 print ("Reverting shelved files.")
2047 else:
2048 print ("Submission cancelled, undoing p4 changes.")
2049 for f in editedFiles | filesToDelete:
2050 p4_revert(f)
2051 for f in filesToAdd:
2052 p4_revert(f)
2053 os.remove(f)
2054
2055 os.remove(fileName)
2056 return submitted
2057
2058 # Export git tags as p4 labels. Create a p4 label and then tag
2059 # with that.
2060 def exportGitTags(self, gitTags):
2061 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2062 if len(validLabelRegexp) == 0:
2063 validLabelRegexp = defaultLabelRegexp
2064 m = re.compile(validLabelRegexp)
2065
2066 for name in gitTags:
2067
2068 if not m.match(name):
2069 if verbose:
2070 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
2071 continue
2072
2073 # Get the p4 commit this corresponds to
2074 logMessage = extractLogMessageFromGitCommit(name)
2075 values = extractSettingsGitLog(logMessage)
2076
2077 if not values.has_key('change'):
2078 # a tag pointing to something not sent to p4; ignore
2079 if verbose:
2080 print "git tag %s does not give a p4 commit" % name
2081 continue
2082 else:
2083 changelist = values['change']
2084
2085 # Get the tag details.
2086 inHeader = True
2087 isAnnotated = False
2088 body = []
2089 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2090 l = l.strip()
2091 if inHeader:
2092 if re.match(r'tag\s+', l):
2093 isAnnotated = True
2094 elif re.match(r'\s*$', l):
2095 inHeader = False
2096 continue
2097 else:
2098 body.append(l)
2099
2100 if not isAnnotated:
2101 body = ["lightweight tag imported by git p4\n"]
2102
2103 # Create the label - use the same view as the client spec we are using
2104 clientSpec = getClientSpec()
2105
2106 labelTemplate = "Label: %s\n" % name
2107 labelTemplate += "Description:\n"
2108 for b in body:
2109 labelTemplate += "\t" + b + "\n"
2110 labelTemplate += "View:\n"
2111 for depot_side in clientSpec.mappings:
2112 labelTemplate += "\t%s\n" % depot_side
2113
2114 if self.dry_run:
2115 print "Would create p4 label %s for tag" % name
2116 elif self.prepare_p4_only:
2117 print "Not creating p4 label %s for tag due to option" \
2118 " --prepare-p4-only" % name
2119 else:
2120 p4_write_pipe(["label", "-i"], labelTemplate)
2121
2122 # Use the label
2123 p4_system(["tag", "-l", name] +
2124 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2125
2126 if verbose:
2127 print "created p4 label for tag %s" % name
2128
2129 def run(self, args):
2130 if len(args) == 0:
2131 self.master = currentGitBranch()
2132 elif len(args) == 1:
2133 self.master = args[0]
2134 if not branchExists(self.master):
2135 die("Branch %s does not exist" % self.master)
2136 else:
2137 return False
2138
2139 for i in self.update_shelve:
2140 if i <= 0:
2141 sys.exit("invalid changelist %d" % i)
2142
2143 if self.master:
2144 allowSubmit = gitConfig("git-p4.allowSubmit")
2145 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2146 die("%s is not in git-p4.allowSubmit" % self.master)
2147
2148 [upstream, settings] = findUpstreamBranchPoint()
2149 self.depotPath = settings['depot-paths'][0]
2150 if len(self.origin) == 0:
2151 self.origin = upstream
2152
2153 if len(self.update_shelve) > 0:
2154 self.shelve = True
2155
2156 if self.preserveUser:
2157 if not self.canChangeChangelists():
2158 die("Cannot preserve user names without p4 super-user or admin permissions")
2159
2160 # if not set from the command line, try the config file
2161 if self.conflict_behavior is None:
2162 val = gitConfig("git-p4.conflict")
2163 if val:
2164 if val not in self.conflict_behavior_choices:
2165 die("Invalid value '%s' for config git-p4.conflict" % val)
2166 else:
2167 val = "ask"
2168 self.conflict_behavior = val
2169
2170 if self.verbose:
2171 print "Origin branch is " + self.origin
2172
2173 if len(self.depotPath) == 0:
2174 print "Internal error: cannot locate perforce depot path from existing branches"
2175 sys.exit(128)
2176
2177 self.useClientSpec = False
2178 if gitConfigBool("git-p4.useclientspec"):
2179 self.useClientSpec = True
2180 if self.useClientSpec:
2181 self.clientSpecDirs = getClientSpec()
2182
2183 # Check for the existence of P4 branches
2184 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2185
2186 if self.useClientSpec and not branchesDetected:
2187 # all files are relative to the client spec
2188 self.clientPath = getClientRoot()
2189 else:
2190 self.clientPath = p4Where(self.depotPath)
2191
2192 if self.clientPath == "":
2193 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2194
2195 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2196 self.oldWorkingDirectory = os.getcwd()
2197
2198 # ensure the clientPath exists
2199 new_client_dir = False
2200 if not os.path.exists(self.clientPath):
2201 new_client_dir = True
2202 os.makedirs(self.clientPath)
2203
2204 chdir(self.clientPath, is_client_path=True)
2205 if self.dry_run:
2206 print "Would synchronize p4 checkout in %s" % self.clientPath
2207 else:
2208 print "Synchronizing p4 checkout..."
2209 if new_client_dir:
2210 # old one was destroyed, and maybe nobody told p4
2211 p4_sync("...", "-f")
2212 else:
2213 p4_sync("...")
2214 self.check()
2215
2216 commits = []
2217 if self.master:
2218 commitish = self.master
2219 else:
2220 commitish = 'HEAD'
2221
2222 if self.commit != "":
2223 if self.commit.find("..") != -1:
2224 limits_ish = self.commit.split("..")
2225 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2226 commits.append(line.strip())
2227 commits.reverse()
2228 else:
2229 commits.append(self.commit)
2230 else:
2231 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2232 commits.append(line.strip())
2233 commits.reverse()
2234
2235 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2236 self.checkAuthorship = False
2237 else:
2238 self.checkAuthorship = True
2239
2240 if self.preserveUser:
2241 self.checkValidP4Users(commits)
2242
2243 #
2244 # Build up a set of options to be passed to diff when
2245 # submitting each commit to p4.
2246 #
2247 if self.detectRenames:
2248 # command-line -M arg
2249 self.diffOpts = "-M"
2250 else:
2251 # If not explicitly set check the config variable
2252 detectRenames = gitConfig("git-p4.detectRenames")
2253
2254 if detectRenames.lower() == "false" or detectRenames == "":
2255 self.diffOpts = ""
2256 elif detectRenames.lower() == "true":
2257 self.diffOpts = "-M"
2258 else:
2259 self.diffOpts = "-M%s" % detectRenames
2260
2261 # no command-line arg for -C or --find-copies-harder, just
2262 # config variables
2263 detectCopies = gitConfig("git-p4.detectCopies")
2264 if detectCopies.lower() == "false" or detectCopies == "":
2265 pass
2266 elif detectCopies.lower() == "true":
2267 self.diffOpts += " -C"
2268 else:
2269 self.diffOpts += " -C%s" % detectCopies
2270
2271 if gitConfigBool("git-p4.detectCopiesHarder"):
2272 self.diffOpts += " --find-copies-harder"
2273
2274 num_shelves = len(self.update_shelve)
2275 if num_shelves > 0 and num_shelves != len(commits):
2276 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2277 (len(commits), num_shelves))
2278
2279 #
2280 # Apply the commits, one at a time. On failure, ask if should
2281 # continue to try the rest of the patches, or quit.
2282 #
2283 if self.dry_run:
2284 print "Would apply"
2285 applied = []
2286 last = len(commits) - 1
2287 for i, commit in enumerate(commits):
2288 if self.dry_run:
2289 print " ", read_pipe(["git", "show", "-s",
2290 "--format=format:%h %s", commit])
2291 ok = True
2292 else:
2293 ok = self.applyCommit(commit)
2294 if ok:
2295 applied.append(commit)
2296 else:
2297 if self.prepare_p4_only and i < last:
2298 print "Processing only the first commit due to option" \
2299 " --prepare-p4-only"
2300 break
2301 if i < last:
2302 quit = False
2303 while True:
2304 # prompt for what to do, or use the option/variable
2305 if self.conflict_behavior == "ask":
2306 print "What do you want to do?"
2307 response = raw_input("[s]kip this commit but apply"
2308 " the rest, or [q]uit? ")
2309 if not response:
2310 continue
2311 elif self.conflict_behavior == "skip":
2312 response = "s"
2313 elif self.conflict_behavior == "quit":
2314 response = "q"
2315 else:
2316 die("Unknown conflict_behavior '%s'" %
2317 self.conflict_behavior)
2318
2319 if response[0] == "s":
2320 print "Skipping this commit, but applying the rest"
2321 break
2322 if response[0] == "q":
2323 print "Quitting"
2324 quit = True
2325 break
2326 if quit:
2327 break
2328
2329 chdir(self.oldWorkingDirectory)
2330 shelved_applied = "shelved" if self.shelve else "applied"
2331 if self.dry_run:
2332 pass
2333 elif self.prepare_p4_only:
2334 pass
2335 elif len(commits) == len(applied):
2336 print ("All commits {0}!".format(shelved_applied))
2337
2338 sync = P4Sync()
2339 if self.branch:
2340 sync.branch = self.branch
2341 if self.disable_p4sync:
2342 sync.sync_origin_only()
2343 else:
2344 sync.run([])
2345
2346 if not self.disable_rebase:
2347 rebase = P4Rebase()
2348 rebase.rebase()
2349
2350 else:
2351 if len(applied) == 0:
2352 print ("No commits {0}.".format(shelved_applied))
2353 else:
2354 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2355 for c in commits:
2356 if c in applied:
2357 star = "*"
2358 else:
2359 star = " "
2360 print star, read_pipe(["git", "show", "-s",
2361 "--format=format:%h %s", c])
2362 print "You will have to do 'git p4 sync' and rebase."
2363
2364 if gitConfigBool("git-p4.exportLabels"):
2365 self.exportLabels = True
2366
2367 if self.exportLabels:
2368 p4Labels = getP4Labels(self.depotPath)
2369 gitTags = getGitTags()
2370
2371 missingGitTags = gitTags - p4Labels
2372 self.exportGitTags(missingGitTags)
2373
2374 # exit with error unless everything applied perfectly
2375 if len(commits) != len(applied):
2376 sys.exit(1)
2377
2378 return True
2379
2380 class View(object):
2381 """Represent a p4 view ("p4 help views"), and map files in a
2382 repo according to the view."""
2383
2384 def __init__(self, client_name):
2385 self.mappings = []
2386 self.client_prefix = "//%s/" % client_name
2387 # cache results of "p4 where" to lookup client file locations
2388 self.client_spec_path_cache = {}
2389
2390 def append(self, view_line):
2391 """Parse a view line, splitting it into depot and client
2392 sides. Append to self.mappings, preserving order. This
2393 is only needed for tag creation."""
2394
2395 # Split the view line into exactly two words. P4 enforces
2396 # structure on these lines that simplifies this quite a bit.
2397 #
2398 # Either or both words may be double-quoted.
2399 # Single quotes do not matter.
2400 # Double-quote marks cannot occur inside the words.
2401 # A + or - prefix is also inside the quotes.
2402 # There are no quotes unless they contain a space.
2403 # The line is already white-space stripped.
2404 # The two words are separated by a single space.
2405 #
2406 if view_line[0] == '"':
2407 # First word is double quoted. Find its end.
2408 close_quote_index = view_line.find('"', 1)
2409 if close_quote_index <= 0:
2410 die("No first-word closing quote found: %s" % view_line)
2411 depot_side = view_line[1:close_quote_index]
2412 # skip closing quote and space
2413 rhs_index = close_quote_index + 1 + 1
2414 else:
2415 space_index = view_line.find(" ")
2416 if space_index <= 0:
2417 die("No word-splitting space found: %s" % view_line)
2418 depot_side = view_line[0:space_index]
2419 rhs_index = space_index + 1
2420
2421 # prefix + means overlay on previous mapping
2422 if depot_side.startswith("+"):
2423 depot_side = depot_side[1:]
2424
2425 # prefix - means exclude this path, leave out of mappings
2426 exclude = False
2427 if depot_side.startswith("-"):
2428 exclude = True
2429 depot_side = depot_side[1:]
2430
2431 if not exclude:
2432 self.mappings.append(depot_side)
2433
2434 def convert_client_path(self, clientFile):
2435 # chop off //client/ part to make it relative
2436 if not clientFile.startswith(self.client_prefix):
2437 die("No prefix '%s' on clientFile '%s'" %
2438 (self.client_prefix, clientFile))
2439 return clientFile[len(self.client_prefix):]
2440
2441 def update_client_spec_path_cache(self, files):
2442 """ Caching file paths by "p4 where" batch query """
2443
2444 # List depot file paths exclude that already cached
2445 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2446
2447 if len(fileArgs) == 0:
2448 return # All files in cache
2449
2450 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2451 for res in where_result:
2452 if "code" in res and res["code"] == "error":
2453 # assume error is "... file(s) not in client view"
2454 continue
2455 if "clientFile" not in res:
2456 die("No clientFile in 'p4 where' output")
2457 if "unmap" in res:
2458 # it will list all of them, but only one not unmap-ped
2459 continue
2460 if gitConfigBool("core.ignorecase"):
2461 res['depotFile'] = res['depotFile'].lower()
2462 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2463
2464 # not found files or unmap files set to ""
2465 for depotFile in fileArgs:
2466 if gitConfigBool("core.ignorecase"):
2467 depotFile = depotFile.lower()
2468 if depotFile not in self.client_spec_path_cache:
2469 self.client_spec_path_cache[depotFile] = ""
2470
2471 def map_in_client(self, depot_path):
2472 """Return the relative location in the client where this
2473 depot file should live. Returns "" if the file should
2474 not be mapped in the client."""
2475
2476 if gitConfigBool("core.ignorecase"):
2477 depot_path = depot_path.lower()
2478
2479 if depot_path in self.client_spec_path_cache:
2480 return self.client_spec_path_cache[depot_path]
2481
2482 die( "Error: %s is not found in client spec path" % depot_path )
2483 return ""
2484
2485 class P4Sync(Command, P4UserMap):
2486 delete_actions = ( "delete", "move/delete", "purge" )
2487
2488 def __init__(self):
2489 Command.__init__(self)
2490 P4UserMap.__init__(self)
2491 self.options = [
2492 optparse.make_option("--branch", dest="branch"),
2493 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2494 optparse.make_option("--changesfile", dest="changesFile"),
2495 optparse.make_option("--silent", dest="silent", action="store_true"),
2496 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2497 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2498 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2499 help="Import into refs/heads/ , not refs/remotes"),
2500 optparse.make_option("--max-changes", dest="maxChanges",
2501 help="Maximum number of changes to import"),
2502 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2503 help="Internal block size to use when iteratively calling p4 changes"),
2504 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2505 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2506 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2507 help="Only sync files that are included in the Perforce Client Spec"),
2508 optparse.make_option("-/", dest="cloneExclude",
2509 action="append", type="string",
2510 help="exclude depot path"),
2511 ]
2512 self.description = """Imports from Perforce into a git repository.\n
2513 example:
2514 //depot/my/project/ -- to import the current head
2515 //depot/my/project/@all -- to import everything
2516 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2517
2518 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2519
2520 self.usage += " //depot/path[@revRange]"
2521 self.silent = False
2522 self.createdBranches = set()
2523 self.committedChanges = set()
2524 self.branch = ""
2525 self.detectBranches = False
2526 self.detectLabels = False
2527 self.importLabels = False
2528 self.changesFile = ""
2529 self.syncWithOrigin = True
2530 self.importIntoRemotes = True
2531 self.maxChanges = ""
2532 self.changes_block_size = None
2533 self.keepRepoPath = False
2534 self.depotPaths = None
2535 self.p4BranchesInGit = []
2536 self.cloneExclude = []
2537 self.useClientSpec = False
2538 self.useClientSpec_from_options = False
2539 self.clientSpecDirs = None
2540 self.tempBranches = []
2541 self.tempBranchLocation = "refs/git-p4-tmp"
2542 self.largeFileSystem = None
2543
2544 if gitConfig('git-p4.largeFileSystem'):
2545 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2546 self.largeFileSystem = largeFileSystemConstructor(
2547 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2548 )
2549
2550 if gitConfig("git-p4.syncFromOrigin") == "false":
2551 self.syncWithOrigin = False
2552
2553 # Force a checkpoint in fast-import and wait for it to finish
2554 def checkpoint(self):
2555 self.gitStream.write("checkpoint\n\n")
2556 self.gitStream.write("progress checkpoint\n\n")
2557 out = self.gitOutput.readline()
2558 if self.verbose:
2559 print "checkpoint finished: " + out
2560
2561 def extractFilesFromCommit(self, commit):
2562 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2563 for path in self.cloneExclude]
2564 files = []
2565 fnum = 0
2566 while commit.has_key("depotFile%s" % fnum):
2567 path = commit["depotFile%s" % fnum]
2568
2569 if [p for p in self.cloneExclude
2570 if p4PathStartsWith(path, p)]:
2571 found = False
2572 else:
2573 found = [p for p in self.depotPaths
2574 if p4PathStartsWith(path, p)]
2575 if not found:
2576 fnum = fnum + 1
2577 continue
2578
2579 file = {}
2580 file["path"] = path
2581 file["rev"] = commit["rev%s" % fnum]
2582 file["action"] = commit["action%s" % fnum]
2583 file["type"] = commit["type%s" % fnum]
2584 files.append(file)
2585 fnum = fnum + 1
2586 return files
2587
2588 def extractJobsFromCommit(self, commit):
2589 jobs = []
2590 jnum = 0
2591 while commit.has_key("job%s" % jnum):
2592 job = commit["job%s" % jnum]
2593 jobs.append(job)
2594 jnum = jnum + 1
2595 return jobs
2596
2597 def stripRepoPath(self, path, prefixes):
2598 """When streaming files, this is called to map a p4 depot path
2599 to where it should go in git. The prefixes are either
2600 self.depotPaths, or self.branchPrefixes in the case of
2601 branch detection."""
2602
2603 if self.useClientSpec:
2604 # branch detection moves files up a level (the branch name)
2605 # from what client spec interpretation gives
2606 path = self.clientSpecDirs.map_in_client(path)
2607 if self.detectBranches:
2608 for b in self.knownBranches:
2609 if path.startswith(b + "/"):
2610 path = path[len(b)+1:]
2611
2612 elif self.keepRepoPath:
2613 # Preserve everything in relative path name except leading
2614 # //depot/; just look at first prefix as they all should
2615 # be in the same depot.
2616 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2617 if p4PathStartsWith(path, depot):
2618 path = path[len(depot):]
2619
2620 else:
2621 for p in prefixes:
2622 if p4PathStartsWith(path, p):
2623 path = path[len(p):]
2624 break
2625
2626 path = wildcard_decode(path)
2627 return path
2628
2629 def splitFilesIntoBranches(self, commit):
2630 """Look at each depotFile in the commit to figure out to what
2631 branch it belongs."""
2632
2633 if self.clientSpecDirs:
2634 files = self.extractFilesFromCommit(commit)
2635 self.clientSpecDirs.update_client_spec_path_cache(files)
2636
2637 branches = {}
2638 fnum = 0
2639 while commit.has_key("depotFile%s" % fnum):
2640 path = commit["depotFile%s" % fnum]
2641 found = [p for p in self.depotPaths
2642 if p4PathStartsWith(path, p)]
2643 if not found:
2644 fnum = fnum + 1
2645 continue
2646
2647 file = {}
2648 file["path"] = path
2649 file["rev"] = commit["rev%s" % fnum]
2650 file["action"] = commit["action%s" % fnum]
2651 file["type"] = commit["type%s" % fnum]
2652 fnum = fnum + 1
2653
2654 # start with the full relative path where this file would
2655 # go in a p4 client
2656 if self.useClientSpec:
2657 relPath = self.clientSpecDirs.map_in_client(path)
2658 else:
2659 relPath = self.stripRepoPath(path, self.depotPaths)
2660
2661 for branch in self.knownBranches.keys():
2662 # add a trailing slash so that a commit into qt/4.2foo
2663 # doesn't end up in qt/4.2, e.g.
2664 if relPath.startswith(branch + "/"):
2665 if branch not in branches:
2666 branches[branch] = []
2667 branches[branch].append(file)
2668 break
2669
2670 return branches
2671
2672 def writeToGitStream(self, gitMode, relPath, contents):
2673 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2674 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2675 for d in contents:
2676 self.gitStream.write(d)
2677 self.gitStream.write('\n')
2678
2679 def encodeWithUTF8(self, path):
2680 try:
2681 path.decode('ascii')
2682 except:
2683 encoding = 'utf8'
2684 if gitConfig('git-p4.pathEncoding'):
2685 encoding = gitConfig('git-p4.pathEncoding')
2686 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2687 if self.verbose:
2688 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2689 return path
2690
2691 # output one file from the P4 stream
2692 # - helper for streamP4Files
2693
2694 def streamOneP4File(self, file, contents):
2695 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2696 relPath = self.encodeWithUTF8(relPath)
2697 if verbose:
2698 size = int(self.stream_file['fileSize'])
2699 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2700 sys.stdout.flush()
2701
2702 (type_base, type_mods) = split_p4_type(file["type"])
2703
2704 git_mode = "100644"
2705 if "x" in type_mods:
2706 git_mode = "100755"
2707 if type_base == "symlink":
2708 git_mode = "120000"
2709 # p4 print on a symlink sometimes contains "target\n";
2710 # if it does, remove the newline
2711 data = ''.join(contents)
2712 if not data:
2713 # Some version of p4 allowed creating a symlink that pointed
2714 # to nothing. This causes p4 errors when checking out such
2715 # a change, and errors here too. Work around it by ignoring
2716 # the bad symlink; hopefully a future change fixes it.
2717 print "\nIgnoring empty symlink in %s" % file['depotFile']
2718 return
2719 elif data[-1] == '\n':
2720 contents = [data[:-1]]
2721 else:
2722 contents = [data]
2723
2724 if type_base == "utf16":
2725 # p4 delivers different text in the python output to -G
2726 # than it does when using "print -o", or normal p4 client
2727 # operations. utf16 is converted to ascii or utf8, perhaps.
2728 # But ascii text saved as -t utf16 is completely mangled.
2729 # Invoke print -o to get the real contents.
2730 #
2731 # On windows, the newlines will always be mangled by print, so put
2732 # them back too. This is not needed to the cygwin windows version,
2733 # just the native "NT" type.
2734 #
2735 try:
2736 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2737 except Exception as e:
2738 if 'Translation of file content failed' in str(e):
2739 type_base = 'binary'
2740 else:
2741 raise e
2742 else:
2743 if p4_version_string().find('/NT') >= 0:
2744 text = text.replace('\r\n', '\n')
2745 contents = [ text ]
2746
2747 if type_base == "apple":
2748 # Apple filetype files will be streamed as a concatenation of
2749 # its appledouble header and the contents. This is useless
2750 # on both macs and non-macs. If using "print -q -o xx", it
2751 # will create "xx" with the data, and "%xx" with the header.
2752 # This is also not very useful.
2753 #
2754 # Ideally, someday, this script can learn how to generate
2755 # appledouble files directly and import those to git, but
2756 # non-mac machines can never find a use for apple filetype.
2757 print "\nIgnoring apple filetype file %s" % file['depotFile']
2758 return
2759
2760 # Note that we do not try to de-mangle keywords on utf16 files,
2761 # even though in theory somebody may want that.
2762 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2763 if pattern:
2764 regexp = re.compile(pattern, re.VERBOSE)
2765 text = ''.join(contents)
2766 text = regexp.sub(r'$\1$', text)
2767 contents = [ text ]
2768
2769 if self.largeFileSystem:
2770 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2771
2772 self.writeToGitStream(git_mode, relPath, contents)
2773
2774 def streamOneP4Deletion(self, file):
2775 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2776 relPath = self.encodeWithUTF8(relPath)
2777 if verbose:
2778 sys.stdout.write("delete %s\n" % relPath)
2779 sys.stdout.flush()
2780 self.gitStream.write("D %s\n" % relPath)
2781
2782 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2783 self.largeFileSystem.removeLargeFile(relPath)
2784
2785 # handle another chunk of streaming data
2786 def streamP4FilesCb(self, marshalled):
2787
2788 # catch p4 errors and complain
2789 err = None
2790 if "code" in marshalled:
2791 if marshalled["code"] == "error":
2792 if "data" in marshalled:
2793 err = marshalled["data"].rstrip()
2794
2795 if not err and 'fileSize' in self.stream_file:
2796 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2797 if required_bytes > 0:
2798 err = 'Not enough space left on %s! Free at least %i MB.' % (
2799 os.getcwd(), required_bytes/1024/1024
2800 )
2801
2802 if err:
2803 f = None
2804 if self.stream_have_file_info:
2805 if "depotFile" in self.stream_file:
2806 f = self.stream_file["depotFile"]
2807 # force a failure in fast-import, else an empty
2808 # commit will be made
2809 self.gitStream.write("\n")
2810 self.gitStream.write("die-now\n")
2811 self.gitStream.close()
2812 # ignore errors, but make sure it exits first
2813 self.importProcess.wait()
2814 if f:
2815 die("Error from p4 print for %s: %s" % (f, err))
2816 else:
2817 die("Error from p4 print: %s" % err)
2818
2819 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2820 # start of a new file - output the old one first
2821 self.streamOneP4File(self.stream_file, self.stream_contents)
2822 self.stream_file = {}
2823 self.stream_contents = []
2824 self.stream_have_file_info = False
2825
2826 # pick up the new file information... for the
2827 # 'data' field we need to append to our array
2828 for k in marshalled.keys():
2829 if k == 'data':
2830 if 'streamContentSize' not in self.stream_file:
2831 self.stream_file['streamContentSize'] = 0
2832 self.stream_file['streamContentSize'] += len(marshalled['data'])
2833 self.stream_contents.append(marshalled['data'])
2834 else:
2835 self.stream_file[k] = marshalled[k]
2836
2837 if (verbose and
2838 'streamContentSize' in self.stream_file and
2839 'fileSize' in self.stream_file and
2840 'depotFile' in self.stream_file):
2841 size = int(self.stream_file["fileSize"])
2842 if size > 0:
2843 progress = 100*self.stream_file['streamContentSize']/size
2844 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2845 sys.stdout.flush()
2846
2847 self.stream_have_file_info = True
2848
2849 # Stream directly from "p4 files" into "git fast-import"
2850 def streamP4Files(self, files):
2851 filesForCommit = []
2852 filesToRead = []
2853 filesToDelete = []
2854
2855 for f in files:
2856 filesForCommit.append(f)
2857 if f['action'] in self.delete_actions:
2858 filesToDelete.append(f)
2859 else:
2860 filesToRead.append(f)
2861
2862 # deleted files...
2863 for f in filesToDelete:
2864 self.streamOneP4Deletion(f)
2865
2866 if len(filesToRead) > 0:
2867 self.stream_file = {}
2868 self.stream_contents = []
2869 self.stream_have_file_info = False
2870
2871 # curry self argument
2872 def streamP4FilesCbSelf(entry):
2873 self.streamP4FilesCb(entry)
2874
2875 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2876
2877 p4CmdList(["-x", "-", "print"],
2878 stdin=fileArgs,
2879 cb=streamP4FilesCbSelf)
2880
2881 # do the last chunk
2882 if self.stream_file.has_key('depotFile'):
2883 self.streamOneP4File(self.stream_file, self.stream_contents)
2884
2885 def make_email(self, userid):
2886 if userid in self.users:
2887 return self.users[userid]
2888 else:
2889 return "%s <a@b>" % userid
2890
2891 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2892 """ Stream a p4 tag.
2893 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2894 """
2895
2896 if verbose:
2897 print "writing tag %s for commit %s" % (labelName, commit)
2898 gitStream.write("tag %s\n" % labelName)
2899 gitStream.write("from %s\n" % commit)
2900
2901 if labelDetails.has_key('Owner'):
2902 owner = labelDetails["Owner"]
2903 else:
2904 owner = None
2905
2906 # Try to use the owner of the p4 label, or failing that,
2907 # the current p4 user id.
2908 if owner:
2909 email = self.make_email(owner)
2910 else:
2911 email = self.make_email(self.p4UserId())
2912 tagger = "%s %s %s" % (email, epoch, self.tz)
2913
2914 gitStream.write("tagger %s\n" % tagger)
2915
2916 print "labelDetails=",labelDetails
2917 if labelDetails.has_key('Description'):
2918 description = labelDetails['Description']
2919 else:
2920 description = 'Label from git p4'
2921
2922 gitStream.write("data %d\n" % len(description))
2923 gitStream.write(description)
2924 gitStream.write("\n")
2925
2926 def inClientSpec(self, path):
2927 if not self.clientSpecDirs:
2928 return True
2929 inClientSpec = self.clientSpecDirs.map_in_client(path)
2930 if not inClientSpec and self.verbose:
2931 print('Ignoring file outside of client spec: {0}'.format(path))
2932 return inClientSpec
2933
2934 def hasBranchPrefix(self, path):
2935 if not self.branchPrefixes:
2936 return True
2937 hasPrefix = [p for p in self.branchPrefixes
2938 if p4PathStartsWith(path, p)]
2939 if not hasPrefix and self.verbose:
2940 print('Ignoring file outside of prefix: {0}'.format(path))
2941 return hasPrefix
2942
2943 def commit(self, details, files, branch, parent = ""):
2944 epoch = details["time"]
2945 author = details["user"]
2946 jobs = self.extractJobsFromCommit(details)
2947
2948 if self.verbose:
2949 print('commit into {0}'.format(branch))
2950
2951 if self.clientSpecDirs:
2952 self.clientSpecDirs.update_client_spec_path_cache(files)
2953
2954 files = [f for f in files
2955 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2956
2957 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2958 print('Ignoring revision {0} as it would produce an empty commit.'
2959 .format(details['change']))
2960 return
2961
2962 self.gitStream.write("commit %s\n" % branch)
2963 self.gitStream.write("mark :%s\n" % details["change"])
2964 self.committedChanges.add(int(details["change"]))
2965 committer = ""
2966 if author not in self.users:
2967 self.getUserMapFromPerforceServer()
2968 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2969
2970 self.gitStream.write("committer %s\n" % committer)
2971
2972 self.gitStream.write("data <<EOT\n")
2973 self.gitStream.write(details["desc"])
2974 if len(jobs) > 0:
2975 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2976 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2977 (','.join(self.branchPrefixes), details["change"]))
2978 if len(details['options']) > 0:
2979 self.gitStream.write(": options = %s" % details['options'])
2980 self.gitStream.write("]\nEOT\n\n")
2981
2982 if len(parent) > 0:
2983 if self.verbose:
2984 print "parent %s" % parent
2985 self.gitStream.write("from %s\n" % parent)
2986
2987 self.streamP4Files(files)
2988 self.gitStream.write("\n")
2989
2990 change = int(details["change"])
2991
2992 if self.labels.has_key(change):
2993 label = self.labels[change]
2994 labelDetails = label[0]
2995 labelRevisions = label[1]
2996 if self.verbose:
2997 print "Change %s is labelled %s" % (change, labelDetails)
2998
2999 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
3000 for p in self.branchPrefixes])
3001
3002 if len(files) == len(labelRevisions):
3003
3004 cleanedFiles = {}
3005 for info in files:
3006 if info["action"] in self.delete_actions:
3007 continue
3008 cleanedFiles[info["depotFile"]] = info["rev"]
3009
3010 if cleanedFiles == labelRevisions:
3011 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
3012
3013 else:
3014 if not self.silent:
3015 print ("Tag %s does not match with change %s: files do not match."
3016 % (labelDetails["label"], change))
3017
3018 else:
3019 if not self.silent:
3020 print ("Tag %s does not match with change %s: file count is different."
3021 % (labelDetails["label"], change))
3022
3023 # Build a dictionary of changelists and labels, for "detect-labels" option.
3024 def getLabels(self):
3025 self.labels = {}
3026
3027 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
3028 if len(l) > 0 and not self.silent:
3029 print "Finding files belonging to labels in %s" % `self.depotPaths`
3030
3031 for output in l:
3032 label = output["label"]
3033 revisions = {}
3034 newestChange = 0
3035 if self.verbose:
3036 print "Querying files for label %s" % label
3037 for file in p4CmdList(["files"] +
3038 ["%s...@%s" % (p, label)
3039 for p in self.depotPaths]):
3040 revisions[file["depotFile"]] = file["rev"]
3041 change = int(file["change"])
3042 if change > newestChange:
3043 newestChange = change
3044
3045 self.labels[newestChange] = [output, revisions]
3046
3047 if self.verbose:
3048 print "Label changes: %s" % self.labels.keys()
3049
3050 # Import p4 labels as git tags. A direct mapping does not
3051 # exist, so assume that if all the files are at the same revision
3052 # then we can use that, or it's something more complicated we should
3053 # just ignore.
3054 def importP4Labels(self, stream, p4Labels):
3055 if verbose:
3056 print "import p4 labels: " + ' '.join(p4Labels)
3057
3058 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
3059 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
3060 if len(validLabelRegexp) == 0:
3061 validLabelRegexp = defaultLabelRegexp
3062 m = re.compile(validLabelRegexp)
3063
3064 for name in p4Labels:
3065 commitFound = False
3066
3067 if not m.match(name):
3068 if verbose:
3069 print "label %s does not match regexp %s" % (name,validLabelRegexp)
3070 continue
3071
3072 if name in ignoredP4Labels:
3073 continue
3074
3075 labelDetails = p4CmdList(['label', "-o", name])[0]
3076
3077 # get the most recent changelist for each file in this label
3078 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3079 for p in self.depotPaths])
3080
3081 if change.has_key('change'):
3082 # find the corresponding git commit; take the oldest commit
3083 changelist = int(change['change'])
3084 if changelist in self.committedChanges:
3085 gitCommit = ":%d" % changelist # use a fast-import mark
3086 commitFound = True
3087 else:
3088 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3089 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3090 if len(gitCommit) == 0:
3091 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3092 else:
3093 commitFound = True
3094 gitCommit = gitCommit.strip()
3095
3096 if commitFound:
3097 # Convert from p4 time format
3098 try:
3099 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3100 except ValueError:
3101 print "Could not convert label time %s" % labelDetails['Update']
3102 tmwhen = 1
3103
3104 when = int(time.mktime(tmwhen))
3105 self.streamTag(stream, name, labelDetails, gitCommit, when)
3106 if verbose:
3107 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3108 else:
3109 if verbose:
3110 print "Label %s has no changelists - possibly deleted?" % name
3111
3112 if not commitFound:
3113 # We can't import this label; don't try again as it will get very
3114 # expensive repeatedly fetching all the files for labels that will
3115 # never be imported. If the label is moved in the future, the
3116 # ignore will need to be removed manually.
3117 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3118
3119 def guessProjectName(self):
3120 for p in self.depotPaths:
3121 if p.endswith("/"):
3122 p = p[:-1]
3123 p = p[p.strip().rfind("/") + 1:]
3124 if not p.endswith("/"):
3125 p += "/"
3126 return p
3127
3128 def getBranchMapping(self):
3129 lostAndFoundBranches = set()
3130
3131 user = gitConfig("git-p4.branchUser")
3132 if len(user) > 0:
3133 command = "branches -u %s" % user
3134 else:
3135 command = "branches"
3136
3137 for info in p4CmdList(command):
3138 details = p4Cmd(["branch", "-o", info["branch"]])
3139 viewIdx = 0
3140 while details.has_key("View%s" % viewIdx):
3141 paths = details["View%s" % viewIdx].split(" ")
3142 viewIdx = viewIdx + 1
3143 # require standard //depot/foo/... //depot/bar/... mapping
3144 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3145 continue
3146 source = paths[0]
3147 destination = paths[1]
3148 ## HACK
3149 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3150 source = source[len(self.depotPaths[0]):-4]
3151 destination = destination[len(self.depotPaths[0]):-4]
3152
3153 if destination in self.knownBranches:
3154 if not self.silent:
3155 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3156 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3157 continue
3158
3159 self.knownBranches[destination] = source
3160
3161 lostAndFoundBranches.discard(destination)
3162
3163 if source not in self.knownBranches:
3164 lostAndFoundBranches.add(source)
3165
3166 # Perforce does not strictly require branches to be defined, so we also
3167 # check git config for a branch list.
3168 #
3169 # Example of branch definition in git config file:
3170 # [git-p4]
3171 # branchList=main:branchA
3172 # branchList=main:branchB
3173 # branchList=branchA:branchC
3174 configBranches = gitConfigList("git-p4.branchList")
3175 for branch in configBranches:
3176 if branch:
3177 (source, destination) = branch.split(":")
3178 self.knownBranches[destination] = source
3179
3180 lostAndFoundBranches.discard(destination)
3181
3182 if source not in self.knownBranches:
3183 lostAndFoundBranches.add(source)
3184
3185
3186 for branch in lostAndFoundBranches:
3187 self.knownBranches[branch] = branch
3188
3189 def getBranchMappingFromGitBranches(self):
3190 branches = p4BranchesInGit(self.importIntoRemotes)
3191 for branch in branches.keys():
3192 if branch == "master":
3193 branch = "main"
3194 else:
3195 branch = branch[len(self.projectName):]
3196 self.knownBranches[branch] = branch
3197
3198 def updateOptionDict(self, d):
3199 option_keys = {}
3200 if self.keepRepoPath:
3201 option_keys['keepRepoPath'] = 1
3202
3203 d["options"] = ' '.join(sorted(option_keys.keys()))
3204
3205 def readOptions(self, d):
3206 self.keepRepoPath = (d.has_key('options')
3207 and ('keepRepoPath' in d['options']))
3208
3209 def gitRefForBranch(self, branch):
3210 if branch == "main":
3211 return self.refPrefix + "master"
3212
3213 if len(branch) <= 0:
3214 return branch
3215
3216 return self.refPrefix + self.projectName + branch
3217
3218 def gitCommitByP4Change(self, ref, change):
3219 if self.verbose:
3220 print "looking in ref " + ref + " for change %s using bisect..." % change
3221
3222 earliestCommit = ""
3223 latestCommit = parseRevision(ref)
3224
3225 while True:
3226 if self.verbose:
3227 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3228 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3229 if len(next) == 0:
3230 if self.verbose:
3231 print "argh"
3232 return ""
3233 log = extractLogMessageFromGitCommit(next)
3234 settings = extractSettingsGitLog(log)
3235 currentChange = int(settings['change'])
3236 if self.verbose:
3237 print "current change %s" % currentChange
3238
3239 if currentChange == change:
3240 if self.verbose:
3241 print "found %s" % next
3242 return next
3243
3244 if currentChange < change:
3245 earliestCommit = "^%s" % next
3246 else:
3247 latestCommit = "%s" % next
3248
3249 return ""
3250
3251 def importNewBranch(self, branch, maxChange):
3252 # make fast-import flush all changes to disk and update the refs using the checkpoint
3253 # command so that we can try to find the branch parent in the git history
3254 self.gitStream.write("checkpoint\n\n");
3255 self.gitStream.flush();
3256 branchPrefix = self.depotPaths[0] + branch + "/"
3257 range = "@1,%s" % maxChange
3258 #print "prefix" + branchPrefix
3259 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3260 if len(changes) <= 0:
3261 return False
3262 firstChange = changes[0]
3263 #print "first change in branch: %s" % firstChange
3264 sourceBranch = self.knownBranches[branch]
3265 sourceDepotPath = self.depotPaths[0] + sourceBranch
3266 sourceRef = self.gitRefForBranch(sourceBranch)
3267 #print "source " + sourceBranch
3268
3269 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3270 #print "branch parent: %s" % branchParentChange
3271 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3272 if len(gitParent) > 0:
3273 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3274 #print "parent git commit: %s" % gitParent
3275
3276 self.importChanges(changes)
3277 return True
3278
3279 def searchParent(self, parent, branch, target):
3280 parentFound = False
3281 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3282 "--no-merges", parent]):
3283 blob = blob.strip()
3284 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3285 parentFound = True
3286 if self.verbose:
3287 print "Found parent of %s in commit %s" % (branch, blob)
3288 break
3289 if parentFound:
3290 return blob
3291 else:
3292 return None
3293
3294 def importChanges(self, changes):
3295 cnt = 1
3296 for change in changes:
3297 description = p4_describe(change)
3298 self.updateOptionDict(description)
3299
3300 if not self.silent:
3301 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3302 sys.stdout.flush()
3303 cnt = cnt + 1
3304
3305 try:
3306 if self.detectBranches:
3307 branches = self.splitFilesIntoBranches(description)
3308 for branch in branches.keys():
3309 ## HACK --hwn
3310 branchPrefix = self.depotPaths[0] + branch + "/"
3311 self.branchPrefixes = [ branchPrefix ]
3312
3313 parent = ""
3314
3315 filesForCommit = branches[branch]
3316
3317 if self.verbose:
3318 print "branch is %s" % branch
3319
3320 self.updatedBranches.add(branch)
3321
3322 if branch not in self.createdBranches:
3323 self.createdBranches.add(branch)
3324 parent = self.knownBranches[branch]
3325 if parent == branch:
3326 parent = ""
3327 else:
3328 fullBranch = self.projectName + branch
3329 if fullBranch not in self.p4BranchesInGit:
3330 if not self.silent:
3331 print("\n Importing new branch %s" % fullBranch);
3332 if self.importNewBranch(branch, change - 1):
3333 parent = ""
3334 self.p4BranchesInGit.append(fullBranch)
3335 if not self.silent:
3336 print("\n Resuming with change %s" % change);
3337
3338 if self.verbose:
3339 print "parent determined through known branches: %s" % parent
3340
3341 branch = self.gitRefForBranch(branch)
3342 parent = self.gitRefForBranch(parent)
3343
3344 if self.verbose:
3345 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3346
3347 if len(parent) == 0 and branch in self.initialParents:
3348 parent = self.initialParents[branch]
3349 del self.initialParents[branch]
3350
3351 blob = None
3352 if len(parent) > 0:
3353 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3354 if self.verbose:
3355 print "Creating temporary branch: " + tempBranch
3356 self.commit(description, filesForCommit, tempBranch)
3357 self.tempBranches.append(tempBranch)
3358 self.checkpoint()
3359 blob = self.searchParent(parent, branch, tempBranch)
3360 if blob:
3361 self.commit(description, filesForCommit, branch, blob)
3362 else:
3363 if self.verbose:
3364 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3365 self.commit(description, filesForCommit, branch, parent)
3366 else:
3367 files = self.extractFilesFromCommit(description)
3368 self.commit(description, files, self.branch,
3369 self.initialParent)
3370 # only needed once, to connect to the previous commit
3371 self.initialParent = ""
3372 except IOError:
3373 print self.gitError.read()
3374 sys.exit(1)
3375
3376 def sync_origin_only(self):
3377 if self.syncWithOrigin:
3378 self.hasOrigin = originP4BranchesExist()
3379 if self.hasOrigin:
3380 if not self.silent:
3381 print 'Syncing with origin first, using "git fetch origin"'
3382 system("git fetch origin")
3383
3384 def importHeadRevision(self, revision):
3385 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3386
3387 details = {}
3388 details["user"] = "git perforce import user"
3389 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3390 % (' '.join(self.depotPaths), revision))
3391 details["change"] = revision
3392 newestRevision = 0
3393
3394 fileCnt = 0
3395 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3396
3397 for info in p4CmdList(["files"] + fileArgs):
3398
3399 if 'code' in info and info['code'] == 'error':
3400 sys.stderr.write("p4 returned an error: %s\n"
3401 % info['data'])
3402 if info['data'].find("must refer to client") >= 0:
3403 sys.stderr.write("This particular p4 error is misleading.\n")
3404 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3405 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3406 sys.exit(1)
3407 if 'p4ExitCode' in info:
3408 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3409 sys.exit(1)
3410
3411
3412 change = int(info["change"])
3413 if change > newestRevision:
3414 newestRevision = change
3415
3416 if info["action"] in self.delete_actions:
3417 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3418 #fileCnt = fileCnt + 1
3419 continue
3420
3421 for prop in ["depotFile", "rev", "action", "type" ]:
3422 details["%s%s" % (prop, fileCnt)] = info[prop]
3423
3424 fileCnt = fileCnt + 1
3425
3426 details["change"] = newestRevision
3427
3428 # Use time from top-most change so that all git p4 clones of
3429 # the same p4 repo have the same commit SHA1s.
3430 res = p4_describe(newestRevision)
3431 details["time"] = res["time"]
3432
3433 self.updateOptionDict(details)
3434 try:
3435 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3436 except IOError:
3437 print "IO error with git fast-import. Is your git version recent enough?"
3438 print self.gitError.read()
3439
3440
3441 def run(self, args):
3442 self.depotPaths = []
3443 self.changeRange = ""
3444 self.previousDepotPaths = []
3445 self.hasOrigin = False
3446
3447 # map from branch depot path to parent branch
3448 self.knownBranches = {}
3449 self.initialParents = {}
3450
3451 if self.importIntoRemotes:
3452 self.refPrefix = "refs/remotes/p4/"
3453 else:
3454 self.refPrefix = "refs/heads/p4/"
3455
3456 self.sync_origin_only()
3457
3458 branch_arg_given = bool(self.branch)
3459 if len(self.branch) == 0:
3460 self.branch = self.refPrefix + "master"
3461 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3462 system("git update-ref %s refs/heads/p4" % self.branch)
3463 system("git branch -D p4")
3464
3465 # accept either the command-line option, or the configuration variable
3466 if self.useClientSpec:
3467 # will use this after clone to set the variable
3468 self.useClientSpec_from_options = True
3469 else:
3470 if gitConfigBool("git-p4.useclientspec"):
3471 self.useClientSpec = True
3472 if self.useClientSpec:
3473 self.clientSpecDirs = getClientSpec()
3474
3475 # TODO: should always look at previous commits,
3476 # merge with previous imports, if possible.
3477 if args == []:
3478 if self.hasOrigin:
3479 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3480
3481 # branches holds mapping from branch name to sha1
3482 branches = p4BranchesInGit(self.importIntoRemotes)
3483
3484 # restrict to just this one, disabling detect-branches
3485 if branch_arg_given:
3486 short = self.branch.split("/")[-1]
3487 if short in branches:
3488 self.p4BranchesInGit = [ short ]
3489 else:
3490 self.p4BranchesInGit = branches.keys()
3491
3492 if len(self.p4BranchesInGit) > 1:
3493 if not self.silent:
3494 print "Importing from/into multiple branches"
3495 self.detectBranches = True
3496 for branch in branches.keys():
3497 self.initialParents[self.refPrefix + branch] = \
3498 branches[branch]
3499
3500 if self.verbose:
3501 print "branches: %s" % self.p4BranchesInGit
3502
3503 p4Change = 0
3504 for branch in self.p4BranchesInGit:
3505 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3506
3507 settings = extractSettingsGitLog(logMsg)
3508
3509 self.readOptions(settings)
3510 if (settings.has_key('depot-paths')
3511 and settings.has_key ('change')):
3512 change = int(settings['change']) + 1
3513 p4Change = max(p4Change, change)
3514
3515 depotPaths = sorted(settings['depot-paths'])
3516 if self.previousDepotPaths == []:
3517 self.previousDepotPaths = depotPaths
3518 else:
3519 paths = []
3520 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3521 prev_list = prev.split("/")
3522 cur_list = cur.split("/")
3523 for i in range(0, min(len(cur_list), len(prev_list))):
3524 if cur_list[i] <> prev_list[i]:
3525 i = i - 1
3526 break
3527
3528 paths.append ("/".join(cur_list[:i + 1]))
3529
3530 self.previousDepotPaths = paths
3531
3532 if p4Change > 0:
3533 self.depotPaths = sorted(self.previousDepotPaths)
3534 self.changeRange = "@%s,#head" % p4Change
3535 if not self.silent and not self.detectBranches:
3536 print "Performing incremental import into %s git branch" % self.branch
3537
3538 # accept multiple ref name abbreviations:
3539 # refs/foo/bar/branch -> use it exactly
3540 # p4/branch -> prepend refs/remotes/ or refs/heads/
3541 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3542 if not self.branch.startswith("refs/"):
3543 if self.importIntoRemotes:
3544 prepend = "refs/remotes/"
3545 else:
3546 prepend = "refs/heads/"
3547 if not self.branch.startswith("p4/"):
3548 prepend += "p4/"
3549 self.branch = prepend + self.branch
3550
3551 if len(args) == 0 and self.depotPaths:
3552 if not self.silent:
3553 print "Depot paths: %s" % ' '.join(self.depotPaths)
3554 else:
3555 if self.depotPaths and self.depotPaths != args:
3556 print ("previous import used depot path %s and now %s was specified. "
3557 "This doesn't work!" % (' '.join (self.depotPaths),
3558 ' '.join (args)))
3559 sys.exit(1)
3560
3561 self.depotPaths = sorted(args)
3562
3563 revision = ""
3564 self.users = {}
3565
3566 # Make sure no revision specifiers are used when --changesfile
3567 # is specified.
3568 bad_changesfile = False
3569 if len(self.changesFile) > 0:
3570 for p in self.depotPaths:
3571 if p.find("@") >= 0 or p.find("#") >= 0:
3572 bad_changesfile = True
3573 break
3574 if bad_changesfile:
3575 die("Option --changesfile is incompatible with revision specifiers")
3576
3577 newPaths = []
3578 for p in self.depotPaths:
3579 if p.find("@") != -1:
3580 atIdx = p.index("@")
3581 self.changeRange = p[atIdx:]
3582 if self.changeRange == "@all":
3583 self.changeRange = ""
3584 elif ',' not in self.changeRange:
3585 revision = self.changeRange
3586 self.changeRange = ""
3587 p = p[:atIdx]
3588 elif p.find("#") != -1:
3589 hashIdx = p.index("#")
3590 revision = p[hashIdx:]
3591 p = p[:hashIdx]
3592 elif self.previousDepotPaths == []:
3593 # pay attention to changesfile, if given, else import
3594 # the entire p4 tree at the head revision
3595 if len(self.changesFile) == 0:
3596 revision = "#head"
3597
3598 p = re.sub ("\.\.\.$", "", p)
3599 if not p.endswith("/"):
3600 p += "/"
3601
3602 newPaths.append(p)
3603
3604 self.depotPaths = newPaths
3605
3606 # --detect-branches may change this for each branch
3607 self.branchPrefixes = self.depotPaths
3608
3609 self.loadUserMapFromCache()
3610 self.labels = {}
3611 if self.detectLabels:
3612 self.getLabels();
3613
3614 if self.detectBranches:
3615 ## FIXME - what's a P4 projectName ?
3616 self.projectName = self.guessProjectName()
3617
3618 if self.hasOrigin:
3619 self.getBranchMappingFromGitBranches()
3620 else