git-p4: do not terminate creating tag for unknown commit
[git/git.git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27 from subprocess import CalledProcessError
28 except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 # Grab changes in blocks of this many revisions, unless otherwise requested
47 defaultBlockSize = 512
48
49 def p4_build_cmd(cmd):
50 """Build a suitable p4 command line.
51
52 This consolidates building and returning a p4 command line into one
53 location. It means that hooking into the environment, or other configuration
54 can be done more easily.
55 """
56 real_cmd = ["p4"]
57
58 user = gitConfig("git-p4.user")
59 if len(user) > 0:
60 real_cmd += ["-u",user]
61
62 password = gitConfig("git-p4.password")
63 if len(password) > 0:
64 real_cmd += ["-P", password]
65
66 port = gitConfig("git-p4.port")
67 if len(port) > 0:
68 real_cmd += ["-p", port]
69
70 host = gitConfig("git-p4.host")
71 if len(host) > 0:
72 real_cmd += ["-H", host]
73
74 client = gitConfig("git-p4.client")
75 if len(client) > 0:
76 real_cmd += ["-c", client]
77
78
79 if isinstance(cmd,basestring):
80 real_cmd = ' '.join(real_cmd) + ' ' + cmd
81 else:
82 real_cmd += cmd
83 return real_cmd
84
85 def chdir(path, is_client_path=False):
86 """Do chdir to the given path, and set the PWD environment
87 variable for use by P4. It does not look at getcwd() output.
88 Since we're not using the shell, it is necessary to set the
89 PWD environment variable explicitly.
90
91 Normally, expand the path to force it to be absolute. This
92 addresses the use of relative path names inside P4 settings,
93 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
94 as given; it looks for .p4config using PWD.
95
96 If is_client_path, the path was handed to us directly by p4,
97 and may be a symbolic link. Do not call os.getcwd() in this
98 case, because it will cause p4 to think that PWD is not inside
99 the client path.
100 """
101
102 os.chdir(path)
103 if not is_client_path:
104 path = os.getcwd()
105 os.environ['PWD'] = path
106
107 def die(msg):
108 if verbose:
109 raise Exception(msg)
110 else:
111 sys.stderr.write(msg + "\n")
112 sys.exit(1)
113
114 def write_pipe(c, stdin):
115 if verbose:
116 sys.stderr.write('Writing pipe: %s\n' % str(c))
117
118 expand = isinstance(c,basestring)
119 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
120 pipe = p.stdin
121 val = pipe.write(stdin)
122 pipe.close()
123 if p.wait():
124 die('Command failed: %s' % str(c))
125
126 return val
127
128 def p4_write_pipe(c, stdin):
129 real_cmd = p4_build_cmd(c)
130 return write_pipe(real_cmd, stdin)
131
132 def read_pipe(c, ignore_error=False):
133 if verbose:
134 sys.stderr.write('Reading pipe: %s\n' % str(c))
135
136 expand = isinstance(c,basestring)
137 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
138 pipe = p.stdout
139 val = pipe.read()
140 if p.wait() and not ignore_error:
141 die('Command failed: %s' % str(c))
142
143 return val
144
145 def p4_read_pipe(c, ignore_error=False):
146 real_cmd = p4_build_cmd(c)
147 return read_pipe(real_cmd, ignore_error)
148
149 def read_pipe_lines(c):
150 if verbose:
151 sys.stderr.write('Reading pipe: %s\n' % str(c))
152
153 expand = isinstance(c, basestring)
154 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
155 pipe = p.stdout
156 val = pipe.readlines()
157 if pipe.close() or p.wait():
158 die('Command failed: %s' % str(c))
159
160 return val
161
162 def p4_read_pipe_lines(c):
163 """Specifically invoke p4 on the command supplied. """
164 real_cmd = p4_build_cmd(c)
165 return read_pipe_lines(real_cmd)
166
167 def p4_has_command(cmd):
168 """Ask p4 for help on this command. If it returns an error, the
169 command does not exist in this version of p4."""
170 real_cmd = p4_build_cmd(["help", cmd])
171 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
172 stderr=subprocess.PIPE)
173 p.communicate()
174 return p.returncode == 0
175
176 def p4_has_move_command():
177 """See if the move command exists, that it supports -k, and that
178 it has not been administratively disabled. The arguments
179 must be correct, but the filenames do not have to exist. Use
180 ones with wildcards so even if they exist, it will fail."""
181
182 if not p4_has_command("move"):
183 return False
184 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
185 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
186 (out, err) = p.communicate()
187 # return code will be 1 in either case
188 if err.find("Invalid option") >= 0:
189 return False
190 if err.find("disabled") >= 0:
191 return False
192 # assume it failed because @... was invalid changelist
193 return True
194
195 def system(cmd):
196 expand = isinstance(cmd,basestring)
197 if verbose:
198 sys.stderr.write("executing %s\n" % str(cmd))
199 retcode = subprocess.call(cmd, shell=expand)
200 if retcode:
201 raise CalledProcessError(retcode, cmd)
202
203 def p4_system(cmd):
204 """Specifically invoke p4 as the system command. """
205 real_cmd = p4_build_cmd(cmd)
206 expand = isinstance(real_cmd, basestring)
207 retcode = subprocess.call(real_cmd, shell=expand)
208 if retcode:
209 raise CalledProcessError(retcode, real_cmd)
210
211 _p4_version_string = None
212 def p4_version_string():
213 """Read the version string, showing just the last line, which
214 hopefully is the interesting version bit.
215
216 $ p4 -V
217 Perforce - The Fast Software Configuration Management System.
218 Copyright 1995-2011 Perforce Software. All rights reserved.
219 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
220 """
221 global _p4_version_string
222 if not _p4_version_string:
223 a = p4_read_pipe_lines(["-V"])
224 _p4_version_string = a[-1].rstrip()
225 return _p4_version_string
226
227 def p4_integrate(src, dest):
228 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
229
230 def p4_sync(f, *options):
231 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
232
233 def p4_add(f):
234 # forcibly add file names with wildcards
235 if wildcard_present(f):
236 p4_system(["add", "-f", f])
237 else:
238 p4_system(["add", f])
239
240 def p4_delete(f):
241 p4_system(["delete", wildcard_encode(f)])
242
243 def p4_edit(f):
244 p4_system(["edit", wildcard_encode(f)])
245
246 def p4_revert(f):
247 p4_system(["revert", wildcard_encode(f)])
248
249 def p4_reopen(type, f):
250 p4_system(["reopen", "-t", type, wildcard_encode(f)])
251
252 def p4_move(src, dest):
253 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
254
255 def p4_last_change():
256 results = p4CmdList(["changes", "-m", "1"])
257 return int(results[0]['change'])
258
259 def p4_describe(change):
260 """Make sure it returns a valid result by checking for
261 the presence of field "time". Return a dict of the
262 results."""
263
264 ds = p4CmdList(["describe", "-s", str(change)])
265 if len(ds) != 1:
266 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
267
268 d = ds[0]
269
270 if "p4ExitCode" in d:
271 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
272 str(d)))
273 if "code" in d:
274 if d["code"] == "error":
275 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
276
277 if "time" not in d:
278 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
279
280 return d
281
282 #
283 # Canonicalize the p4 type and return a tuple of the
284 # base type, plus any modifiers. See "p4 help filetypes"
285 # for a list and explanation.
286 #
287 def split_p4_type(p4type):
288
289 p4_filetypes_historical = {
290 "ctempobj": "binary+Sw",
291 "ctext": "text+C",
292 "cxtext": "text+Cx",
293 "ktext": "text+k",
294 "kxtext": "text+kx",
295 "ltext": "text+F",
296 "tempobj": "binary+FSw",
297 "ubinary": "binary+F",
298 "uresource": "resource+F",
299 "uxbinary": "binary+Fx",
300 "xbinary": "binary+x",
301 "xltext": "text+Fx",
302 "xtempobj": "binary+Swx",
303 "xtext": "text+x",
304 "xunicode": "unicode+x",
305 "xutf16": "utf16+x",
306 }
307 if p4type in p4_filetypes_historical:
308 p4type = p4_filetypes_historical[p4type]
309 mods = ""
310 s = p4type.split("+")
311 base = s[0]
312 mods = ""
313 if len(s) > 1:
314 mods = s[1]
315 return (base, mods)
316
317 #
318 # return the raw p4 type of a file (text, text+ko, etc)
319 #
320 def p4_type(f):
321 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
322 return results[0]['headType']
323
324 #
325 # Given a type base and modifier, return a regexp matching
326 # the keywords that can be expanded in the file
327 #
328 def p4_keywords_regexp_for_type(base, type_mods):
329 if base in ("text", "unicode", "binary"):
330 kwords = None
331 if "ko" in type_mods:
332 kwords = 'Id|Header'
333 elif "k" in type_mods:
334 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
335 else:
336 return None
337 pattern = r"""
338 \$ # Starts with a dollar, followed by...
339 (%s) # one of the keywords, followed by...
340 (:[^$\n]+)? # possibly an old expansion, followed by...
341 \$ # another dollar
342 """ % kwords
343 return pattern
344 else:
345 return None
346
347 #
348 # Given a file, return a regexp matching the possible
349 # RCS keywords that will be expanded, or None for files
350 # with kw expansion turned off.
351 #
352 def p4_keywords_regexp_for_file(file):
353 if not os.path.exists(file):
354 return None
355 else:
356 (type_base, type_mods) = split_p4_type(p4_type(file))
357 return p4_keywords_regexp_for_type(type_base, type_mods)
358
359 def setP4ExecBit(file, mode):
360 # Reopens an already open file and changes the execute bit to match
361 # the execute bit setting in the passed in mode.
362
363 p4Type = "+x"
364
365 if not isModeExec(mode):
366 p4Type = getP4OpenedType(file)
367 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
368 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
369 if p4Type[-1] == "+":
370 p4Type = p4Type[0:-1]
371
372 p4_reopen(p4Type, file)
373
374 def getP4OpenedType(file):
375 # Returns the perforce file type for the given file.
376
377 result = p4_read_pipe(["opened", wildcard_encode(file)])
378 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
379 if match:
380 return match.group(1)
381 else:
382 die("Could not determine file type for %s (result: '%s')" % (file, result))
383
384 # Return the set of all p4 labels
385 def getP4Labels(depotPaths):
386 labels = set()
387 if isinstance(depotPaths,basestring):
388 depotPaths = [depotPaths]
389
390 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
391 label = l['label']
392 labels.add(label)
393
394 return labels
395
396 # Return the set of all git tags
397 def getGitTags():
398 gitTags = set()
399 for line in read_pipe_lines(["git", "tag"]):
400 tag = line.strip()
401 gitTags.add(tag)
402 return gitTags
403
404 def diffTreePattern():
405 # This is a simple generator for the diff tree regex pattern. This could be
406 # a class variable if this and parseDiffTreeEntry were a part of a class.
407 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
408 while True:
409 yield pattern
410
411 def parseDiffTreeEntry(entry):
412 """Parses a single diff tree entry into its component elements.
413
414 See git-diff-tree(1) manpage for details about the format of the diff
415 output. This method returns a dictionary with the following elements:
416
417 src_mode - The mode of the source file
418 dst_mode - The mode of the destination file
419 src_sha1 - The sha1 for the source file
420 dst_sha1 - The sha1 fr the destination file
421 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
422 status_score - The score for the status (applicable for 'C' and 'R'
423 statuses). This is None if there is no score.
424 src - The path for the source file.
425 dst - The path for the destination file. This is only present for
426 copy or renames. If it is not present, this is None.
427
428 If the pattern is not matched, None is returned."""
429
430 match = diffTreePattern().next().match(entry)
431 if match:
432 return {
433 'src_mode': match.group(1),
434 'dst_mode': match.group(2),
435 'src_sha1': match.group(3),
436 'dst_sha1': match.group(4),
437 'status': match.group(5),
438 'status_score': match.group(6),
439 'src': match.group(7),
440 'dst': match.group(10)
441 }
442 return None
443
444 def isModeExec(mode):
445 # Returns True if the given git mode represents an executable file,
446 # otherwise False.
447 return mode[-3:] == "755"
448
449 def isModeExecChanged(src_mode, dst_mode):
450 return isModeExec(src_mode) != isModeExec(dst_mode)
451
452 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
453
454 if isinstance(cmd,basestring):
455 cmd = "-G " + cmd
456 expand = True
457 else:
458 cmd = ["-G"] + cmd
459 expand = False
460
461 cmd = p4_build_cmd(cmd)
462 if verbose:
463 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
464
465 # Use a temporary file to avoid deadlocks without
466 # subprocess.communicate(), which would put another copy
467 # of stdout into memory.
468 stdin_file = None
469 if stdin is not None:
470 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
471 if isinstance(stdin,basestring):
472 stdin_file.write(stdin)
473 else:
474 for i in stdin:
475 stdin_file.write(i + '\n')
476 stdin_file.flush()
477 stdin_file.seek(0)
478
479 p4 = subprocess.Popen(cmd,
480 shell=expand,
481 stdin=stdin_file,
482 stdout=subprocess.PIPE)
483
484 result = []
485 try:
486 while True:
487 entry = marshal.load(p4.stdout)
488 if cb is not None:
489 cb(entry)
490 else:
491 result.append(entry)
492 except EOFError:
493 pass
494 exitCode = p4.wait()
495 if exitCode != 0:
496 entry = {}
497 entry["p4ExitCode"] = exitCode
498 result.append(entry)
499
500 return result
501
502 def p4Cmd(cmd):
503 list = p4CmdList(cmd)
504 result = {}
505 for entry in list:
506 result.update(entry)
507 return result;
508
509 def p4Where(depotPath):
510 if not depotPath.endswith("/"):
511 depotPath += "/"
512 depotPathLong = depotPath + "..."
513 outputList = p4CmdList(["where", depotPathLong])
514 output = None
515 for entry in outputList:
516 if "depotFile" in entry:
517 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
518 # The base path always ends with "/...".
519 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
520 output = entry
521 break
522 elif "data" in entry:
523 data = entry.get("data")
524 space = data.find(" ")
525 if data[:space] == depotPath:
526 output = entry
527 break
528 if output == None:
529 return ""
530 if output["code"] == "error":
531 return ""
532 clientPath = ""
533 if "path" in output:
534 clientPath = output.get("path")
535 elif "data" in output:
536 data = output.get("data")
537 lastSpace = data.rfind(" ")
538 clientPath = data[lastSpace + 1:]
539
540 if clientPath.endswith("..."):
541 clientPath = clientPath[:-3]
542 return clientPath
543
544 def currentGitBranch():
545 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
546
547 def isValidGitDir(path):
548 if (os.path.exists(path + "/HEAD")
549 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
550 return True;
551 return False
552
553 def parseRevision(ref):
554 return read_pipe("git rev-parse %s" % ref).strip()
555
556 def branchExists(ref):
557 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
558 ignore_error=True)
559 return len(rev) > 0
560
561 def extractLogMessageFromGitCommit(commit):
562 logMessage = ""
563
564 ## fixme: title is first line of commit, not 1st paragraph.
565 foundTitle = False
566 for log in read_pipe_lines("git cat-file commit %s" % commit):
567 if not foundTitle:
568 if len(log) == 1:
569 foundTitle = True
570 continue
571
572 logMessage += log
573 return logMessage
574
575 def extractSettingsGitLog(log):
576 values = {}
577 for line in log.split("\n"):
578 line = line.strip()
579 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
580 if not m:
581 continue
582
583 assignments = m.group(1).split (':')
584 for a in assignments:
585 vals = a.split ('=')
586 key = vals[0].strip()
587 val = ('='.join (vals[1:])).strip()
588 if val.endswith ('\"') and val.startswith('"'):
589 val = val[1:-1]
590
591 values[key] = val
592
593 paths = values.get("depot-paths")
594 if not paths:
595 paths = values.get("depot-path")
596 if paths:
597 values['depot-paths'] = paths.split(',')
598 return values
599
600 def gitBranchExists(branch):
601 proc = subprocess.Popen(["git", "rev-parse", branch],
602 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
603 return proc.wait() == 0;
604
605 _gitConfig = {}
606
607 def gitConfig(key):
608 if not _gitConfig.has_key(key):
609 cmd = [ "git", "config", key ]
610 s = read_pipe(cmd, ignore_error=True)
611 _gitConfig[key] = s.strip()
612 return _gitConfig[key]
613
614 def gitConfigBool(key):
615 """Return a bool, using git config --bool. It is True only if the
616 variable is set to true, and False if set to false or not present
617 in the config."""
618
619 if not _gitConfig.has_key(key):
620 cmd = [ "git", "config", "--bool", key ]
621 s = read_pipe(cmd, ignore_error=True)
622 v = s.strip()
623 _gitConfig[key] = v == "true"
624 return _gitConfig[key]
625
626 def gitConfigList(key):
627 if not _gitConfig.has_key(key):
628 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
629 _gitConfig[key] = s.strip().split(os.linesep)
630 return _gitConfig[key]
631
632 def p4BranchesInGit(branchesAreInRemotes=True):
633 """Find all the branches whose names start with "p4/", looking
634 in remotes or heads as specified by the argument. Return
635 a dictionary of { branch: revision } for each one found.
636 The branch names are the short names, without any
637 "p4/" prefix."""
638
639 branches = {}
640
641 cmdline = "git rev-parse --symbolic "
642 if branchesAreInRemotes:
643 cmdline += "--remotes"
644 else:
645 cmdline += "--branches"
646
647 for line in read_pipe_lines(cmdline):
648 line = line.strip()
649
650 # only import to p4/
651 if not line.startswith('p4/'):
652 continue
653 # special symbolic ref to p4/master
654 if line == "p4/HEAD":
655 continue
656
657 # strip off p4/ prefix
658 branch = line[len("p4/"):]
659
660 branches[branch] = parseRevision(line)
661
662 return branches
663
664 def branch_exists(branch):
665 """Make sure that the given ref name really exists."""
666
667 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
668 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
669 out, _ = p.communicate()
670 if p.returncode:
671 return False
672 # expect exactly one line of output: the branch name
673 return out.rstrip() == branch
674
675 def findUpstreamBranchPoint(head = "HEAD"):
676 branches = p4BranchesInGit()
677 # map from depot-path to branch name
678 branchByDepotPath = {}
679 for branch in branches.keys():
680 tip = branches[branch]
681 log = extractLogMessageFromGitCommit(tip)
682 settings = extractSettingsGitLog(log)
683 if settings.has_key("depot-paths"):
684 paths = ",".join(settings["depot-paths"])
685 branchByDepotPath[paths] = "remotes/p4/" + branch
686
687 settings = None
688 parent = 0
689 while parent < 65535:
690 commit = head + "~%s" % parent
691 log = extractLogMessageFromGitCommit(commit)
692 settings = extractSettingsGitLog(log)
693 if settings.has_key("depot-paths"):
694 paths = ",".join(settings["depot-paths"])
695 if branchByDepotPath.has_key(paths):
696 return [branchByDepotPath[paths], settings]
697
698 parent = parent + 1
699
700 return ["", settings]
701
702 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
703 if not silent:
704 print ("Creating/updating branch(es) in %s based on origin branch(es)"
705 % localRefPrefix)
706
707 originPrefix = "origin/p4/"
708
709 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
710 line = line.strip()
711 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
712 continue
713
714 headName = line[len(originPrefix):]
715 remoteHead = localRefPrefix + headName
716 originHead = line
717
718 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
719 if (not original.has_key('depot-paths')
720 or not original.has_key('change')):
721 continue
722
723 update = False
724 if not gitBranchExists(remoteHead):
725 if verbose:
726 print "creating %s" % remoteHead
727 update = True
728 else:
729 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
730 if settings.has_key('change') > 0:
731 if settings['depot-paths'] == original['depot-paths']:
732 originP4Change = int(original['change'])
733 p4Change = int(settings['change'])
734 if originP4Change > p4Change:
735 print ("%s (%s) is newer than %s (%s). "
736 "Updating p4 branch from origin."
737 % (originHead, originP4Change,
738 remoteHead, p4Change))
739 update = True
740 else:
741 print ("Ignoring: %s was imported from %s while "
742 "%s was imported from %s"
743 % (originHead, ','.join(original['depot-paths']),
744 remoteHead, ','.join(settings['depot-paths'])))
745
746 if update:
747 system("git update-ref %s %s" % (remoteHead, originHead))
748
749 def originP4BranchesExist():
750 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
751
752
753 def p4ParseNumericChangeRange(parts):
754 changeStart = int(parts[0][1:])
755 if parts[1] == '#head':
756 changeEnd = p4_last_change()
757 else:
758 changeEnd = int(parts[1])
759
760 return (changeStart, changeEnd)
761
762 def chooseBlockSize(blockSize):
763 if blockSize:
764 return blockSize
765 else:
766 return defaultBlockSize
767
768 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
769 assert depotPaths
770
771 # Parse the change range into start and end. Try to find integer
772 # revision ranges as these can be broken up into blocks to avoid
773 # hitting server-side limits (maxrows, maxscanresults). But if
774 # that doesn't work, fall back to using the raw revision specifier
775 # strings, without using block mode.
776
777 if changeRange is None or changeRange == '':
778 changeStart = 1
779 changeEnd = p4_last_change()
780 block_size = chooseBlockSize(requestedBlockSize)
781 else:
782 parts = changeRange.split(',')
783 assert len(parts) == 2
784 try:
785 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
786 block_size = chooseBlockSize(requestedBlockSize)
787 except:
788 changeStart = parts[0][1:]
789 changeEnd = parts[1]
790 if requestedBlockSize:
791 die("cannot use --changes-block-size with non-numeric revisions")
792 block_size = None
793
794 # Accumulate change numbers in a dictionary to avoid duplicates
795 changes = {}
796
797 for p in depotPaths:
798 # Retrieve changes a block at a time, to prevent running
799 # into a MaxResults/MaxScanRows error from the server.
800
801 while True:
802 cmd = ['changes']
803
804 if block_size:
805 end = min(changeEnd, changeStart + block_size)
806 revisionRange = "%d,%d" % (changeStart, end)
807 else:
808 revisionRange = "%s,%s" % (changeStart, changeEnd)
809
810 cmd += ["%s...@%s" % (p, revisionRange)]
811
812 for line in p4_read_pipe_lines(cmd):
813 changeNum = int(line.split(" ")[1])
814 changes[changeNum] = True
815
816 if not block_size:
817 break
818
819 if end >= changeEnd:
820 break
821
822 changeStart = end + 1
823
824 changelist = changes.keys()
825 changelist.sort()
826 return changelist
827
828 def p4PathStartsWith(path, prefix):
829 # This method tries to remedy a potential mixed-case issue:
830 #
831 # If UserA adds //depot/DirA/file1
832 # and UserB adds //depot/dira/file2
833 #
834 # we may or may not have a problem. If you have core.ignorecase=true,
835 # we treat DirA and dira as the same directory
836 if gitConfigBool("core.ignorecase"):
837 return path.lower().startswith(prefix.lower())
838 return path.startswith(prefix)
839
840 def getClientSpec():
841 """Look at the p4 client spec, create a View() object that contains
842 all the mappings, and return it."""
843
844 specList = p4CmdList("client -o")
845 if len(specList) != 1:
846 die('Output from "client -o" is %d lines, expecting 1' %
847 len(specList))
848
849 # dictionary of all client parameters
850 entry = specList[0]
851
852 # the //client/ name
853 client_name = entry["Client"]
854
855 # just the keys that start with "View"
856 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
857
858 # hold this new View
859 view = View(client_name)
860
861 # append the lines, in order, to the view
862 for view_num in range(len(view_keys)):
863 k = "View%d" % view_num
864 if k not in view_keys:
865 die("Expected view key %s missing" % k)
866 view.append(entry[k])
867
868 return view
869
870 def getClientRoot():
871 """Grab the client directory."""
872
873 output = p4CmdList("client -o")
874 if len(output) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' % len(output))
876
877 entry = output[0]
878 if "Root" not in entry:
879 die('Client has no "Root"')
880
881 return entry["Root"]
882
883 #
884 # P4 wildcards are not allowed in filenames. P4 complains
885 # if you simply add them, but you can force it with "-f", in
886 # which case it translates them into %xx encoding internally.
887 #
888 def wildcard_decode(path):
889 # Search for and fix just these four characters. Do % last so
890 # that fixing it does not inadvertently create new %-escapes.
891 # Cannot have * in a filename in windows; untested as to
892 # what p4 would do in such a case.
893 if not platform.system() == "Windows":
894 path = path.replace("%2A", "*")
895 path = path.replace("%23", "#") \
896 .replace("%40", "@") \
897 .replace("%25", "%")
898 return path
899
900 def wildcard_encode(path):
901 # do % first to avoid double-encoding the %s introduced here
902 path = path.replace("%", "%25") \
903 .replace("*", "%2A") \
904 .replace("#", "%23") \
905 .replace("@", "%40")
906 return path
907
908 def wildcard_present(path):
909 m = re.search("[*#@%]", path)
910 return m is not None
911
912 class Command:
913 def __init__(self):
914 self.usage = "usage: %prog [options]"
915 self.needsGit = True
916 self.verbose = False
917
918 class P4UserMap:
919 def __init__(self):
920 self.userMapFromPerforceServer = False
921 self.myP4UserId = None
922
923 def p4UserId(self):
924 if self.myP4UserId:
925 return self.myP4UserId
926
927 results = p4CmdList("user -o")
928 for r in results:
929 if r.has_key('User'):
930 self.myP4UserId = r['User']
931 return r['User']
932 die("Could not find your p4 user id")
933
934 def p4UserIsMe(self, p4User):
935 # return True if the given p4 user is actually me
936 me = self.p4UserId()
937 if not p4User or p4User != me:
938 return False
939 else:
940 return True
941
942 def getUserCacheFilename(self):
943 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
944 return home + "/.gitp4-usercache.txt"
945
946 def getUserMapFromPerforceServer(self):
947 if self.userMapFromPerforceServer:
948 return
949 self.users = {}
950 self.emails = {}
951
952 for output in p4CmdList("users"):
953 if not output.has_key("User"):
954 continue
955 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
956 self.emails[output["Email"]] = output["User"]
957
958
959 s = ''
960 for (key, val) in self.users.items():
961 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
962
963 open(self.getUserCacheFilename(), "wb").write(s)
964 self.userMapFromPerforceServer = True
965
966 def loadUserMapFromCache(self):
967 self.users = {}
968 self.userMapFromPerforceServer = False
969 try:
970 cache = open(self.getUserCacheFilename(), "rb")
971 lines = cache.readlines()
972 cache.close()
973 for line in lines:
974 entry = line.strip().split("\t")
975 self.users[entry[0]] = entry[1]
976 except IOError:
977 self.getUserMapFromPerforceServer()
978
979 class P4Debug(Command):
980 def __init__(self):
981 Command.__init__(self)
982 self.options = []
983 self.description = "A tool to debug the output of p4 -G."
984 self.needsGit = False
985
986 def run(self, args):
987 j = 0
988 for output in p4CmdList(args):
989 print 'Element: %d' % j
990 j += 1
991 print output
992 return True
993
994 class P4RollBack(Command):
995 def __init__(self):
996 Command.__init__(self)
997 self.options = [
998 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
999 ]
1000 self.description = "A tool to debug the multi-branch import. Don't use :)"
1001 self.rollbackLocalBranches = False
1002
1003 def run(self, args):
1004 if len(args) != 1:
1005 return False
1006 maxChange = int(args[0])
1007
1008 if "p4ExitCode" in p4Cmd("changes -m 1"):
1009 die("Problems executing p4");
1010
1011 if self.rollbackLocalBranches:
1012 refPrefix = "refs/heads/"
1013 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1014 else:
1015 refPrefix = "refs/remotes/"
1016 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1017
1018 for line in lines:
1019 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1020 line = line.strip()
1021 ref = refPrefix + line
1022 log = extractLogMessageFromGitCommit(ref)
1023 settings = extractSettingsGitLog(log)
1024
1025 depotPaths = settings['depot-paths']
1026 change = settings['change']
1027
1028 changed = False
1029
1030 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1031 for p in depotPaths]))) == 0:
1032 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1033 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1034 continue
1035
1036 while change and int(change) > maxChange:
1037 changed = True
1038 if self.verbose:
1039 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1040 system("git update-ref %s \"%s^\"" % (ref, ref))
1041 log = extractLogMessageFromGitCommit(ref)
1042 settings = extractSettingsGitLog(log)
1043
1044
1045 depotPaths = settings['depot-paths']
1046 change = settings['change']
1047
1048 if changed:
1049 print "%s rewound to %s" % (ref, change)
1050
1051 return True
1052
1053 class P4Submit(Command, P4UserMap):
1054
1055 conflict_behavior_choices = ("ask", "skip", "quit")
1056
1057 def __init__(self):
1058 Command.__init__(self)
1059 P4UserMap.__init__(self)
1060 self.options = [
1061 optparse.make_option("--origin", dest="origin"),
1062 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1063 # preserve the user, requires relevant p4 permissions
1064 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1065 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1066 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1067 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1068 optparse.make_option("--conflict", dest="conflict_behavior",
1069 choices=self.conflict_behavior_choices),
1070 optparse.make_option("--branch", dest="branch"),
1071 ]
1072 self.description = "Submit changes from git to the perforce depot."
1073 self.usage += " [name of git branch to submit into perforce depot]"
1074 self.origin = ""
1075 self.detectRenames = False
1076 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1077 self.dry_run = False
1078 self.prepare_p4_only = False
1079 self.conflict_behavior = None
1080 self.isWindows = (platform.system() == "Windows")
1081 self.exportLabels = False
1082 self.p4HasMoveCommand = p4_has_move_command()
1083 self.branch = None
1084
1085 def check(self):
1086 if len(p4CmdList("opened ...")) > 0:
1087 die("You have files opened with perforce! Close them before starting the sync.")
1088
1089 def separate_jobs_from_description(self, message):
1090 """Extract and return a possible Jobs field in the commit
1091 message. It goes into a separate section in the p4 change
1092 specification.
1093
1094 A jobs line starts with "Jobs:" and looks like a new field
1095 in a form. Values are white-space separated on the same
1096 line or on following lines that start with a tab.
1097
1098 This does not parse and extract the full git commit message
1099 like a p4 form. It just sees the Jobs: line as a marker
1100 to pass everything from then on directly into the p4 form,
1101 but outside the description section.
1102
1103 Return a tuple (stripped log message, jobs string)."""
1104
1105 m = re.search(r'^Jobs:', message, re.MULTILINE)
1106 if m is None:
1107 return (message, None)
1108
1109 jobtext = message[m.start():]
1110 stripped_message = message[:m.start()].rstrip()
1111 return (stripped_message, jobtext)
1112
1113 def prepareLogMessage(self, template, message, jobs):
1114 """Edits the template returned from "p4 change -o" to insert
1115 the message in the Description field, and the jobs text in
1116 the Jobs field."""
1117 result = ""
1118
1119 inDescriptionSection = False
1120
1121 for line in template.split("\n"):
1122 if line.startswith("#"):
1123 result += line + "\n"
1124 continue
1125
1126 if inDescriptionSection:
1127 if line.startswith("Files:") or line.startswith("Jobs:"):
1128 inDescriptionSection = False
1129 # insert Jobs section
1130 if jobs:
1131 result += jobs + "\n"
1132 else:
1133 continue
1134 else:
1135 if line.startswith("Description:"):
1136 inDescriptionSection = True
1137 line += "\n"
1138 for messageLine in message.split("\n"):
1139 line += "\t" + messageLine + "\n"
1140
1141 result += line + "\n"
1142
1143 return result
1144
1145 def patchRCSKeywords(self, file, pattern):
1146 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1147 (handle, outFileName) = tempfile.mkstemp(dir='.')
1148 try:
1149 outFile = os.fdopen(handle, "w+")
1150 inFile = open(file, "r")
1151 regexp = re.compile(pattern, re.VERBOSE)
1152 for line in inFile.readlines():
1153 line = regexp.sub(r'$\1$', line)
1154 outFile.write(line)
1155 inFile.close()
1156 outFile.close()
1157 # Forcibly overwrite the original file
1158 os.unlink(file)
1159 shutil.move(outFileName, file)
1160 except:
1161 # cleanup our temporary file
1162 os.unlink(outFileName)
1163 print "Failed to strip RCS keywords in %s" % file
1164 raise
1165
1166 print "Patched up RCS keywords in %s" % file
1167
1168 def p4UserForCommit(self,id):
1169 # Return the tuple (perforce user,git email) for a given git commit id
1170 self.getUserMapFromPerforceServer()
1171 gitEmail = read_pipe(["git", "log", "--max-count=1",
1172 "--format=%ae", id])
1173 gitEmail = gitEmail.strip()
1174 if not self.emails.has_key(gitEmail):
1175 return (None,gitEmail)
1176 else:
1177 return (self.emails[gitEmail],gitEmail)
1178
1179 def checkValidP4Users(self,commits):
1180 # check if any git authors cannot be mapped to p4 users
1181 for id in commits:
1182 (user,email) = self.p4UserForCommit(id)
1183 if not user:
1184 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1185 if gitConfigBool("git-p4.allowMissingP4Users"):
1186 print "%s" % msg
1187 else:
1188 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1189
1190 def lastP4Changelist(self):
1191 # Get back the last changelist number submitted in this client spec. This
1192 # then gets used to patch up the username in the change. If the same
1193 # client spec is being used by multiple processes then this might go
1194 # wrong.
1195 results = p4CmdList("client -o") # find the current client
1196 client = None
1197 for r in results:
1198 if r.has_key('Client'):
1199 client = r['Client']
1200 break
1201 if not client:
1202 die("could not get client spec")
1203 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1204 for r in results:
1205 if r.has_key('change'):
1206 return r['change']
1207 die("Could not get changelist number for last submit - cannot patch up user details")
1208
1209 def modifyChangelistUser(self, changelist, newUser):
1210 # fixup the user field of a changelist after it has been submitted.
1211 changes = p4CmdList("change -o %s" % changelist)
1212 if len(changes) != 1:
1213 die("Bad output from p4 change modifying %s to user %s" %
1214 (changelist, newUser))
1215
1216 c = changes[0]
1217 if c['User'] == newUser: return # nothing to do
1218 c['User'] = newUser
1219 input = marshal.dumps(c)
1220
1221 result = p4CmdList("change -f -i", stdin=input)
1222 for r in result:
1223 if r.has_key('code'):
1224 if r['code'] == 'error':
1225 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1226 if r.has_key('data'):
1227 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1228 return
1229 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1230
1231 def canChangeChangelists(self):
1232 # check to see if we have p4 admin or super-user permissions, either of
1233 # which are required to modify changelists.
1234 results = p4CmdList(["protects", self.depotPath])
1235 for r in results:
1236 if r.has_key('perm'):
1237 if r['perm'] == 'admin':
1238 return 1
1239 if r['perm'] == 'super':
1240 return 1
1241 return 0
1242
1243 def prepareSubmitTemplate(self):
1244 """Run "p4 change -o" to grab a change specification template.
1245 This does not use "p4 -G", as it is nice to keep the submission
1246 template in original order, since a human might edit it.
1247
1248 Remove lines in the Files section that show changes to files
1249 outside the depot path we're committing into."""
1250
1251 template = ""
1252 inFilesSection = False
1253 for line in p4_read_pipe_lines(['change', '-o']):
1254 if line.endswith("\r\n"):
1255 line = line[:-2] + "\n"
1256 if inFilesSection:
1257 if line.startswith("\t"):
1258 # path starts and ends with a tab
1259 path = line[1:]
1260 lastTab = path.rfind("\t")
1261 if lastTab != -1:
1262 path = path[:lastTab]
1263 if not p4PathStartsWith(path, self.depotPath):
1264 continue
1265 else:
1266 inFilesSection = False
1267 else:
1268 if line.startswith("Files:"):
1269 inFilesSection = True
1270
1271 template += line
1272
1273 return template
1274
1275 def edit_template(self, template_file):
1276 """Invoke the editor to let the user change the submission
1277 message. Return true if okay to continue with the submit."""
1278
1279 # if configured to skip the editing part, just submit
1280 if gitConfigBool("git-p4.skipSubmitEdit"):
1281 return True
1282
1283 # look at the modification time, to check later if the user saved
1284 # the file
1285 mtime = os.stat(template_file).st_mtime
1286
1287 # invoke the editor
1288 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1289 editor = os.environ.get("P4EDITOR")
1290 else:
1291 editor = read_pipe("git var GIT_EDITOR").strip()
1292 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1293
1294 # If the file was not saved, prompt to see if this patch should
1295 # be skipped. But skip this verification step if configured so.
1296 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1297 return True
1298
1299 # modification time updated means user saved the file
1300 if os.stat(template_file).st_mtime > mtime:
1301 return True
1302
1303 while True:
1304 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1305 if response == 'y':
1306 return True
1307 if response == 'n':
1308 return False
1309
1310 def get_diff_description(self, editedFiles, filesToAdd):
1311 # diff
1312 if os.environ.has_key("P4DIFF"):
1313 del(os.environ["P4DIFF"])
1314 diff = ""
1315 for editedFile in editedFiles:
1316 diff += p4_read_pipe(['diff', '-du',
1317 wildcard_encode(editedFile)])
1318
1319 # new file diff
1320 newdiff = ""
1321 for newFile in filesToAdd:
1322 newdiff += "==== new file ====\n"
1323 newdiff += "--- /dev/null\n"
1324 newdiff += "+++ %s\n" % newFile
1325 f = open(newFile, "r")
1326 for line in f.readlines():
1327 newdiff += "+" + line
1328 f.close()
1329
1330 return (diff + newdiff).replace('\r\n', '\n')
1331
1332 def applyCommit(self, id):
1333 """Apply one commit, return True if it succeeded."""
1334
1335 print "Applying", read_pipe(["git", "show", "-s",
1336 "--format=format:%h %s", id])
1337
1338 (p4User, gitEmail) = self.p4UserForCommit(id)
1339
1340 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1341 filesToAdd = set()
1342 filesToDelete = set()
1343 editedFiles = set()
1344 pureRenameCopy = set()
1345 filesToChangeExecBit = {}
1346
1347 for line in diff:
1348 diff = parseDiffTreeEntry(line)
1349 modifier = diff['status']
1350 path = diff['src']
1351 if modifier == "M":
1352 p4_edit(path)
1353 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1354 filesToChangeExecBit[path] = diff['dst_mode']
1355 editedFiles.add(path)
1356 elif modifier == "A":
1357 filesToAdd.add(path)
1358 filesToChangeExecBit[path] = diff['dst_mode']
1359 if path in filesToDelete:
1360 filesToDelete.remove(path)
1361 elif modifier == "D":
1362 filesToDelete.add(path)
1363 if path in filesToAdd:
1364 filesToAdd.remove(path)
1365 elif modifier == "C":
1366 src, dest = diff['src'], diff['dst']
1367 p4_integrate(src, dest)
1368 pureRenameCopy.add(dest)
1369 if diff['src_sha1'] != diff['dst_sha1']:
1370 p4_edit(dest)
1371 pureRenameCopy.discard(dest)
1372 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1373 p4_edit(dest)
1374 pureRenameCopy.discard(dest)
1375 filesToChangeExecBit[dest] = diff['dst_mode']
1376 if self.isWindows:
1377 # turn off read-only attribute
1378 os.chmod(dest, stat.S_IWRITE)
1379 os.unlink(dest)
1380 editedFiles.add(dest)
1381 elif modifier == "R":
1382 src, dest = diff['src'], diff['dst']
1383 if self.p4HasMoveCommand:
1384 p4_edit(src) # src must be open before move
1385 p4_move(src, dest) # opens for (move/delete, move/add)
1386 else:
1387 p4_integrate(src, dest)
1388 if diff['src_sha1'] != diff['dst_sha1']:
1389 p4_edit(dest)
1390 else:
1391 pureRenameCopy.add(dest)
1392 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1393 if not self.p4HasMoveCommand:
1394 p4_edit(dest) # with move: already open, writable
1395 filesToChangeExecBit[dest] = diff['dst_mode']
1396 if not self.p4HasMoveCommand:
1397 if self.isWindows:
1398 os.chmod(dest, stat.S_IWRITE)
1399 os.unlink(dest)
1400 filesToDelete.add(src)
1401 editedFiles.add(dest)
1402 else:
1403 die("unknown modifier %s for %s" % (modifier, path))
1404
1405 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1406 patchcmd = diffcmd + " | git apply "
1407 tryPatchCmd = patchcmd + "--check -"
1408 applyPatchCmd = patchcmd + "--check --apply -"
1409 patch_succeeded = True
1410
1411 if os.system(tryPatchCmd) != 0:
1412 fixed_rcs_keywords = False
1413 patch_succeeded = False
1414 print "Unfortunately applying the change failed!"
1415
1416 # Patch failed, maybe it's just RCS keyword woes. Look through
1417 # the patch to see if that's possible.
1418 if gitConfigBool("git-p4.attemptRCSCleanup"):
1419 file = None
1420 pattern = None
1421 kwfiles = {}
1422 for file in editedFiles | filesToDelete:
1423 # did this file's delta contain RCS keywords?
1424 pattern = p4_keywords_regexp_for_file(file)
1425
1426 if pattern:
1427 # this file is a possibility...look for RCS keywords.
1428 regexp = re.compile(pattern, re.VERBOSE)
1429 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1430 if regexp.search(line):
1431 if verbose:
1432 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1433 kwfiles[file] = pattern
1434 break
1435
1436 for file in kwfiles:
1437 if verbose:
1438 print "zapping %s with %s" % (line,pattern)
1439 # File is being deleted, so not open in p4. Must
1440 # disable the read-only bit on windows.
1441 if self.isWindows and file not in editedFiles:
1442 os.chmod(file, stat.S_IWRITE)
1443 self.patchRCSKeywords(file, kwfiles[file])
1444 fixed_rcs_keywords = True
1445
1446 if fixed_rcs_keywords:
1447 print "Retrying the patch with RCS keywords cleaned up"
1448 if os.system(tryPatchCmd) == 0:
1449 patch_succeeded = True
1450
1451 if not patch_succeeded:
1452 for f in editedFiles:
1453 p4_revert(f)
1454 return False
1455
1456 #
1457 # Apply the patch for real, and do add/delete/+x handling.
1458 #
1459 system(applyPatchCmd)
1460
1461 for f in filesToAdd:
1462 p4_add(f)
1463 for f in filesToDelete:
1464 p4_revert(f)
1465 p4_delete(f)
1466
1467 # Set/clear executable bits
1468 for f in filesToChangeExecBit.keys():
1469 mode = filesToChangeExecBit[f]
1470 setP4ExecBit(f, mode)
1471
1472 #
1473 # Build p4 change description, starting with the contents
1474 # of the git commit message.
1475 #
1476 logMessage = extractLogMessageFromGitCommit(id)
1477 logMessage = logMessage.strip()
1478 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1479
1480 template = self.prepareSubmitTemplate()
1481 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1482
1483 if self.preserveUser:
1484 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1485
1486 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1487 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1488 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1489 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1490
1491 separatorLine = "######## everything below this line is just the diff #######\n"
1492 if not self.prepare_p4_only:
1493 submitTemplate += separatorLine
1494 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1495
1496 (handle, fileName) = tempfile.mkstemp()
1497 tmpFile = os.fdopen(handle, "w+b")
1498 if self.isWindows:
1499 submitTemplate = submitTemplate.replace("\n", "\r\n")
1500 tmpFile.write(submitTemplate)
1501 tmpFile.close()
1502
1503 if self.prepare_p4_only:
1504 #
1505 # Leave the p4 tree prepared, and the submit template around
1506 # and let the user decide what to do next
1507 #
1508 print
1509 print "P4 workspace prepared for submission."
1510 print "To submit or revert, go to client workspace"
1511 print " " + self.clientPath
1512 print
1513 print "To submit, use \"p4 submit\" to write a new description,"
1514 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1515 " \"git p4\"." % fileName
1516 print "You can delete the file \"%s\" when finished." % fileName
1517
1518 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1519 print "To preserve change ownership by user %s, you must\n" \
1520 "do \"p4 change -f <change>\" after submitting and\n" \
1521 "edit the User field."
1522 if pureRenameCopy:
1523 print "After submitting, renamed files must be re-synced."
1524 print "Invoke \"p4 sync -f\" on each of these files:"
1525 for f in pureRenameCopy:
1526 print " " + f
1527
1528 print
1529 print "To revert the changes, use \"p4 revert ...\", and delete"
1530 print "the submit template file \"%s\"" % fileName
1531 if filesToAdd:
1532 print "Since the commit adds new files, they must be deleted:"
1533 for f in filesToAdd:
1534 print " " + f
1535 print
1536 return True
1537
1538 #
1539 # Let the user edit the change description, then submit it.
1540 #
1541 if self.edit_template(fileName):
1542 # read the edited message and submit
1543 ret = True
1544 tmpFile = open(fileName, "rb")
1545 message = tmpFile.read()
1546 tmpFile.close()
1547 if self.isWindows:
1548 message = message.replace("\r\n", "\n")
1549 submitTemplate = message[:message.index(separatorLine)]
1550 p4_write_pipe(['submit', '-i'], submitTemplate)
1551
1552 if self.preserveUser:
1553 if p4User:
1554 # Get last changelist number. Cannot easily get it from
1555 # the submit command output as the output is
1556 # unmarshalled.
1557 changelist = self.lastP4Changelist()
1558 self.modifyChangelistUser(changelist, p4User)
1559
1560 # The rename/copy happened by applying a patch that created a
1561 # new file. This leaves it writable, which confuses p4.
1562 for f in pureRenameCopy:
1563 p4_sync(f, "-f")
1564
1565 else:
1566 # skip this patch
1567 ret = False
1568 print "Submission cancelled, undoing p4 changes."
1569 for f in editedFiles:
1570 p4_revert(f)
1571 for f in filesToAdd:
1572 p4_revert(f)
1573 os.remove(f)
1574 for f in filesToDelete:
1575 p4_revert(f)
1576
1577 os.remove(fileName)
1578 return ret
1579
1580 # Export git tags as p4 labels. Create a p4 label and then tag
1581 # with that.
1582 def exportGitTags(self, gitTags):
1583 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1584 if len(validLabelRegexp) == 0:
1585 validLabelRegexp = defaultLabelRegexp
1586 m = re.compile(validLabelRegexp)
1587
1588 for name in gitTags:
1589
1590 if not m.match(name):
1591 if verbose:
1592 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1593 continue
1594
1595 # Get the p4 commit this corresponds to
1596 logMessage = extractLogMessageFromGitCommit(name)
1597 values = extractSettingsGitLog(logMessage)
1598
1599 if not values.has_key('change'):
1600 # a tag pointing to something not sent to p4; ignore
1601 if verbose:
1602 print "git tag %s does not give a p4 commit" % name
1603 continue
1604 else:
1605 changelist = values['change']
1606
1607 # Get the tag details.
1608 inHeader = True
1609 isAnnotated = False
1610 body = []
1611 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1612 l = l.strip()
1613 if inHeader:
1614 if re.match(r'tag\s+', l):
1615 isAnnotated = True
1616 elif re.match(r'\s*$', l):
1617 inHeader = False
1618 continue
1619 else:
1620 body.append(l)
1621
1622 if not isAnnotated:
1623 body = ["lightweight tag imported by git p4\n"]
1624
1625 # Create the label - use the same view as the client spec we are using
1626 clientSpec = getClientSpec()
1627
1628 labelTemplate = "Label: %s\n" % name
1629 labelTemplate += "Description:\n"
1630 for b in body:
1631 labelTemplate += "\t" + b + "\n"
1632 labelTemplate += "View:\n"
1633 for depot_side in clientSpec.mappings:
1634 labelTemplate += "\t%s\n" % depot_side
1635
1636 if self.dry_run:
1637 print "Would create p4 label %s for tag" % name
1638 elif self.prepare_p4_only:
1639 print "Not creating p4 label %s for tag due to option" \
1640 " --prepare-p4-only" % name
1641 else:
1642 p4_write_pipe(["label", "-i"], labelTemplate)
1643
1644 # Use the label
1645 p4_system(["tag", "-l", name] +
1646 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1647
1648 if verbose:
1649 print "created p4 label for tag %s" % name
1650
1651 def run(self, args):
1652 if len(args) == 0:
1653 self.master = currentGitBranch()
1654 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1655 die("Detecting current git branch failed!")
1656 elif len(args) == 1:
1657 self.master = args[0]
1658 if not branchExists(self.master):
1659 die("Branch %s does not exist" % self.master)
1660 else:
1661 return False
1662
1663 allowSubmit = gitConfig("git-p4.allowSubmit")
1664 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1665 die("%s is not in git-p4.allowSubmit" % self.master)
1666
1667 [upstream, settings] = findUpstreamBranchPoint()
1668 self.depotPath = settings['depot-paths'][0]
1669 if len(self.origin) == 0:
1670 self.origin = upstream
1671
1672 if self.preserveUser:
1673 if not self.canChangeChangelists():
1674 die("Cannot preserve user names without p4 super-user or admin permissions")
1675
1676 # if not set from the command line, try the config file
1677 if self.conflict_behavior is None:
1678 val = gitConfig("git-p4.conflict")
1679 if val:
1680 if val not in self.conflict_behavior_choices:
1681 die("Invalid value '%s' for config git-p4.conflict" % val)
1682 else:
1683 val = "ask"
1684 self.conflict_behavior = val
1685
1686 if self.verbose:
1687 print "Origin branch is " + self.origin
1688
1689 if len(self.depotPath) == 0:
1690 print "Internal error: cannot locate perforce depot path from existing branches"
1691 sys.exit(128)
1692
1693 self.useClientSpec = False
1694 if gitConfigBool("git-p4.useclientspec"):
1695 self.useClientSpec = True
1696 if self.useClientSpec:
1697 self.clientSpecDirs = getClientSpec()
1698
1699 # Check for the existance of P4 branches
1700 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1701
1702 if self.useClientSpec and not branchesDetected:
1703 # all files are relative to the client spec
1704 self.clientPath = getClientRoot()
1705 else:
1706 self.clientPath = p4Where(self.depotPath)
1707
1708 if self.clientPath == "":
1709 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1710
1711 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1712 self.oldWorkingDirectory = os.getcwd()
1713
1714 # ensure the clientPath exists
1715 new_client_dir = False
1716 if not os.path.exists(self.clientPath):
1717 new_client_dir = True
1718 os.makedirs(self.clientPath)
1719
1720 chdir(self.clientPath, is_client_path=True)
1721 if self.dry_run:
1722 print "Would synchronize p4 checkout in %s" % self.clientPath
1723 else:
1724 print "Synchronizing p4 checkout..."
1725 if new_client_dir:
1726 # old one was destroyed, and maybe nobody told p4
1727 p4_sync("...", "-f")
1728 else:
1729 p4_sync("...")
1730 self.check()
1731
1732 commits = []
1733 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1734 commits.append(line.strip())
1735 commits.reverse()
1736
1737 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1738 self.checkAuthorship = False
1739 else:
1740 self.checkAuthorship = True
1741
1742 if self.preserveUser:
1743 self.checkValidP4Users(commits)
1744
1745 #
1746 # Build up a set of options to be passed to diff when
1747 # submitting each commit to p4.
1748 #
1749 if self.detectRenames:
1750 # command-line -M arg
1751 self.diffOpts = "-M"
1752 else:
1753 # If not explicitly set check the config variable
1754 detectRenames = gitConfig("git-p4.detectRenames")
1755
1756 if detectRenames.lower() == "false" or detectRenames == "":
1757 self.diffOpts = ""
1758 elif detectRenames.lower() == "true":
1759 self.diffOpts = "-M"
1760 else:
1761 self.diffOpts = "-M%s" % detectRenames
1762
1763 # no command-line arg for -C or --find-copies-harder, just
1764 # config variables
1765 detectCopies = gitConfig("git-p4.detectCopies")
1766 if detectCopies.lower() == "false" or detectCopies == "":
1767 pass
1768 elif detectCopies.lower() == "true":
1769 self.diffOpts += " -C"
1770 else:
1771 self.diffOpts += " -C%s" % detectCopies
1772
1773 if gitConfigBool("git-p4.detectCopiesHarder"):
1774 self.diffOpts += " --find-copies-harder"
1775
1776 #
1777 # Apply the commits, one at a time. On failure, ask if should
1778 # continue to try the rest of the patches, or quit.
1779 #
1780 if self.dry_run:
1781 print "Would apply"
1782 applied = []
1783 last = len(commits) - 1
1784 for i, commit in enumerate(commits):
1785 if self.dry_run:
1786 print " ", read_pipe(["git", "show", "-s",
1787 "--format=format:%h %s", commit])
1788 ok = True
1789 else:
1790 ok = self.applyCommit(commit)
1791 if ok:
1792 applied.append(commit)
1793 else:
1794 if self.prepare_p4_only and i < last:
1795 print "Processing only the first commit due to option" \
1796 " --prepare-p4-only"
1797 break
1798 if i < last:
1799 quit = False
1800 while True:
1801 # prompt for what to do, or use the option/variable
1802 if self.conflict_behavior == "ask":
1803 print "What do you want to do?"
1804 response = raw_input("[s]kip this commit but apply"
1805 " the rest, or [q]uit? ")
1806 if not response:
1807 continue
1808 elif self.conflict_behavior == "skip":
1809 response = "s"
1810 elif self.conflict_behavior == "quit":
1811 response = "q"
1812 else:
1813 die("Unknown conflict_behavior '%s'" %
1814 self.conflict_behavior)
1815
1816 if response[0] == "s":
1817 print "Skipping this commit, but applying the rest"
1818 break
1819 if response[0] == "q":
1820 print "Quitting"
1821 quit = True
1822 break
1823 if quit:
1824 break
1825
1826 chdir(self.oldWorkingDirectory)
1827
1828 if self.dry_run:
1829 pass
1830 elif self.prepare_p4_only:
1831 pass
1832 elif len(commits) == len(applied):
1833 print "All commits applied!"
1834
1835 sync = P4Sync()
1836 if self.branch:
1837 sync.branch = self.branch
1838 sync.run([])
1839
1840 rebase = P4Rebase()
1841 rebase.rebase()
1842
1843 else:
1844 if len(applied) == 0:
1845 print "No commits applied."
1846 else:
1847 print "Applied only the commits marked with '*':"
1848 for c in commits:
1849 if c in applied:
1850 star = "*"
1851 else:
1852 star = " "
1853 print star, read_pipe(["git", "show", "-s",
1854 "--format=format:%h %s", c])
1855 print "You will have to do 'git p4 sync' and rebase."
1856
1857 if gitConfigBool("git-p4.exportLabels"):
1858 self.exportLabels = True
1859
1860 if self.exportLabels:
1861 p4Labels = getP4Labels(self.depotPath)
1862 gitTags = getGitTags()
1863
1864 missingGitTags = gitTags - p4Labels
1865 self.exportGitTags(missingGitTags)
1866
1867 # exit with error unless everything applied perfectly
1868 if len(commits) != len(applied):
1869 sys.exit(1)
1870
1871 return True
1872
1873 class View(object):
1874 """Represent a p4 view ("p4 help views"), and map files in a
1875 repo according to the view."""
1876
1877 def __init__(self, client_name):
1878 self.mappings = []
1879 self.client_prefix = "//%s/" % client_name
1880 # cache results of "p4 where" to lookup client file locations
1881 self.client_spec_path_cache = {}
1882
1883 def append(self, view_line):
1884 """Parse a view line, splitting it into depot and client
1885 sides. Append to self.mappings, preserving order. This
1886 is only needed for tag creation."""
1887
1888 # Split the view line into exactly two words. P4 enforces
1889 # structure on these lines that simplifies this quite a bit.
1890 #
1891 # Either or both words may be double-quoted.
1892 # Single quotes do not matter.
1893 # Double-quote marks cannot occur inside the words.
1894 # A + or - prefix is also inside the quotes.
1895 # There are no quotes unless they contain a space.
1896 # The line is already white-space stripped.
1897 # The two words are separated by a single space.
1898 #
1899 if view_line[0] == '"':
1900 # First word is double quoted. Find its end.
1901 close_quote_index = view_line.find('"', 1)
1902 if close_quote_index <= 0:
1903 die("No first-word closing quote found: %s" % view_line)
1904 depot_side = view_line[1:close_quote_index]
1905 # skip closing quote and space
1906 rhs_index = close_quote_index + 1 + 1
1907 else:
1908 space_index = view_line.find(" ")
1909 if space_index <= 0:
1910 die("No word-splitting space found: %s" % view_line)
1911 depot_side = view_line[0:space_index]
1912 rhs_index = space_index + 1
1913
1914 # prefix + means overlay on previous mapping
1915 if depot_side.startswith("+"):
1916 depot_side = depot_side[1:]
1917
1918 # prefix - means exclude this path, leave out of mappings
1919 exclude = False
1920 if depot_side.startswith("-"):
1921 exclude = True
1922 depot_side = depot_side[1:]
1923
1924 if not exclude:
1925 self.mappings.append(depot_side)
1926
1927 def convert_client_path(self, clientFile):
1928 # chop off //client/ part to make it relative
1929 if not clientFile.startswith(self.client_prefix):
1930 die("No prefix '%s' on clientFile '%s'" %
1931 (self.client_prefix, clientFile))
1932 return clientFile[len(self.client_prefix):]
1933
1934 def update_client_spec_path_cache(self, files):
1935 """ Caching file paths by "p4 where" batch query """
1936
1937 # List depot file paths exclude that already cached
1938 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1939
1940 if len(fileArgs) == 0:
1941 return # All files in cache
1942
1943 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1944 for res in where_result:
1945 if "code" in res and res["code"] == "error":
1946 # assume error is "... file(s) not in client view"
1947 continue
1948 if "clientFile" not in res:
1949 die("No clientFile in 'p4 where' output")
1950 if "unmap" in res:
1951 # it will list all of them, but only one not unmap-ped
1952 continue
1953 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1954
1955 # not found files or unmap files set to ""
1956 for depotFile in fileArgs:
1957 if depotFile not in self.client_spec_path_cache:
1958 self.client_spec_path_cache[depotFile] = ""
1959
1960 def map_in_client(self, depot_path):
1961 """Return the relative location in the client where this
1962 depot file should live. Returns "" if the file should
1963 not be mapped in the client."""
1964
1965 if depot_path in self.client_spec_path_cache:
1966 return self.client_spec_path_cache[depot_path]
1967
1968 die( "Error: %s is not found in client spec path" % depot_path )
1969 return ""
1970
1971 class P4Sync(Command, P4UserMap):
1972 delete_actions = ( "delete", "move/delete", "purge" )
1973
1974 def __init__(self):
1975 Command.__init__(self)
1976 P4UserMap.__init__(self)
1977 self.options = [
1978 optparse.make_option("--branch", dest="branch"),
1979 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1980 optparse.make_option("--changesfile", dest="changesFile"),
1981 optparse.make_option("--silent", dest="silent", action="store_true"),
1982 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1983 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1984 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1985 help="Import into refs/heads/ , not refs/remotes"),
1986 optparse.make_option("--max-changes", dest="maxChanges",
1987 help="Maximum number of changes to import"),
1988 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1989 help="Internal block size to use when iteratively calling p4 changes"),
1990 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1991 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1992 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1993 help="Only sync files that are included in the Perforce Client Spec"),
1994 optparse.make_option("-/", dest="cloneExclude",
1995 action="append", type="string",
1996 help="exclude depot path"),
1997 ]
1998 self.description = """Imports from Perforce into a git repository.\n
1999 example:
2000 //depot/my/project/ -- to import the current head
2001 //depot/my/project/@all -- to import everything
2002 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2003
2004 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2005
2006 self.usage += " //depot/path[@revRange]"
2007 self.silent = False
2008 self.createdBranches = set()
2009 self.committedChanges = set()
2010 self.branch = ""
2011 self.detectBranches = False
2012 self.detectLabels = False
2013 self.importLabels = False
2014 self.changesFile = ""
2015 self.syncWithOrigin = True
2016 self.importIntoRemotes = True
2017 self.maxChanges = ""
2018 self.changes_block_size = None
2019 self.keepRepoPath = False
2020 self.depotPaths = None
2021 self.p4BranchesInGit = []
2022 self.cloneExclude = []
2023 self.useClientSpec = False
2024 self.useClientSpec_from_options = False
2025 self.clientSpecDirs = None
2026 self.tempBranches = []
2027 self.tempBranchLocation = "git-p4-tmp"
2028
2029 if gitConfig("git-p4.syncFromOrigin") == "false":
2030 self.syncWithOrigin = False
2031
2032 # This is required for the "append" cloneExclude action
2033 def ensure_value(self, attr, value):
2034 if not hasattr(self, attr) or getattr(self, attr) is None:
2035 setattr(self, attr, value)
2036 return getattr(self, attr)
2037
2038 # Force a checkpoint in fast-import and wait for it to finish
2039 def checkpoint(self):
2040 self.gitStream.write("checkpoint\n\n")
2041 self.gitStream.write("progress checkpoint\n\n")
2042 out = self.gitOutput.readline()
2043 if self.verbose:
2044 print "checkpoint finished: " + out
2045
2046 def extractFilesFromCommit(self, commit):
2047 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2048 for path in self.cloneExclude]
2049 files = []
2050 fnum = 0
2051 while commit.has_key("depotFile%s" % fnum):
2052 path = commit["depotFile%s" % fnum]
2053
2054 if [p for p in self.cloneExclude
2055 if p4PathStartsWith(path, p)]:
2056 found = False
2057 else:
2058 found = [p for p in self.depotPaths
2059 if p4PathStartsWith(path, p)]
2060 if not found:
2061 fnum = fnum + 1
2062 continue
2063
2064 file = {}
2065 file["path"] = path
2066 file["rev"] = commit["rev%s" % fnum]
2067 file["action"] = commit["action%s" % fnum]
2068 file["type"] = commit["type%s" % fnum]
2069 files.append(file)
2070 fnum = fnum + 1
2071 return files
2072
2073 def stripRepoPath(self, path, prefixes):
2074 """When streaming files, this is called to map a p4 depot path
2075 to where it should go in git. The prefixes are either
2076 self.depotPaths, or self.branchPrefixes in the case of
2077 branch detection."""
2078
2079 if self.useClientSpec:
2080 # branch detection moves files up a level (the branch name)
2081 # from what client spec interpretation gives
2082 path = self.clientSpecDirs.map_in_client(path)
2083 if self.detectBranches:
2084 for b in self.knownBranches:
2085 if path.startswith(b + "/"):
2086 path = path[len(b)+1:]
2087
2088 elif self.keepRepoPath:
2089 # Preserve everything in relative path name except leading
2090 # //depot/; just look at first prefix as they all should
2091 # be in the same depot.
2092 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2093 if p4PathStartsWith(path, depot):
2094 path = path[len(depot):]
2095
2096 else:
2097 for p in prefixes:
2098 if p4PathStartsWith(path, p):
2099 path = path[len(p):]
2100 break
2101
2102 path = wildcard_decode(path)
2103 return path
2104
2105 def splitFilesIntoBranches(self, commit):
2106 """Look at each depotFile in the commit to figure out to what
2107 branch it belongs."""
2108
2109 if self.clientSpecDirs:
2110 files = self.extractFilesFromCommit(commit)
2111 self.clientSpecDirs.update_client_spec_path_cache(files)
2112
2113 branches = {}
2114 fnum = 0
2115 while commit.has_key("depotFile%s" % fnum):
2116 path = commit["depotFile%s" % fnum]
2117 found = [p for p in self.depotPaths
2118 if p4PathStartsWith(path, p)]
2119 if not found:
2120 fnum = fnum + 1
2121 continue
2122
2123 file = {}
2124 file["path"] = path
2125 file["rev"] = commit["rev%s" % fnum]
2126 file["action"] = commit["action%s" % fnum]
2127 file["type"] = commit["type%s" % fnum]
2128 fnum = fnum + 1
2129
2130 # start with the full relative path where this file would
2131 # go in a p4 client
2132 if self.useClientSpec:
2133 relPath = self.clientSpecDirs.map_in_client(path)
2134 else:
2135 relPath = self.stripRepoPath(path, self.depotPaths)
2136
2137 for branch in self.knownBranches.keys():
2138 # add a trailing slash so that a commit into qt/4.2foo
2139 # doesn't end up in qt/4.2, e.g.
2140 if relPath.startswith(branch + "/"):
2141 if branch not in branches:
2142 branches[branch] = []
2143 branches[branch].append(file)
2144 break
2145
2146 return branches
2147
2148 # output one file from the P4 stream
2149 # - helper for streamP4Files
2150
2151 def streamOneP4File(self, file, contents):
2152 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2153 if verbose:
2154 sys.stderr.write("%s\n" % relPath)
2155
2156 (type_base, type_mods) = split_p4_type(file["type"])
2157
2158 git_mode = "100644"
2159 if "x" in type_mods:
2160 git_mode = "100755"
2161 if type_base == "symlink":
2162 git_mode = "120000"
2163 # p4 print on a symlink sometimes contains "target\n";
2164 # if it does, remove the newline
2165 data = ''.join(contents)
2166 if not data:
2167 # Some version of p4 allowed creating a symlink that pointed
2168 # to nothing. This causes p4 errors when checking out such
2169 # a change, and errors here too. Work around it by ignoring
2170 # the bad symlink; hopefully a future change fixes it.
2171 print "\nIgnoring empty symlink in %s" % file['depotFile']
2172 return
2173 elif data[-1] == '\n':
2174 contents = [data[:-1]]
2175 else:
2176 contents = [data]
2177
2178 if type_base == "utf16":
2179 # p4 delivers different text in the python output to -G
2180 # than it does when using "print -o", or normal p4 client
2181 # operations. utf16 is converted to ascii or utf8, perhaps.
2182 # But ascii text saved as -t utf16 is completely mangled.
2183 # Invoke print -o to get the real contents.
2184 #
2185 # On windows, the newlines will always be mangled by print, so put
2186 # them back too. This is not needed to the cygwin windows version,
2187 # just the native "NT" type.
2188 #
2189 text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
2190 if p4_version_string().find("/NT") >= 0:
2191 text = text.replace("\r\n", "\n")
2192 contents = [ text ]
2193
2194 if type_base == "apple":
2195 # Apple filetype files will be streamed as a concatenation of
2196 # its appledouble header and the contents. This is useless
2197 # on both macs and non-macs. If using "print -q -o xx", it
2198 # will create "xx" with the data, and "%xx" with the header.
2199 # This is also not very useful.
2200 #
2201 # Ideally, someday, this script can learn how to generate
2202 # appledouble files directly and import those to git, but
2203 # non-mac machines can never find a use for apple filetype.
2204 print "\nIgnoring apple filetype file %s" % file['depotFile']
2205 return
2206
2207 # Note that we do not try to de-mangle keywords on utf16 files,
2208 # even though in theory somebody may want that.
2209 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2210 if pattern:
2211 regexp = re.compile(pattern, re.VERBOSE)
2212 text = ''.join(contents)
2213 text = regexp.sub(r'$\1$', text)
2214 contents = [ text ]
2215
2216 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2217
2218 # total length...
2219 length = 0
2220 for d in contents:
2221 length = length + len(d)
2222
2223 self.gitStream.write("data %d\n" % length)
2224 for d in contents:
2225 self.gitStream.write(d)
2226 self.gitStream.write("\n")
2227
2228 def streamOneP4Deletion(self, file):
2229 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2230 if verbose:
2231 sys.stderr.write("delete %s\n" % relPath)
2232 self.gitStream.write("D %s\n" % relPath)
2233
2234 # handle another chunk of streaming data
2235 def streamP4FilesCb(self, marshalled):
2236
2237 # catch p4 errors and complain
2238 err = None
2239 if "code" in marshalled:
2240 if marshalled["code"] == "error":
2241 if "data" in marshalled:
2242 err = marshalled["data"].rstrip()
2243 if err:
2244 f = None
2245 if self.stream_have_file_info:
2246 if "depotFile" in self.stream_file:
2247 f = self.stream_file["depotFile"]
2248 # force a failure in fast-import, else an empty
2249 # commit will be made
2250 self.gitStream.write("\n")
2251 self.gitStream.write("die-now\n")
2252 self.gitStream.close()
2253 # ignore errors, but make sure it exits first
2254 self.importProcess.wait()
2255 if f:
2256 die("Error from p4 print for %s: %s" % (f, err))
2257 else:
2258 die("Error from p4 print: %s" % err)
2259
2260 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2261 # start of a new file - output the old one first
2262 self.streamOneP4File(self.stream_file, self.stream_contents)
2263 self.stream_file = {}
2264 self.stream_contents = []
2265 self.stream_have_file_info = False
2266
2267 # pick up the new file information... for the
2268 # 'data' field we need to append to our array
2269 for k in marshalled.keys():
2270 if k == 'data':
2271 self.stream_contents.append(marshalled['data'])
2272 else:
2273 self.stream_file[k] = marshalled[k]
2274
2275 self.stream_have_file_info = True
2276
2277 # Stream directly from "p4 files" into "git fast-import"
2278 def streamP4Files(self, files):
2279 filesForCommit = []
2280 filesToRead = []
2281 filesToDelete = []
2282
2283 for f in files:
2284 # if using a client spec, only add the files that have
2285 # a path in the client
2286 if self.clientSpecDirs:
2287 if self.clientSpecDirs.map_in_client(f['path']) == "":
2288 continue
2289
2290 filesForCommit.append(f)
2291 if f['action'] in self.delete_actions:
2292 filesToDelete.append(f)
2293 else:
2294 filesToRead.append(f)
2295
2296 # deleted files...
2297 for f in filesToDelete:
2298 self.streamOneP4Deletion(f)
2299
2300 if len(filesToRead) > 0:
2301 self.stream_file = {}
2302 self.stream_contents = []
2303 self.stream_have_file_info = False
2304
2305 # curry self argument
2306 def streamP4FilesCbSelf(entry):
2307 self.streamP4FilesCb(entry)
2308
2309 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2310
2311 p4CmdList(["-x", "-", "print"],
2312 stdin=fileArgs,
2313 cb=streamP4FilesCbSelf)
2314
2315 # do the last chunk
2316 if self.stream_file.has_key('depotFile'):
2317 self.streamOneP4File(self.stream_file, self.stream_contents)
2318
2319 def make_email(self, userid):
2320 if userid in self.users:
2321 return self.users[userid]
2322 else:
2323 return "%s <a@b>" % userid
2324
2325 # Stream a p4 tag
2326 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2327 if verbose:
2328 print "writing tag %s for commit %s" % (labelName, commit)
2329 gitStream.write("tag %s\n" % labelName)
2330 gitStream.write("from %s\n" % commit)
2331
2332 if labelDetails.has_key('Owner'):
2333 owner = labelDetails["Owner"]
2334 else:
2335 owner = None
2336
2337 # Try to use the owner of the p4 label, or failing that,
2338 # the current p4 user id.
2339 if owner:
2340 email = self.make_email(owner)
2341 else:
2342 email = self.make_email(self.p4UserId())
2343 tagger = "%s %s %s" % (email, epoch, self.tz)
2344
2345 gitStream.write("tagger %s\n" % tagger)
2346
2347 print "labelDetails=",labelDetails
2348 if labelDetails.has_key('Description'):
2349 description = labelDetails['Description']
2350 else:
2351 description = 'Label from git p4'
2352
2353 gitStream.write("data %d\n" % len(description))
2354 gitStream.write(description)
2355 gitStream.write("\n")
2356
2357 def commit(self, details, files, branch, parent = ""):
2358 epoch = details["time"]
2359 author = details["user"]
2360
2361 if self.verbose:
2362 print "commit into %s" % branch
2363
2364 # start with reading files; if that fails, we should not
2365 # create a commit.
2366 new_files = []
2367 for f in files:
2368 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2369 new_files.append (f)
2370 else:
2371 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2372
2373 if self.clientSpecDirs:
2374 self.clientSpecDirs.update_client_spec_path_cache(files)
2375
2376 self.gitStream.write("commit %s\n" % branch)
2377 # gitStream.write("mark :%s\n" % details["change"])
2378 self.committedChanges.add(int(details["change"]))
2379 committer = ""
2380 if author not in self.users:
2381 self.getUserMapFromPerforceServer()
2382 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2383
2384 self.gitStream.write("committer %s\n" % committer)
2385
2386 self.gitStream.write("data <<EOT\n")
2387 self.gitStream.write(details["desc"])
2388 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2389 (','.join(self.branchPrefixes), details["change"]))
2390 if len(details['options']) > 0:
2391 self.gitStream.write(": options = %s" % details['options'])
2392 self.gitStream.write("]\nEOT\n\n")
2393
2394 if len(parent) > 0:
2395 if self.verbose:
2396 print "parent %s" % parent
2397 self.gitStream.write("from %s\n" % parent)
2398
2399 self.streamP4Files(new_files)
2400 self.gitStream.write("\n")
2401
2402 change = int(details["change"])
2403
2404 if self.labels.has_key(change):
2405 label = self.labels[change]
2406 labelDetails = label[0]
2407 labelRevisions = label[1]
2408 if self.verbose:
2409 print "Change %s is labelled %s" % (change, labelDetails)
2410
2411 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2412 for p in self.branchPrefixes])
2413
2414 if len(files) == len(labelRevisions):
2415
2416 cleanedFiles = {}
2417 for info in files:
2418 if info["action"] in self.delete_actions:
2419 continue
2420 cleanedFiles[info["depotFile"]] = info["rev"]
2421
2422 if cleanedFiles == labelRevisions:
2423 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2424
2425 else:
2426 if not self.silent:
2427 print ("Tag %s does not match with change %s: files do not match."
2428 % (labelDetails["label"], change))
2429
2430 else:
2431 if not self.silent:
2432 print ("Tag %s does not match with change %s: file count is different."
2433 % (labelDetails["label"], change))
2434
2435 # Build a dictionary of changelists and labels, for "detect-labels" option.
2436 def getLabels(self):
2437 self.labels = {}
2438
2439 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2440 if len(l) > 0 and not self.silent:
2441 print "Finding files belonging to labels in %s" % `self.depotPaths`
2442
2443 for output in l:
2444 label = output["label"]
2445 revisions = {}
2446 newestChange = 0
2447 if self.verbose:
2448 print "Querying files for label %s" % label
2449 for file in p4CmdList(["files"] +
2450 ["%s...@%s" % (p, label)
2451 for p in self.depotPaths]):
2452 revisions[file["depotFile"]] = file["rev"]
2453 change = int(file["change"])
2454 if change > newestChange:
2455 newestChange = change
2456
2457 self.labels[newestChange] = [output, revisions]
2458
2459 if self.verbose:
2460 print "Label changes: %s" % self.labels.keys()
2461
2462 # Import p4 labels as git tags. A direct mapping does not
2463 # exist, so assume that if all the files are at the same revision
2464 # then we can use that, or it's something more complicated we should
2465 # just ignore.
2466 def importP4Labels(self, stream, p4Labels):
2467 if verbose:
2468 print "import p4 labels: " + ' '.join(p4Labels)
2469
2470 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2471 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2472 if len(validLabelRegexp) == 0:
2473 validLabelRegexp = defaultLabelRegexp
2474 m = re.compile(validLabelRegexp)
2475
2476 for name in p4Labels:
2477 commitFound = False
2478
2479 if not m.match(name):
2480 if verbose:
2481 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2482 continue
2483
2484 if name in ignoredP4Labels:
2485 continue
2486
2487 labelDetails = p4CmdList(['label', "-o", name])[0]
2488
2489 # get the most recent changelist for each file in this label
2490 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2491 for p in self.depotPaths])
2492
2493 if change.has_key('change'):
2494 # find the corresponding git commit; take the oldest commit
2495 changelist = int(change['change'])
2496 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2497 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2498 if len(gitCommit) == 0:
2499 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2500 else:
2501 gitCommit = gitCommit.strip()
2502 commitFound = True
2503 # Convert from p4 time format
2504 try:
2505 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2506 except ValueError:
2507 print "Could not convert label time %s" % labelDetails['Update']
2508 tmwhen = 1
2509
2510 when = int(time.mktime(tmwhen))
2511 self.streamTag(stream, name, labelDetails, gitCommit, when)
2512 if verbose:
2513 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2514 else:
2515 if verbose:
2516 print "Label %s has no changelists - possibly deleted?" % name
2517
2518 if not commitFound:
2519 # We can't import this label; don't try again as it will get very
2520 # expensive repeatedly fetching all the files for labels that will
2521 # never be imported. If the label is moved in the future, the
2522 # ignore will need to be removed manually.
2523 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2524
2525 def guessProjectName(self):
2526 for p in self.depotPaths:
2527 if p.endswith("/"):
2528 p = p[:-1]
2529 p = p[p.strip().rfind("/") + 1:]
2530 if not p.endswith("/"):
2531 p += "/"
2532 return p
2533
2534 def getBranchMapping(self):
2535 lostAndFoundBranches = set()
2536
2537 user = gitConfig("git-p4.branchUser")
2538 if len(user) > 0:
2539 command = "branches -u %s" % user
2540 else:
2541 command = "branches"
2542
2543 for info in p4CmdList(command):
2544 details = p4Cmd(["branch", "-o", info["branch"]])
2545 viewIdx = 0
2546 while details.has_key("View%s" % viewIdx):
2547 paths = details["View%s" % viewIdx].split(" ")
2548 viewIdx = viewIdx + 1
2549 # require standard //depot/foo/... //depot/bar/... mapping
2550 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2551 continue
2552 source = paths[0]
2553 destination = paths[1]
2554 ## HACK
2555 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2556 source = source[len(self.depotPaths[0]):-4]
2557 destination = destination[len(self.depotPaths[0]):-4]
2558
2559 if destination in self.knownBranches:
2560 if not self.silent:
2561 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2562 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2563 continue
2564
2565 self.knownBranches[destination] = source
2566
2567 lostAndFoundBranches.discard(destination)
2568
2569 if source not in self.knownBranches:
2570 lostAndFoundBranches.add(source)
2571
2572 # Perforce does not strictly require branches to be defined, so we also
2573 # check git config for a branch list.
2574 #
2575 # Example of branch definition in git config file:
2576 # [git-p4]
2577 # branchList=main:branchA
2578 # branchList=main:branchB
2579 # branchList=branchA:branchC
2580 configBranches = gitConfigList("git-p4.branchList")
2581 for branch in configBranches:
2582 if branch:
2583 (source, destination) = branch.split(":")
2584 self.knownBranches[destination] = source
2585
2586 lostAndFoundBranches.discard(destination)
2587
2588 if source not in self.knownBranches:
2589 lostAndFoundBranches.add(source)
2590
2591
2592 for branch in lostAndFoundBranches:
2593 self.knownBranches[branch] = branch
2594
2595 def getBranchMappingFromGitBranches(self):
2596 branches = p4BranchesInGit(self.importIntoRemotes)
2597 for branch in branches.keys():
2598 if branch == "master":
2599 branch = "main"
2600 else:
2601 branch = branch[len(self.projectName):]
2602 self.knownBranches[branch] = branch
2603
2604 def updateOptionDict(self, d):
2605 option_keys = {}
2606 if self.keepRepoPath:
2607 option_keys['keepRepoPath'] = 1
2608
2609 d["options"] = ' '.join(sorted(option_keys.keys()))
2610
2611 def readOptions(self, d):
2612 self.keepRepoPath = (d.has_key('options')
2613 and ('keepRepoPath' in d['options']))
2614
2615 def gitRefForBranch(self, branch):
2616 if branch == "main":
2617 return self.refPrefix + "master"
2618
2619 if len(branch) <= 0:
2620 return branch
2621
2622 return self.refPrefix + self.projectName + branch
2623
2624 def gitCommitByP4Change(self, ref, change):
2625 if self.verbose:
2626 print "looking in ref " + ref + " for change %s using bisect..." % change
2627
2628 earliestCommit = ""
2629 latestCommit = parseRevision(ref)
2630
2631 while True:
2632 if self.verbose:
2633 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2634 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2635 if len(next) == 0:
2636 if self.verbose:
2637 print "argh"
2638 return ""
2639 log = extractLogMessageFromGitCommit(next)
2640 settings = extractSettingsGitLog(log)
2641 currentChange = int(settings['change'])
2642 if self.verbose:
2643 print "current change %s" % currentChange
2644
2645 if currentChange == change:
2646 if self.verbose:
2647 print "found %s" % next
2648 return next
2649
2650 if currentChange < change:
2651 earliestCommit = "^%s" % next
2652 else:
2653 latestCommit = "%s" % next
2654
2655 return ""
2656
2657 def importNewBranch(self, branch, maxChange):
2658 # make fast-import flush all changes to disk and update the refs using the checkpoint
2659 # command so that we can try to find the branch parent in the git history
2660 self.gitStream.write("checkpoint\n\n");
2661 self.gitStream.flush();
2662 branchPrefix = self.depotPaths[0] + branch + "/"
2663 range = "@1,%s" % maxChange
2664 #print "prefix" + branchPrefix
2665 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2666 if len(changes) <= 0:
2667 return False
2668 firstChange = changes[0]
2669 #print "first change in branch: %s" % firstChange
2670 sourceBranch = self.knownBranches[branch]
2671 sourceDepotPath = self.depotPaths[0] + sourceBranch
2672 sourceRef = self.gitRefForBranch(sourceBranch)
2673 #print "source " + sourceBranch
2674
2675 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2676 #print "branch parent: %s" % branchParentChange
2677 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2678 if len(gitParent) > 0:
2679 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2680 #print "parent git commit: %s" % gitParent
2681
2682 self.importChanges(changes)
2683 return True
2684
2685 def searchParent(self, parent, branch, target):
2686 parentFound = False
2687 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2688 "--no-merges", parent]):
2689 blob = blob.strip()
2690 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2691 parentFound = True
2692 if self.verbose:
2693 print "Found parent of %s in commit %s" % (branch, blob)
2694 break
2695 if parentFound:
2696 return blob
2697 else:
2698 return None
2699
2700 def importChanges(self, changes):
2701 cnt = 1
2702 for change in changes:
2703 description = p4_describe(change)
2704 self.updateOptionDict(description)
2705
2706 if not self.silent:
2707 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2708 sys.stdout.flush()
2709 cnt = cnt + 1
2710
2711 try:
2712 if self.detectBranches:
2713 branches = self.splitFilesIntoBranches(description)
2714 for branch in branches.keys():
2715 ## HACK --hwn
2716 branchPrefix = self.depotPaths[0] + branch + "/"
2717 self.branchPrefixes = [ branchPrefix ]
2718
2719 parent = ""
2720
2721 filesForCommit = branches[branch]
2722
2723 if self.verbose:
2724 print "branch is %s" % branch
2725
2726 self.updatedBranches.add(branch)
2727
2728 if branch not in self.createdBranches:
2729 self.createdBranches.add(branch)
2730 parent = self.knownBranches[branch]
2731 if parent == branch:
2732 parent = ""
2733 else:
2734 fullBranch = self.projectName + branch
2735 if fullBranch not in self.p4BranchesInGit:
2736 if not self.silent:
2737 print("\n Importing new branch %s" % fullBranch);
2738 if self.importNewBranch(branch, change - 1):
2739 parent = ""
2740 self.p4BranchesInGit.append(fullBranch)
2741 if not self.silent:
2742 print("\n Resuming with change %s" % change);
2743
2744 if self.verbose:
2745 print "parent determined through known branches: %s" % parent
2746
2747 branch = self.gitRefForBranch(branch)
2748 parent = self.gitRefForBranch(parent)
2749
2750 if self.verbose:
2751 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2752
2753 if len(parent) == 0 and branch in self.initialParents:
2754 parent = self.initialParents[branch]
2755 del self.initialParents[branch]
2756
2757 blob = None
2758 if len(parent) > 0:
2759 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2760 if self.verbose:
2761 print "Creating temporary branch: " + tempBranch
2762 self.commit(description, filesForCommit, tempBranch)
2763 self.tempBranches.append(tempBranch)
2764 self.checkpoint()
2765 blob = self.searchParent(parent, branch, tempBranch)
2766 if blob:
2767 self.commit(description, filesForCommit, branch, blob)
2768 else:
2769 if self.verbose:
2770 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2771 self.commit(description, filesForCommit, branch, parent)
2772 else:
2773 files = self.extractFilesFromCommit(description)
2774 self.commit(description, files, self.branch,
2775 self.initialParent)
2776 # only needed once, to connect to the previous commit
2777 self.initialParent = ""
2778 except IOError:
2779 print self.gitError.read()
2780 sys.exit(1)
2781
2782 def importHeadRevision(self, revision):
2783 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2784
2785 details = {}
2786 details["user"] = "git perforce import user"
2787 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2788 % (' '.join(self.depotPaths), revision))
2789 details["change"] = revision
2790 newestRevision = 0
2791
2792 fileCnt = 0
2793 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2794
2795 for info in p4CmdList(["files"] + fileArgs):
2796
2797 if 'code' in info and info['code'] == 'error':
2798 sys.stderr.write("p4 returned an error: %s\n"
2799 % info['data'])
2800 if info['data'].find("must refer to client") >= 0:
2801 sys.stderr.write("This particular p4 error is misleading.\n")
2802 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2803 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2804 sys.exit(1)
2805 if 'p4ExitCode' in info:
2806 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2807 sys.exit(1)
2808
2809
2810 change = int(info["change"])
2811 if change > newestRevision:
2812 newestRevision = change
2813
2814 if info["action"] in self.delete_actions:
2815 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2816 #fileCnt = fileCnt + 1
2817 continue
2818
2819 for prop in ["depotFile", "rev", "action", "type" ]:
2820 details["%s%s" % (prop, fileCnt)] = info[prop]
2821
2822 fileCnt = fileCnt + 1
2823
2824 details["change"] = newestRevision
2825
2826 # Use time from top-most change so that all git p4 clones of
2827 # the same p4 repo have the same commit SHA1s.
2828 res = p4_describe(newestRevision)
2829 details["time"] = res["time"]
2830
2831 self.updateOptionDict(details)
2832 try:
2833 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2834 except IOError:
2835 print "IO error with git fast-import. Is your git version recent enough?"
2836 print self.gitError.read()
2837
2838
2839 def run(self, args):
2840 self.depotPaths = []
2841 self.changeRange = ""
2842 self.previousDepotPaths = []
2843 self.hasOrigin = False
2844
2845 # map from branch depot path to parent branch
2846 self.knownBranches = {}
2847 self.initialParents = {}
2848
2849 if self.importIntoRemotes:
2850 self.refPrefix = "refs/remotes/p4/"
2851 else:
2852 self.refPrefix = "refs/heads/p4/"
2853
2854 if self.syncWithOrigin:
2855 self.hasOrigin = originP4BranchesExist()
2856 if self.hasOrigin:
2857 if not self.silent:
2858 print 'Syncing with origin first, using "git fetch origin"'
2859 system("git fetch origin")
2860
2861 branch_arg_given = bool(self.branch)
2862 if len(self.branch) == 0:
2863 self.branch = self.refPrefix + "master"
2864 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2865 system("git update-ref %s refs/heads/p4" % self.branch)
2866 system("git branch -D p4")
2867
2868 # accept either the command-line option, or the configuration variable
2869 if self.useClientSpec:
2870 # will use this after clone to set the variable
2871 self.useClientSpec_from_options = True
2872 else:
2873 if gitConfigBool("git-p4.useclientspec"):
2874 self.useClientSpec = True
2875 if self.useClientSpec:
2876 self.clientSpecDirs = getClientSpec()
2877
2878 # TODO: should always look at previous commits,
2879 # merge with previous imports, if possible.
2880 if args == []:
2881 if self.hasOrigin:
2882 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2883
2884 # branches holds mapping from branch name to sha1
2885 branches = p4BranchesInGit(self.importIntoRemotes)
2886
2887 # restrict to just this one, disabling detect-branches
2888 if branch_arg_given:
2889 short = self.branch.split("/")[-1]
2890 if short in branches:
2891 self.p4BranchesInGit = [ short ]
2892 else:
2893 self.p4BranchesInGit = branches.keys()
2894
2895 if len(self.p4BranchesInGit) > 1:
2896 if not self.silent:
2897 print "Importing from/into multiple branches"
2898 self.detectBranches = True
2899 for branch in branches.keys():
2900 self.initialParents[self.refPrefix + branch] = \
2901 branches[branch]
2902
2903 if self.verbose:
2904 print "branches: %s" % self.p4BranchesInGit
2905
2906 p4Change = 0
2907 for branch in self.p4BranchesInGit:
2908 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2909
2910 settings = extractSettingsGitLog(logMsg)
2911
2912 self.readOptions(settings)
2913 if (settings.has_key('depot-paths')
2914 and settings.has_key ('change')):
2915 change = int(settings['change']) + 1
2916 p4Change = max(p4Change, change)
2917
2918 depotPaths = sorted(settings['depot-paths'])
2919 if self.previousDepotPaths == []:
2920 self.previousDepotPaths = depotPaths
2921 else:
2922 paths = []
2923 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2924 prev_list = prev.split("/")
2925 cur_list = cur.split("/")
2926 for i in range(0, min(len(cur_list), len(prev_list))):
2927 if cur_list[i] <> prev_list[i]:
2928 i = i - 1
2929 break
2930
2931 paths.append ("/".join(cur_list[:i + 1]))
2932
2933 self.previousDepotPaths = paths
2934
2935 if p4Change > 0:
2936 self.depotPaths = sorted(self.previousDepotPaths)
2937 self.changeRange = "@%s,#head" % p4Change
2938 if not self.silent and not self.detectBranches:
2939 print "Performing incremental import into %s git branch" % self.branch
2940
2941 # accept multiple ref name abbreviations:
2942 # refs/foo/bar/branch -> use it exactly
2943 # p4/branch -> prepend refs/remotes/ or refs/heads/
2944 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2945 if not self.branch.startswith("refs/"):
2946 if self.importIntoRemotes:
2947 prepend = "refs/remotes/"
2948 else:
2949 prepend = "refs/heads/"
2950 if not self.branch.startswith("p4/"):
2951 prepend += "p4/"
2952 self.branch = prepend + self.branch
2953
2954 if len(args) == 0 and self.depotPaths:
2955 if not self.silent:
2956 print "Depot paths: %s" % ' '.join(self.depotPaths)
2957 else:
2958 if self.depotPaths and self.depotPaths != args:
2959 print ("previous import used depot path %s and now %s was specified. "
2960 "This doesn't work!" % (' '.join (self.depotPaths),
2961 ' '.join (args)))
2962 sys.exit(1)
2963
2964 self.depotPaths = sorted(args)
2965
2966 revision = ""
2967 self.users = {}
2968
2969 # Make sure no revision specifiers are used when --changesfile
2970 # is specified.
2971 bad_changesfile = False
2972 if len(self.changesFile) > 0:
2973 for p in self.depotPaths:
2974 if p.find("@") >= 0 or p.find("#") >= 0:
2975 bad_changesfile = True
2976 break
2977 if bad_changesfile:
2978 die("Option --changesfile is incompatible with revision specifiers")
2979
2980 newPaths = []
2981 for p in self.depotPaths:
2982 if p.find("@") != -1:
2983 atIdx = p.index("@")
2984 self.changeRange = p[atIdx:]
2985 if self.changeRange == "@all":
2986 self.changeRange = ""
2987 elif ',' not in self.changeRange:
2988 revision = self.changeRange
2989 self.changeRange = ""
2990 p = p[:atIdx]
2991 elif p.find("#") != -1:
2992 hashIdx = p.index("#")
2993 revision = p[hashIdx:]
2994 p = p[:hashIdx]
2995 elif self.previousDepotPaths == []:
2996 # pay attention to changesfile, if given, else import
2997 # the entire p4 tree at the head revision
2998 if len(self.changesFile) == 0:
2999 revision = "#head"
3000
3001 p = re.sub ("\.\.\.$", "", p)
3002 if not p.endswith("/"):
3003 p += "/"
3004
3005 newPaths.append(p)
3006
3007 self.depotPaths = newPaths
3008
3009 # --detect-branches may change this for each branch
3010 self.branchPrefixes = self.depotPaths
3011
3012 self.loadUserMapFromCache()
3013 self.labels = {}
3014 if self.detectLabels:
3015 self.getLabels();
3016
3017 if self.detectBranches:
3018 ## FIXME - what's a P4 projectName ?
3019 self.projectName = self.guessProjectName()
3020
3021 if self.hasOrigin:
3022 self.getBranchMappingFromGitBranches()
3023 else:
3024 self.getBranchMapping()
3025 if self.verbose:
3026 print "p4-git branches: %s" % self.p4BranchesInGit
3027 print "initial parents: %s" % self.initialParents
3028 for b in self.p4BranchesInGit:
3029 if b != "master":
3030
3031 ## FIXME
3032 b = b[len(self.projectName):]
3033 self.createdBranches.add(b)
3034
3035 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3036
3037 self.importProcess = subprocess.Popen(["git", "fast-import"],
3038 stdin=subprocess.PIPE,
3039 stdout=subprocess.PIPE,
3040 stderr=subprocess.PIPE);
3041 self.gitOutput = self.importProcess.stdout
3042 self.gitStream = self.importProcess.stdin
3043 self.gitError = self.importProcess.stderr
3044
3045 if revision:
3046 self.importHeadRevision(revision)
3047 else:
3048 changes = []
3049
3050 if len(self.changesFile) > 0:
3051 output = open(self.changesFile).readlines()
3052 changeSet = set()
3053 for line in output:
3054 changeSet.add(int(line))
3055
3056 for change in changeSet:
3057 changes.append(change)
3058
3059 changes.sort()
3060 else:
3061 # catch "git p4 sync" with no new branches, in a repo that
3062 # does not have any existing p4 branches
3063 if len(args) == 0:
3064 if not self.p4BranchesInGit:
3065 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3066
3067 # The default branch is master, unless --branch is used to
3068 # specify something else. Make sure it exists, or complain
3069 # nicely about how to use --branch.
3070 if not self.detectBranches:
3071 if not branch_exists(self.branch):
3072 if branch_arg_given:
3073 die("Error: branch %s does not exist." % self.branch)
3074 else:
3075 die("Error: no branch %s; perhaps specify one with --branch." %
3076 self.branch)
3077
3078 if self.verbose:
3079 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3080 self.changeRange)
3081 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3082
3083 if len(self.maxChanges) > 0:
3084 changes = changes[:min(int(self.maxChanges), len(changes))]
3085
3086 if len(changes) == 0:
3087 if not self.silent:
3088 print "No changes to import!"
3089 else:
3090 if not self.silent and not self.detectBranches:
3091 print "Import destination: %s" % self.branch
3092
3093 self.updatedBranches = set()
3094
3095 if not self.detectBranches:
3096 if args:
3097 # start a new branch
3098 self.initialParent = ""
3099 else:
3100 # build on a previous revision
3101 self.initialParent = parseRevision(self.branch)
3102
3103 self.importChanges(changes)
3104
3105 if not self.silent:
3106 print ""
3107 if len(self.updatedBranches) > 0:
3108 sys.stdout.write("Updated branches: ")
3109 for b in self.updatedBranches:
3110 sys.stdout.write("%s " % b)
3111 sys.stdout.write("\n")
3112
3113 if gitConfigBool("git-p4.importLabels"):
3114 self.importLabels = True
3115
3116 if self.importLabels:
3117 p4Labels = getP4Labels(self.depotPaths)
3118 gitTags = getGitTags()
3119
3120 missingP4Labels = p4Labels - gitTags
3121 self.importP4Labels(self.gitStream, missingP4Labels)
3122
3123 self.gitStream.close()
3124 if self.importProcess.wait() != 0:
3125 die("fast-import failed: %s" % self.gitError.read())
3126 self.gitOutput.close()
3127 self.gitError.close()
3128
3129 # Cleanup temporary branches created during import
3130 if self.tempBranches != []:
3131 for branch in self.tempBranches:
3132 read_pipe("git update-ref -d %s" % branch)
3133 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3134
3135 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3136 # a convenient shortcut refname "p4".
3137 if self.importIntoRemotes:
3138 head_ref = self.refPrefix + "HEAD"
3139 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3140 system(["git", "symbolic-ref", head_ref, self.branch])
3141
3142 return True
3143
3144 class P4Rebase(Command):
3145 def __init__(self):
3146 Command.__init__(self)
3147 self.options = [
3148 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3149 ]
3150 self.importLabels = False
3151 self.description = ("Fetches the latest revision from perforce and "
3152 + "rebases the current work (branch) against it")
3153
3154 def run(self, args):
3155 sync = P4Sync()
3156 sync.importLabels = self.importLabels
3157 sync.run([])
3158
3159 return self.rebase()
3160
3161 def rebase(self):
3162 if os.system("git update-index --refresh") != 0:
3163 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3164 if len(read_pipe("git diff-index HEAD --")) > 0:
3165 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3166
3167 [upstream, settings] = findUpstreamBranchPoint()
3168 if len(upstream) == 0:
3169 die("Cannot find upstream branchpoint for rebase")
3170
3171 # the branchpoint may be p4/foo~3, so strip off the parent
3172 upstream = re.sub("~[0-9]+$", "", upstream)
3173
3174 print "Rebasing the current branch onto %s" % upstream
3175 oldHead = read_pipe("git rev-parse HEAD").strip()
3176 system("git rebase %s" % upstream)
3177 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3178 return True
3179
3180 class P4Clone(P4Sync):
3181 def __init__(self):
3182 P4Sync.__init__(self)
3183 self.description = "Creates a new git repository and imports from Perforce into it"
3184 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3185 self.options += [
3186 optparse.make_option("--destination", dest="cloneDestination",
3187 action='store', default=None,
3188 help="where to leave result of the clone"),
3189 optparse.make_option("--bare", dest="cloneBare",
3190 action="store_true", default=False),
3191 ]
3192 self.cloneDestination = None
3193 self.needsGit = False
3194 self.cloneBare = False
3195
3196 def defaultDestination(self, args):
3197 ## TODO: use common prefix of args?
3198 depotPath = args[0]
3199 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3200 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3201 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3202 depotDir = re.sub(r"/$", "", depotDir)
3203 return os.path.split(depotDir)[1]
3204
3205 def run(self, args):
3206 if len(args) < 1:
3207 return False
3208
3209 if self.keepRepoPath and not self.cloneDestination:
3210 sys.stderr.write("Must specify destination for --keep-path\n")
3211 sys.exit(1)
3212
3213 depotPaths = args
3214
3215 if not self.cloneDestination and len(depotPaths) > 1:
3216 self.cloneDestination = depotPaths[-1]
3217 depotPaths = depotPaths[:-1]
3218
3219 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3220 for p in depotPaths:
3221 if not p.startswith("//"):
3222 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3223 return False
3224
3225 if not self.cloneDestination:
3226 self.cloneDestination = self.defaultDestination(args)
3227
3228 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3229
3230 if not os.path.exists(self.cloneDestination):
3231 os.makedirs(self.cloneDestination)
3232 chdir(self.cloneDestination)
3233
3234 init_cmd = [ "git", "init" ]
3235 if self.cloneBare:
3236 init_cmd.append("--bare")
3237 retcode = subprocess.call(init_cmd)
3238 if retcode:
3239 raise CalledProcessError(retcode, init_cmd)
3240
3241 if not P4Sync.run(self, depotPaths):
3242 return False
3243
3244 # create a master branch and check out a work tree
3245 if gitBranchExists(self.branch):
3246 system([ "git", "branch", "master", self.branch ])
3247 if not self.cloneBare:
3248 system([ "git", "checkout", "-f" ])
3249 else:
3250 print 'Not checking out any branch, use ' \
3251 '"git checkout -q -b master <branch>"'
3252
3253 # auto-set this variable if invoked with --use-client-spec
3254 if self.useClientSpec_from_options:
3255 system("git config --bool git-p4.useclientspec true")
3256
3257 return True
3258
3259 class P4Branches(Command):
3260 def __init__(self):
3261 Command.__init__(self)
3262 self.options = [ ]
3263 self.description = ("Shows the git branches that hold imports and their "
3264 + "corresponding perforce depot paths")
3265 self.verbose = False
3266
3267 def run(self, args):
3268 if originP4BranchesExist():
3269 createOrUpdateBranchesFromOrigin()
3270
3271 cmdline = "git rev-parse --symbolic "
3272 cmdline += " --remotes"
3273
3274 for line in read_pipe_lines(cmdline):
3275 line = line.strip()
3276
3277 if not line.startswith('p4/') or line == "p4/HEAD":
3278 continue
3279 branch = line
3280
3281 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3282 settings = extractSettingsGitLog(log)
3283
3284 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3285 return True
3286
3287 class HelpFormatter(optparse.IndentedHelpFormatter):
3288 def __init__(self):
3289 optparse.IndentedHelpFormatter.__init__(self)
3290
3291 def format_description(self, description):
3292 if description:
3293 return description + "\n"
3294 else:
3295 return ""
3296
3297 def printUsage(commands):
3298 print "usage: %s <command> [options]" % sys.argv[0]
3299 print ""
3300 print "valid commands: %s" % ", ".join(commands)
3301 print ""
3302 print "Try %s <command> --help for command specific help." % sys.argv[0]
3303 print ""
3304
3305 commands = {
3306 "debug" : P4Debug,
3307 "submit" : P4Submit,
3308 "commit" : P4Submit,
3309 "sync" : P4Sync,
3310 "rebase" : P4Rebase,
3311 "clone" : P4Clone,
3312 "rollback" : P4RollBack,
3313 "branches" : P4Branches
3314 }
3315
3316
3317 def main():
3318 if len(sys.argv[1:]) == 0:
3319 printUsage(commands.keys())
3320 sys.exit(2)
3321
3322 cmdName = sys.argv[1]
3323 try:
3324 klass = commands[cmdName]
3325 cmd = klass()
3326 except KeyError:
3327 print "unknown command %s" % cmdName
3328 print ""
3329 printUsage(commands.keys())
3330 sys.exit(2)
3331
3332 options = cmd.options
3333 cmd.gitdir = os.environ.get("GIT_DIR", None)
3334
3335 args = sys.argv[2:]
3336
3337 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3338 if cmd.needsGit:
3339 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3340
3341 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3342 options,
3343 description = cmd.description,
3344 formatter = HelpFormatter())
3345
3346 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3347 global verbose
3348 verbose = cmd.verbose
3349 if cmd.needsGit:
3350 if cmd.gitdir == None:
3351 cmd.gitdir = os.path.abspath(".git")
3352 if not isValidGitDir(cmd.gitdir):
3353 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3354 if os.path.exists(cmd.gitdir):
3355 cdup = read_pipe("git rev-parse --show-cdup").strip()
3356 if len(cdup) > 0:
3357 chdir(cdup);
3358
3359 if not isValidGitDir(cmd.gitdir):
3360 if isValidGitDir(cmd.gitdir + "/.git"):
3361 cmd.gitdir += "/.git"
3362 else:
3363 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3364
3365 os.environ["GIT_DIR"] = cmd.gitdir
3366
3367 if not cmd.run(args):
3368 parser.print_help()
3369 sys.exit(2)
3370
3371
3372 if __name__ == '__main__':
3373 main()