Updates for dirty submodules in release notes and user manual
[git/git.git] / contrib / fast-import / git-p4
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 import re
14
15 verbose = False
16
17
18 def p4_build_cmd(cmd):
19 """Build a suitable p4 command line.
20
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
24 """
25 real_cmd = "%s " % "p4"
26
27 user = gitConfig("git-p4.user")
28 if len(user) > 0:
29 real_cmd += "-u %s " % user
30
31 password = gitConfig("git-p4.password")
32 if len(password) > 0:
33 real_cmd += "-P %s " % password
34
35 port = gitConfig("git-p4.port")
36 if len(port) > 0:
37 real_cmd += "-p %s " % port
38
39 host = gitConfig("git-p4.host")
40 if len(host) > 0:
41 real_cmd += "-h %s " % host
42
43 client = gitConfig("git-p4.client")
44 if len(client) > 0:
45 real_cmd += "-c %s " % client
46
47 real_cmd += "%s" % (cmd)
48 if verbose:
49 print real_cmd
50 return real_cmd
51
52 def chdir(dir):
53 if os.name == 'nt':
54 os.environ['PWD']=dir
55 os.chdir(dir)
56
57 def die(msg):
58 if verbose:
59 raise Exception(msg)
60 else:
61 sys.stderr.write(msg + "\n")
62 sys.exit(1)
63
64 def write_pipe(c, str):
65 if verbose:
66 sys.stderr.write('Writing pipe: %s\n' % c)
67
68 pipe = os.popen(c, 'w')
69 val = pipe.write(str)
70 if pipe.close():
71 die('Command failed: %s' % c)
72
73 return val
74
75 def p4_write_pipe(c, str):
76 real_cmd = p4_build_cmd(c)
77 return write_pipe(real_cmd, str)
78
79 def read_pipe(c, ignore_error=False):
80 if verbose:
81 sys.stderr.write('Reading pipe: %s\n' % c)
82
83 pipe = os.popen(c, 'rb')
84 val = pipe.read()
85 if pipe.close() and not ignore_error:
86 die('Command failed: %s' % c)
87
88 return val
89
90 def p4_read_pipe(c, ignore_error=False):
91 real_cmd = p4_build_cmd(c)
92 return read_pipe(real_cmd, ignore_error)
93
94 def read_pipe_lines(c):
95 if verbose:
96 sys.stderr.write('Reading pipe: %s\n' % c)
97 ## todo: check return status
98 pipe = os.popen(c, 'rb')
99 val = pipe.readlines()
100 if pipe.close():
101 die('Command failed: %s' % c)
102
103 return val
104
105 def p4_read_pipe_lines(c):
106 """Specifically invoke p4 on the command supplied. """
107 real_cmd = p4_build_cmd(c)
108 return read_pipe_lines(real_cmd)
109
110 def system(cmd):
111 if verbose:
112 sys.stderr.write("executing %s\n" % cmd)
113 if os.system(cmd) != 0:
114 die("command failed: %s" % cmd)
115
116 def p4_system(cmd):
117 """Specifically invoke p4 as the system command. """
118 real_cmd = p4_build_cmd(cmd)
119 return system(real_cmd)
120
121 def isP4Exec(kind):
122 """Determine if a Perforce 'kind' should have execute permission
123
124 'p4 help filetypes' gives a list of the types. If it starts with 'x',
125 or x follows one of a few letters. Otherwise, if there is an 'x' after
126 a plus sign, it is also executable"""
127 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
128
129 def setP4ExecBit(file, mode):
130 # Reopens an already open file and changes the execute bit to match
131 # the execute bit setting in the passed in mode.
132
133 p4Type = "+x"
134
135 if not isModeExec(mode):
136 p4Type = getP4OpenedType(file)
137 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
138 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
139 if p4Type[-1] == "+":
140 p4Type = p4Type[0:-1]
141
142 p4_system("reopen -t %s %s" % (p4Type, file))
143
144 def getP4OpenedType(file):
145 # Returns the perforce file type for the given file.
146
147 result = p4_read_pipe("opened %s" % file)
148 match = re.match(".*\((.+)\)\r?$", result)
149 if match:
150 return match.group(1)
151 else:
152 die("Could not determine file type for %s (result: '%s')" % (file, result))
153
154 def diffTreePattern():
155 # This is a simple generator for the diff tree regex pattern. This could be
156 # a class variable if this and parseDiffTreeEntry were a part of a class.
157 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
158 while True:
159 yield pattern
160
161 def parseDiffTreeEntry(entry):
162 """Parses a single diff tree entry into its component elements.
163
164 See git-diff-tree(1) manpage for details about the format of the diff
165 output. This method returns a dictionary with the following elements:
166
167 src_mode - The mode of the source file
168 dst_mode - The mode of the destination file
169 src_sha1 - The sha1 for the source file
170 dst_sha1 - The sha1 fr the destination file
171 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
172 status_score - The score for the status (applicable for 'C' and 'R'
173 statuses). This is None if there is no score.
174 src - The path for the source file.
175 dst - The path for the destination file. This is only present for
176 copy or renames. If it is not present, this is None.
177
178 If the pattern is not matched, None is returned."""
179
180 match = diffTreePattern().next().match(entry)
181 if match:
182 return {
183 'src_mode': match.group(1),
184 'dst_mode': match.group(2),
185 'src_sha1': match.group(3),
186 'dst_sha1': match.group(4),
187 'status': match.group(5),
188 'status_score': match.group(6),
189 'src': match.group(7),
190 'dst': match.group(10)
191 }
192 return None
193
194 def isModeExec(mode):
195 # Returns True if the given git mode represents an executable file,
196 # otherwise False.
197 return mode[-3:] == "755"
198
199 def isModeExecChanged(src_mode, dst_mode):
200 return isModeExec(src_mode) != isModeExec(dst_mode)
201
202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
203 cmd = p4_build_cmd("-G %s" % (cmd))
204 if verbose:
205 sys.stderr.write("Opening pipe: %s\n" % cmd)
206
207 # Use a temporary file to avoid deadlocks without
208 # subprocess.communicate(), which would put another copy
209 # of stdout into memory.
210 stdin_file = None
211 if stdin is not None:
212 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
213 stdin_file.write(stdin)
214 stdin_file.flush()
215 stdin_file.seek(0)
216
217 p4 = subprocess.Popen(cmd, shell=True,
218 stdin=stdin_file,
219 stdout=subprocess.PIPE)
220
221 result = []
222 try:
223 while True:
224 entry = marshal.load(p4.stdout)
225 if cb is not None:
226 cb(entry)
227 else:
228 result.append(entry)
229 except EOFError:
230 pass
231 exitCode = p4.wait()
232 if exitCode != 0:
233 entry = {}
234 entry["p4ExitCode"] = exitCode
235 result.append(entry)
236
237 return result
238
239 def p4Cmd(cmd):
240 list = p4CmdList(cmd)
241 result = {}
242 for entry in list:
243 result.update(entry)
244 return result;
245
246 def p4Where(depotPath):
247 if not depotPath.endswith("/"):
248 depotPath += "/"
249 depotPath = depotPath + "..."
250 outputList = p4CmdList("where %s" % depotPath)
251 output = None
252 for entry in outputList:
253 if "depotFile" in entry:
254 if entry["depotFile"] == depotPath:
255 output = entry
256 break
257 elif "data" in entry:
258 data = entry.get("data")
259 space = data.find(" ")
260 if data[:space] == depotPath:
261 output = entry
262 break
263 if output == None:
264 return ""
265 if output["code"] == "error":
266 return ""
267 clientPath = ""
268 if "path" in output:
269 clientPath = output.get("path")
270 elif "data" in output:
271 data = output.get("data")
272 lastSpace = data.rfind(" ")
273 clientPath = data[lastSpace + 1:]
274
275 if clientPath.endswith("..."):
276 clientPath = clientPath[:-3]
277 return clientPath
278
279 def currentGitBranch():
280 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
281
282 def isValidGitDir(path):
283 if (os.path.exists(path + "/HEAD")
284 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
285 return True;
286 return False
287
288 def parseRevision(ref):
289 return read_pipe("git rev-parse %s" % ref).strip()
290
291 def extractLogMessageFromGitCommit(commit):
292 logMessage = ""
293
294 ## fixme: title is first line of commit, not 1st paragraph.
295 foundTitle = False
296 for log in read_pipe_lines("git cat-file commit %s" % commit):
297 if not foundTitle:
298 if len(log) == 1:
299 foundTitle = True
300 continue
301
302 logMessage += log
303 return logMessage
304
305 def extractSettingsGitLog(log):
306 values = {}
307 for line in log.split("\n"):
308 line = line.strip()
309 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
310 if not m:
311 continue
312
313 assignments = m.group(1).split (':')
314 for a in assignments:
315 vals = a.split ('=')
316 key = vals[0].strip()
317 val = ('='.join (vals[1:])).strip()
318 if val.endswith ('\"') and val.startswith('"'):
319 val = val[1:-1]
320
321 values[key] = val
322
323 paths = values.get("depot-paths")
324 if not paths:
325 paths = values.get("depot-path")
326 if paths:
327 values['depot-paths'] = paths.split(',')
328 return values
329
330 def gitBranchExists(branch):
331 proc = subprocess.Popen(["git", "rev-parse", branch],
332 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
333 return proc.wait() == 0;
334
335 _gitConfig = {}
336 def gitConfig(key):
337 if not _gitConfig.has_key(key):
338 _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
339 return _gitConfig[key]
340
341 def p4BranchesInGit(branchesAreInRemotes = True):
342 branches = {}
343
344 cmdline = "git rev-parse --symbolic "
345 if branchesAreInRemotes:
346 cmdline += " --remotes"
347 else:
348 cmdline += " --branches"
349
350 for line in read_pipe_lines(cmdline):
351 line = line.strip()
352
353 ## only import to p4/
354 if not line.startswith('p4/') or line == "p4/HEAD":
355 continue
356 branch = line
357
358 # strip off p4
359 branch = re.sub ("^p4/", "", line)
360
361 branches[branch] = parseRevision(line)
362 return branches
363
364 def findUpstreamBranchPoint(head = "HEAD"):
365 branches = p4BranchesInGit()
366 # map from depot-path to branch name
367 branchByDepotPath = {}
368 for branch in branches.keys():
369 tip = branches[branch]
370 log = extractLogMessageFromGitCommit(tip)
371 settings = extractSettingsGitLog(log)
372 if settings.has_key("depot-paths"):
373 paths = ",".join(settings["depot-paths"])
374 branchByDepotPath[paths] = "remotes/p4/" + branch
375
376 settings = None
377 parent = 0
378 while parent < 65535:
379 commit = head + "~%s" % parent
380 log = extractLogMessageFromGitCommit(commit)
381 settings = extractSettingsGitLog(log)
382 if settings.has_key("depot-paths"):
383 paths = ",".join(settings["depot-paths"])
384 if branchByDepotPath.has_key(paths):
385 return [branchByDepotPath[paths], settings]
386
387 parent = parent + 1
388
389 return ["", settings]
390
391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
392 if not silent:
393 print ("Creating/updating branch(es) in %s based on origin branch(es)"
394 % localRefPrefix)
395
396 originPrefix = "origin/p4/"
397
398 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
399 line = line.strip()
400 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
401 continue
402
403 headName = line[len(originPrefix):]
404 remoteHead = localRefPrefix + headName
405 originHead = line
406
407 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
408 if (not original.has_key('depot-paths')
409 or not original.has_key('change')):
410 continue
411
412 update = False
413 if not gitBranchExists(remoteHead):
414 if verbose:
415 print "creating %s" % remoteHead
416 update = True
417 else:
418 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
419 if settings.has_key('change') > 0:
420 if settings['depot-paths'] == original['depot-paths']:
421 originP4Change = int(original['change'])
422 p4Change = int(settings['change'])
423 if originP4Change > p4Change:
424 print ("%s (%s) is newer than %s (%s). "
425 "Updating p4 branch from origin."
426 % (originHead, originP4Change,
427 remoteHead, p4Change))
428 update = True
429 else:
430 print ("Ignoring: %s was imported from %s while "
431 "%s was imported from %s"
432 % (originHead, ','.join(original['depot-paths']),
433 remoteHead, ','.join(settings['depot-paths'])))
434
435 if update:
436 system("git update-ref %s %s" % (remoteHead, originHead))
437
438 def originP4BranchesExist():
439 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
440
441 def p4ChangesForPaths(depotPaths, changeRange):
442 assert depotPaths
443 output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
444 for p in depotPaths]))
445
446 changes = {}
447 for line in output:
448 changeNum = int(line.split(" ")[1])
449 changes[changeNum] = True
450
451 changelist = changes.keys()
452 changelist.sort()
453 return changelist
454
455 class Command:
456 def __init__(self):
457 self.usage = "usage: %prog [options]"
458 self.needsGit = True
459
460 class P4Debug(Command):
461 def __init__(self):
462 Command.__init__(self)
463 self.options = [
464 optparse.make_option("--verbose", dest="verbose", action="store_true",
465 default=False),
466 ]
467 self.description = "A tool to debug the output of p4 -G."
468 self.needsGit = False
469 self.verbose = False
470
471 def run(self, args):
472 j = 0
473 for output in p4CmdList(" ".join(args)):
474 print 'Element: %d' % j
475 j += 1
476 print output
477 return True
478
479 class P4RollBack(Command):
480 def __init__(self):
481 Command.__init__(self)
482 self.options = [
483 optparse.make_option("--verbose", dest="verbose", action="store_true"),
484 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
485 ]
486 self.description = "A tool to debug the multi-branch import. Don't use :)"
487 self.verbose = False
488 self.rollbackLocalBranches = False
489
490 def run(self, args):
491 if len(args) != 1:
492 return False
493 maxChange = int(args[0])
494
495 if "p4ExitCode" in p4Cmd("changes -m 1"):
496 die("Problems executing p4");
497
498 if self.rollbackLocalBranches:
499 refPrefix = "refs/heads/"
500 lines = read_pipe_lines("git rev-parse --symbolic --branches")
501 else:
502 refPrefix = "refs/remotes/"
503 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
504
505 for line in lines:
506 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
507 line = line.strip()
508 ref = refPrefix + line
509 log = extractLogMessageFromGitCommit(ref)
510 settings = extractSettingsGitLog(log)
511
512 depotPaths = settings['depot-paths']
513 change = settings['change']
514
515 changed = False
516
517 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
518 for p in depotPaths]))) == 0:
519 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
520 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
521 continue
522
523 while change and int(change) > maxChange:
524 changed = True
525 if self.verbose:
526 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
527 system("git update-ref %s \"%s^\"" % (ref, ref))
528 log = extractLogMessageFromGitCommit(ref)
529 settings = extractSettingsGitLog(log)
530
531
532 depotPaths = settings['depot-paths']
533 change = settings['change']
534
535 if changed:
536 print "%s rewound to %s" % (ref, change)
537
538 return True
539
540 class P4Submit(Command):
541 def __init__(self):
542 Command.__init__(self)
543 self.options = [
544 optparse.make_option("--verbose", dest="verbose", action="store_true"),
545 optparse.make_option("--origin", dest="origin"),
546 optparse.make_option("-M", dest="detectRename", action="store_true"),
547 ]
548 self.description = "Submit changes from git to the perforce depot."
549 self.usage += " [name of git branch to submit into perforce depot]"
550 self.interactive = True
551 self.origin = ""
552 self.detectRename = False
553 self.verbose = False
554 self.isWindows = (platform.system() == "Windows")
555
556 def check(self):
557 if len(p4CmdList("opened ...")) > 0:
558 die("You have files opened with perforce! Close them before starting the sync.")
559
560 # replaces everything between 'Description:' and the next P4 submit template field with the
561 # commit message
562 def prepareLogMessage(self, template, message):
563 result = ""
564
565 inDescriptionSection = False
566
567 for line in template.split("\n"):
568 if line.startswith("#"):
569 result += line + "\n"
570 continue
571
572 if inDescriptionSection:
573 if line.startswith("Files:"):
574 inDescriptionSection = False
575 else:
576 continue
577 else:
578 if line.startswith("Description:"):
579 inDescriptionSection = True
580 line += "\n"
581 for messageLine in message.split("\n"):
582 line += "\t" + messageLine + "\n"
583
584 result += line + "\n"
585
586 return result
587
588 def prepareSubmitTemplate(self):
589 # remove lines in the Files section that show changes to files outside the depot path we're committing into
590 template = ""
591 inFilesSection = False
592 for line in p4_read_pipe_lines("change -o"):
593 if line.endswith("\r\n"):
594 line = line[:-2] + "\n"
595 if inFilesSection:
596 if line.startswith("\t"):
597 # path starts and ends with a tab
598 path = line[1:]
599 lastTab = path.rfind("\t")
600 if lastTab != -1:
601 path = path[:lastTab]
602 if not path.startswith(self.depotPath):
603 continue
604 else:
605 inFilesSection = False
606 else:
607 if line.startswith("Files:"):
608 inFilesSection = True
609
610 template += line
611
612 return template
613
614 def applyCommit(self, id):
615 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
616 diffOpts = ("", "-M")[self.detectRename]
617 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
618 filesToAdd = set()
619 filesToDelete = set()
620 editedFiles = set()
621 filesToChangeExecBit = {}
622 for line in diff:
623 diff = parseDiffTreeEntry(line)
624 modifier = diff['status']
625 path = diff['src']
626 if modifier == "M":
627 p4_system("edit \"%s\"" % path)
628 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
629 filesToChangeExecBit[path] = diff['dst_mode']
630 editedFiles.add(path)
631 elif modifier == "A":
632 filesToAdd.add(path)
633 filesToChangeExecBit[path] = diff['dst_mode']
634 if path in filesToDelete:
635 filesToDelete.remove(path)
636 elif modifier == "D":
637 filesToDelete.add(path)
638 if path in filesToAdd:
639 filesToAdd.remove(path)
640 elif modifier == "R":
641 src, dest = diff['src'], diff['dst']
642 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
643 p4_system("edit \"%s\"" % (dest))
644 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
645 filesToChangeExecBit[dest] = diff['dst_mode']
646 os.unlink(dest)
647 editedFiles.add(dest)
648 filesToDelete.add(src)
649 else:
650 die("unknown modifier %s for %s" % (modifier, path))
651
652 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
653 patchcmd = diffcmd + " | git apply "
654 tryPatchCmd = patchcmd + "--check -"
655 applyPatchCmd = patchcmd + "--check --apply -"
656
657 if os.system(tryPatchCmd) != 0:
658 print "Unfortunately applying the change failed!"
659 print "What do you want to do?"
660 response = "x"
661 while response != "s" and response != "a" and response != "w":
662 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
663 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
664 if response == "s":
665 print "Skipping! Good luck with the next patches..."
666 for f in editedFiles:
667 p4_system("revert \"%s\"" % f);
668 for f in filesToAdd:
669 system("rm %s" %f)
670 return
671 elif response == "a":
672 os.system(applyPatchCmd)
673 if len(filesToAdd) > 0:
674 print "You may also want to call p4 add on the following files:"
675 print " ".join(filesToAdd)
676 if len(filesToDelete):
677 print "The following files should be scheduled for deletion with p4 delete:"
678 print " ".join(filesToDelete)
679 die("Please resolve and submit the conflict manually and "
680 + "continue afterwards with git-p4 submit --continue")
681 elif response == "w":
682 system(diffcmd + " > patch.txt")
683 print "Patch saved to patch.txt in %s !" % self.clientPath
684 die("Please resolve and submit the conflict manually and "
685 "continue afterwards with git-p4 submit --continue")
686
687 system(applyPatchCmd)
688
689 for f in filesToAdd:
690 p4_system("add \"%s\"" % f)
691 for f in filesToDelete:
692 p4_system("revert \"%s\"" % f)
693 p4_system("delete \"%s\"" % f)
694
695 # Set/clear executable bits
696 for f in filesToChangeExecBit.keys():
697 mode = filesToChangeExecBit[f]
698 setP4ExecBit(f, mode)
699
700 logMessage = extractLogMessageFromGitCommit(id)
701 logMessage = logMessage.strip()
702
703 template = self.prepareSubmitTemplate()
704
705 if self.interactive:
706 submitTemplate = self.prepareLogMessage(template, logMessage)
707 if os.environ.has_key("P4DIFF"):
708 del(os.environ["P4DIFF"])
709 diff = p4_read_pipe("diff -du ...")
710
711 newdiff = ""
712 for newFile in filesToAdd:
713 newdiff += "==== new file ====\n"
714 newdiff += "--- /dev/null\n"
715 newdiff += "+++ %s\n" % newFile
716 f = open(newFile, "r")
717 for line in f.readlines():
718 newdiff += "+" + line
719 f.close()
720
721 separatorLine = "######## everything below this line is just the diff #######\n"
722
723 [handle, fileName] = tempfile.mkstemp()
724 tmpFile = os.fdopen(handle, "w+")
725 if self.isWindows:
726 submitTemplate = submitTemplate.replace("\n", "\r\n")
727 separatorLine = separatorLine.replace("\n", "\r\n")
728 newdiff = newdiff.replace("\n", "\r\n")
729 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
730 tmpFile.close()
731 mtime = os.stat(fileName).st_mtime
732 if os.environ.has_key("P4EDITOR"):
733 editor = os.environ.get("P4EDITOR")
734 else:
735 editor = read_pipe("git var GIT_EDITOR").strip()
736 system(editor + " " + fileName)
737
738 response = "y"
739 if os.stat(fileName).st_mtime <= mtime:
740 response = "x"
741 while response != "y" and response != "n":
742 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
743
744 if response == "y":
745 tmpFile = open(fileName, "rb")
746 message = tmpFile.read()
747 tmpFile.close()
748 submitTemplate = message[:message.index(separatorLine)]
749 if self.isWindows:
750 submitTemplate = submitTemplate.replace("\r\n", "\n")
751 p4_write_pipe("submit -i", submitTemplate)
752 else:
753 for f in editedFiles:
754 p4_system("revert \"%s\"" % f);
755 for f in filesToAdd:
756 p4_system("revert \"%s\"" % f);
757 system("rm %s" %f)
758
759 os.remove(fileName)
760 else:
761 fileName = "submit.txt"
762 file = open(fileName, "w+")
763 file.write(self.prepareLogMessage(template, logMessage))
764 file.close()
765 print ("Perforce submit template written as %s. "
766 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
767 % (fileName, fileName))
768
769 def run(self, args):
770 if len(args) == 0:
771 self.master = currentGitBranch()
772 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
773 die("Detecting current git branch failed!")
774 elif len(args) == 1:
775 self.master = args[0]
776 else:
777 return False
778
779 allowSubmit = gitConfig("git-p4.allowSubmit")
780 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
781 die("%s is not in git-p4.allowSubmit" % self.master)
782
783 [upstream, settings] = findUpstreamBranchPoint()
784 self.depotPath = settings['depot-paths'][0]
785 if len(self.origin) == 0:
786 self.origin = upstream
787
788 if self.verbose:
789 print "Origin branch is " + self.origin
790
791 if len(self.depotPath) == 0:
792 print "Internal error: cannot locate perforce depot path from existing branches"
793 sys.exit(128)
794
795 self.clientPath = p4Where(self.depotPath)
796
797 if len(self.clientPath) == 0:
798 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
799 sys.exit(128)
800
801 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
802 self.oldWorkingDirectory = os.getcwd()
803
804 chdir(self.clientPath)
805 print "Syncronizing p4 checkout..."
806 p4_system("sync ...")
807
808 self.check()
809
810 commits = []
811 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
812 commits.append(line.strip())
813 commits.reverse()
814
815 while len(commits) > 0:
816 commit = commits[0]
817 commits = commits[1:]
818 self.applyCommit(commit)
819 if not self.interactive:
820 break
821
822 if len(commits) == 0:
823 print "All changes applied!"
824 chdir(self.oldWorkingDirectory)
825
826 sync = P4Sync()
827 sync.run([])
828
829 rebase = P4Rebase()
830 rebase.rebase()
831
832 return True
833
834 class P4Sync(Command):
835 def __init__(self):
836 Command.__init__(self)
837 self.options = [
838 optparse.make_option("--branch", dest="branch"),
839 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
840 optparse.make_option("--changesfile", dest="changesFile"),
841 optparse.make_option("--silent", dest="silent", action="store_true"),
842 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
843 optparse.make_option("--verbose", dest="verbose", action="store_true"),
844 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
845 help="Import into refs/heads/ , not refs/remotes"),
846 optparse.make_option("--max-changes", dest="maxChanges"),
847 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
848 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
849 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
850 help="Only sync files that are included in the Perforce Client Spec")
851 ]
852 self.description = """Imports from Perforce into a git repository.\n
853 example:
854 //depot/my/project/ -- to import the current head
855 //depot/my/project/@all -- to import everything
856 //depot/my/project/@1,6 -- to import only from revision 1 to 6
857
858 (a ... is not needed in the path p4 specification, it's added implicitly)"""
859
860 self.usage += " //depot/path[@revRange]"
861 self.silent = False
862 self.createdBranches = set()
863 self.committedChanges = set()
864 self.branch = ""
865 self.detectBranches = False
866 self.detectLabels = False
867 self.changesFile = ""
868 self.syncWithOrigin = True
869 self.verbose = False
870 self.importIntoRemotes = True
871 self.maxChanges = ""
872 self.isWindows = (platform.system() == "Windows")
873 self.keepRepoPath = False
874 self.depotPaths = None
875 self.p4BranchesInGit = []
876 self.cloneExclude = []
877 self.useClientSpec = False
878 self.clientSpecDirs = []
879
880 if gitConfig("git-p4.syncFromOrigin") == "false":
881 self.syncWithOrigin = False
882
883 def extractFilesFromCommit(self, commit):
884 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
885 for path in self.cloneExclude]
886 files = []
887 fnum = 0
888 while commit.has_key("depotFile%s" % fnum):
889 path = commit["depotFile%s" % fnum]
890
891 if [p for p in self.cloneExclude
892 if path.startswith (p)]:
893 found = False
894 else:
895 found = [p for p in self.depotPaths
896 if path.startswith (p)]
897 if not found:
898 fnum = fnum + 1
899 continue
900
901 file = {}
902 file["path"] = path
903 file["rev"] = commit["rev%s" % fnum]
904 file["action"] = commit["action%s" % fnum]
905 file["type"] = commit["type%s" % fnum]
906 files.append(file)
907 fnum = fnum + 1
908 return files
909
910 def stripRepoPath(self, path, prefixes):
911 if self.keepRepoPath:
912 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
913
914 for p in prefixes:
915 if path.startswith(p):
916 path = path[len(p):]
917
918 return path
919
920 def splitFilesIntoBranches(self, commit):
921 branches = {}
922 fnum = 0
923 while commit.has_key("depotFile%s" % fnum):
924 path = commit["depotFile%s" % fnum]
925 found = [p for p in self.depotPaths
926 if path.startswith (p)]
927 if not found:
928 fnum = fnum + 1
929 continue
930
931 file = {}
932 file["path"] = path
933 file["rev"] = commit["rev%s" % fnum]
934 file["action"] = commit["action%s" % fnum]
935 file["type"] = commit["type%s" % fnum]
936 fnum = fnum + 1
937
938 relPath = self.stripRepoPath(path, self.depotPaths)
939
940 for branch in self.knownBranches.keys():
941
942 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
943 if relPath.startswith(branch + "/"):
944 if branch not in branches:
945 branches[branch] = []
946 branches[branch].append(file)
947 break
948
949 return branches
950
951 # output one file from the P4 stream
952 # - helper for streamP4Files
953
954 def streamOneP4File(self, file, contents):
955 if file["type"] == "apple":
956 print "\nfile %s is a strange apple file that forks. Ignoring" % \
957 file['depotFile']
958 return
959
960 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
961 if verbose:
962 sys.stderr.write("%s\n" % relPath)
963
964 mode = "644"
965 if isP4Exec(file["type"]):
966 mode = "755"
967 elif file["type"] == "symlink":
968 mode = "120000"
969 # p4 print on a symlink contains "target\n", so strip it off
970 last = contents.pop()
971 last = last[:-1]
972 contents.append(last)
973
974 if self.isWindows and file["type"].endswith("text"):
975 mangled = []
976 for data in contents:
977 data = data.replace("\r\n", "\n")
978 mangled.append(data)
979 contents = mangled
980
981 if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
982 contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
983 elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
984 contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
985
986 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
987
988 # total length...
989 length = 0
990 for d in contents:
991 length = length + len(d)
992
993 self.gitStream.write("data %d\n" % length)
994 for d in contents:
995 self.gitStream.write(d)
996 self.gitStream.write("\n")
997
998 def streamOneP4Deletion(self, file):
999 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1000 if verbose:
1001 sys.stderr.write("delete %s\n" % relPath)
1002 self.gitStream.write("D %s\n" % relPath)
1003
1004 # handle another chunk of streaming data
1005 def streamP4FilesCb(self, marshalled):
1006
1007 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1008 # start of a new file - output the old one first
1009 self.streamOneP4File(self.stream_file, self.stream_contents)
1010 self.stream_file = {}
1011 self.stream_contents = []
1012 self.stream_have_file_info = False
1013
1014 # pick up the new file information... for the
1015 # 'data' field we need to append to our array
1016 for k in marshalled.keys():
1017 if k == 'data':
1018 self.stream_contents.append(marshalled['data'])
1019 else:
1020 self.stream_file[k] = marshalled[k]
1021
1022 self.stream_have_file_info = True
1023
1024 # Stream directly from "p4 files" into "git fast-import"
1025 def streamP4Files(self, files):
1026 filesForCommit = []
1027 filesToRead = []
1028 filesToDelete = []
1029
1030 for f in files:
1031 includeFile = True
1032 for val in self.clientSpecDirs:
1033 if f['path'].startswith(val[0]):
1034 if val[1] <= 0:
1035 includeFile = False
1036 break
1037
1038 if includeFile:
1039 filesForCommit.append(f)
1040 if f['action'] not in ('delete', 'purge'):
1041 filesToRead.append(f)
1042 else:
1043 filesToDelete.append(f)
1044
1045 # deleted files...
1046 for f in filesToDelete:
1047 self.streamOneP4Deletion(f)
1048
1049 if len(filesToRead) > 0:
1050 self.stream_file = {}
1051 self.stream_contents = []
1052 self.stream_have_file_info = False
1053
1054 # curry self argument
1055 def streamP4FilesCbSelf(entry):
1056 self.streamP4FilesCb(entry)
1057
1058 p4CmdList("-x - print",
1059 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1060 for f in filesToRead]),
1061 cb=streamP4FilesCbSelf)
1062
1063 # do the last chunk
1064 if self.stream_file.has_key('depotFile'):
1065 self.streamOneP4File(self.stream_file, self.stream_contents)
1066
1067 def commit(self, details, files, branch, branchPrefixes, parent = ""):
1068 epoch = details["time"]
1069 author = details["user"]
1070 self.branchPrefixes = branchPrefixes
1071
1072 if self.verbose:
1073 print "commit into %s" % branch
1074
1075 # start with reading files; if that fails, we should not
1076 # create a commit.
1077 new_files = []
1078 for f in files:
1079 if [p for p in branchPrefixes if f['path'].startswith(p)]:
1080 new_files.append (f)
1081 else:
1082 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1083
1084 self.gitStream.write("commit %s\n" % branch)
1085 # gitStream.write("mark :%s\n" % details["change"])
1086 self.committedChanges.add(int(details["change"]))
1087 committer = ""
1088 if author not in self.users:
1089 self.getUserMapFromPerforceServer()
1090 if author in self.users:
1091 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1092 else:
1093 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1094
1095 self.gitStream.write("committer %s\n" % committer)
1096
1097 self.gitStream.write("data <<EOT\n")
1098 self.gitStream.write(details["desc"])
1099 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1100 % (','.join (branchPrefixes), details["change"]))
1101 if len(details['options']) > 0:
1102 self.gitStream.write(": options = %s" % details['options'])
1103 self.gitStream.write("]\nEOT\n\n")
1104
1105 if len(parent) > 0:
1106 if self.verbose:
1107 print "parent %s" % parent
1108 self.gitStream.write("from %s\n" % parent)
1109
1110 self.streamP4Files(new_files)
1111 self.gitStream.write("\n")
1112
1113 change = int(details["change"])
1114
1115 if self.labels.has_key(change):
1116 label = self.labels[change]
1117 labelDetails = label[0]
1118 labelRevisions = label[1]
1119 if self.verbose:
1120 print "Change %s is labelled %s" % (change, labelDetails)
1121
1122 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1123 for p in branchPrefixes]))
1124
1125 if len(files) == len(labelRevisions):
1126
1127 cleanedFiles = {}
1128 for info in files:
1129 if info["action"] in ("delete", "purge"):
1130 continue
1131 cleanedFiles[info["depotFile"]] = info["rev"]
1132
1133 if cleanedFiles == labelRevisions:
1134 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1135 self.gitStream.write("from %s\n" % branch)
1136
1137 owner = labelDetails["Owner"]
1138 tagger = ""
1139 if author in self.users:
1140 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1141 else:
1142 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1143 self.gitStream.write("tagger %s\n" % tagger)
1144 self.gitStream.write("data <<EOT\n")
1145 self.gitStream.write(labelDetails["Description"])
1146 self.gitStream.write("EOT\n\n")
1147
1148 else:
1149 if not self.silent:
1150 print ("Tag %s does not match with change %s: files do not match."
1151 % (labelDetails["label"], change))
1152
1153 else:
1154 if not self.silent:
1155 print ("Tag %s does not match with change %s: file count is different."
1156 % (labelDetails["label"], change))
1157
1158 def getUserCacheFilename(self):
1159 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1160 return home + "/.gitp4-usercache.txt"
1161
1162 def getUserMapFromPerforceServer(self):
1163 if self.userMapFromPerforceServer:
1164 return
1165 self.users = {}
1166
1167 for output in p4CmdList("users"):
1168 if not output.has_key("User"):
1169 continue
1170 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1171
1172
1173 s = ''
1174 for (key, val) in self.users.items():
1175 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1176
1177 open(self.getUserCacheFilename(), "wb").write(s)
1178 self.userMapFromPerforceServer = True
1179
1180 def loadUserMapFromCache(self):
1181 self.users = {}
1182 self.userMapFromPerforceServer = False
1183 try:
1184 cache = open(self.getUserCacheFilename(), "rb")
1185 lines = cache.readlines()
1186 cache.close()
1187 for line in lines:
1188 entry = line.strip().split("\t")
1189 self.users[entry[0]] = entry[1]
1190 except IOError:
1191 self.getUserMapFromPerforceServer()
1192
1193 def getLabels(self):
1194 self.labels = {}
1195
1196 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1197 if len(l) > 0 and not self.silent:
1198 print "Finding files belonging to labels in %s" % `self.depotPaths`
1199
1200 for output in l:
1201 label = output["label"]
1202 revisions = {}
1203 newestChange = 0
1204 if self.verbose:
1205 print "Querying files for label %s" % label
1206 for file in p4CmdList("files "
1207 + ' '.join (["%s...@%s" % (p, label)
1208 for p in self.depotPaths])):
1209 revisions[file["depotFile"]] = file["rev"]
1210 change = int(file["change"])
1211 if change > newestChange:
1212 newestChange = change
1213
1214 self.labels[newestChange] = [output, revisions]
1215
1216 if self.verbose:
1217 print "Label changes: %s" % self.labels.keys()
1218
1219 def guessProjectName(self):
1220 for p in self.depotPaths:
1221 if p.endswith("/"):
1222 p = p[:-1]
1223 p = p[p.strip().rfind("/") + 1:]
1224 if not p.endswith("/"):
1225 p += "/"
1226 return p
1227
1228 def getBranchMapping(self):
1229 lostAndFoundBranches = set()
1230
1231 for info in p4CmdList("branches"):
1232 details = p4Cmd("branch -o %s" % info["branch"])
1233 viewIdx = 0
1234 while details.has_key("View%s" % viewIdx):
1235 paths = details["View%s" % viewIdx].split(" ")
1236 viewIdx = viewIdx + 1
1237 # require standard //depot/foo/... //depot/bar/... mapping
1238 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1239 continue
1240 source = paths[0]
1241 destination = paths[1]
1242 ## HACK
1243 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1244 source = source[len(self.depotPaths[0]):-4]
1245 destination = destination[len(self.depotPaths[0]):-4]
1246
1247 if destination in self.knownBranches:
1248 if not self.silent:
1249 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1250 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1251 continue
1252
1253 self.knownBranches[destination] = source
1254
1255 lostAndFoundBranches.discard(destination)
1256
1257 if source not in self.knownBranches:
1258 lostAndFoundBranches.add(source)
1259
1260
1261 for branch in lostAndFoundBranches:
1262 self.knownBranches[branch] = branch
1263
1264 def getBranchMappingFromGitBranches(self):
1265 branches = p4BranchesInGit(self.importIntoRemotes)
1266 for branch in branches.keys():
1267 if branch == "master":
1268 branch = "main"
1269 else:
1270 branch = branch[len(self.projectName):]
1271 self.knownBranches[branch] = branch
1272
1273 def listExistingP4GitBranches(self):
1274 # branches holds mapping from name to commit
1275 branches = p4BranchesInGit(self.importIntoRemotes)
1276 self.p4BranchesInGit = branches.keys()
1277 for branch in branches.keys():
1278 self.initialParents[self.refPrefix + branch] = branches[branch]
1279
1280 def updateOptionDict(self, d):
1281 option_keys = {}
1282 if self.keepRepoPath:
1283 option_keys['keepRepoPath'] = 1
1284
1285 d["options"] = ' '.join(sorted(option_keys.keys()))
1286
1287 def readOptions(self, d):
1288 self.keepRepoPath = (d.has_key('options')
1289 and ('keepRepoPath' in d['options']))
1290
1291 def gitRefForBranch(self, branch):
1292 if branch == "main":
1293 return self.refPrefix + "master"
1294
1295 if len(branch) <= 0:
1296 return branch
1297
1298 return self.refPrefix + self.projectName + branch
1299
1300 def gitCommitByP4Change(self, ref, change):
1301 if self.verbose:
1302 print "looking in ref " + ref + " for change %s using bisect..." % change
1303
1304 earliestCommit = ""
1305 latestCommit = parseRevision(ref)
1306
1307 while True:
1308 if self.verbose:
1309 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1310 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1311 if len(next) == 0:
1312 if self.verbose:
1313 print "argh"
1314 return ""
1315 log = extractLogMessageFromGitCommit(next)
1316 settings = extractSettingsGitLog(log)
1317 currentChange = int(settings['change'])
1318 if self.verbose:
1319 print "current change %s" % currentChange
1320
1321 if currentChange == change:
1322 if self.verbose:
1323 print "found %s" % next
1324 return next
1325
1326 if currentChange < change:
1327 earliestCommit = "^%s" % next
1328 else:
1329 latestCommit = "%s" % next
1330
1331 return ""
1332
1333 def importNewBranch(self, branch, maxChange):
1334 # make fast-import flush all changes to disk and update the refs using the checkpoint
1335 # command so that we can try to find the branch parent in the git history
1336 self.gitStream.write("checkpoint\n\n");
1337 self.gitStream.flush();
1338 branchPrefix = self.depotPaths[0] + branch + "/"
1339 range = "@1,%s" % maxChange
1340 #print "prefix" + branchPrefix
1341 changes = p4ChangesForPaths([branchPrefix], range)
1342 if len(changes) <= 0:
1343 return False
1344 firstChange = changes[0]
1345 #print "first change in branch: %s" % firstChange
1346 sourceBranch = self.knownBranches[branch]
1347 sourceDepotPath = self.depotPaths[0] + sourceBranch
1348 sourceRef = self.gitRefForBranch(sourceBranch)
1349 #print "source " + sourceBranch
1350
1351 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1352 #print "branch parent: %s" % branchParentChange
1353 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1354 if len(gitParent) > 0:
1355 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1356 #print "parent git commit: %s" % gitParent
1357
1358 self.importChanges(changes)
1359 return True
1360
1361 def importChanges(self, changes):
1362 cnt = 1
1363 for change in changes:
1364 description = p4Cmd("describe %s" % change)
1365 self.updateOptionDict(description)
1366
1367 if not self.silent:
1368 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1369 sys.stdout.flush()
1370 cnt = cnt + 1
1371
1372 try:
1373 if self.detectBranches:
1374 branches = self.splitFilesIntoBranches(description)
1375 for branch in branches.keys():
1376 ## HACK --hwn
1377 branchPrefix = self.depotPaths[0] + branch + "/"
1378
1379 parent = ""
1380
1381 filesForCommit = branches[branch]
1382
1383 if self.verbose:
1384 print "branch is %s" % branch
1385
1386 self.updatedBranches.add(branch)
1387
1388 if branch not in self.createdBranches:
1389 self.createdBranches.add(branch)
1390 parent = self.knownBranches[branch]
1391 if parent == branch:
1392 parent = ""
1393 else:
1394 fullBranch = self.projectName + branch
1395 if fullBranch not in self.p4BranchesInGit:
1396 if not self.silent:
1397 print("\n Importing new branch %s" % fullBranch);
1398 if self.importNewBranch(branch, change - 1):
1399 parent = ""
1400 self.p4BranchesInGit.append(fullBranch)
1401 if not self.silent:
1402 print("\n Resuming with change %s" % change);
1403
1404 if self.verbose:
1405 print "parent determined through known branches: %s" % parent
1406
1407 branch = self.gitRefForBranch(branch)
1408 parent = self.gitRefForBranch(parent)
1409
1410 if self.verbose:
1411 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1412
1413 if len(parent) == 0 and branch in self.initialParents:
1414 parent = self.initialParents[branch]
1415 del self.initialParents[branch]
1416
1417 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1418 else:
1419 files = self.extractFilesFromCommit(description)
1420 self.commit(description, files, self.branch, self.depotPaths,
1421 self.initialParent)
1422 self.initialParent = ""
1423 except IOError:
1424 print self.gitError.read()
1425 sys.exit(1)
1426
1427 def importHeadRevision(self, revision):
1428 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1429
1430 details = { "user" : "git perforce import user", "time" : int(time.time()) }
1431 details["desc"] = ("Initial import of %s from the state at revision %s"
1432 % (' '.join(self.depotPaths), revision))
1433 details["change"] = revision
1434 newestRevision = 0
1435
1436 fileCnt = 0
1437 for info in p4CmdList("files "
1438 + ' '.join(["%s...%s"
1439 % (p, revision)
1440 for p in self.depotPaths])):
1441
1442 if info['code'] == 'error':
1443 sys.stderr.write("p4 returned an error: %s\n"
1444 % info['data'])
1445 sys.exit(1)
1446
1447
1448 change = int(info["change"])
1449 if change > newestRevision:
1450 newestRevision = change
1451
1452 if info["action"] in ("delete", "purge"):
1453 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1454 #fileCnt = fileCnt + 1
1455 continue
1456
1457 for prop in ["depotFile", "rev", "action", "type" ]:
1458 details["%s%s" % (prop, fileCnt)] = info[prop]
1459
1460 fileCnt = fileCnt + 1
1461
1462 details["change"] = newestRevision
1463 self.updateOptionDict(details)
1464 try:
1465 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1466 except IOError:
1467 print "IO error with git fast-import. Is your git version recent enough?"
1468 print self.gitError.read()
1469
1470
1471 def getClientSpec(self):
1472 specList = p4CmdList( "client -o" )
1473 temp = {}
1474 for entry in specList:
1475 for k,v in entry.iteritems():
1476 if k.startswith("View"):
1477 if v.startswith('"'):
1478 start = 1
1479 else:
1480 start = 0
1481 index = v.find("...")
1482 v = v[start:index]
1483 if v.startswith("-"):
1484 v = v[1:]
1485 temp[v] = -len(v)
1486 else:
1487 temp[v] = len(v)
1488 self.clientSpecDirs = temp.items()
1489 self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1490
1491 def run(self, args):
1492 self.depotPaths = []
1493 self.changeRange = ""
1494 self.initialParent = ""
1495 self.previousDepotPaths = []
1496
1497 # map from branch depot path to parent branch
1498 self.knownBranches = {}
1499 self.initialParents = {}
1500 self.hasOrigin = originP4BranchesExist()
1501 if not self.syncWithOrigin:
1502 self.hasOrigin = False
1503
1504 if self.importIntoRemotes:
1505 self.refPrefix = "refs/remotes/p4/"
1506 else:
1507 self.refPrefix = "refs/heads/p4/"
1508
1509 if self.syncWithOrigin and self.hasOrigin:
1510 if not self.silent:
1511 print "Syncing with origin first by calling git fetch origin"
1512 system("git fetch origin")
1513
1514 if len(self.branch) == 0:
1515 self.branch = self.refPrefix + "master"
1516 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1517 system("git update-ref %s refs/heads/p4" % self.branch)
1518 system("git branch -D p4");
1519 # create it /after/ importing, when master exists
1520 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1521 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1522
1523 if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1524 self.getClientSpec()
1525
1526 # TODO: should always look at previous commits,
1527 # merge with previous imports, if possible.
1528 if args == []:
1529 if self.hasOrigin:
1530 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1531 self.listExistingP4GitBranches()
1532
1533 if len(self.p4BranchesInGit) > 1:
1534 if not self.silent:
1535 print "Importing from/into multiple branches"
1536 self.detectBranches = True
1537
1538 if self.verbose:
1539 print "branches: %s" % self.p4BranchesInGit
1540
1541 p4Change = 0
1542 for branch in self.p4BranchesInGit:
1543 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
1544
1545 settings = extractSettingsGitLog(logMsg)
1546
1547 self.readOptions(settings)
1548 if (settings.has_key('depot-paths')
1549 and settings.has_key ('change')):
1550 change = int(settings['change']) + 1
1551 p4Change = max(p4Change, change)
1552
1553 depotPaths = sorted(settings['depot-paths'])
1554 if self.previousDepotPaths == []:
1555 self.previousDepotPaths = depotPaths
1556 else:
1557 paths = []
1558 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1559 for i in range(0, min(len(cur), len(prev))):
1560 if cur[i] <> prev[i]:
1561 i = i - 1
1562 break
1563
1564 paths.append (cur[:i + 1])
1565
1566 self.previousDepotPaths = paths
1567
1568 if p4Change > 0:
1569 self.depotPaths = sorted(self.previousDepotPaths)
1570 self.changeRange = "@%s,#head" % p4Change
1571 if not self.detectBranches:
1572 self.initialParent = parseRevision(self.branch)
1573 if not self.silent and not self.detectBranches:
1574 print "Performing incremental import into %s git branch" % self.branch
1575
1576 if not self.branch.startswith("refs/"):
1577 self.branch = "refs/heads/" + self.branch
1578
1579 if len(args) == 0 and self.depotPaths:
1580 if not self.silent:
1581 print "Depot paths: %s" % ' '.join(self.depotPaths)
1582 else:
1583 if self.depotPaths and self.depotPaths != args:
1584 print ("previous import used depot path %s and now %s was specified. "
1585 "This doesn't work!" % (' '.join (self.depotPaths),
1586 ' '.join (args)))
1587 sys.exit(1)
1588
1589 self.depotPaths = sorted(args)
1590
1591 revision = ""
1592 self.users = {}
1593
1594 newPaths = []
1595 for p in self.depotPaths:
1596 if p.find("@") != -1:
1597 atIdx = p.index("@")
1598 self.changeRange = p[atIdx:]
1599 if self.changeRange == "@all":
1600 self.changeRange = ""
1601 elif ',' not in self.changeRange:
1602 revision = self.changeRange
1603 self.changeRange = ""
1604 p = p[:atIdx]
1605 elif p.find("#") != -1:
1606 hashIdx = p.index("#")
1607 revision = p[hashIdx:]
1608 p = p[:hashIdx]
1609 elif self.previousDepotPaths == []:
1610 revision = "#head"
1611
1612 p = re.sub ("\.\.\.$", "", p)
1613 if not p.endswith("/"):
1614 p += "/"
1615
1616 newPaths.append(p)
1617
1618 self.depotPaths = newPaths
1619
1620
1621 self.loadUserMapFromCache()
1622 self.labels = {}
1623 if self.detectLabels:
1624 self.getLabels();
1625
1626 if self.detectBranches:
1627 ## FIXME - what's a P4 projectName ?
1628 self.projectName = self.guessProjectName()
1629
1630 if self.hasOrigin:
1631 self.getBranchMappingFromGitBranches()
1632 else:
1633 self.getBranchMapping()
1634 if self.verbose:
1635 print "p4-git branches: %s" % self.p4BranchesInGit
1636 print "initial parents: %s" % self.initialParents
1637 for b in self.p4BranchesInGit:
1638 if b != "master":
1639
1640 ## FIXME
1641 b = b[len(self.projectName):]
1642 self.createdBranches.add(b)
1643
1644 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1645
1646 importProcess = subprocess.Popen(["git", "fast-import"],
1647 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1648 stderr=subprocess.PIPE);
1649 self.gitOutput = importProcess.stdout
1650 self.gitStream = importProcess.stdin
1651 self.gitError = importProcess.stderr
1652
1653 if revision:
1654 self.importHeadRevision(revision)
1655 else:
1656 changes = []
1657
1658 if len(self.changesFile) > 0:
1659 output = open(self.changesFile).readlines()
1660 changeSet = set()
1661 for line in output:
1662 changeSet.add(int(line))
1663
1664 for change in changeSet:
1665 changes.append(change)
1666
1667 changes.sort()
1668 else:
1669 if self.verbose:
1670 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1671 self.changeRange)
1672 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1673
1674 if len(self.maxChanges) > 0:
1675 changes = changes[:min(int(self.maxChanges), len(changes))]
1676
1677 if len(changes) == 0:
1678 if not self.silent:
1679 print "No changes to import!"
1680 return True
1681
1682 if not self.silent and not self.detectBranches:
1683 print "Import destination: %s" % self.branch
1684
1685 self.updatedBranches = set()
1686
1687 self.importChanges(changes)
1688
1689 if not self.silent:
1690 print ""
1691 if len(self.updatedBranches) > 0:
1692 sys.stdout.write("Updated branches: ")
1693 for b in self.updatedBranches:
1694 sys.stdout.write("%s " % b)
1695 sys.stdout.write("\n")
1696
1697 self.gitStream.close()
1698 if importProcess.wait() != 0:
1699 die("fast-import failed: %s" % self.gitError.read())
1700 self.gitOutput.close()
1701 self.gitError.close()
1702
1703 return True
1704
1705 class P4Rebase(Command):
1706 def __init__(self):
1707 Command.__init__(self)
1708 self.options = [ ]
1709 self.description = ("Fetches the latest revision from perforce and "
1710 + "rebases the current work (branch) against it")
1711 self.verbose = False
1712
1713 def run(self, args):
1714 sync = P4Sync()
1715 sync.run([])
1716
1717 return self.rebase()
1718
1719 def rebase(self):
1720 if os.system("git update-index --refresh") != 0:
1721 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1722 if len(read_pipe("git diff-index HEAD --")) > 0:
1723 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1724
1725 [upstream, settings] = findUpstreamBranchPoint()
1726 if len(upstream) == 0:
1727 die("Cannot find upstream branchpoint for rebase")
1728
1729 # the branchpoint may be p4/foo~3, so strip off the parent
1730 upstream = re.sub("~[0-9]+$", "", upstream)
1731
1732 print "Rebasing the current branch onto %s" % upstream
1733 oldHead = read_pipe("git rev-parse HEAD").strip()
1734 system("git rebase %s" % upstream)
1735 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1736 return True
1737
1738 class P4Clone(P4Sync):
1739 def __init__(self):
1740 P4Sync.__init__(self)
1741 self.description = "Creates a new git repository and imports from Perforce into it"
1742 self.usage = "usage: %prog [options] //depot/path[@revRange]"
1743 self.options += [
1744 optparse.make_option("--destination", dest="cloneDestination",
1745 action='store', default=None,
1746 help="where to leave result of the clone"),
1747 optparse.make_option("-/", dest="cloneExclude",
1748 action="append", type="string",
1749 help="exclude depot path")
1750 ]
1751 self.cloneDestination = None
1752 self.needsGit = False
1753
1754 # This is required for the "append" cloneExclude action
1755 def ensure_value(self, attr, value):
1756 if not hasattr(self, attr) or getattr(self, attr) is None:
1757 setattr(self, attr, value)
1758 return getattr(self, attr)
1759
1760 def defaultDestination(self, args):
1761 ## TODO: use common prefix of args?
1762 depotPath = args[0]
1763 depotDir = re.sub("(@[^@]*)$", "", depotPath)
1764 depotDir = re.sub("(#[^#]*)$", "", depotDir)
1765 depotDir = re.sub(r"\.\.\.$", "", depotDir)
1766 depotDir = re.sub(r"/$", "", depotDir)
1767 return os.path.split(depotDir)[1]
1768
1769 def run(self, args):
1770 if len(args) < 1:
1771 return False
1772
1773 if self.keepRepoPath and not self.cloneDestination:
1774 sys.stderr.write("Must specify destination for --keep-path\n")
1775 sys.exit(1)
1776
1777 depotPaths = args
1778
1779 if not self.cloneDestination and len(depotPaths) > 1:
1780 self.cloneDestination = depotPaths[-1]
1781 depotPaths = depotPaths[:-1]
1782
1783 self.cloneExclude = ["/"+p for p in self.cloneExclude]
1784 for p in depotPaths:
1785 if not p.startswith("//"):
1786 return False
1787
1788 if not self.cloneDestination:
1789 self.cloneDestination = self.defaultDestination(args)
1790
1791 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1792 if not os.path.exists(self.cloneDestination):
1793 os.makedirs(self.cloneDestination)
1794 chdir(self.cloneDestination)
1795 system("git init")
1796 self.gitdir = os.getcwd() + "/.git"
1797 if not P4Sync.run(self, depotPaths):
1798 return False
1799 if self.branch != "master":
1800 if self.importIntoRemotes:
1801 masterbranch = "refs/remotes/p4/master"
1802 else:
1803 masterbranch = "refs/heads/p4/master"
1804 if gitBranchExists(masterbranch):
1805 system("git branch master %s" % masterbranch)
1806 system("git checkout -f")
1807 else:
1808 print "Could not detect main branch. No checkout/master branch created."
1809
1810 return True
1811
1812 class P4Branches(Command):
1813 def __init__(self):
1814 Command.__init__(self)
1815 self.options = [ ]
1816 self.description = ("Shows the git branches that hold imports and their "
1817 + "corresponding perforce depot paths")
1818 self.verbose = False
1819
1820 def run(self, args):
1821 if originP4BranchesExist():
1822 createOrUpdateBranchesFromOrigin()
1823
1824 cmdline = "git rev-parse --symbolic "
1825 cmdline += " --remotes"
1826
1827 for line in read_pipe_lines(cmdline):
1828 line = line.strip()
1829
1830 if not line.startswith('p4/') or line == "p4/HEAD":
1831 continue
1832 branch = line
1833
1834 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1835 settings = extractSettingsGitLog(log)
1836
1837 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1838 return True
1839
1840 class HelpFormatter(optparse.IndentedHelpFormatter):
1841 def __init__(self):
1842 optparse.IndentedHelpFormatter.__init__(self)
1843
1844 def format_description(self, description):
1845 if description:
1846 return description + "\n"
1847 else:
1848 return ""
1849
1850 def printUsage(commands):
1851 print "usage: %s <command> [options]" % sys.argv[0]
1852 print ""
1853 print "valid commands: %s" % ", ".join(commands)
1854 print ""
1855 print "Try %s <command> --help for command specific help." % sys.argv[0]
1856 print ""
1857
1858 commands = {
1859 "debug" : P4Debug,
1860 "submit" : P4Submit,
1861 "commit" : P4Submit,
1862 "sync" : P4Sync,
1863 "rebase" : P4Rebase,
1864 "clone" : P4Clone,
1865 "rollback" : P4RollBack,
1866 "branches" : P4Branches
1867 }
1868
1869
1870 def main():
1871 if len(sys.argv[1:]) == 0:
1872 printUsage(commands.keys())
1873 sys.exit(2)
1874
1875 cmd = ""
1876 cmdName = sys.argv[1]
1877 try:
1878 klass = commands[cmdName]
1879 cmd = klass()
1880 except KeyError:
1881 print "unknown command %s" % cmdName
1882 print ""
1883 printUsage(commands.keys())
1884 sys.exit(2)
1885
1886 options = cmd.options
1887 cmd.gitdir = os.environ.get("GIT_DIR", None)
1888
1889 args = sys.argv[2:]
1890
1891 if len(options) > 0:
1892 options.append(optparse.make_option("--git-dir", dest="gitdir"))
1893
1894 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1895 options,
1896 description = cmd.description,
1897 formatter = HelpFormatter())
1898
1899 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1900 global verbose
1901 verbose = cmd.verbose
1902 if cmd.needsGit:
1903 if cmd.gitdir == None:
1904 cmd.gitdir = os.path.abspath(".git")
1905 if not isValidGitDir(cmd.gitdir):
1906 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1907 if os.path.exists(cmd.gitdir):
1908 cdup = read_pipe("git rev-parse --show-cdup").strip()
1909 if len(cdup) > 0:
1910 chdir(cdup);
1911
1912 if not isValidGitDir(cmd.gitdir):
1913 if isValidGitDir(cmd.gitdir + "/.git"):
1914 cmd.gitdir += "/.git"
1915 else:
1916 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1917
1918 os.environ["GIT_DIR"] = cmd.gitdir
1919
1920 if not cmd.run(args):
1921 parser.print_help()
1922
1923
1924 if __name__ == '__main__':
1925 main()