Merge branch 'jn/gitweb-grep'
[git/git.git] / contrib / fast-import / git-p4
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10
11 import optparse, sys, os, marshal, popen2, subprocess, shelve
12 import tempfile, getopt, sha, os.path, time, platform
13 import re
14
15 from sets import Set;
16
17 verbose = False
18
19 def die(msg):
20 if verbose:
21 raise Exception(msg)
22 else:
23 sys.stderr.write(msg + "\n")
24 sys.exit(1)
25
26 def write_pipe(c, str):
27 if verbose:
28 sys.stderr.write('Writing pipe: %s\n' % c)
29
30 pipe = os.popen(c, 'w')
31 val = pipe.write(str)
32 if pipe.close():
33 die('Command failed: %s' % c)
34
35 return val
36
37 def read_pipe(c, ignore_error=False):
38 if verbose:
39 sys.stderr.write('Reading pipe: %s\n' % c)
40
41 pipe = os.popen(c, 'rb')
42 val = pipe.read()
43 if pipe.close() and not ignore_error:
44 die('Command failed: %s' % c)
45
46 return val
47
48
49 def read_pipe_lines(c):
50 if verbose:
51 sys.stderr.write('Reading pipe: %s\n' % c)
52 ## todo: check return status
53 pipe = os.popen(c, 'rb')
54 val = pipe.readlines()
55 if pipe.close():
56 die('Command failed: %s' % c)
57
58 return val
59
60 def system(cmd):
61 if verbose:
62 sys.stderr.write("executing %s\n" % cmd)
63 if os.system(cmd) != 0:
64 die("command failed: %s" % cmd)
65
66 def isP4Exec(kind):
67 """Determine if a Perforce 'kind' should have execute permission
68
69 'p4 help filetypes' gives a list of the types. If it starts with 'x',
70 or x follows one of a few letters. Otherwise, if there is an 'x' after
71 a plus sign, it is also executable"""
72 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
73
74 def setP4ExecBit(file, mode):
75 # Reopens an already open file and changes the execute bit to match
76 # the execute bit setting in the passed in mode.
77
78 p4Type = "+x"
79
80 if not isModeExec(mode):
81 p4Type = getP4OpenedType(file)
82 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
83 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
84 if p4Type[-1] == "+":
85 p4Type = p4Type[0:-1]
86
87 system("p4 reopen -t %s %s" % (p4Type, file))
88
89 def getP4OpenedType(file):
90 # Returns the perforce file type for the given file.
91
92 result = read_pipe("p4 opened %s" % file)
93 match = re.match(".*\((.+)\)$", result)
94 if match:
95 return match.group(1)
96 else:
97 die("Could not determine file type for %s" % file)
98
99 def diffTreePattern():
100 # This is a simple generator for the diff tree regex pattern. This could be
101 # a class variable if this and parseDiffTreeEntry were a part of a class.
102 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
103 while True:
104 yield pattern
105
106 def parseDiffTreeEntry(entry):
107 """Parses a single diff tree entry into its component elements.
108
109 See git-diff-tree(1) manpage for details about the format of the diff
110 output. This method returns a dictionary with the following elements:
111
112 src_mode - The mode of the source file
113 dst_mode - The mode of the destination file
114 src_sha1 - The sha1 for the source file
115 dst_sha1 - The sha1 fr the destination file
116 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
117 status_score - The score for the status (applicable for 'C' and 'R'
118 statuses). This is None if there is no score.
119 src - The path for the source file.
120 dst - The path for the destination file. This is only present for
121 copy or renames. If it is not present, this is None.
122
123 If the pattern is not matched, None is returned."""
124
125 match = diffTreePattern().next().match(entry)
126 if match:
127 return {
128 'src_mode': match.group(1),
129 'dst_mode': match.group(2),
130 'src_sha1': match.group(3),
131 'dst_sha1': match.group(4),
132 'status': match.group(5),
133 'status_score': match.group(6),
134 'src': match.group(7),
135 'dst': match.group(10)
136 }
137 return None
138
139 def isModeExec(mode):
140 # Returns True if the given git mode represents an executable file,
141 # otherwise False.
142 return mode[-3:] == "755"
143
144 def isModeExecChanged(src_mode, dst_mode):
145 return isModeExec(src_mode) != isModeExec(dst_mode)
146
147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
148 cmd = "p4 -G %s" % cmd
149 if verbose:
150 sys.stderr.write("Opening pipe: %s\n" % cmd)
151
152 # Use a temporary file to avoid deadlocks without
153 # subprocess.communicate(), which would put another copy
154 # of stdout into memory.
155 stdin_file = None
156 if stdin is not None:
157 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
158 stdin_file.write(stdin)
159 stdin_file.flush()
160 stdin_file.seek(0)
161
162 p4 = subprocess.Popen(cmd, shell=True,
163 stdin=stdin_file,
164 stdout=subprocess.PIPE)
165
166 result = []
167 try:
168 while True:
169 entry = marshal.load(p4.stdout)
170 result.append(entry)
171 except EOFError:
172 pass
173 exitCode = p4.wait()
174 if exitCode != 0:
175 entry = {}
176 entry["p4ExitCode"] = exitCode
177 result.append(entry)
178
179 return result
180
181 def p4Cmd(cmd):
182 list = p4CmdList(cmd)
183 result = {}
184 for entry in list:
185 result.update(entry)
186 return result;
187
188 def p4Where(depotPath):
189 if not depotPath.endswith("/"):
190 depotPath += "/"
191 output = p4Cmd("where %s..." % depotPath)
192 if output["code"] == "error":
193 return ""
194 clientPath = ""
195 if "path" in output:
196 clientPath = output.get("path")
197 elif "data" in output:
198 data = output.get("data")
199 lastSpace = data.rfind(" ")
200 clientPath = data[lastSpace + 1:]
201
202 if clientPath.endswith("..."):
203 clientPath = clientPath[:-3]
204 return clientPath
205
206 def currentGitBranch():
207 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
208
209 def isValidGitDir(path):
210 if (os.path.exists(path + "/HEAD")
211 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
212 return True;
213 return False
214
215 def parseRevision(ref):
216 return read_pipe("git rev-parse %s" % ref).strip()
217
218 def extractLogMessageFromGitCommit(commit):
219 logMessage = ""
220
221 ## fixme: title is first line of commit, not 1st paragraph.
222 foundTitle = False
223 for log in read_pipe_lines("git cat-file commit %s" % commit):
224 if not foundTitle:
225 if len(log) == 1:
226 foundTitle = True
227 continue
228
229 logMessage += log
230 return logMessage
231
232 def extractSettingsGitLog(log):
233 values = {}
234 for line in log.split("\n"):
235 line = line.strip()
236 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
237 if not m:
238 continue
239
240 assignments = m.group(1).split (':')
241 for a in assignments:
242 vals = a.split ('=')
243 key = vals[0].strip()
244 val = ('='.join (vals[1:])).strip()
245 if val.endswith ('\"') and val.startswith('"'):
246 val = val[1:-1]
247
248 values[key] = val
249
250 paths = values.get("depot-paths")
251 if not paths:
252 paths = values.get("depot-path")
253 if paths:
254 values['depot-paths'] = paths.split(',')
255 return values
256
257 def gitBranchExists(branch):
258 proc = subprocess.Popen(["git", "rev-parse", branch],
259 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
260 return proc.wait() == 0;
261
262 def gitConfig(key):
263 return read_pipe("git config %s" % key, ignore_error=True).strip()
264
265 def p4BranchesInGit(branchesAreInRemotes = True):
266 branches = {}
267
268 cmdline = "git rev-parse --symbolic "
269 if branchesAreInRemotes:
270 cmdline += " --remotes"
271 else:
272 cmdline += " --branches"
273
274 for line in read_pipe_lines(cmdline):
275 line = line.strip()
276
277 ## only import to p4/
278 if not line.startswith('p4/') or line == "p4/HEAD":
279 continue
280 branch = line
281
282 # strip off p4
283 branch = re.sub ("^p4/", "", line)
284
285 branches[branch] = parseRevision(line)
286 return branches
287
288 def findUpstreamBranchPoint(head = "HEAD"):
289 branches = p4BranchesInGit()
290 # map from depot-path to branch name
291 branchByDepotPath = {}
292 for branch in branches.keys():
293 tip = branches[branch]
294 log = extractLogMessageFromGitCommit(tip)
295 settings = extractSettingsGitLog(log)
296 if settings.has_key("depot-paths"):
297 paths = ",".join(settings["depot-paths"])
298 branchByDepotPath[paths] = "remotes/p4/" + branch
299
300 settings = None
301 parent = 0
302 while parent < 65535:
303 commit = head + "~%s" % parent
304 log = extractLogMessageFromGitCommit(commit)
305 settings = extractSettingsGitLog(log)
306 if settings.has_key("depot-paths"):
307 paths = ",".join(settings["depot-paths"])
308 if branchByDepotPath.has_key(paths):
309 return [branchByDepotPath[paths], settings]
310
311 parent = parent + 1
312
313 return ["", settings]
314
315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
316 if not silent:
317 print ("Creating/updating branch(es) in %s based on origin branch(es)"
318 % localRefPrefix)
319
320 originPrefix = "origin/p4/"
321
322 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
323 line = line.strip()
324 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
325 continue
326
327 headName = line[len(originPrefix):]
328 remoteHead = localRefPrefix + headName
329 originHead = line
330
331 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
332 if (not original.has_key('depot-paths')
333 or not original.has_key('change')):
334 continue
335
336 update = False
337 if not gitBranchExists(remoteHead):
338 if verbose:
339 print "creating %s" % remoteHead
340 update = True
341 else:
342 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
343 if settings.has_key('change') > 0:
344 if settings['depot-paths'] == original['depot-paths']:
345 originP4Change = int(original['change'])
346 p4Change = int(settings['change'])
347 if originP4Change > p4Change:
348 print ("%s (%s) is newer than %s (%s). "
349 "Updating p4 branch from origin."
350 % (originHead, originP4Change,
351 remoteHead, p4Change))
352 update = True
353 else:
354 print ("Ignoring: %s was imported from %s while "
355 "%s was imported from %s"
356 % (originHead, ','.join(original['depot-paths']),
357 remoteHead, ','.join(settings['depot-paths'])))
358
359 if update:
360 system("git update-ref %s %s" % (remoteHead, originHead))
361
362 def originP4BranchesExist():
363 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
364
365 def p4ChangesForPaths(depotPaths, changeRange):
366 assert depotPaths
367 output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
368 for p in depotPaths]))
369
370 changes = []
371 for line in output:
372 changeNum = line.split(" ")[1]
373 changes.append(int(changeNum))
374
375 changes.sort()
376 return changes
377
378 class Command:
379 def __init__(self):
380 self.usage = "usage: %prog [options]"
381 self.needsGit = True
382
383 class P4Debug(Command):
384 def __init__(self):
385 Command.__init__(self)
386 self.options = [
387 optparse.make_option("--verbose", dest="verbose", action="store_true",
388 default=False),
389 ]
390 self.description = "A tool to debug the output of p4 -G."
391 self.needsGit = False
392 self.verbose = False
393
394 def run(self, args):
395 j = 0
396 for output in p4CmdList(" ".join(args)):
397 print 'Element: %d' % j
398 j += 1
399 print output
400 return True
401
402 class P4RollBack(Command):
403 def __init__(self):
404 Command.__init__(self)
405 self.options = [
406 optparse.make_option("--verbose", dest="verbose", action="store_true"),
407 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
408 ]
409 self.description = "A tool to debug the multi-branch import. Don't use :)"
410 self.verbose = False
411 self.rollbackLocalBranches = False
412
413 def run(self, args):
414 if len(args) != 1:
415 return False
416 maxChange = int(args[0])
417
418 if "p4ExitCode" in p4Cmd("changes -m 1"):
419 die("Problems executing p4");
420
421 if self.rollbackLocalBranches:
422 refPrefix = "refs/heads/"
423 lines = read_pipe_lines("git rev-parse --symbolic --branches")
424 else:
425 refPrefix = "refs/remotes/"
426 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
427
428 for line in lines:
429 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
430 line = line.strip()
431 ref = refPrefix + line
432 log = extractLogMessageFromGitCommit(ref)
433 settings = extractSettingsGitLog(log)
434
435 depotPaths = settings['depot-paths']
436 change = settings['change']
437
438 changed = False
439
440 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
441 for p in depotPaths]))) == 0:
442 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
443 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
444 continue
445
446 while change and int(change) > maxChange:
447 changed = True
448 if self.verbose:
449 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
450 system("git update-ref %s \"%s^\"" % (ref, ref))
451 log = extractLogMessageFromGitCommit(ref)
452 settings = extractSettingsGitLog(log)
453
454
455 depotPaths = settings['depot-paths']
456 change = settings['change']
457
458 if changed:
459 print "%s rewound to %s" % (ref, change)
460
461 return True
462
463 class P4Submit(Command):
464 def __init__(self):
465 Command.__init__(self)
466 self.options = [
467 optparse.make_option("--verbose", dest="verbose", action="store_true"),
468 optparse.make_option("--origin", dest="origin"),
469 optparse.make_option("-M", dest="detectRename", action="store_true"),
470 ]
471 self.description = "Submit changes from git to the perforce depot."
472 self.usage += " [name of git branch to submit into perforce depot]"
473 self.interactive = True
474 self.origin = ""
475 self.detectRename = False
476 self.verbose = False
477 self.isWindows = (platform.system() == "Windows")
478
479 def check(self):
480 if len(p4CmdList("opened ...")) > 0:
481 die("You have files opened with perforce! Close them before starting the sync.")
482
483 # replaces everything between 'Description:' and the next P4 submit template field with the
484 # commit message
485 def prepareLogMessage(self, template, message):
486 result = ""
487
488 inDescriptionSection = False
489
490 for line in template.split("\n"):
491 if line.startswith("#"):
492 result += line + "\n"
493 continue
494
495 if inDescriptionSection:
496 if line.startswith("Files:"):
497 inDescriptionSection = False
498 else:
499 continue
500 else:
501 if line.startswith("Description:"):
502 inDescriptionSection = True
503 line += "\n"
504 for messageLine in message.split("\n"):
505 line += "\t" + messageLine + "\n"
506
507 result += line + "\n"
508
509 return result
510
511 def prepareSubmitTemplate(self):
512 # remove lines in the Files section that show changes to files outside the depot path we're committing into
513 template = ""
514 inFilesSection = False
515 for line in read_pipe_lines("p4 change -o"):
516 if inFilesSection:
517 if line.startswith("\t"):
518 # path starts and ends with a tab
519 path = line[1:]
520 lastTab = path.rfind("\t")
521 if lastTab != -1:
522 path = path[:lastTab]
523 if not path.startswith(self.depotPath):
524 continue
525 else:
526 inFilesSection = False
527 else:
528 if line.startswith("Files:"):
529 inFilesSection = True
530
531 template += line
532
533 return template
534
535 def applyCommit(self, id):
536 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
537 diffOpts = ("", "-M")[self.detectRename]
538 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
539 filesToAdd = set()
540 filesToDelete = set()
541 editedFiles = set()
542 filesToChangeExecBit = {}
543 for line in diff:
544 diff = parseDiffTreeEntry(line)
545 modifier = diff['status']
546 path = diff['src']
547 if modifier == "M":
548 system("p4 edit \"%s\"" % path)
549 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
550 filesToChangeExecBit[path] = diff['dst_mode']
551 editedFiles.add(path)
552 elif modifier == "A":
553 filesToAdd.add(path)
554 filesToChangeExecBit[path] = diff['dst_mode']
555 if path in filesToDelete:
556 filesToDelete.remove(path)
557 elif modifier == "D":
558 filesToDelete.add(path)
559 if path in filesToAdd:
560 filesToAdd.remove(path)
561 elif modifier == "R":
562 src, dest = diff['src'], diff['dst']
563 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
564 system("p4 edit \"%s\"" % (dest))
565 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
566 filesToChangeExecBit[dest] = diff['dst_mode']
567 os.unlink(dest)
568 editedFiles.add(dest)
569 filesToDelete.add(src)
570 else:
571 die("unknown modifier %s for %s" % (modifier, path))
572
573 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
574 patchcmd = diffcmd + " | git apply "
575 tryPatchCmd = patchcmd + "--check -"
576 applyPatchCmd = patchcmd + "--check --apply -"
577
578 if os.system(tryPatchCmd) != 0:
579 print "Unfortunately applying the change failed!"
580 print "What do you want to do?"
581 response = "x"
582 while response != "s" and response != "a" and response != "w":
583 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
584 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
585 if response == "s":
586 print "Skipping! Good luck with the next patches..."
587 for f in editedFiles:
588 system("p4 revert \"%s\"" % f);
589 for f in filesToAdd:
590 system("rm %s" %f)
591 return
592 elif response == "a":
593 os.system(applyPatchCmd)
594 if len(filesToAdd) > 0:
595 print "You may also want to call p4 add on the following files:"
596 print " ".join(filesToAdd)
597 if len(filesToDelete):
598 print "The following files should be scheduled for deletion with p4 delete:"
599 print " ".join(filesToDelete)
600 die("Please resolve and submit the conflict manually and "
601 + "continue afterwards with git-p4 submit --continue")
602 elif response == "w":
603 system(diffcmd + " > patch.txt")
604 print "Patch saved to patch.txt in %s !" % self.clientPath
605 die("Please resolve and submit the conflict manually and "
606 "continue afterwards with git-p4 submit --continue")
607
608 system(applyPatchCmd)
609
610 for f in filesToAdd:
611 system("p4 add \"%s\"" % f)
612 for f in filesToDelete:
613 system("p4 revert \"%s\"" % f)
614 system("p4 delete \"%s\"" % f)
615
616 # Set/clear executable bits
617 for f in filesToChangeExecBit.keys():
618 mode = filesToChangeExecBit[f]
619 setP4ExecBit(f, mode)
620
621 logMessage = extractLogMessageFromGitCommit(id)
622 if self.isWindows:
623 logMessage = logMessage.replace("\n", "\r\n")
624 logMessage = logMessage.strip()
625
626 template = self.prepareSubmitTemplate()
627
628 if self.interactive:
629 submitTemplate = self.prepareLogMessage(template, logMessage)
630 diff = read_pipe("p4 diff -du ...")
631
632 for newFile in filesToAdd:
633 diff += "==== new file ====\n"
634 diff += "--- /dev/null\n"
635 diff += "+++ %s\n" % newFile
636 f = open(newFile, "r")
637 for line in f.readlines():
638 diff += "+" + line
639 f.close()
640
641 separatorLine = "######## everything below this line is just the diff #######"
642 if platform.system() == "Windows":
643 separatorLine += "\r"
644 separatorLine += "\n"
645
646 [handle, fileName] = tempfile.mkstemp()
647 tmpFile = os.fdopen(handle, "w+")
648 tmpFile.write(submitTemplate + separatorLine + diff)
649 tmpFile.close()
650 defaultEditor = "vi"
651 if platform.system() == "Windows":
652 defaultEditor = "notepad"
653 editor = os.environ.get("EDITOR", defaultEditor);
654 system(editor + " " + fileName)
655 tmpFile = open(fileName, "rb")
656 message = tmpFile.read()
657 tmpFile.close()
658 os.remove(fileName)
659 submitTemplate = message[:message.index(separatorLine)]
660 if self.isWindows:
661 submitTemplate = submitTemplate.replace("\r\n", "\n")
662
663 write_pipe("p4 submit -i", submitTemplate)
664 else:
665 fileName = "submit.txt"
666 file = open(fileName, "w+")
667 file.write(self.prepareLogMessage(template, logMessage))
668 file.close()
669 print ("Perforce submit template written as %s. "
670 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
671 % (fileName, fileName))
672
673 def run(self, args):
674 if len(args) == 0:
675 self.master = currentGitBranch()
676 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
677 die("Detecting current git branch failed!")
678 elif len(args) == 1:
679 self.master = args[0]
680 else:
681 return False
682
683 [upstream, settings] = findUpstreamBranchPoint()
684 self.depotPath = settings['depot-paths'][0]
685 if len(self.origin) == 0:
686 self.origin = upstream
687
688 if self.verbose:
689 print "Origin branch is " + self.origin
690
691 if len(self.depotPath) == 0:
692 print "Internal error: cannot locate perforce depot path from existing branches"
693 sys.exit(128)
694
695 self.clientPath = p4Where(self.depotPath)
696
697 if len(self.clientPath) == 0:
698 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
699 sys.exit(128)
700
701 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
702 self.oldWorkingDirectory = os.getcwd()
703
704 os.chdir(self.clientPath)
705 print "Syncronizing p4 checkout..."
706 system("p4 sync ...")
707
708 self.check()
709
710 commits = []
711 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
712 commits.append(line.strip())
713 commits.reverse()
714
715 while len(commits) > 0:
716 commit = commits[0]
717 commits = commits[1:]
718 self.applyCommit(commit)
719 if not self.interactive:
720 break
721
722 if len(commits) == 0:
723 print "All changes applied!"
724 os.chdir(self.oldWorkingDirectory)
725
726 sync = P4Sync()
727 sync.run([])
728
729 rebase = P4Rebase()
730 rebase.rebase()
731
732 return True
733
734 class P4Sync(Command):
735 def __init__(self):
736 Command.__init__(self)
737 self.options = [
738 optparse.make_option("--branch", dest="branch"),
739 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
740 optparse.make_option("--changesfile", dest="changesFile"),
741 optparse.make_option("--silent", dest="silent", action="store_true"),
742 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
743 optparse.make_option("--verbose", dest="verbose", action="store_true"),
744 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
745 help="Import into refs/heads/ , not refs/remotes"),
746 optparse.make_option("--max-changes", dest="maxChanges"),
747 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
748 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
749 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
750 help="Only sync files that are included in the Perforce Client Spec")
751 ]
752 self.description = """Imports from Perforce into a git repository.\n
753 example:
754 //depot/my/project/ -- to import the current head
755 //depot/my/project/@all -- to import everything
756 //depot/my/project/@1,6 -- to import only from revision 1 to 6
757
758 (a ... is not needed in the path p4 specification, it's added implicitly)"""
759
760 self.usage += " //depot/path[@revRange]"
761 self.silent = False
762 self.createdBranches = Set()
763 self.committedChanges = Set()
764 self.branch = ""
765 self.detectBranches = False
766 self.detectLabels = False
767 self.changesFile = ""
768 self.syncWithOrigin = True
769 self.verbose = False
770 self.importIntoRemotes = True
771 self.maxChanges = ""
772 self.isWindows = (platform.system() == "Windows")
773 self.keepRepoPath = False
774 self.depotPaths = None
775 self.p4BranchesInGit = []
776 self.cloneExclude = []
777 self.useClientSpec = False
778 self.clientSpecDirs = []
779
780 if gitConfig("git-p4.syncFromOrigin") == "false":
781 self.syncWithOrigin = False
782
783 def extractFilesFromCommit(self, commit):
784 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
785 for path in self.cloneExclude]
786 files = []
787 fnum = 0
788 while commit.has_key("depotFile%s" % fnum):
789 path = commit["depotFile%s" % fnum]
790
791 if [p for p in self.cloneExclude
792 if path.startswith (p)]:
793 found = False
794 else:
795 found = [p for p in self.depotPaths
796 if path.startswith (p)]
797 if not found:
798 fnum = fnum + 1
799 continue
800
801 file = {}
802 file["path"] = path
803 file["rev"] = commit["rev%s" % fnum]
804 file["action"] = commit["action%s" % fnum]
805 file["type"] = commit["type%s" % fnum]
806 files.append(file)
807 fnum = fnum + 1
808 return files
809
810 def stripRepoPath(self, path, prefixes):
811 if self.keepRepoPath:
812 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
813
814 for p in prefixes:
815 if path.startswith(p):
816 path = path[len(p):]
817
818 return path
819
820 def splitFilesIntoBranches(self, commit):
821 branches = {}
822 fnum = 0
823 while commit.has_key("depotFile%s" % fnum):
824 path = commit["depotFile%s" % fnum]
825 found = [p for p in self.depotPaths
826 if path.startswith (p)]
827 if not found:
828 fnum = fnum + 1
829 continue
830
831 file = {}
832 file["path"] = path
833 file["rev"] = commit["rev%s" % fnum]
834 file["action"] = commit["action%s" % fnum]
835 file["type"] = commit["type%s" % fnum]
836 fnum = fnum + 1
837
838 relPath = self.stripRepoPath(path, self.depotPaths)
839
840 for branch in self.knownBranches.keys():
841
842 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
843 if relPath.startswith(branch + "/"):
844 if branch not in branches:
845 branches[branch] = []
846 branches[branch].append(file)
847 break
848
849 return branches
850
851 ## Should move this out, doesn't use SELF.
852 def readP4Files(self, files):
853 for f in files:
854 for val in self.clientSpecDirs:
855 if f['path'].startswith(val[0]):
856 if val[1] > 0:
857 f['include'] = True
858 else:
859 f['include'] = False
860 break
861
862 files = [f for f in files
863 if f['action'] != 'delete' and
864 (f.has_key('include') == False or f['include'] == True)]
865
866 if not files:
867 return []
868
869 filedata = p4CmdList('-x - print',
870 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
871 for f in files]),
872 stdin_mode='w+')
873 if "p4ExitCode" in filedata[0]:
874 die("Problems executing p4. Error: [%d]."
875 % (filedata[0]['p4ExitCode']));
876
877 j = 0;
878 contents = {}
879 while j < len(filedata):
880 stat = filedata[j]
881 j += 1
882 text = ''
883 while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
884 tmp = filedata[j]['data']
885 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
886 tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
887 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
888 tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
889 text += tmp
890 j += 1
891
892
893 if not stat.has_key('depotFile'):
894 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
895 continue
896
897 contents[stat['depotFile']] = text
898
899 for f in files:
900 assert not f.has_key('data')
901 f['data'] = contents[f['path']]
902 return files
903
904 def commit(self, details, files, branch, branchPrefixes, parent = ""):
905 epoch = details["time"]
906 author = details["user"]
907
908 if self.verbose:
909 print "commit into %s" % branch
910
911 # start with reading files; if that fails, we should not
912 # create a commit.
913 new_files = []
914 for f in files:
915 if [p for p in branchPrefixes if f['path'].startswith(p)]:
916 new_files.append (f)
917 else:
918 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
919 files = self.readP4Files(new_files)
920
921 self.gitStream.write("commit %s\n" % branch)
922 # gitStream.write("mark :%s\n" % details["change"])
923 self.committedChanges.add(int(details["change"]))
924 committer = ""
925 if author not in self.users:
926 self.getUserMapFromPerforceServer()
927 if author in self.users:
928 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
929 else:
930 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
931
932 self.gitStream.write("committer %s\n" % committer)
933
934 self.gitStream.write("data <<EOT\n")
935 self.gitStream.write(details["desc"])
936 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
937 % (','.join (branchPrefixes), details["change"]))
938 if len(details['options']) > 0:
939 self.gitStream.write(": options = %s" % details['options'])
940 self.gitStream.write("]\nEOT\n\n")
941
942 if len(parent) > 0:
943 if self.verbose:
944 print "parent %s" % parent
945 self.gitStream.write("from %s\n" % parent)
946
947 for file in files:
948 if file["type"] == "apple":
949 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
950 continue
951
952 relPath = self.stripRepoPath(file['path'], branchPrefixes)
953 if file["action"] == "delete":
954 self.gitStream.write("D %s\n" % relPath)
955 else:
956 data = file['data']
957
958 mode = "644"
959 if isP4Exec(file["type"]):
960 mode = "755"
961 elif file["type"] == "symlink":
962 mode = "120000"
963 # p4 print on a symlink contains "target\n", so strip it off
964 data = data[:-1]
965
966 if self.isWindows and file["type"].endswith("text"):
967 data = data.replace("\r\n", "\n")
968
969 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
970 self.gitStream.write("data %s\n" % len(data))
971 self.gitStream.write(data)
972 self.gitStream.write("\n")
973
974 self.gitStream.write("\n")
975
976 change = int(details["change"])
977
978 if self.labels.has_key(change):
979 label = self.labels[change]
980 labelDetails = label[0]
981 labelRevisions = label[1]
982 if self.verbose:
983 print "Change %s is labelled %s" % (change, labelDetails)
984
985 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
986 for p in branchPrefixes]))
987
988 if len(files) == len(labelRevisions):
989
990 cleanedFiles = {}
991 for info in files:
992 if info["action"] == "delete":
993 continue
994 cleanedFiles[info["depotFile"]] = info["rev"]
995
996 if cleanedFiles == labelRevisions:
997 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
998 self.gitStream.write("from %s\n" % branch)
999
1000 owner = labelDetails["Owner"]
1001 tagger = ""
1002 if author in self.users:
1003 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1004 else:
1005 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1006 self.gitStream.write("tagger %s\n" % tagger)
1007 self.gitStream.write("data <<EOT\n")
1008 self.gitStream.write(labelDetails["Description"])
1009 self.gitStream.write("EOT\n\n")
1010
1011 else:
1012 if not self.silent:
1013 print ("Tag %s does not match with change %s: files do not match."
1014 % (labelDetails["label"], change))
1015
1016 else:
1017 if not self.silent:
1018 print ("Tag %s does not match with change %s: file count is different."
1019 % (labelDetails["label"], change))
1020
1021 def getUserCacheFilename(self):
1022 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1023 return home + "/.gitp4-usercache.txt"
1024
1025 def getUserMapFromPerforceServer(self):
1026 if self.userMapFromPerforceServer:
1027 return
1028 self.users = {}
1029
1030 for output in p4CmdList("users"):
1031 if not output.has_key("User"):
1032 continue
1033 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1034
1035
1036 s = ''
1037 for (key, val) in self.users.items():
1038 s += "%s\t%s\n" % (key, val)
1039
1040 open(self.getUserCacheFilename(), "wb").write(s)
1041 self.userMapFromPerforceServer = True
1042
1043 def loadUserMapFromCache(self):
1044 self.users = {}
1045 self.userMapFromPerforceServer = False
1046 try:
1047 cache = open(self.getUserCacheFilename(), "rb")
1048 lines = cache.readlines()
1049 cache.close()
1050 for line in lines:
1051 entry = line.strip().split("\t")
1052 self.users[entry[0]] = entry[1]
1053 except IOError:
1054 self.getUserMapFromPerforceServer()
1055
1056 def getLabels(self):
1057 self.labels = {}
1058
1059 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1060 if len(l) > 0 and not self.silent:
1061 print "Finding files belonging to labels in %s" % `self.depotPaths`
1062
1063 for output in l:
1064 label = output["label"]
1065 revisions = {}
1066 newestChange = 0
1067 if self.verbose:
1068 print "Querying files for label %s" % label
1069 for file in p4CmdList("files "
1070 + ' '.join (["%s...@%s" % (p, label)
1071 for p in self.depotPaths])):
1072 revisions[file["depotFile"]] = file["rev"]
1073 change = int(file["change"])
1074 if change > newestChange:
1075 newestChange = change
1076
1077 self.labels[newestChange] = [output, revisions]
1078
1079 if self.verbose:
1080 print "Label changes: %s" % self.labels.keys()
1081
1082 def guessProjectName(self):
1083 for p in self.depotPaths:
1084 if p.endswith("/"):
1085 p = p[:-1]
1086 p = p[p.strip().rfind("/") + 1:]
1087 if not p.endswith("/"):
1088 p += "/"
1089 return p
1090
1091 def getBranchMapping(self):
1092 lostAndFoundBranches = set()
1093
1094 for info in p4CmdList("branches"):
1095 details = p4Cmd("branch -o %s" % info["branch"])
1096 viewIdx = 0
1097 while details.has_key("View%s" % viewIdx):
1098 paths = details["View%s" % viewIdx].split(" ")
1099 viewIdx = viewIdx + 1
1100 # require standard //depot/foo/... //depot/bar/... mapping
1101 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1102 continue
1103 source = paths[0]
1104 destination = paths[1]
1105 ## HACK
1106 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1107 source = source[len(self.depotPaths[0]):-4]
1108 destination = destination[len(self.depotPaths[0]):-4]
1109
1110 if destination in self.knownBranches:
1111 if not self.silent:
1112 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1113 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1114 continue
1115
1116 self.knownBranches[destination] = source
1117
1118 lostAndFoundBranches.discard(destination)
1119
1120 if source not in self.knownBranches:
1121 lostAndFoundBranches.add(source)
1122
1123
1124 for branch in lostAndFoundBranches:
1125 self.knownBranches[branch] = branch
1126
1127 def getBranchMappingFromGitBranches(self):
1128 branches = p4BranchesInGit(self.importIntoRemotes)
1129 for branch in branches.keys():
1130 if branch == "master":
1131 branch = "main"
1132 else:
1133 branch = branch[len(self.projectName):]
1134 self.knownBranches[branch] = branch
1135
1136 def listExistingP4GitBranches(self):
1137 # branches holds mapping from name to commit
1138 branches = p4BranchesInGit(self.importIntoRemotes)
1139 self.p4BranchesInGit = branches.keys()
1140 for branch in branches.keys():
1141 self.initialParents[self.refPrefix + branch] = branches[branch]
1142
1143 def updateOptionDict(self, d):
1144 option_keys = {}
1145 if self.keepRepoPath:
1146 option_keys['keepRepoPath'] = 1
1147
1148 d["options"] = ' '.join(sorted(option_keys.keys()))
1149
1150 def readOptions(self, d):
1151 self.keepRepoPath = (d.has_key('options')
1152 and ('keepRepoPath' in d['options']))
1153
1154 def gitRefForBranch(self, branch):
1155 if branch == "main":
1156 return self.refPrefix + "master"
1157
1158 if len(branch) <= 0:
1159 return branch
1160
1161 return self.refPrefix + self.projectName + branch
1162
1163 def gitCommitByP4Change(self, ref, change):
1164 if self.verbose:
1165 print "looking in ref " + ref + " for change %s using bisect..." % change
1166
1167 earliestCommit = ""
1168 latestCommit = parseRevision(ref)
1169
1170 while True:
1171 if self.verbose:
1172 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1173 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1174 if len(next) == 0:
1175 if self.verbose:
1176 print "argh"
1177 return ""
1178 log = extractLogMessageFromGitCommit(next)
1179 settings = extractSettingsGitLog(log)
1180 currentChange = int(settings['change'])
1181 if self.verbose:
1182 print "current change %s" % currentChange
1183
1184 if currentChange == change:
1185 if self.verbose:
1186 print "found %s" % next
1187 return next
1188
1189 if currentChange < change:
1190 earliestCommit = "^%s" % next
1191 else:
1192 latestCommit = "%s" % next
1193
1194 return ""
1195
1196 def importNewBranch(self, branch, maxChange):
1197 # make fast-import flush all changes to disk and update the refs using the checkpoint
1198 # command so that we can try to find the branch parent in the git history
1199 self.gitStream.write("checkpoint\n\n");
1200 self.gitStream.flush();
1201 branchPrefix = self.depotPaths[0] + branch + "/"
1202 range = "@1,%s" % maxChange
1203 #print "prefix" + branchPrefix
1204 changes = p4ChangesForPaths([branchPrefix], range)
1205 if len(changes) <= 0:
1206 return False
1207 firstChange = changes[0]
1208 #print "first change in branch: %s" % firstChange
1209 sourceBranch = self.knownBranches[branch]
1210 sourceDepotPath = self.depotPaths[0] + sourceBranch
1211 sourceRef = self.gitRefForBranch(sourceBranch)
1212 #print "source " + sourceBranch
1213
1214 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1215 #print "branch parent: %s" % branchParentChange
1216 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1217 if len(gitParent) > 0:
1218 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1219 #print "parent git commit: %s" % gitParent
1220
1221 self.importChanges(changes)
1222 return True
1223
1224 def importChanges(self, changes):
1225 cnt = 1
1226 for change in changes:
1227 description = p4Cmd("describe %s" % change)
1228 self.updateOptionDict(description)
1229
1230 if not self.silent:
1231 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1232 sys.stdout.flush()
1233 cnt = cnt + 1
1234
1235 try:
1236 if self.detectBranches:
1237 branches = self.splitFilesIntoBranches(description)
1238 for branch in branches.keys():
1239 ## HACK --hwn
1240 branchPrefix = self.depotPaths[0] + branch + "/"
1241
1242 parent = ""
1243
1244 filesForCommit = branches[branch]
1245
1246 if self.verbose:
1247 print "branch is %s" % branch
1248
1249 self.updatedBranches.add(branch)
1250
1251 if branch not in self.createdBranches:
1252 self.createdBranches.add(branch)
1253 parent = self.knownBranches[branch]
1254 if parent == branch:
1255 parent = ""
1256 else:
1257 fullBranch = self.projectName + branch
1258 if fullBranch not in self.p4BranchesInGit:
1259 if not self.silent:
1260 print("\n Importing new branch %s" % fullBranch);
1261 if self.importNewBranch(branch, change - 1):
1262 parent = ""
1263 self.p4BranchesInGit.append(fullBranch)
1264 if not self.silent:
1265 print("\n Resuming with change %s" % change);
1266
1267 if self.verbose:
1268 print "parent determined through known branches: %s" % parent
1269
1270 branch = self.gitRefForBranch(branch)
1271 parent = self.gitRefForBranch(parent)
1272
1273 if self.verbose:
1274 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1275
1276 if len(parent) == 0 and branch in self.initialParents:
1277 parent = self.initialParents[branch]
1278 del self.initialParents[branch]
1279
1280 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1281 else:
1282 files = self.extractFilesFromCommit(description)
1283 self.commit(description, files, self.branch, self.depotPaths,
1284 self.initialParent)
1285 self.initialParent = ""
1286 except IOError:
1287 print self.gitError.read()
1288 sys.exit(1)
1289
1290 def importHeadRevision(self, revision):
1291 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1292
1293 details = { "user" : "git perforce import user", "time" : int(time.time()) }
1294 details["desc"] = ("Initial import of %s from the state at revision %s"
1295 % (' '.join(self.depotPaths), revision))
1296 details["change"] = revision
1297 newestRevision = 0
1298
1299 fileCnt = 0
1300 for info in p4CmdList("files "
1301 + ' '.join(["%s...%s"
1302 % (p, revision)
1303 for p in self.depotPaths])):
1304
1305 if info['code'] == 'error':
1306 sys.stderr.write("p4 returned an error: %s\n"
1307 % info['data'])
1308 sys.exit(1)
1309
1310
1311 change = int(info["change"])
1312 if change > newestRevision:
1313 newestRevision = change
1314
1315 if info["action"] == "delete":
1316 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1317 #fileCnt = fileCnt + 1
1318 continue
1319
1320 for prop in ["depotFile", "rev", "action", "type" ]:
1321 details["%s%s" % (prop, fileCnt)] = info[prop]
1322
1323 fileCnt = fileCnt + 1
1324
1325 details["change"] = newestRevision
1326 self.updateOptionDict(details)
1327 try:
1328 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1329 except IOError:
1330 print "IO error with git fast-import. Is your git version recent enough?"
1331 print self.gitError.read()
1332
1333
1334 def getClientSpec(self):
1335 specList = p4CmdList( "client -o" )
1336 temp = {}
1337 for entry in specList:
1338 for k,v in entry.iteritems():
1339 if k.startswith("View"):
1340 if v.startswith('"'):
1341 start = 1
1342 else:
1343 start = 0
1344 index = v.find("...")
1345 v = v[start:index]
1346 if v.startswith("-"):
1347 v = v[1:]
1348 temp[v] = -len(v)
1349 else:
1350 temp[v] = len(v)
1351 self.clientSpecDirs = temp.items()
1352 self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1353
1354 def run(self, args):
1355 self.depotPaths = []
1356 self.changeRange = ""
1357 self.initialParent = ""
1358 self.previousDepotPaths = []
1359
1360 # map from branch depot path to parent branch
1361 self.knownBranches = {}
1362 self.initialParents = {}
1363 self.hasOrigin = originP4BranchesExist()
1364 if not self.syncWithOrigin:
1365 self.hasOrigin = False
1366
1367 if self.importIntoRemotes:
1368 self.refPrefix = "refs/remotes/p4/"
1369 else:
1370 self.refPrefix = "refs/heads/p4/"
1371
1372 if self.syncWithOrigin and self.hasOrigin:
1373 if not self.silent:
1374 print "Syncing with origin first by calling git fetch origin"
1375 system("git fetch origin")
1376
1377 if len(self.branch) == 0:
1378 self.branch = self.refPrefix + "master"
1379 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1380 system("git update-ref %s refs/heads/p4" % self.branch)
1381 system("git branch -D p4");
1382 # create it /after/ importing, when master exists
1383 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1384 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1385
1386 if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1387 self.getClientSpec()
1388
1389 # TODO: should always look at previous commits,
1390 # merge with previous imports, if possible.
1391 if args == []:
1392 if self.hasOrigin:
1393 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1394 self.listExistingP4GitBranches()
1395
1396 if len(self.p4BranchesInGit) > 1:
1397 if not self.silent:
1398 print "Importing from/into multiple branches"
1399 self.detectBranches = True
1400
1401 if self.verbose:
1402 print "branches: %s" % self.p4BranchesInGit
1403
1404 p4Change = 0
1405 for branch in self.p4BranchesInGit:
1406 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
1407
1408 settings = extractSettingsGitLog(logMsg)
1409
1410 self.readOptions(settings)
1411 if (settings.has_key('depot-paths')
1412 and settings.has_key ('change')):
1413 change = int(settings['change']) + 1
1414 p4Change = max(p4Change, change)
1415
1416 depotPaths = sorted(settings['depot-paths'])
1417 if self.previousDepotPaths == []:
1418 self.previousDepotPaths = depotPaths
1419 else:
1420 paths = []
1421 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1422 for i in range(0, min(len(cur), len(prev))):
1423 if cur[i] <> prev[i]:
1424 i = i - 1
1425 break
1426
1427 paths.append (cur[:i + 1])
1428
1429 self.previousDepotPaths = paths
1430
1431 if p4Change > 0:
1432 self.depotPaths = sorted(self.previousDepotPaths)
1433 self.changeRange = "@%s,#head" % p4Change
1434 if not self.detectBranches:
1435 self.initialParent = parseRevision(self.branch)
1436 if not self.silent and not self.detectBranches:
1437 print "Performing incremental import into %s git branch" % self.branch
1438
1439 if not self.branch.startswith("refs/"):
1440 self.branch = "refs/heads/" + self.branch
1441
1442 if len(args) == 0 and self.depotPaths:
1443 if not self.silent:
1444 print "Depot paths: %s" % ' '.join(self.depotPaths)
1445 else:
1446 if self.depotPaths and self.depotPaths != args:
1447 print ("previous import used depot path %s and now %s was specified. "
1448 "This doesn't work!" % (' '.join (self.depotPaths),
1449 ' '.join (args)))
1450 sys.exit(1)
1451
1452 self.depotPaths = sorted(args)
1453
1454 revision = ""
1455 self.users = {}
1456
1457 newPaths = []
1458 for p in self.depotPaths:
1459 if p.find("@") != -1:
1460 atIdx = p.index("@")
1461 self.changeRange = p[atIdx:]
1462 if self.changeRange == "@all":
1463 self.changeRange = ""
1464 elif ',' not in self.changeRange:
1465 revision = self.changeRange
1466 self.changeRange = ""
1467 p = p[:atIdx]
1468 elif p.find("#") != -1:
1469 hashIdx = p.index("#")
1470 revision = p[hashIdx:]
1471 p = p[:hashIdx]
1472 elif self.previousDepotPaths == []:
1473 revision = "#head"
1474
1475 p = re.sub ("\.\.\.$", "", p)
1476 if not p.endswith("/"):
1477 p += "/"
1478
1479 newPaths.append(p)
1480
1481 self.depotPaths = newPaths
1482
1483
1484 self.loadUserMapFromCache()
1485 self.labels = {}
1486 if self.detectLabels:
1487 self.getLabels();
1488
1489 if self.detectBranches:
1490 ## FIXME - what's a P4 projectName ?
1491 self.projectName = self.guessProjectName()
1492
1493 if self.hasOrigin:
1494 self.getBranchMappingFromGitBranches()
1495 else:
1496 self.getBranchMapping()
1497 if self.verbose:
1498 print "p4-git branches: %s" % self.p4BranchesInGit
1499 print "initial parents: %s" % self.initialParents
1500 for b in self.p4BranchesInGit:
1501 if b != "master":
1502
1503 ## FIXME
1504 b = b[len(self.projectName):]
1505 self.createdBranches.add(b)
1506
1507 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1508
1509 importProcess = subprocess.Popen(["git", "fast-import"],
1510 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1511 stderr=subprocess.PIPE);
1512 self.gitOutput = importProcess.stdout
1513 self.gitStream = importProcess.stdin
1514 self.gitError = importProcess.stderr
1515
1516 if revision:
1517 self.importHeadRevision(revision)
1518 else:
1519 changes = []
1520
1521 if len(self.changesFile) > 0:
1522 output = open(self.changesFile).readlines()
1523 changeSet = Set()
1524 for line in output:
1525 changeSet.add(int(line))
1526
1527 for change in changeSet:
1528 changes.append(change)
1529
1530 changes.sort()
1531 else:
1532 if self.verbose:
1533 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1534 self.changeRange)
1535 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1536
1537 if len(self.maxChanges) > 0:
1538 changes = changes[:min(int(self.maxChanges), len(changes))]
1539
1540 if len(changes) == 0:
1541 if not self.silent:
1542 print "No changes to import!"
1543 return True
1544
1545 if not self.silent and not self.detectBranches:
1546 print "Import destination: %s" % self.branch
1547
1548 self.updatedBranches = set()
1549
1550 self.importChanges(changes)
1551
1552 if not self.silent:
1553 print ""
1554 if len(self.updatedBranches) > 0:
1555 sys.stdout.write("Updated branches: ")
1556 for b in self.updatedBranches:
1557 sys.stdout.write("%s " % b)
1558 sys.stdout.write("\n")
1559
1560 self.gitStream.close()
1561 if importProcess.wait() != 0:
1562 die("fast-import failed: %s" % self.gitError.read())
1563 self.gitOutput.close()
1564 self.gitError.close()
1565
1566 return True
1567
1568 class P4Rebase(Command):
1569 def __init__(self):
1570 Command.__init__(self)
1571 self.options = [ ]
1572 self.description = ("Fetches the latest revision from perforce and "
1573 + "rebases the current work (branch) against it")
1574 self.verbose = False
1575
1576 def run(self, args):
1577 sync = P4Sync()
1578 sync.run([])
1579
1580 return self.rebase()
1581
1582 def rebase(self):
1583 if os.system("git update-index --refresh") != 0:
1584 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1585 if len(read_pipe("git diff-index HEAD --")) > 0:
1586 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1587
1588 [upstream, settings] = findUpstreamBranchPoint()
1589 if len(upstream) == 0:
1590 die("Cannot find upstream branchpoint for rebase")
1591
1592 # the branchpoint may be p4/foo~3, so strip off the parent
1593 upstream = re.sub("~[0-9]+$", "", upstream)
1594
1595 print "Rebasing the current branch onto %s" % upstream
1596 oldHead = read_pipe("git rev-parse HEAD").strip()
1597 system("git rebase %s" % upstream)
1598 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1599 return True
1600
1601 class P4Clone(P4Sync):
1602 def __init__(self):
1603 P4Sync.__init__(self)
1604 self.description = "Creates a new git repository and imports from Perforce into it"
1605 self.usage = "usage: %prog [options] //depot/path[@revRange]"
1606 self.options += [
1607 optparse.make_option("--destination", dest="cloneDestination",
1608 action='store', default=None,
1609 help="where to leave result of the clone"),
1610 optparse.make_option("-/", dest="cloneExclude",
1611 action="append", type="string",
1612 help="exclude depot path")
1613 ]
1614 self.cloneDestination = None
1615 self.needsGit = False
1616
1617 # This is required for the "append" cloneExclude action
1618 def ensure_value(self, attr, value):
1619 if not hasattr(self, attr) or getattr(self, attr) is None:
1620 setattr(self, attr, value)
1621 return getattr(self, attr)
1622
1623 def defaultDestination(self, args):
1624 ## TODO: use common prefix of args?
1625 depotPath = args[0]
1626 depotDir = re.sub("(@[^@]*)$", "", depotPath)
1627 depotDir = re.sub("(#[^#]*)$", "", depotDir)
1628 depotDir = re.sub(r"\.\.\.$", "", depotDir)
1629 depotDir = re.sub(r"/$", "", depotDir)
1630 return os.path.split(depotDir)[1]
1631
1632 def run(self, args):
1633 if len(args) < 1:
1634 return False
1635
1636 if self.keepRepoPath and not self.cloneDestination:
1637 sys.stderr.write("Must specify destination for --keep-path\n")
1638 sys.exit(1)
1639
1640 depotPaths = args
1641
1642 if not self.cloneDestination and len(depotPaths) > 1:
1643 self.cloneDestination = depotPaths[-1]
1644 depotPaths = depotPaths[:-1]
1645
1646 self.cloneExclude = ["/"+p for p in self.cloneExclude]
1647 for p in depotPaths:
1648 if not p.startswith("//"):
1649 return False
1650
1651 if not self.cloneDestination:
1652 self.cloneDestination = self.defaultDestination(args)
1653
1654 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1655 if not os.path.exists(self.cloneDestination):
1656 os.makedirs(self.cloneDestination)
1657 os.chdir(self.cloneDestination)
1658 system("git init")
1659 self.gitdir = os.getcwd() + "/.git"
1660 if not P4Sync.run(self, depotPaths):
1661 return False
1662 if self.branch != "master":
1663 if gitBranchExists("refs/remotes/p4/master"):
1664 system("git branch master refs/remotes/p4/master")
1665 system("git checkout -f")
1666 else:
1667 print "Could not detect main branch. No checkout/master branch created."
1668
1669 return True
1670
1671 class P4Branches(Command):
1672 def __init__(self):
1673 Command.__init__(self)
1674 self.options = [ ]
1675 self.description = ("Shows the git branches that hold imports and their "
1676 + "corresponding perforce depot paths")
1677 self.verbose = False
1678
1679 def run(self, args):
1680 if originP4BranchesExist():
1681 createOrUpdateBranchesFromOrigin()
1682
1683 cmdline = "git rev-parse --symbolic "
1684 cmdline += " --remotes"
1685
1686 for line in read_pipe_lines(cmdline):
1687 line = line.strip()
1688
1689 if not line.startswith('p4/') or line == "p4/HEAD":
1690 continue
1691 branch = line
1692
1693 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1694 settings = extractSettingsGitLog(log)
1695
1696 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1697 return True
1698
1699 class HelpFormatter(optparse.IndentedHelpFormatter):
1700 def __init__(self):
1701 optparse.IndentedHelpFormatter.__init__(self)
1702
1703 def format_description(self, description):
1704 if description:
1705 return description + "\n"
1706 else:
1707 return ""
1708
1709 def printUsage(commands):
1710 print "usage: %s <command> [options]" % sys.argv[0]
1711 print ""
1712 print "valid commands: %s" % ", ".join(commands)
1713 print ""
1714 print "Try %s <command> --help for command specific help." % sys.argv[0]
1715 print ""
1716
1717 commands = {
1718 "debug" : P4Debug,
1719 "submit" : P4Submit,
1720 "commit" : P4Submit,
1721 "sync" : P4Sync,
1722 "rebase" : P4Rebase,
1723 "clone" : P4Clone,
1724 "rollback" : P4RollBack,
1725 "branches" : P4Branches
1726 }
1727
1728
1729 def main():
1730 if len(sys.argv[1:]) == 0:
1731 printUsage(commands.keys())
1732 sys.exit(2)
1733
1734 cmd = ""
1735 cmdName = sys.argv[1]
1736 try:
1737 klass = commands[cmdName]
1738 cmd = klass()
1739 except KeyError:
1740 print "unknown command %s" % cmdName
1741 print ""
1742 printUsage(commands.keys())
1743 sys.exit(2)
1744
1745 options = cmd.options
1746 cmd.gitdir = os.environ.get("GIT_DIR", None)
1747
1748 args = sys.argv[2:]
1749
1750 if len(options) > 0:
1751 options.append(optparse.make_option("--git-dir", dest="gitdir"))
1752
1753 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1754 options,
1755 description = cmd.description,
1756 formatter = HelpFormatter())
1757
1758 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1759 global verbose
1760 verbose = cmd.verbose
1761 if cmd.needsGit:
1762 if cmd.gitdir == None:
1763 cmd.gitdir = os.path.abspath(".git")
1764 if not isValidGitDir(cmd.gitdir):
1765 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1766 if os.path.exists(cmd.gitdir):
1767 cdup = read_pipe("git rev-parse --show-cdup").strip()
1768 if len(cdup) > 0:
1769 os.chdir(cdup);
1770
1771 if not isValidGitDir(cmd.gitdir):
1772 if isValidGitDir(cmd.gitdir + "/.git"):
1773 cmd.gitdir += "/.git"
1774 else:
1775 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1776
1777 os.environ["GIT_DIR"] = cmd.gitdir
1778
1779 if not cmd.run(args):
1780 parser.print_help()
1781
1782
1783 if __name__ == '__main__':
1784 main()