Document some implementation details, for the curious... :)
[git/git.git] / contrib / fast-import / git-p4
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
5# Author: Simon Hausmann <hausmann@kde.org>
83dce55a
SH
6# Copyright: 2007 Simon Hausmann <hausmann@kde.org>
7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
4f5cf76a 11import optparse, sys, os, marshal, popen2, shelve
b984733c
SH
12import tempfile, getopt, sha, os.path, time
13from sets import Set;
4f5cf76a
SH
14
15gitdir = os.environ.get("GIT_DIR", "")
86949eef
SH
16
17def p4CmdList(cmd):
18 cmd = "p4 -G %s" % cmd
19 pipe = os.popen(cmd, "rb")
20
21 result = []
22 try:
23 while True:
24 entry = marshal.load(pipe)
25 result.append(entry)
26 except EOFError:
27 pass
28 pipe.close()
29
30 return result
31
32def p4Cmd(cmd):
33 list = p4CmdList(cmd)
34 result = {}
35 for entry in list:
36 result.update(entry)
37 return result;
38
cb2c9db5
SH
39def p4Where(depotPath):
40 if not depotPath.endswith("/"):
41 depotPath += "/"
42 output = p4Cmd("where %s..." % depotPath)
43 clientPath = ""
44 if "path" in output:
45 clientPath = output.get("path")
46 elif "data" in output:
47 data = output.get("data")
48 lastSpace = data.rfind(" ")
49 clientPath = data[lastSpace + 1:]
50
51 if clientPath.endswith("..."):
52 clientPath = clientPath[:-3]
53 return clientPath
54
86949eef
SH
55def die(msg):
56 sys.stderr.write(msg + "\n")
57 sys.exit(1)
58
59def currentGitBranch():
9863f405 60 return os.popen("git name-rev HEAD").read().split(" ")[1][:-1]
86949eef 61
4f5cf76a
SH
62def isValidGitDir(path):
63 if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
64 return True;
65 return False
66
67def system(cmd):
68 if os.system(cmd) != 0:
69 die("command failed: %s" % cmd)
70
6ae8de88
SH
71def extractLogMessageFromGitCommit(commit):
72 logMessage = ""
73 foundTitle = False
9863f405 74 for log in os.popen("git cat-file commit %s" % commit).readlines():
6ae8de88
SH
75 if not foundTitle:
76 if len(log) == 1:
1c094184 77 foundTitle = True
6ae8de88
SH
78 continue
79
80 logMessage += log
81 return logMessage
82
83def extractDepotPathAndChangeFromGitLog(log):
84 values = {}
85 for line in log.split("\n"):
86 line = line.strip()
87 if line.startswith("[git-p4:") and line.endswith("]"):
88 line = line[8:-1].strip()
89 for assignment in line.split(":"):
90 variable = assignment.strip()
91 value = ""
92 equalPos = assignment.find("=")
93 if equalPos != -1:
94 variable = assignment[:equalPos].strip()
95 value = assignment[equalPos + 1:].strip()
96 if value.startswith("\"") and value.endswith("\""):
97 value = value[1:-1]
98 values[variable] = value
99
100 return values.get("depot-path"), values.get("change")
101
8136a639 102def gitBranchExists(branch):
9863f405 103 if os.system("git rev-parse %s 2>/dev/null >/dev/null" % branch) == 0:
179caebf
SH
104 return True
105 return False
8136a639 106
b984733c
SH
107class Command:
108 def __init__(self):
109 self.usage = "usage: %prog [options]"
8910ac0e 110 self.needsGit = True
b984733c
SH
111
112class P4Debug(Command):
86949eef 113 def __init__(self):
6ae8de88 114 Command.__init__(self)
86949eef
SH
115 self.options = [
116 ]
c8c39116 117 self.description = "A tool to debug the output of p4 -G."
8910ac0e 118 self.needsGit = False
86949eef
SH
119
120 def run(self, args):
121 for output in p4CmdList(" ".join(args)):
122 print output
b984733c 123 return True
86949eef 124
b984733c 125class P4CleanTags(Command):
86949eef 126 def __init__(self):
b984733c 127 Command.__init__(self)
86949eef
SH
128 self.options = [
129# optparse.make_option("--branch", dest="branch", default="refs/heads/master")
130 ]
c8c39116 131 self.description = "A tool to remove stale unused tags from incremental perforce imports."
86949eef
SH
132 def run(self, args):
133 branch = currentGitBranch()
134 print "Cleaning out stale p4 import tags..."
9863f405 135 sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % branch)
86949eef
SH
136 output = sout.read()
137 try:
138 tagIdx = output.index(" tags/p4/")
139 except:
140 print "Cannot find any p4/* tag. Nothing to do."
141 sys.exit(0)
142
143 try:
144 caretIdx = output.index("^")
145 except:
146 caretIdx = len(output) - 1
147 rev = int(output[tagIdx + 9 : caretIdx])
148
149 allTags = os.popen("git tag -l p4/").readlines()
150 for i in range(len(allTags)):
151 allTags[i] = int(allTags[i][3:-1])
152
153 allTags.sort()
154
155 allTags.remove(rev)
156
157 for rev in allTags:
158 print os.popen("git tag -d p4/%s" % rev).read()
159
160 print "%s tags removed." % len(allTags)
b984733c 161 return True
86949eef 162
711544b0 163class P4Submit(Command):
4f5cf76a 164 def __init__(self):
b984733c 165 Command.__init__(self)
4f5cf76a
SH
166 self.options = [
167 optparse.make_option("--continue", action="store_false", dest="firstTime"),
168 optparse.make_option("--origin", dest="origin"),
169 optparse.make_option("--reset", action="store_true", dest="reset"),
4f5cf76a
SH
170 optparse.make_option("--log-substitutions", dest="substFile"),
171 optparse.make_option("--noninteractive", action="store_false"),
04219c04 172 optparse.make_option("--dry-run", action="store_true"),
4f5cf76a
SH
173 ]
174 self.description = "Submit changes from git to the perforce depot."
c9b50e63 175 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a
SH
176 self.firstTime = True
177 self.reset = False
178 self.interactive = True
179 self.dryRun = False
180 self.substFile = ""
181 self.firstTime = True
9512497b 182 self.origin = ""
4f5cf76a
SH
183
184 self.logSubstitutions = {}
185 self.logSubstitutions["<enter description here>"] = "%log%"
186 self.logSubstitutions["\tDetails:"] = "\tDetails: %log%"
187
188 def check(self):
189 if len(p4CmdList("opened ...")) > 0:
190 die("You have files opened with perforce! Close them before starting the sync.")
191
192 def start(self):
193 if len(self.config) > 0 and not self.reset:
194 die("Cannot start sync. Previous sync config found at %s" % self.configFile)
195
196 commits = []
9863f405 197 for line in os.popen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
4f5cf76a
SH
198 commits.append(line[:-1])
199 commits.reverse()
200
201 self.config["commits"] = commits
202
4f5cf76a
SH
203 def prepareLogMessage(self, template, message):
204 result = ""
205
206 for line in template.split("\n"):
207 if line.startswith("#"):
208 result += line + "\n"
209 continue
210
211 substituted = False
212 for key in self.logSubstitutions.keys():
213 if line.find(key) != -1:
214 value = self.logSubstitutions[key]
215 value = value.replace("%log%", message)
216 if value != "@remove@":
217 result += line.replace(key, value) + "\n"
218 substituted = True
219 break
220
221 if not substituted:
222 result += line + "\n"
223
224 return result
225
226 def apply(self, id):
9863f405 227 print "Applying %s" % (os.popen("git log --max-count=1 --pretty=oneline %s" % id).read())
4f5cf76a
SH
228 diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
229 filesToAdd = set()
230 filesToDelete = set()
231 for line in diff:
232 modifier = line[0]
233 path = line[1:].strip()
234 if modifier == "M":
235 system("p4 edit %s" % path)
236 elif modifier == "A":
237 filesToAdd.add(path)
238 if path in filesToDelete:
239 filesToDelete.remove(path)
240 elif modifier == "D":
241 filesToDelete.add(path)
242 if path in filesToAdd:
243 filesToAdd.remove(path)
244 else:
245 die("unknown modifier %s for %s" % (modifier, path))
246
51a2640a
SH
247 diffcmd = "git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\"" % (id, id)
248 patchcmd = diffcmd + " | patch -p1"
249
250 if os.system(patchcmd + " --dry-run --silent") != 0:
251 print "Unfortunately applying the change failed!"
252 print "What do you want to do?"
253 response = "x"
254 while response != "s" and response != "a" and response != "w":
255 response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ")
256 if response == "s":
257 print "Skipping! Good luck with the next patches..."
258 return
259 elif response == "a":
260 os.system(patchcmd)
261 if len(filesToAdd) > 0:
262 print "You may also want to call p4 add on the following files:"
263 print " ".join(filesToAdd)
264 if len(filesToDelete):
265 print "The following files should be scheduled for deletion with p4 delete:"
266 print " ".join(filesToDelete)
267 die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
268 elif response == "w":
269 system(diffcmd + " > patch.txt")
270 print "Patch saved to patch.txt in %s !" % self.clientPath
271 die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
272
273 system(patchcmd)
4f5cf76a
SH
274
275 for f in filesToAdd:
276 system("p4 add %s" % f)
277 for f in filesToDelete:
278 system("p4 revert %s" % f)
279 system("p4 delete %s" % f)
280
6ae8de88
SH
281 logMessage = extractLogMessageFromGitCommit(id)
282 logMessage = logMessage.replace("\n", "\n\t")
283 logMessage = logMessage[:-1]
4f5cf76a
SH
284
285 template = os.popen("p4 change -o").read()
286
287 if self.interactive:
288 submitTemplate = self.prepareLogMessage(template, logMessage)
289 diff = os.popen("p4 diff -du ...").read()
290
291 for newFile in filesToAdd:
292 diff += "==== new file ====\n"
293 diff += "--- /dev/null\n"
294 diff += "+++ %s\n" % newFile
295 f = open(newFile, "r")
296 for line in f.readlines():
297 diff += "+" + line
298 f.close()
299
53150250 300 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a
SH
301
302 response = "e"
53150250 303 firstIteration = True
4f5cf76a 304 while response == "e":
53150250 305 if not firstIteration:
5e80dd4d 306 response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o ")
53150250 307 firstIteration = False
4f5cf76a
SH
308 if response == "e":
309 [handle, fileName] = tempfile.mkstemp()
310 tmpFile = os.fdopen(handle, "w+")
53150250 311 tmpFile.write(submitTemplate + separatorLine + diff)
4f5cf76a
SH
312 tmpFile.close()
313 editor = os.environ.get("EDITOR", "vi")
314 system(editor + " " + fileName)
315 tmpFile = open(fileName, "r")
53150250 316 message = tmpFile.read()
4f5cf76a
SH
317 tmpFile.close()
318 os.remove(fileName)
53150250 319 submitTemplate = message[:message.index(separatorLine)]
4f5cf76a
SH
320
321 if response == "y" or response == "yes":
322 if self.dryRun:
323 print submitTemplate
324 raw_input("Press return to continue...")
325 else:
326 pipe = os.popen("p4 submit -i", "w")
327 pipe.write(submitTemplate)
328 pipe.close()
329 else:
330 print "Not submitting!"
331 self.interactive = False
332 else:
333 fileName = "submit.txt"
334 file = open(fileName, "w+")
335 file.write(self.prepareLogMessage(template, logMessage))
336 file.close()
337 print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
338
339 def run(self, args):
9512497b
SH
340 global gitdir
341 # make gitdir absolute so we can cd out into the perforce checkout
342 gitdir = os.path.abspath(gitdir)
343 os.environ["GIT_DIR"] = gitdir
c9b50e63
SH
344
345 if len(args) == 0:
346 self.master = currentGitBranch()
347 if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
348 die("Detecting current git branch failed!")
349 elif len(args) == 1:
350 self.master = args[0]
351 else:
352 return False
353
9512497b
SH
354 depotPath = ""
355 if gitBranchExists("p4"):
356 [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4"))
357 if len(depotPath) == 0 and gitBranchExists("origin"):
358 [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin"))
359
360 if len(depotPath) == 0:
361 print "Internal error: cannot locate perforce depot path from existing branches"
362 sys.exit(128)
363
51a2640a 364 self.clientPath = p4Where(depotPath)
9512497b 365
51a2640a 366 if len(self.clientPath) == 0:
9512497b
SH
367 print "Error: Cannot locate perforce checkout of %s in client view" % depotPath
368 sys.exit(128)
369
51a2640a 370 print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath)
80b5910f 371 oldWorkingDirectory = os.getcwd()
51a2640a
SH
372 os.chdir(self.clientPath)
373 response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath)
9512497b
SH
374 if response == "y" or response == "yes":
375 system("p4 sync ...")
376
377 if len(self.origin) == 0:
378 if gitBranchExists("p4"):
379 self.origin = "p4"
380 else:
381 self.origin = "origin"
382
4f5cf76a
SH
383 if self.reset:
384 self.firstTime = True
385
386 if len(self.substFile) > 0:
387 for line in open(self.substFile, "r").readlines():
388 tokens = line[:-1].split("=")
389 self.logSubstitutions[tokens[0]] = tokens[1]
390
4f5cf76a
SH
391 self.check()
392 self.configFile = gitdir + "/p4-git-sync.cfg"
393 self.config = shelve.open(self.configFile, writeback=True)
394
395 if self.firstTime:
396 self.start()
397
398 commits = self.config.get("commits", [])
399
400 while len(commits) > 0:
401 self.firstTime = False
402 commit = commits[0]
403 commits = commits[1:]
404 self.config["commits"] = commits
405 self.apply(commit)
406 if not self.interactive:
407 break
408
409 self.config.close()
410
411 if len(commits) == 0:
412 if self.firstTime:
413 print "No changes found to apply between %s and current HEAD" % self.origin
414 else:
415 print "All changes applied!"
5e80dd4d 416 response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ")
80b5910f
SH
417 if response == "y" or response == "yes":
418 os.chdir(oldWorkingDirectory)
419 rebase = P4Rebase()
420 rebase.run([])
4f5cf76a
SH
421 os.remove(self.configFile)
422
b984733c
SH
423 return True
424
711544b0 425class P4Sync(Command):
b984733c
SH
426 def __init__(self):
427 Command.__init__(self)
428 self.options = [
429 optparse.make_option("--branch", dest="branch"),
430 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
431 optparse.make_option("--changesfile", dest="changesFile"),
432 optparse.make_option("--silent", dest="silent", action="store_true"),
433 optparse.make_option("--known-branches", dest="knownBranches"),
2a9489c0 434 optparse.make_option("--data-cache", dest="dataCache", action="store_true"),
cb53e1f8
SH
435 optparse.make_option("--command-cache", dest="commandCache", action="store_true"),
436 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true")
b984733c
SH
437 ]
438 self.description = """Imports from Perforce into a git repository.\n
439 example:
440 //depot/my/project/ -- to import the current head
441 //depot/my/project/@all -- to import everything
442 //depot/my/project/@1,6 -- to import only from revision 1 to 6
443
444 (a ... is not needed in the path p4 specification, it's added implicitly)"""
445
446 self.usage += " //depot/path[@revRange]"
447
448 self.dataCache = False
449 self.commandCache = False
450 self.silent = False
451 self.knownBranches = Set()
452 self.createdBranches = Set()
453 self.committedChanges = Set()
569d1bd4 454 self.branch = ""
b984733c 455 self.detectBranches = False
cb53e1f8 456 self.detectLabels = False
b984733c
SH
457 self.changesFile = ""
458
459 def p4File(self, depotPath):
460 return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
461
462 def extractFilesFromCommit(self, commit):
463 files = []
464 fnum = 0
465 while commit.has_key("depotFile%s" % fnum):
466 path = commit["depotFile%s" % fnum]
8f872531 467 if not path.startswith(self.depotPath):
b984733c 468 # if not self.silent:
8f872531 469 # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change)
b984733c
SH
470 fnum = fnum + 1
471 continue
472
473 file = {}
474 file["path"] = path
475 file["rev"] = commit["rev%s" % fnum]
476 file["action"] = commit["action%s" % fnum]
477 file["type"] = commit["type%s" % fnum]
478 files.append(file)
479 fnum = fnum + 1
480 return files
481
482 def isSubPathOf(self, first, second):
483 if not first.startswith(second):
484 return False
485 if first == second:
486 return True
487 return first[len(second)] == "/"
488
489 def branchesForCommit(self, files):
490 branches = Set()
491
492 for file in files:
8f872531 493 relativePath = file["path"][len(self.depotPath):]
b984733c
SH
494 # strip off the filename
495 relativePath = relativePath[0:relativePath.rfind("/")]
496
497 # if len(branches) == 0:
498 # branches.add(relativePath)
499 # knownBranches.add(relativePath)
500 # continue
501
502 ###### this needs more testing :)
503 knownBranch = False
504 for branch in branches:
505 if relativePath == branch:
506 knownBranch = True
507 break
508 # if relativePath.startswith(branch):
509 if self.isSubPathOf(relativePath, branch):
510 knownBranch = True
511 break
512 # if branch.startswith(relativePath):
513 if self.isSubPathOf(branch, relativePath):
514 branches.remove(branch)
515 break
516
517 if knownBranch:
518 continue
519
2a9489c0 520 for branch in self.knownBranches:
b984733c
SH
521 #if relativePath.startswith(branch):
522 if self.isSubPathOf(relativePath, branch):
523 if len(branches) == 0:
524 relativePath = branch
525 else:
526 knownBranch = True
527 break
528
529 if knownBranch:
530 continue
531
532 branches.add(relativePath)
533 self.knownBranches.add(relativePath)
534
535 return branches
536
537 def findBranchParent(self, branchPrefix, files):
538 for file in files:
539 path = file["path"]
540 if not path.startswith(branchPrefix):
541 continue
542 action = file["action"]
543 if action != "integrate" and action != "branch":
544 continue
545 rev = file["rev"]
546 depotPath = path + "#" + rev
547
548 log = p4CmdList("filelog \"%s\"" % depotPath)
549 if len(log) != 1:
550 print "eek! I got confused by the filelog of %s" % depotPath
551 sys.exit(1);
552
553 log = log[0]
554 if log["action0"] != action:
555 print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
556 sys.exit(1);
557
558 branchAction = log["how0,0"]
559 # if branchAction == "branch into" or branchAction == "ignored":
560 # continue # ignore for branching
561
562 if not branchAction.endswith(" from"):
563 continue # ignore for branching
564 # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
565 # sys.exit(1);
566
567 source = log["file0,0"]
568 if source.startswith(branchPrefix):
569 continue
570
571 lastSourceRev = log["erev0,0"]
572
573 sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
574 if len(sourceLog) != 1:
575 print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
576 sys.exit(1);
577 sourceLog = sourceLog[0]
578
8f872531 579 relPath = source[len(self.depotPath):]
b984733c
SH
580 # strip off the filename
581 relPath = relPath[0:relPath.rfind("/")]
582
583 for branch in self.knownBranches:
584 if self.isSubPathOf(relPath, branch):
585 # print "determined parent branch branch %s due to change in file %s" % (branch, source)
586 return branch
587 # else:
588 # print "%s is not a subpath of branch %s" % (relPath, branch)
589
590 return ""
591
c715706b 592 def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""):
b984733c
SH
593 epoch = details["time"]
594 author = details["user"]
595
596 self.gitStream.write("commit %s\n" % branch)
597 # gitStream.write("mark :%s\n" % details["change"])
598 self.committedChanges.add(int(details["change"]))
599 committer = ""
600 if author in self.users:
0828ab14 601 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
b984733c 602 else:
0828ab14 603 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
b984733c
SH
604
605 self.gitStream.write("committer %s\n" % committer)
606
607 self.gitStream.write("data <<EOT\n")
608 self.gitStream.write(details["desc"])
6ae8de88 609 self.gitStream.write("\n[git-p4: depot-path = \"%s\": change = %s]\n" % (branchPrefix, details["change"]))
b984733c
SH
610 self.gitStream.write("EOT\n\n")
611
612 if len(parent) > 0:
613 self.gitStream.write("from %s\n" % parent)
614
615 if len(merged) > 0:
616 self.gitStream.write("merge %s\n" % merged)
617
618 for file in files:
619 path = file["path"]
620 if not path.startswith(branchPrefix):
621 # if not silent:
622 # print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
623 continue
624 rev = file["rev"]
625 depotPath = path + "#" + rev
626 relPath = path[len(branchPrefix):]
627 action = file["action"]
628
629 if file["type"] == "apple":
630 print "\nfile %s is a strange apple file that forks. Ignoring!" % path
631 continue
632
633 if action == "delete":
634 self.gitStream.write("D %s\n" % relPath)
635 else:
636 mode = 644
637 if file["type"].startswith("x"):
638 mode = 755
639
640 data = self.p4File(depotPath)
641
642 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
643 self.gitStream.write("data %s\n" % len(data))
644 self.gitStream.write(data)
645 self.gitStream.write("\n")
646
647 self.gitStream.write("\n")
648
1f4ba1cb
SH
649 change = int(details["change"])
650
651 self.lastChange = change
652
653 if change in self.labels:
654 label = self.labels[change]
655 labelDetails = label[0]
656 labelRevisions = label[1]
657
658 files = p4CmdList("files %s...@%s" % (branchPrefix, change))
659
660 if len(files) == len(labelRevisions):
661
662 cleanedFiles = {}
663 for info in files:
664 if info["action"] == "delete":
665 continue
666 cleanedFiles[info["depotFile"]] = info["rev"]
667
668 if cleanedFiles == labelRevisions:
669 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
670 self.gitStream.write("from %s\n" % branch)
671
672 owner = labelDetails["Owner"]
673 tagger = ""
674 if author in self.users:
675 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
676 else:
677 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
678 self.gitStream.write("tagger %s\n" % tagger)
679 self.gitStream.write("data <<EOT\n")
680 self.gitStream.write(labelDetails["Description"])
681 self.gitStream.write("EOT\n\n")
682
683 else:
a46668fa 684 if not self.silent:
1f4ba1cb
SH
685 print "Tag %s does not match with change %s: files do not match." % (labelDetails["label"], change)
686
687 else:
a46668fa 688 if not self.silent:
1f4ba1cb 689 print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)
b984733c
SH
690
691 def extractFilesInCommitToBranch(self, files, branchPrefix):
692 newFiles = []
693
694 for file in files:
695 path = file["path"]
696 if path.startswith(branchPrefix):
697 newFiles.append(file)
698
699 return newFiles
700
701 def findBranchSourceHeuristic(self, files, branch, branchPrefix):
702 for file in files:
703 action = file["action"]
704 if action != "integrate" and action != "branch":
705 continue
706 path = file["path"]
707 rev = file["rev"]
708 depotPath = path + "#" + rev
709
710 log = p4CmdList("filelog \"%s\"" % depotPath)
711 if len(log) != 1:
712 print "eek! I got confused by the filelog of %s" % depotPath
713 sys.exit(1);
714
715 log = log[0]
716 if log["action0"] != action:
717 print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
718 sys.exit(1);
719
720 branchAction = log["how0,0"]
721
722 if not branchAction.endswith(" from"):
723 continue # ignore for branching
724 # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
725 # sys.exit(1);
726
727 source = log["file0,0"]
728 if source.startswith(branchPrefix):
729 continue
730
731 lastSourceRev = log["erev0,0"]
732
733 sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
734 if len(sourceLog) != 1:
735 print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
736 sys.exit(1);
737 sourceLog = sourceLog[0]
738
8f872531 739 relPath = source[len(self.depotPath):]
b984733c
SH
740 # strip off the filename
741 relPath = relPath[0:relPath.rfind("/")]
742
743 for candidate in self.knownBranches:
744 if self.isSubPathOf(relPath, candidate) and candidate != branch:
745 return candidate
746
747 return ""
748
749 def changeIsBranchMerge(self, sourceBranch, destinationBranch, change):
750 sourceFiles = {}
8f872531 751 for file in p4CmdList("files %s...@%s" % (self.depotPath + sourceBranch + "/", change)):
b984733c
SH
752 if file["action"] == "delete":
753 continue
754 sourceFiles[file["depotFile"]] = file
755
756 destinationFiles = {}
8f872531 757 for file in p4CmdList("files %s...@%s" % (self.depotPath + destinationBranch + "/", change)):
b984733c
SH
758 destinationFiles[file["depotFile"]] = file
759
760 for fileName in sourceFiles.keys():
761 integrations = []
762 deleted = False
763 integrationCount = 0
764 for integration in p4CmdList("integrated \"%s\"" % fileName):
765 toFile = integration["fromFile"] # yes, it's true, it's fromFile
766 if not toFile in destinationFiles:
767 continue
768 destFile = destinationFiles[toFile]
769 if destFile["action"] == "delete":
770 # print "file %s has been deleted in %s" % (fileName, toFile)
771 deleted = True
772 break
773 integrationCount += 1
774 if integration["how"] == "branch from":
775 continue
776
777 if int(integration["change"]) == change:
778 integrations.append(integration)
779 continue
780 if int(integration["change"]) > change:
781 continue
782
783 destRev = int(destFile["rev"])
784
785 startRev = integration["startFromRev"][1:]
786 if startRev == "none":
787 startRev = 0
788 else:
789 startRev = int(startRev)
790
791 endRev = integration["endFromRev"][1:]
792 if endRev == "none":
793 endRev = 0
794 else:
795 endRev = int(endRev)
796
797 initialBranch = (destRev == 1 and integration["how"] != "branch into")
798 inRange = (destRev >= startRev and destRev <= endRev)
799 newer = (destRev > startRev and destRev > endRev)
800
801 if initialBranch or inRange or newer:
802 integrations.append(integration)
803
804 if deleted:
805 continue
806
807 if len(integrations) == 0 and integrationCount > 1:
808 print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch)
809 return False
810
811 return True
812
813 def getUserMap(self):
814 self.users = {}
815
816 for output in p4CmdList("users"):
817 if not output.has_key("User"):
818 continue
819 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
820
1f4ba1cb
SH
821 def getLabels(self):
822 self.labels = {}
823
8f872531 824 l = p4CmdList("labels %s..." % self.depotPath)
10c3211b 825 if len(l) > 0 and not self.silent:
8f872531 826 print "Finding files belonging to labels in %s" % self.depotPath
01ce1fe9
SH
827
828 for output in l:
1f4ba1cb
SH
829 label = output["label"]
830 revisions = {}
831 newestChange = 0
832 for file in p4CmdList("files //...@%s" % label):
833 revisions[file["depotFile"]] = file["rev"]
834 change = int(file["change"])
835 if change > newestChange:
836 newestChange = change
837
838 self.labels[newestChange] = [output, revisions]
839
b984733c 840 def run(self, args):
8f872531 841 self.depotPath = ""
179caebf
SH
842 self.changeRange = ""
843 self.initialParent = ""
179caebf 844
569d1bd4
SH
845 if len(self.branch) == 0:
846 self.branch = "p4"
967f72e2
SH
847
848 if len(args) == 0:
849 if not gitBranchExists(self.branch) and gitBranchExists("origin"):
850 if not self.silent:
851 print "Creating %s branch in git repository based on origin" % self.branch
852 system("git branch %s origin" % self.branch)
853
854 [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch))
855 if len(self.previousDepotPath) > 0 and len(p4Change) > 0:
856 p4Change = int(p4Change) + 1
8f872531 857 self.depotPath = self.previousDepotPath
967f72e2
SH
858 self.changeRange = "@%s,#head" % p4Change
859 self.initialParent = self.branch
967f72e2
SH
860 if not self.silent:
861 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 862
b984733c 863 self.branch = "refs/heads/" + self.branch
179caebf 864
8f872531
SH
865 if len(self.depotPath) != 0:
866 self.depotPath = self.depotPath[:-1]
b984733c 867
8f872531 868 if len(args) == 0 and len(self.depotPath) != 0:
b984733c 869 if not self.silent:
8f872531 870 print "Depot path: %s" % self.depotPath
b984733c
SH
871 elif len(args) != 1:
872 return False
873 else:
8f872531
SH
874 if len(self.depotPath) != 0 and self.depotPath != args[0]:
875 print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0])
b984733c 876 sys.exit(1)
8f872531 877 self.depotPath = args[0]
b984733c 878
b984733c
SH
879 self.revision = ""
880 self.users = {}
b984733c 881 self.lastChange = 0
b984733c 882
8f872531
SH
883 if self.depotPath.find("@") != -1:
884 atIdx = self.depotPath.index("@")
885 self.changeRange = self.depotPath[atIdx:]
b984733c
SH
886 if self.changeRange == "@all":
887 self.changeRange = ""
888 elif self.changeRange.find(",") == -1:
889 self.revision = self.changeRange
890 self.changeRange = ""
8f872531
SH
891 self.depotPath = self.depotPath[0:atIdx]
892 elif self.depotPath.find("#") != -1:
893 hashIdx = self.depotPath.index("#")
894 self.revision = self.depotPath[hashIdx:]
895 self.depotPath = self.depotPath[0:hashIdx]
b984733c
SH
896 elif len(self.previousDepotPath) == 0:
897 self.revision = "#head"
898
8f872531
SH
899 if self.depotPath.endswith("..."):
900 self.depotPath = self.depotPath[:-3]
b984733c 901
8f872531
SH
902 if not self.depotPath.endswith("/"):
903 self.depotPath += "/"
b984733c
SH
904
905 self.getUserMap()
cb53e1f8
SH
906 self.labels = {}
907 if self.detectLabels:
908 self.getLabels();
b984733c
SH
909
910 if len(self.changeRange) == 0:
911 try:
9863f405 912 sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % self.branch)
b984733c
SH
913 output = sout.read()
914 if output.endswith("\n"):
915 output = output[:-1]
916 tagIdx = output.index(" tags/p4/")
917 caretIdx = output.find("^")
918 endPos = len(output)
919 if caretIdx != -1:
920 endPos = caretIdx
921 self.rev = int(output[tagIdx + 9 : endPos]) + 1
922 self.changeRange = "@%s,#head" % self.rev
9863f405 923 self.initialParent = os.popen("git rev-parse %s" % self.branch).read()[:-1]
b984733c
SH
924 except:
925 pass
926
f291b4e3 927 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 928
90865adc
SH
929 importProcess = popen2.Popen3("git fast-import", capturestderr = True)
930 self.gitOutput = importProcess.fromchild
931 self.gitStream = importProcess.tochild
932 self.gitError = importProcess.childerr
b984733c
SH
933
934 if len(self.revision) > 0:
8f872531 935 print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision)
b984733c
SH
936
937 details = { "user" : "git perforce import user", "time" : int(time.time()) }
8f872531 938 details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision)
b984733c
SH
939 details["change"] = self.revision
940 newestRevision = 0
941
942 fileCnt = 0
8f872531 943 for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)):
b984733c
SH
944 change = int(info["change"])
945 if change > newestRevision:
946 newestRevision = change
947
948 if info["action"] == "delete":
c45b1cfe
SH
949 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
950 #fileCnt = fileCnt + 1
b984733c
SH
951 continue
952
953 for prop in [ "depotFile", "rev", "action", "type" ]:
954 details["%s%s" % (prop, fileCnt)] = info[prop]
955
956 fileCnt = fileCnt + 1
957
958 details["change"] = newestRevision
959
960 try:
8f872531 961 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath)
c715706b 962 except IOError:
fd4ca86a 963 print "IO error with git fast-import. Is your git version recent enough?"
b984733c
SH
964 print self.gitError.read()
965
966 else:
967 changes = []
968
0828ab14 969 if len(self.changesFile) > 0:
b984733c
SH
970 output = open(self.changesFile).readlines()
971 changeSet = Set()
972 for line in output:
973 changeSet.add(int(line))
974
975 for change in changeSet:
976 changes.append(change)
977
978 changes.sort()
979 else:
8f872531 980 output = os.popen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines()
b984733c
SH
981
982 for line in output:
983 changeNum = line.split(" ")[1]
984 changes.append(changeNum)
985
986 changes.reverse()
987
988 if len(changes) == 0:
0828ab14 989 if not self.silent:
b984733c 990 print "no changes to import!"
1f52af6c 991 return True
b984733c
SH
992
993 cnt = 1
994 for change in changes:
995 description = p4Cmd("describe %s" % change)
996
0828ab14 997 if not self.silent:
b984733c
SH
998 sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
999 sys.stdout.flush()
1000 cnt = cnt + 1
1001
1002 try:
1003 files = self.extractFilesFromCommit(description)
1004 if self.detectBranches:
1005 for branch in self.branchesForCommit(files):
1006 self.knownBranches.add(branch)
8f872531 1007 branchPrefix = self.depotPath + branch + "/"
b984733c
SH
1008
1009 filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix)
1010
1011 merged = ""
1012 parent = ""
1013 ########### remove cnt!!!
1014 if branch not in self.createdBranches and cnt > 2:
1015 self.createdBranches.add(branch)
1016 parent = self.findBranchParent(branchPrefix, files)
1017 if parent == branch:
1018 parent = ""
1019 # elif len(parent) > 0:
1020 # print "%s branched off of %s" % (branch, parent)
1021
1022 if len(parent) == 0:
1023 merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix)
1024 if len(merged) > 0:
1025 print "change %s could be a merge from %s into %s" % (description["change"], merged, branch)
1026 if not self.changeIsBranchMerge(merged, branch, int(description["change"])):
1027 merged = ""
1028
1029 branch = "refs/heads/" + branch
1030 if len(parent) > 0:
1031 parent = "refs/heads/" + parent
1032 if len(merged) > 0:
1033 merged = "refs/heads/" + merged
1034 self.commit(description, files, branch, branchPrefix, parent, merged)
1035 else:
8f872531 1036 self.commit(description, files, self.branch, self.depotPath, self.initialParent)
b984733c
SH
1037 self.initialParent = ""
1038 except IOError:
1039 print self.gitError.read()
1040 sys.exit(1)
1041
1042 if not self.silent:
1043 print ""
1044
b984733c
SH
1045
1046 self.gitStream.close()
1047 self.gitOutput.close()
1048 self.gitError.close()
90865adc 1049 importProcess.wait()
b984733c 1050
b984733c
SH
1051 return True
1052
01ce1fe9
SH
1053class P4Rebase(Command):
1054 def __init__(self):
1055 Command.__init__(self)
1056 self.options = [ ]
1057 self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it"
1058
1059 def run(self, args):
1060 sync = P4Sync()
1061 sync.run([])
1062 print "Rebasing the current branch"
1f52af6c 1063 oldHead = os.popen("git rev-parse HEAD").read()[:-1]
01ce1fe9 1064 system("git rebase p4")
1f52af6c 1065 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
1066 return True
1067
f9a3a4f7
SH
1068class P4Clone(P4Sync):
1069 def __init__(self):
1070 P4Sync.__init__(self)
1071 self.description = "Creates a new git repository and imports from Perforce into it"
1072 self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]"
1073 self.needsGit = False
f9a3a4f7
SH
1074
1075 def run(self, args):
1076 if len(args) < 1:
1077 return False
1078 depotPath = args[0]
1079 dir = ""
1080 if len(args) == 2:
1081 dir = args[1]
1082 elif len(args) > 2:
1083 return False
1084
1085 if not depotPath.startswith("//"):
1086 return False
1087
1088 if len(dir) == 0:
1089 dir = depotPath
1090 atPos = dir.rfind("@")
1091 if atPos != -1:
1092 dir = dir[0:atPos]
1093 hashPos = dir.rfind("#")
1094 if hashPos != -1:
1095 dir = dir[0:hashPos]
1096
1097 if dir.endswith("..."):
1098 dir = dir[:-3]
1099
1100 if dir.endswith("/"):
1101 dir = dir[:-1]
1102
1103 slashPos = dir.rfind("/")
1104 if slashPos != -1:
1105 dir = dir[slashPos + 1:]
1106
1107 print "Importing from %s into %s" % (depotPath, dir)
1108 os.makedirs(dir)
1109 os.chdir(dir)
1110 system("git init")
1111 if not P4Sync.run(self, [depotPath]):
1112 return False
f9a3a4f7
SH
1113 if self.branch != "master":
1114 system("git branch master p4")
1115 system("git checkout -f")
1116 return True
1117
b984733c
SH
1118class HelpFormatter(optparse.IndentedHelpFormatter):
1119 def __init__(self):
1120 optparse.IndentedHelpFormatter.__init__(self)
1121
1122 def format_description(self, description):
1123 if description:
1124 return description + "\n"
1125 else:
1126 return ""
4f5cf76a 1127
86949eef
SH
1128def printUsage(commands):
1129 print "usage: %s <command> [options]" % sys.argv[0]
1130 print ""
1131 print "valid commands: %s" % ", ".join(commands)
1132 print ""
1133 print "Try %s <command> --help for command specific help." % sys.argv[0]
1134 print ""
1135
1136commands = {
1137 "debug" : P4Debug(),
4f5cf76a 1138 "clean-tags" : P4CleanTags(),
711544b0 1139 "submit" : P4Submit(),
01ce1fe9 1140 "sync" : P4Sync(),
f9a3a4f7
SH
1141 "rebase" : P4Rebase(),
1142 "clone" : P4Clone()
86949eef
SH
1143}
1144
1145if len(sys.argv[1:]) == 0:
1146 printUsage(commands.keys())
1147 sys.exit(2)
1148
1149cmd = ""
1150cmdName = sys.argv[1]
1151try:
1152 cmd = commands[cmdName]
1153except KeyError:
1154 print "unknown command %s" % cmdName
1155 print ""
1156 printUsage(commands.keys())
1157 sys.exit(2)
1158
4f5cf76a
SH
1159options = cmd.options
1160cmd.gitdir = gitdir
4f5cf76a 1161
e20a9e53 1162args = sys.argv[2:]
86949eef 1163
e20a9e53
SH
1164if len(options) > 0:
1165 options.append(optparse.make_option("--git-dir", dest="gitdir"))
1166
1167 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1168 options,
1169 description = cmd.description,
1170 formatter = HelpFormatter())
1171
1172 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
86949eef 1173
8910ac0e
SH
1174if cmd.needsGit:
1175 gitdir = cmd.gitdir
1176 if len(gitdir) == 0:
1177 gitdir = ".git"
1178 if not isValidGitDir(gitdir):
1179 cdup = os.popen("git rev-parse --show-cdup").read()[:-1]
1180 if isValidGitDir(cdup + "/" + gitdir):
1181 os.chdir(cdup)
4f5cf76a 1182
8910ac0e
SH
1183 if not isValidGitDir(gitdir):
1184 if isValidGitDir(gitdir + "/.git"):
1185 gitdir += "/.git"
1186 else:
1187 die("fatal: cannot locate git repository at %s" % gitdir)
4f5cf76a 1188
8910ac0e 1189 os.environ["GIT_DIR"] = gitdir
4f5cf76a 1190
b984733c
SH
1191if not cmd.run(args):
1192 parser.print_help()
1193