git-p4: Add a helper function to parse the full git diff-tree output.
[git/git.git] / contrib / fast-import / git-p4
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
08483580 11import optparse, sys, os, marshal, popen2, subprocess, shelve
25df95cc 12import tempfile, getopt, sha, os.path, time, platform
ce6f33c8 13import re
8b41a97f 14
b984733c 15from sets import Set;
4f5cf76a 16
4addad22 17verbose = False
86949eef 18
86dff6b6
HWN
19def die(msg):
20 if verbose:
21 raise Exception(msg)
22 else:
23 sys.stderr.write(msg + "\n")
24 sys.exit(1)
25
bce4c5fc 26def write_pipe(c, str):
4addad22 27 if verbose:
86dff6b6 28 sys.stderr.write('Writing pipe: %s\n' % c)
b016d397 29
bce4c5fc 30 pipe = os.popen(c, 'w')
b016d397 31 val = pipe.write(str)
bce4c5fc 32 if pipe.close():
86dff6b6 33 die('Command failed: %s' % c)
b016d397
HWN
34
35 return val
36
4addad22
HWN
37def read_pipe(c, ignore_error=False):
38 if verbose:
86dff6b6 39 sys.stderr.write('Reading pipe: %s\n' % c)
8b41a97f 40
bce4c5fc 41 pipe = os.popen(c, 'rb')
b016d397 42 val = pipe.read()
4addad22 43 if pipe.close() and not ignore_error:
86dff6b6 44 die('Command failed: %s' % c)
b016d397
HWN
45
46 return val
47
48
bce4c5fc 49def read_pipe_lines(c):
4addad22 50 if verbose:
86dff6b6 51 sys.stderr.write('Reading pipe: %s\n' % c)
b016d397 52 ## todo: check return status
bce4c5fc 53 pipe = os.popen(c, 'rb')
b016d397 54 val = pipe.readlines()
bce4c5fc 55 if pipe.close():
86dff6b6 56 die('Command failed: %s' % c)
b016d397
HWN
57
58 return val
caace111 59
6754a299 60def system(cmd):
4addad22 61 if verbose:
bb6e09b2 62 sys.stderr.write("executing %s\n" % cmd)
6754a299
HWN
63 if os.system(cmd) != 0:
64 die("command failed: %s" % cmd)
65
b9fc6ea9
DB
66def isP4Exec(kind):
67 """Determine if a Perforce 'kind' should have execute permission
68
69 'p4 help filetypes' gives a list of the types. If it starts with 'x',
70 or x follows one of a few letters. Otherwise, if there is an 'x' after
71 a plus sign, it is also executable"""
72 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
73
b43b0a3c
CP
74def diffTreePattern():
75 # This is a simple generator for the diff tree regex pattern. This could be
76 # a class variable if this and parseDiffTreeEntry were a part of a class.
77 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
78 while True:
79 yield pattern
80
81def parseDiffTreeEntry(entry):
82 """Parses a single diff tree entry into its component elements.
83
84 See git-diff-tree(1) manpage for details about the format of the diff
85 output. This method returns a dictionary with the following elements:
86
87 src_mode - The mode of the source file
88 dst_mode - The mode of the destination file
89 src_sha1 - The sha1 for the source file
90 dst_sha1 - The sha1 fr the destination file
91 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
92 status_score - The score for the status (applicable for 'C' and 'R'
93 statuses). This is None if there is no score.
94 src - The path for the source file.
95 dst - The path for the destination file. This is only present for
96 copy or renames. If it is not present, this is None.
97
98 If the pattern is not matched, None is returned."""
99
100 match = diffTreePattern().next().match(entry)
101 if match:
102 return {
103 'src_mode': match.group(1),
104 'dst_mode': match.group(2),
105 'src_sha1': match.group(3),
106 'dst_sha1': match.group(4),
107 'status': match.group(5),
108 'status_score': match.group(6),
109 'src': match.group(7),
110 'dst': match.group(10)
111 }
112 return None
113
9f90c733 114def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
86949eef 115 cmd = "p4 -G %s" % cmd
6a49f8e2
HWN
116 if verbose:
117 sys.stderr.write("Opening pipe: %s\n" % cmd)
9f90c733
SL
118
119 # Use a temporary file to avoid deadlocks without
120 # subprocess.communicate(), which would put another copy
121 # of stdout into memory.
122 stdin_file = None
123 if stdin is not None:
124 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
125 stdin_file.write(stdin)
126 stdin_file.flush()
127 stdin_file.seek(0)
128
129 p4 = subprocess.Popen(cmd, shell=True,
130 stdin=stdin_file,
131 stdout=subprocess.PIPE)
86949eef
SH
132
133 result = []
134 try:
135 while True:
9f90c733 136 entry = marshal.load(p4.stdout)
86949eef
SH
137 result.append(entry)
138 except EOFError:
139 pass
9f90c733
SL
140 exitCode = p4.wait()
141 if exitCode != 0:
ac3e0d79
SH
142 entry = {}
143 entry["p4ExitCode"] = exitCode
144 result.append(entry)
86949eef
SH
145
146 return result
147
148def p4Cmd(cmd):
149 list = p4CmdList(cmd)
150 result = {}
151 for entry in list:
152 result.update(entry)
153 return result;
154
cb2c9db5
SH
155def p4Where(depotPath):
156 if not depotPath.endswith("/"):
157 depotPath += "/"
158 output = p4Cmd("where %s..." % depotPath)
dc524036
SH
159 if output["code"] == "error":
160 return ""
cb2c9db5
SH
161 clientPath = ""
162 if "path" in output:
163 clientPath = output.get("path")
164 elif "data" in output:
165 data = output.get("data")
166 lastSpace = data.rfind(" ")
167 clientPath = data[lastSpace + 1:]
168
169 if clientPath.endswith("..."):
170 clientPath = clientPath[:-3]
171 return clientPath
172
86949eef 173def currentGitBranch():
b25b2065 174 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 175
4f5cf76a 176def isValidGitDir(path):
bb6e09b2
HWN
177 if (os.path.exists(path + "/HEAD")
178 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
179 return True;
180 return False
181
463e8af6 182def parseRevision(ref):
b25b2065 183 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 184
6ae8de88
SH
185def extractLogMessageFromGitCommit(commit):
186 logMessage = ""
b016d397
HWN
187
188 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 189 foundTitle = False
b016d397 190 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
191 if not foundTitle:
192 if len(log) == 1:
1c094184 193 foundTitle = True
6ae8de88
SH
194 continue
195
196 logMessage += log
197 return logMessage
198
bb6e09b2 199def extractSettingsGitLog(log):
6ae8de88
SH
200 values = {}
201 for line in log.split("\n"):
202 line = line.strip()
6326aa58
HWN
203 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
204 if not m:
205 continue
206
207 assignments = m.group(1).split (':')
208 for a in assignments:
209 vals = a.split ('=')
210 key = vals[0].strip()
211 val = ('='.join (vals[1:])).strip()
212 if val.endswith ('\"') and val.startswith('"'):
213 val = val[1:-1]
214
215 values[key] = val
216
845b42cb
SH
217 paths = values.get("depot-paths")
218 if not paths:
219 paths = values.get("depot-path")
a3fdd579
SH
220 if paths:
221 values['depot-paths'] = paths.split(',')
bb6e09b2 222 return values
6ae8de88 223
8136a639 224def gitBranchExists(branch):
bb6e09b2
HWN
225 proc = subprocess.Popen(["git", "rev-parse", branch],
226 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 227 return proc.wait() == 0;
8136a639 228
01265103 229def gitConfig(key):
4addad22 230 return read_pipe("git config %s" % key, ignore_error=True).strip()
01265103 231
062410bb
SH
232def p4BranchesInGit(branchesAreInRemotes = True):
233 branches = {}
234
235 cmdline = "git rev-parse --symbolic "
236 if branchesAreInRemotes:
237 cmdline += " --remotes"
238 else:
239 cmdline += " --branches"
240
241 for line in read_pipe_lines(cmdline):
242 line = line.strip()
243
244 ## only import to p4/
245 if not line.startswith('p4/') or line == "p4/HEAD":
246 continue
247 branch = line
248
249 # strip off p4
250 branch = re.sub ("^p4/", "", line)
251
252 branches[branch] = parseRevision(line)
253 return branches
254
9ceab363 255def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
256 branches = p4BranchesInGit()
257 # map from depot-path to branch name
258 branchByDepotPath = {}
259 for branch in branches.keys():
260 tip = branches[branch]
261 log = extractLogMessageFromGitCommit(tip)
262 settings = extractSettingsGitLog(log)
263 if settings.has_key("depot-paths"):
264 paths = ",".join(settings["depot-paths"])
265 branchByDepotPath[paths] = "remotes/p4/" + branch
266
27d2d811 267 settings = None
27d2d811
SH
268 parent = 0
269 while parent < 65535:
9ceab363 270 commit = head + "~%s" % parent
27d2d811
SH
271 log = extractLogMessageFromGitCommit(commit)
272 settings = extractSettingsGitLog(log)
86506fe5
SH
273 if settings.has_key("depot-paths"):
274 paths = ",".join(settings["depot-paths"])
275 if branchByDepotPath.has_key(paths):
276 return [branchByDepotPath[paths], settings]
27d2d811 277
86506fe5 278 parent = parent + 1
27d2d811 279
86506fe5 280 return ["", settings]
27d2d811 281
5ca44617
SH
282def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
283 if not silent:
284 print ("Creating/updating branch(es) in %s based on origin branch(es)"
285 % localRefPrefix)
286
287 originPrefix = "origin/p4/"
288
289 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
290 line = line.strip()
291 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
292 continue
293
294 headName = line[len(originPrefix):]
295 remoteHead = localRefPrefix + headName
296 originHead = line
297
298 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
299 if (not original.has_key('depot-paths')
300 or not original.has_key('change')):
301 continue
302
303 update = False
304 if not gitBranchExists(remoteHead):
305 if verbose:
306 print "creating %s" % remoteHead
307 update = True
308 else:
309 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
310 if settings.has_key('change') > 0:
311 if settings['depot-paths'] == original['depot-paths']:
312 originP4Change = int(original['change'])
313 p4Change = int(settings['change'])
314 if originP4Change > p4Change:
315 print ("%s (%s) is newer than %s (%s). "
316 "Updating p4 branch from origin."
317 % (originHead, originP4Change,
318 remoteHead, p4Change))
319 update = True
320 else:
321 print ("Ignoring: %s was imported from %s while "
322 "%s was imported from %s"
323 % (originHead, ','.join(original['depot-paths']),
324 remoteHead, ','.join(settings['depot-paths'])))
325
326 if update:
327 system("git update-ref %s %s" % (remoteHead, originHead))
328
329def originP4BranchesExist():
330 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
331
4f6432d8
SH
332def p4ChangesForPaths(depotPaths, changeRange):
333 assert depotPaths
334 output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
335 for p in depotPaths]))
336
337 changes = []
338 for line in output:
339 changeNum = line.split(" ")[1]
340 changes.append(int(changeNum))
341
342 changes.sort()
343 return changes
344
b984733c
SH
345class Command:
346 def __init__(self):
347 self.usage = "usage: %prog [options]"
8910ac0e 348 self.needsGit = True
b984733c
SH
349
350class P4Debug(Command):
86949eef 351 def __init__(self):
6ae8de88 352 Command.__init__(self)
86949eef 353 self.options = [
b1ce9447
HWN
354 optparse.make_option("--verbose", dest="verbose", action="store_true",
355 default=False),
4addad22 356 ]
c8c39116 357 self.description = "A tool to debug the output of p4 -G."
8910ac0e 358 self.needsGit = False
b1ce9447 359 self.verbose = False
86949eef
SH
360
361 def run(self, args):
b1ce9447 362 j = 0
86949eef 363 for output in p4CmdList(" ".join(args)):
b1ce9447
HWN
364 print 'Element: %d' % j
365 j += 1
86949eef 366 print output
b984733c 367 return True
86949eef 368
5834684d
SH
369class P4RollBack(Command):
370 def __init__(self):
371 Command.__init__(self)
372 self.options = [
0c66a783
SH
373 optparse.make_option("--verbose", dest="verbose", action="store_true"),
374 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
375 ]
376 self.description = "A tool to debug the multi-branch import. Don't use :)"
52102d47 377 self.verbose = False
0c66a783 378 self.rollbackLocalBranches = False
5834684d
SH
379
380 def run(self, args):
381 if len(args) != 1:
382 return False
383 maxChange = int(args[0])
0c66a783 384
ad192f28 385 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
386 die("Problems executing p4");
387
0c66a783
SH
388 if self.rollbackLocalBranches:
389 refPrefix = "refs/heads/"
b016d397 390 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
391 else:
392 refPrefix = "refs/remotes/"
b016d397 393 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
394
395 for line in lines:
396 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
397 line = line.strip()
398 ref = refPrefix + line
5834684d 399 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
400 settings = extractSettingsGitLog(log)
401
402 depotPaths = settings['depot-paths']
403 change = settings['change']
404
5834684d 405 changed = False
52102d47 406
6326aa58
HWN
407 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
408 for p in depotPaths]))) == 0:
52102d47
SH
409 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
410 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
411 continue
412
bb6e09b2 413 while change and int(change) > maxChange:
5834684d 414 changed = True
52102d47
SH
415 if self.verbose:
416 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
417 system("git update-ref %s \"%s^\"" % (ref, ref))
418 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
419 settings = extractSettingsGitLog(log)
420
421
422 depotPaths = settings['depot-paths']
423 change = settings['change']
5834684d
SH
424
425 if changed:
52102d47 426 print "%s rewound to %s" % (ref, change)
5834684d
SH
427
428 return True
429
711544b0 430class P4Submit(Command):
4f5cf76a 431 def __init__(self):
b984733c 432 Command.__init__(self)
4f5cf76a
SH
433 self.options = [
434 optparse.make_option("--continue", action="store_false", dest="firstTime"),
4addad22 435 optparse.make_option("--verbose", dest="verbose", action="store_true"),
4f5cf76a
SH
436 optparse.make_option("--origin", dest="origin"),
437 optparse.make_option("--reset", action="store_true", dest="reset"),
4f5cf76a 438 optparse.make_option("--log-substitutions", dest="substFile"),
04219c04 439 optparse.make_option("--dry-run", action="store_true"),
c1b296b9 440 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
cb4f1280 441 optparse.make_option("--trust-me-like-a-fool", dest="trustMeLikeAFool", action="store_true"),
d9a5f25b 442 optparse.make_option("-M", dest="detectRename", action="store_true"),
4f5cf76a
SH
443 ]
444 self.description = "Submit changes from git to the perforce depot."
c9b50e63 445 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a
SH
446 self.firstTime = True
447 self.reset = False
448 self.interactive = True
449 self.dryRun = False
450 self.substFile = ""
451 self.firstTime = True
9512497b 452 self.origin = ""
c1b296b9 453 self.directSubmit = False
cb4f1280 454 self.trustMeLikeAFool = False
d9a5f25b 455 self.detectRename = False
b0d10df7 456 self.verbose = False
f7baba8b 457 self.isWindows = (platform.system() == "Windows")
4f5cf76a
SH
458
459 self.logSubstitutions = {}
460 self.logSubstitutions["<enter description here>"] = "%log%"
461 self.logSubstitutions["\tDetails:"] = "\tDetails: %log%"
462
463 def check(self):
464 if len(p4CmdList("opened ...")) > 0:
465 die("You have files opened with perforce! Close them before starting the sync.")
466
467 def start(self):
468 if len(self.config) > 0 and not self.reset:
cebdf5af
HWN
469 die("Cannot start sync. Previous sync config found at %s\n"
470 "If you want to start submitting again from scratch "
471 "maybe you want to call git-p4 submit --reset" % self.configFile)
4f5cf76a
SH
472
473 commits = []
c1b296b9
SH
474 if self.directSubmit:
475 commits.append("0")
476 else:
b016d397 477 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
b25b2065 478 commits.append(line.strip())
c1b296b9 479 commits.reverse()
4f5cf76a
SH
480
481 self.config["commits"] = commits
482
4f5cf76a
SH
483 def prepareLogMessage(self, template, message):
484 result = ""
485
486 for line in template.split("\n"):
487 if line.startswith("#"):
488 result += line + "\n"
489 continue
490
491 substituted = False
492 for key in self.logSubstitutions.keys():
493 if line.find(key) != -1:
494 value = self.logSubstitutions[key]
495 value = value.replace("%log%", message)
496 if value != "@remove@":
497 result += line.replace(key, value) + "\n"
498 substituted = True
499 break
500
501 if not substituted:
502 result += line + "\n"
503
504 return result
505
ea99c3ae
SH
506 def prepareSubmitTemplate(self):
507 # remove lines in the Files section that show changes to files outside the depot path we're committing into
508 template = ""
509 inFilesSection = False
510 for line in read_pipe_lines("p4 change -o"):
511 if inFilesSection:
512 if line.startswith("\t"):
513 # path starts and ends with a tab
514 path = line[1:]
515 lastTab = path.rfind("\t")
516 if lastTab != -1:
517 path = path[:lastTab]
518 if not path.startswith(self.depotPath):
519 continue
520 else:
521 inFilesSection = False
522 else:
523 if line.startswith("Files:"):
524 inFilesSection = True
525
526 template += line
527
528 return template
529
7cb5cbef 530 def applyCommit(self, id):
c1b296b9
SH
531 if self.directSubmit:
532 print "Applying local change in working directory/index"
533 diff = self.diffStatus
534 else:
b016d397 535 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
d9a5f25b 536 diffOpts = ("", "-M")[self.detectRename]
b43b0a3c 537 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
538 filesToAdd = set()
539 filesToDelete = set()
d336c158 540 editedFiles = set()
4f5cf76a 541 for line in diff:
b43b0a3c
CP
542 diff = parseDiffTreeEntry(line)
543 modifier = diff['status']
544 path = diff['src']
4f5cf76a 545 if modifier == "M":
d336c158
SH
546 system("p4 edit \"%s\"" % path)
547 editedFiles.add(path)
4f5cf76a
SH
548 elif modifier == "A":
549 filesToAdd.add(path)
550 if path in filesToDelete:
551 filesToDelete.remove(path)
552 elif modifier == "D":
553 filesToDelete.add(path)
554 if path in filesToAdd:
555 filesToAdd.remove(path)
d9a5f25b 556 elif modifier == "R":
b43b0a3c 557 src, dest = diff['src'], diff['dst']
d9a5f25b
CP
558 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
559 system("p4 edit \"%s\"" % (dest))
560 os.unlink(dest)
561 editedFiles.add(dest)
562 filesToDelete.add(src)
4f5cf76a
SH
563 else:
564 die("unknown modifier %s for %s" % (modifier, path))
565
c1b296b9
SH
566 if self.directSubmit:
567 diffcmd = "cat \"%s\"" % self.diffFile
568 else:
569 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 570 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
571 tryPatchCmd = patchcmd + "--check -"
572 applyPatchCmd = patchcmd + "--check --apply -"
51a2640a 573
47a130b7 574 if os.system(tryPatchCmd) != 0:
51a2640a
SH
575 print "Unfortunately applying the change failed!"
576 print "What do you want to do?"
577 response = "x"
578 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
579 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
580 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
581 if response == "s":
582 print "Skipping! Good luck with the next patches..."
20947149
SH
583 for f in editedFiles:
584 system("p4 revert \"%s\"" % f);
585 for f in filesToAdd:
586 system("rm %s" %f)
51a2640a
SH
587 return
588 elif response == "a":
47a130b7 589 os.system(applyPatchCmd)
51a2640a
SH
590 if len(filesToAdd) > 0:
591 print "You may also want to call p4 add on the following files:"
592 print " ".join(filesToAdd)
593 if len(filesToDelete):
594 print "The following files should be scheduled for deletion with p4 delete:"
595 print " ".join(filesToDelete)
cebdf5af
HWN
596 die("Please resolve and submit the conflict manually and "
597 + "continue afterwards with git-p4 submit --continue")
51a2640a
SH
598 elif response == "w":
599 system(diffcmd + " > patch.txt")
600 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af
HWN
601 die("Please resolve and submit the conflict manually and "
602 "continue afterwards with git-p4 submit --continue")
51a2640a 603
47a130b7 604 system(applyPatchCmd)
4f5cf76a
SH
605
606 for f in filesToAdd:
e6b711f0 607 system("p4 add \"%s\"" % f)
4f5cf76a 608 for f in filesToDelete:
e6b711f0
SH
609 system("p4 revert \"%s\"" % f)
610 system("p4 delete \"%s\"" % f)
4f5cf76a 611
c1b296b9
SH
612 logMessage = ""
613 if not self.directSubmit:
614 logMessage = extractLogMessageFromGitCommit(id)
615 logMessage = logMessage.replace("\n", "\n\t")
f7baba8b
MSO
616 if self.isWindows:
617 logMessage = logMessage.replace("\n", "\r\n")
b25b2065 618 logMessage = logMessage.strip()
4f5cf76a 619
ea99c3ae 620 template = self.prepareSubmitTemplate()
4f5cf76a
SH
621
622 if self.interactive:
623 submitTemplate = self.prepareLogMessage(template, logMessage)
b016d397 624 diff = read_pipe("p4 diff -du ...")
4f5cf76a
SH
625
626 for newFile in filesToAdd:
627 diff += "==== new file ====\n"
628 diff += "--- /dev/null\n"
629 diff += "+++ %s\n" % newFile
630 f = open(newFile, "r")
631 for line in f.readlines():
632 diff += "+" + line
633 f.close()
634
25df95cc
SH
635 separatorLine = "######## everything below this line is just the diff #######"
636 if platform.system() == "Windows":
637 separatorLine += "\r"
638 separatorLine += "\n"
4f5cf76a
SH
639
640 response = "e"
cb4f1280
SH
641 if self.trustMeLikeAFool:
642 response = "y"
643
53150250 644 firstIteration = True
4f5cf76a 645 while response == "e":
53150250 646 if not firstIteration:
d336c158 647 response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
53150250 648 firstIteration = False
4f5cf76a
SH
649 if response == "e":
650 [handle, fileName] = tempfile.mkstemp()
651 tmpFile = os.fdopen(handle, "w+")
53150250 652 tmpFile.write(submitTemplate + separatorLine + diff)
4f5cf76a 653 tmpFile.close()
25df95cc
SH
654 defaultEditor = "vi"
655 if platform.system() == "Windows":
656 defaultEditor = "notepad"
657 editor = os.environ.get("EDITOR", defaultEditor);
4f5cf76a 658 system(editor + " " + fileName)
25df95cc 659 tmpFile = open(fileName, "rb")
53150250 660 message = tmpFile.read()
4f5cf76a
SH
661 tmpFile.close()
662 os.remove(fileName)
53150250 663 submitTemplate = message[:message.index(separatorLine)]
f7baba8b
MSO
664 if self.isWindows:
665 submitTemplate = submitTemplate.replace("\r\n", "\n")
4f5cf76a
SH
666
667 if response == "y" or response == "yes":
668 if self.dryRun:
669 print submitTemplate
670 raw_input("Press return to continue...")
671 else:
7944f142
SH
672 if self.directSubmit:
673 print "Submitting to git first"
674 os.chdir(self.oldWorkingDirectory)
b016d397 675 write_pipe("git commit -a -F -", submitTemplate)
7944f142
SH
676 os.chdir(self.clientPath)
677
b016d397 678 write_pipe("p4 submit -i", submitTemplate)
d336c158
SH
679 elif response == "s":
680 for f in editedFiles:
681 system("p4 revert \"%s\"" % f);
682 for f in filesToAdd:
683 system("p4 revert \"%s\"" % f);
684 system("rm %s" %f)
685 for f in filesToDelete:
686 system("p4 delete \"%s\"" % f);
687 return
4f5cf76a
SH
688 else:
689 print "Not submitting!"
690 self.interactive = False
691 else:
692 fileName = "submit.txt"
693 file = open(fileName, "w+")
694 file.write(self.prepareLogMessage(template, logMessage))
695 file.close()
cebdf5af
HWN
696 print ("Perforce submit template written as %s. "
697 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
698 % (fileName, fileName))
4f5cf76a
SH
699
700 def run(self, args):
c9b50e63
SH
701 if len(args) == 0:
702 self.master = currentGitBranch()
4280e533 703 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
704 die("Detecting current git branch failed!")
705 elif len(args) == 1:
706 self.master = args[0]
707 else:
708 return False
709
27d2d811 710 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 711 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
712 if len(self.origin) == 0:
713 self.origin = upstream
a3fdd579
SH
714
715 if self.verbose:
716 print "Origin branch is " + self.origin
9512497b 717
ea99c3ae 718 if len(self.depotPath) == 0:
9512497b
SH
719 print "Internal error: cannot locate perforce depot path from existing branches"
720 sys.exit(128)
721
ea99c3ae 722 self.clientPath = p4Where(self.depotPath)
9512497b 723
51a2640a 724 if len(self.clientPath) == 0:
ea99c3ae 725 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
9512497b
SH
726 sys.exit(128)
727
ea99c3ae 728 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 729 self.oldWorkingDirectory = os.getcwd()
c1b296b9
SH
730
731 if self.directSubmit:
b016d397 732 self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
cbf5efa6
SH
733 if len(self.diffStatus) == 0:
734 print "No changes in working directory to submit."
735 return True
b016d397 736 patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
b86f7378 737 self.diffFile = self.gitdir + "/p4-git-diff"
c1b296b9
SH
738 f = open(self.diffFile, "wb")
739 f.write(patch)
740 f.close();
741
51a2640a 742 os.chdir(self.clientPath)
31f9ec12
SH
743 print "Syncronizing p4 checkout..."
744 system("p4 sync ...")
9512497b 745
4f5cf76a
SH
746 if self.reset:
747 self.firstTime = True
748
749 if len(self.substFile) > 0:
750 for line in open(self.substFile, "r").readlines():
b25b2065 751 tokens = line.strip().split("=")
4f5cf76a
SH
752 self.logSubstitutions[tokens[0]] = tokens[1]
753
4f5cf76a 754 self.check()
b86f7378 755 self.configFile = self.gitdir + "/p4-git-sync.cfg"
4f5cf76a
SH
756 self.config = shelve.open(self.configFile, writeback=True)
757
758 if self.firstTime:
759 self.start()
760
761 commits = self.config.get("commits", [])
762
763 while len(commits) > 0:
764 self.firstTime = False
765 commit = commits[0]
766 commits = commits[1:]
767 self.config["commits"] = commits
7cb5cbef 768 self.applyCommit(commit)
4f5cf76a
SH
769 if not self.interactive:
770 break
771
772 self.config.close()
773
c1b296b9
SH
774 if self.directSubmit:
775 os.remove(self.diffFile)
776
4f5cf76a
SH
777 if len(commits) == 0:
778 if self.firstTime:
779 print "No changes found to apply between %s and current HEAD" % self.origin
780 else:
781 print "All changes applied!"
7944f142 782 os.chdir(self.oldWorkingDirectory)
14594f4b
SH
783
784 sync = P4Sync()
785 sync.run([])
786
787 response = raw_input("Do you want to rebase current HEAD from Perforce now using git-p4 rebase? [y]es/[n]o ")
80b5910f 788 if response == "y" or response == "yes":
80b5910f 789 rebase = P4Rebase()
14594f4b 790 rebase.rebase()
4f5cf76a
SH
791 os.remove(self.configFile)
792
b984733c
SH
793 return True
794
711544b0 795class P4Sync(Command):
b984733c
SH
796 def __init__(self):
797 Command.__init__(self)
798 self.options = [
799 optparse.make_option("--branch", dest="branch"),
800 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
801 optparse.make_option("--changesfile", dest="changesFile"),
802 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 803 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
a028a98e 804 optparse.make_option("--verbose", dest="verbose", action="store_true"),
d2c6dd30
HWN
805 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
806 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 807 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6
HWN
808 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
809 help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
b984733c
SH
810 ]
811 self.description = """Imports from Perforce into a git repository.\n
812 example:
813 //depot/my/project/ -- to import the current head
814 //depot/my/project/@all -- to import everything
815 //depot/my/project/@1,6 -- to import only from revision 1 to 6
816
817 (a ... is not needed in the path p4 specification, it's added implicitly)"""
818
819 self.usage += " //depot/path[@revRange]"
b984733c 820 self.silent = False
b984733c
SH
821 self.createdBranches = Set()
822 self.committedChanges = Set()
569d1bd4 823 self.branch = ""
b984733c 824 self.detectBranches = False
cb53e1f8 825 self.detectLabels = False
b984733c 826 self.changesFile = ""
01265103 827 self.syncWithOrigin = True
4b97ffb1 828 self.verbose = False
a028a98e 829 self.importIntoRemotes = True
01a9c9c5 830 self.maxChanges = ""
c1f9197f 831 self.isWindows = (platform.system() == "Windows")
8b41a97f 832 self.keepRepoPath = False
6326aa58 833 self.depotPaths = None
3c699645 834 self.p4BranchesInGit = []
b984733c 835
01265103
SH
836 if gitConfig("git-p4.syncFromOrigin") == "false":
837 self.syncWithOrigin = False
838
b984733c
SH
839 def extractFilesFromCommit(self, commit):
840 files = []
841 fnum = 0
842 while commit.has_key("depotFile%s" % fnum):
843 path = commit["depotFile%s" % fnum]
6326aa58
HWN
844
845 found = [p for p in self.depotPaths
846 if path.startswith (p)]
847 if not found:
b984733c
SH
848 fnum = fnum + 1
849 continue
850
851 file = {}
852 file["path"] = path
853 file["rev"] = commit["rev%s" % fnum]
854 file["action"] = commit["action%s" % fnum]
855 file["type"] = commit["type%s" % fnum]
856 files.append(file)
857 fnum = fnum + 1
858 return files
859
6326aa58 860 def stripRepoPath(self, path, prefixes):
8b41a97f 861 if self.keepRepoPath:
6326aa58
HWN
862 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
863
864 for p in prefixes:
865 if path.startswith(p):
866 path = path[len(p):]
8b41a97f 867
6326aa58 868 return path
6754a299 869
71b112d4 870 def splitFilesIntoBranches(self, commit):
d5904674 871 branches = {}
71b112d4
SH
872 fnum = 0
873 while commit.has_key("depotFile%s" % fnum):
874 path = commit["depotFile%s" % fnum]
6326aa58
HWN
875 found = [p for p in self.depotPaths
876 if path.startswith (p)]
877 if not found:
71b112d4
SH
878 fnum = fnum + 1
879 continue
880
881 file = {}
882 file["path"] = path
883 file["rev"] = commit["rev%s" % fnum]
884 file["action"] = commit["action%s" % fnum]
885 file["type"] = commit["type%s" % fnum]
886 fnum = fnum + 1
887
6326aa58 888 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 889
4b97ffb1 890 for branch in self.knownBranches.keys():
6754a299
HWN
891
892 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
893 if relPath.startswith(branch + "/"):
d5904674
SH
894 if branch not in branches:
895 branches[branch] = []
71b112d4 896 branches[branch].append(file)
6555b2cc 897 break
b984733c
SH
898
899 return branches
900
6a49f8e2
HWN
901 ## Should move this out, doesn't use SELF.
902 def readP4Files(self, files):
b1ce9447 903 files = [f for f in files
982bb8a3 904 if f['action'] != 'delete']
6a49f8e2 905
b1ce9447 906 if not files:
f2eda79f
HWN
907 return
908
78800190
SL
909 filedata = p4CmdList('-x - print',
910 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
911 for f in files]),
912 stdin_mode='w+')
913 if "p4ExitCode" in filedata[0]:
914 die("Problems executing p4. Error: [%d]."
915 % (filedata[0]['p4ExitCode']));
6a49f8e2 916
d2c6dd30
HWN
917 j = 0;
918 contents = {}
b1ce9447 919 while j < len(filedata):
d2c6dd30 920 stat = filedata[j]
b1ce9447
HWN
921 j += 1
922 text = ''
7530a40c
HWN
923 while j < len(filedata) and filedata[j]['code'] in ('text',
924 'binary'):
b1ce9447
HWN
925 text += filedata[j]['data']
926 j += 1
6a49f8e2 927
1b9a4684
HWN
928
929 if not stat.has_key('depotFile'):
930 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
931 continue
932
b1ce9447 933 contents[stat['depotFile']] = text
6a49f8e2 934
d2c6dd30
HWN
935 for f in files:
936 assert not f.has_key('data')
937 f['data'] = contents[f['path']]
6a49f8e2 938
6326aa58 939 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
940 epoch = details["time"]
941 author = details["user"]
942
4b97ffb1
SH
943 if self.verbose:
944 print "commit into %s" % branch
945
96e07dd2
HWN
946 # start with reading files; if that fails, we should not
947 # create a commit.
948 new_files = []
949 for f in files:
950 if [p for p in branchPrefixes if f['path'].startswith(p)]:
951 new_files.append (f)
952 else:
953 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
954 files = new_files
955 self.readP4Files(files)
956
957
958
959
b984733c 960 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 961# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
962 self.committedChanges.add(int(details["change"]))
963 committer = ""
b607e71e
SH
964 if author not in self.users:
965 self.getUserMapFromPerforceServer()
b984733c 966 if author in self.users:
0828ab14 967 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
b984733c 968 else:
0828ab14 969 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
b984733c
SH
970
971 self.gitStream.write("committer %s\n" % committer)
972
973 self.gitStream.write("data <<EOT\n")
974 self.gitStream.write(details["desc"])
6581de09
SH
975 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
976 % (','.join (branchPrefixes), details["change"]))
977 if len(details['options']) > 0:
978 self.gitStream.write(": options = %s" % details['options'])
979 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
980
981 if len(parent) > 0:
4b97ffb1
SH
982 if self.verbose:
983 print "parent %s" % parent
b984733c
SH
984 self.gitStream.write("from %s\n" % parent)
985
6a49f8e2 986 for file in files:
b984733c 987 if file["type"] == "apple":
6a49f8e2 988 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
b984733c
SH
989 continue
990
6a49f8e2
HWN
991 relPath = self.stripRepoPath(file['path'], branchPrefixes)
992 if file["action"] == "delete":
b984733c
SH
993 self.gitStream.write("D %s\n" % relPath)
994 else:
6a49f8e2 995 data = file['data']
b984733c 996
74276ec6 997 mode = "644"
b9fc6ea9 998 if isP4Exec(file["type"]):
74276ec6
SH
999 mode = "755"
1000 elif file["type"] == "symlink":
1001 mode = "120000"
1002 # p4 print on a symlink contains "target\n", so strip it off
1003 data = data[:-1]
1004
c1f9197f
MSO
1005 if self.isWindows and file["type"].endswith("text"):
1006 data = data.replace("\r\n", "\n")
1007
74276ec6 1008 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
b984733c
SH
1009 self.gitStream.write("data %s\n" % len(data))
1010 self.gitStream.write(data)
1011 self.gitStream.write("\n")
1012
1013 self.gitStream.write("\n")
1014
1f4ba1cb
SH
1015 change = int(details["change"])
1016
9bda3a85 1017 if self.labels.has_key(change):
1f4ba1cb
SH
1018 label = self.labels[change]
1019 labelDetails = label[0]
1020 labelRevisions = label[1]
71b112d4
SH
1021 if self.verbose:
1022 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 1023
6326aa58
HWN
1024 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1025 for p in branchPrefixes]))
1f4ba1cb
SH
1026
1027 if len(files) == len(labelRevisions):
1028
1029 cleanedFiles = {}
1030 for info in files:
1031 if info["action"] == "delete":
1032 continue
1033 cleanedFiles[info["depotFile"]] = info["rev"]
1034
1035 if cleanedFiles == labelRevisions:
1036 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1037 self.gitStream.write("from %s\n" % branch)
1038
1039 owner = labelDetails["Owner"]
1040 tagger = ""
1041 if author in self.users:
1042 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1043 else:
1044 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1045 self.gitStream.write("tagger %s\n" % tagger)
1046 self.gitStream.write("data <<EOT\n")
1047 self.gitStream.write(labelDetails["Description"])
1048 self.gitStream.write("EOT\n\n")
1049
1050 else:
a46668fa 1051 if not self.silent:
cebdf5af
HWN
1052 print ("Tag %s does not match with change %s: files do not match."
1053 % (labelDetails["label"], change))
1f4ba1cb
SH
1054
1055 else:
a46668fa 1056 if not self.silent:
cebdf5af
HWN
1057 print ("Tag %s does not match with change %s: file count is different."
1058 % (labelDetails["label"], change))
b984733c 1059
183b8ef8 1060 def getUserCacheFilename(self):
b2d2d16a
SH
1061 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1062 return home + "/.gitp4-usercache.txt"
183b8ef8 1063
b607e71e 1064 def getUserMapFromPerforceServer(self):
ebd81168
SH
1065 if self.userMapFromPerforceServer:
1066 return
b984733c
SH
1067 self.users = {}
1068
1069 for output in p4CmdList("users"):
1070 if not output.has_key("User"):
1071 continue
1072 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1073
183b8ef8
HWN
1074
1075 s = ''
1076 for (key, val) in self.users.items():
1077 s += "%s\t%s\n" % (key, val)
1078
1079 open(self.getUserCacheFilename(), "wb").write(s)
ebd81168 1080 self.userMapFromPerforceServer = True
b607e71e
SH
1081
1082 def loadUserMapFromCache(self):
1083 self.users = {}
ebd81168 1084 self.userMapFromPerforceServer = False
b607e71e 1085 try:
183b8ef8 1086 cache = open(self.getUserCacheFilename(), "rb")
b607e71e
SH
1087 lines = cache.readlines()
1088 cache.close()
1089 for line in lines:
b25b2065 1090 entry = line.strip().split("\t")
b607e71e
SH
1091 self.users[entry[0]] = entry[1]
1092 except IOError:
1093 self.getUserMapFromPerforceServer()
1094
1f4ba1cb
SH
1095 def getLabels(self):
1096 self.labels = {}
1097
6326aa58 1098 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
10c3211b 1099 if len(l) > 0 and not self.silent:
6326aa58 1100 print "Finding files belonging to labels in %s" % `self.depotPath`
01ce1fe9
SH
1101
1102 for output in l:
1f4ba1cb
SH
1103 label = output["label"]
1104 revisions = {}
1105 newestChange = 0
71b112d4
SH
1106 if self.verbose:
1107 print "Querying files for label %s" % label
6326aa58
HWN
1108 for file in p4CmdList("files "
1109 + ' '.join (["%s...@%s" % (p, label)
1110 for p in self.depotPaths])):
1f4ba1cb
SH
1111 revisions[file["depotFile"]] = file["rev"]
1112 change = int(file["change"])
1113 if change > newestChange:
1114 newestChange = change
1115
9bda3a85
SH
1116 self.labels[newestChange] = [output, revisions]
1117
1118 if self.verbose:
1119 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 1120
86dff6b6
HWN
1121 def guessProjectName(self):
1122 for p in self.depotPaths:
6e5295c4
SH
1123 if p.endswith("/"):
1124 p = p[:-1]
1125 p = p[p.strip().rfind("/") + 1:]
1126 if not p.endswith("/"):
1127 p += "/"
1128 return p
86dff6b6 1129
4b97ffb1 1130 def getBranchMapping(self):
6555b2cc
SH
1131 lostAndFoundBranches = set()
1132
4b97ffb1
SH
1133 for info in p4CmdList("branches"):
1134 details = p4Cmd("branch -o %s" % info["branch"])
1135 viewIdx = 0
1136 while details.has_key("View%s" % viewIdx):
1137 paths = details["View%s" % viewIdx].split(" ")
1138 viewIdx = viewIdx + 1
1139 # require standard //depot/foo/... //depot/bar/... mapping
1140 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1141 continue
1142 source = paths[0]
1143 destination = paths[1]
6509e19c
SH
1144 ## HACK
1145 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1146 source = source[len(self.depotPaths[0]):-4]
1147 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 1148
1a2edf4e
SH
1149 if destination in self.knownBranches:
1150 if not self.silent:
1151 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1152 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1153 continue
1154
6555b2cc
SH
1155 self.knownBranches[destination] = source
1156
1157 lostAndFoundBranches.discard(destination)
1158
29bdbac1 1159 if source not in self.knownBranches:
6555b2cc
SH
1160 lostAndFoundBranches.add(source)
1161
1162
1163 for branch in lostAndFoundBranches:
1164 self.knownBranches[branch] = branch
29bdbac1
SH
1165
1166 def listExistingP4GitBranches(self):
144ff46b
SH
1167 # branches holds mapping from name to commit
1168 branches = p4BranchesInGit(self.importIntoRemotes)
1169 self.p4BranchesInGit = branches.keys()
1170 for branch in branches.keys():
1171 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 1172
bb6e09b2
HWN
1173 def updateOptionDict(self, d):
1174 option_keys = {}
1175 if self.keepRepoPath:
1176 option_keys['keepRepoPath'] = 1
1177
1178 d["options"] = ' '.join(sorted(option_keys.keys()))
1179
1180 def readOptions(self, d):
1181 self.keepRepoPath = (d.has_key('options')
1182 and ('keepRepoPath' in d['options']))
6326aa58 1183
8134f69c
SH
1184 def gitRefForBranch(self, branch):
1185 if branch == "main":
1186 return self.refPrefix + "master"
1187
1188 if len(branch) <= 0:
1189 return branch
1190
1191 return self.refPrefix + self.projectName + branch
1192
1ca3d710
SH
1193 def gitCommitByP4Change(self, ref, change):
1194 if self.verbose:
1195 print "looking in ref " + ref + " for change %s using bisect..." % change
1196
1197 earliestCommit = ""
1198 latestCommit = parseRevision(ref)
1199
1200 while True:
1201 if self.verbose:
1202 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1203 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1204 if len(next) == 0:
1205 if self.verbose:
1206 print "argh"
1207 return ""
1208 log = extractLogMessageFromGitCommit(next)
1209 settings = extractSettingsGitLog(log)
1210 currentChange = int(settings['change'])
1211 if self.verbose:
1212 print "current change %s" % currentChange
1213
1214 if currentChange == change:
1215 if self.verbose:
1216 print "found %s" % next
1217 return next
1218
1219 if currentChange < change:
1220 earliestCommit = "^%s" % next
1221 else:
1222 latestCommit = "%s" % next
1223
1224 return ""
1225
1226 def importNewBranch(self, branch, maxChange):
1227 # make fast-import flush all changes to disk and update the refs using the checkpoint
1228 # command so that we can try to find the branch parent in the git history
1229 self.gitStream.write("checkpoint\n\n");
1230 self.gitStream.flush();
1231 branchPrefix = self.depotPaths[0] + branch + "/"
1232 range = "@1,%s" % maxChange
1233 #print "prefix" + branchPrefix
1234 changes = p4ChangesForPaths([branchPrefix], range)
1235 if len(changes) <= 0:
1236 return False
1237 firstChange = changes[0]
1238 #print "first change in branch: %s" % firstChange
1239 sourceBranch = self.knownBranches[branch]
1240 sourceDepotPath = self.depotPaths[0] + sourceBranch
1241 sourceRef = self.gitRefForBranch(sourceBranch)
1242 #print "source " + sourceBranch
1243
1244 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1245 #print "branch parent: %s" % branchParentChange
1246 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1247 if len(gitParent) > 0:
1248 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1249 #print "parent git commit: %s" % gitParent
1250
1251 self.importChanges(changes)
1252 return True
1253
e87f37ae
SH
1254 def importChanges(self, changes):
1255 cnt = 1
1256 for change in changes:
1257 description = p4Cmd("describe %s" % change)
1258 self.updateOptionDict(description)
1259
1260 if not self.silent:
1261 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1262 sys.stdout.flush()
1263 cnt = cnt + 1
1264
1265 try:
1266 if self.detectBranches:
1267 branches = self.splitFilesIntoBranches(description)
1268 for branch in branches.keys():
1269 ## HACK --hwn
1270 branchPrefix = self.depotPaths[0] + branch + "/"
1271
1272 parent = ""
1273
1274 filesForCommit = branches[branch]
1275
1276 if self.verbose:
1277 print "branch is %s" % branch
1278
1279 self.updatedBranches.add(branch)
1280
1281 if branch not in self.createdBranches:
1282 self.createdBranches.add(branch)
1283 parent = self.knownBranches[branch]
1284 if parent == branch:
1285 parent = ""
1ca3d710
SH
1286 else:
1287 fullBranch = self.projectName + branch
1288 if fullBranch not in self.p4BranchesInGit:
1289 if not self.silent:
1290 print("\n Importing new branch %s" % fullBranch);
1291 if self.importNewBranch(branch, change - 1):
1292 parent = ""
1293 self.p4BranchesInGit.append(fullBranch)
1294 if not self.silent:
1295 print("\n Resuming with change %s" % change);
1296
1297 if self.verbose:
1298 print "parent determined through known branches: %s" % parent
e87f37ae 1299
8134f69c
SH
1300 branch = self.gitRefForBranch(branch)
1301 parent = self.gitRefForBranch(parent)
e87f37ae
SH
1302
1303 if self.verbose:
1304 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1305
1306 if len(parent) == 0 and branch in self.initialParents:
1307 parent = self.initialParents[branch]
1308 del self.initialParents[branch]
1309
1310 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1311 else:
1312 files = self.extractFilesFromCommit(description)
1313 self.commit(description, files, self.branch, self.depotPaths,
1314 self.initialParent)
1315 self.initialParent = ""
1316 except IOError:
1317 print self.gitError.read()
1318 sys.exit(1)
1319
c208a243
SH
1320 def importHeadRevision(self, revision):
1321 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1322
1323 details = { "user" : "git perforce import user", "time" : int(time.time()) }
1324 details["desc"] = ("Initial import of %s from the state at revision %s"
1325 % (' '.join(self.depotPaths), revision))
1326 details["change"] = revision
1327 newestRevision = 0
1328
1329 fileCnt = 0
1330 for info in p4CmdList("files "
1331 + ' '.join(["%s...%s"
1332 % (p, revision)
1333 for p in self.depotPaths])):
1334
1335 if info['code'] == 'error':
1336 sys.stderr.write("p4 returned an error: %s\n"
1337 % info['data'])
1338 sys.exit(1)
1339
1340
1341 change = int(info["change"])
1342 if change > newestRevision:
1343 newestRevision = change
1344
1345 if info["action"] == "delete":
1346 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1347 #fileCnt = fileCnt + 1
1348 continue
1349
1350 for prop in ["depotFile", "rev", "action", "type" ]:
1351 details["%s%s" % (prop, fileCnt)] = info[prop]
1352
1353 fileCnt = fileCnt + 1
1354
1355 details["change"] = newestRevision
1356 self.updateOptionDict(details)
1357 try:
1358 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1359 except IOError:
1360 print "IO error with git fast-import. Is your git version recent enough?"
1361 print self.gitError.read()
1362
1363
b984733c 1364 def run(self, args):
6326aa58 1365 self.depotPaths = []
179caebf
SH
1366 self.changeRange = ""
1367 self.initialParent = ""
6326aa58 1368 self.previousDepotPaths = []
ce6f33c8 1369
29bdbac1
SH
1370 # map from branch depot path to parent branch
1371 self.knownBranches = {}
1372 self.initialParents = {}
5ca44617 1373 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
1374 if not self.syncWithOrigin:
1375 self.hasOrigin = False
29bdbac1 1376
a028a98e
SH
1377 if self.importIntoRemotes:
1378 self.refPrefix = "refs/remotes/p4/"
1379 else:
db775559 1380 self.refPrefix = "refs/heads/p4/"
a028a98e 1381
cebdf5af
HWN
1382 if self.syncWithOrigin and self.hasOrigin:
1383 if not self.silent:
1384 print "Syncing with origin first by calling git fetch origin"
1385 system("git fetch origin")
10f880f8 1386
569d1bd4 1387 if len(self.branch) == 0:
db775559 1388 self.branch = self.refPrefix + "master"
a028a98e 1389 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 1390 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 1391 system("git branch -D p4");
faf1bd20 1392 # create it /after/ importing, when master exists
0058a33a 1393 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 1394 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 1395
6a49f8e2
HWN
1396 # TODO: should always look at previous commits,
1397 # merge with previous imports, if possible.
1398 if args == []:
d414c74a 1399 if self.hasOrigin:
5ca44617 1400 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
1401 self.listExistingP4GitBranches()
1402
1403 if len(self.p4BranchesInGit) > 1:
1404 if not self.silent:
1405 print "Importing from/into multiple branches"
1406 self.detectBranches = True
967f72e2 1407
29bdbac1
SH
1408 if self.verbose:
1409 print "branches: %s" % self.p4BranchesInGit
1410
1411 p4Change = 0
1412 for branch in self.p4BranchesInGit:
cebdf5af 1413 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
1414
1415 settings = extractSettingsGitLog(logMsg)
29bdbac1 1416
bb6e09b2
HWN
1417 self.readOptions(settings)
1418 if (settings.has_key('depot-paths')
1419 and settings.has_key ('change')):
1420 change = int(settings['change']) + 1
29bdbac1
SH
1421 p4Change = max(p4Change, change)
1422
bb6e09b2
HWN
1423 depotPaths = sorted(settings['depot-paths'])
1424 if self.previousDepotPaths == []:
6326aa58 1425 self.previousDepotPaths = depotPaths
29bdbac1 1426 else:
6326aa58
HWN
1427 paths = []
1428 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
583e1707 1429 for i in range(0, min(len(cur), len(prev))):
6326aa58 1430 if cur[i] <> prev[i]:
583e1707 1431 i = i - 1
6326aa58
HWN
1432 break
1433
583e1707 1434 paths.append (cur[:i + 1])
6326aa58
HWN
1435
1436 self.previousDepotPaths = paths
29bdbac1
SH
1437
1438 if p4Change > 0:
bb6e09b2 1439 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 1440 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
1441 if not self.detectBranches:
1442 self.initialParent = parseRevision(self.branch)
341dc1c1 1443 if not self.silent and not self.detectBranches:
967f72e2 1444 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 1445
f9162f6a
SH
1446 if not self.branch.startswith("refs/"):
1447 self.branch = "refs/heads/" + self.branch
179caebf 1448
6326aa58 1449 if len(args) == 0 and self.depotPaths:
b984733c 1450 if not self.silent:
6326aa58 1451 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 1452 else:
6326aa58 1453 if self.depotPaths and self.depotPaths != args:
cebdf5af 1454 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
1455 "This doesn't work!" % (' '.join (self.depotPaths),
1456 ' '.join (args)))
b984733c 1457 sys.exit(1)
6326aa58 1458
bb6e09b2 1459 self.depotPaths = sorted(args)
b984733c 1460
1c49fc19 1461 revision = ""
b984733c 1462 self.users = {}
b984733c 1463
6326aa58
HWN
1464 newPaths = []
1465 for p in self.depotPaths:
1466 if p.find("@") != -1:
1467 atIdx = p.index("@")
1468 self.changeRange = p[atIdx:]
1469 if self.changeRange == "@all":
1470 self.changeRange = ""
6a49f8e2 1471 elif ',' not in self.changeRange:
1c49fc19 1472 revision = self.changeRange
6326aa58 1473 self.changeRange = ""
7fcff9de 1474 p = p[:atIdx]
6326aa58
HWN
1475 elif p.find("#") != -1:
1476 hashIdx = p.index("#")
1c49fc19 1477 revision = p[hashIdx:]
7fcff9de 1478 p = p[:hashIdx]
6326aa58 1479 elif self.previousDepotPaths == []:
1c49fc19 1480 revision = "#head"
6326aa58
HWN
1481
1482 p = re.sub ("\.\.\.$", "", p)
1483 if not p.endswith("/"):
1484 p += "/"
1485
1486 newPaths.append(p)
1487
1488 self.depotPaths = newPaths
1489
b984733c 1490
b607e71e 1491 self.loadUserMapFromCache()
cb53e1f8
SH
1492 self.labels = {}
1493 if self.detectLabels:
1494 self.getLabels();
b984733c 1495
4b97ffb1 1496 if self.detectBranches:
df450923
SH
1497 ## FIXME - what's a P4 projectName ?
1498 self.projectName = self.guessProjectName()
1499
1500 if not self.hasOrigin:
1501 self.getBranchMapping();
29bdbac1
SH
1502 if self.verbose:
1503 print "p4-git branches: %s" % self.p4BranchesInGit
1504 print "initial parents: %s" % self.initialParents
1505 for b in self.p4BranchesInGit:
1506 if b != "master":
6326aa58
HWN
1507
1508 ## FIXME
29bdbac1
SH
1509 b = b[len(self.projectName):]
1510 self.createdBranches.add(b)
4b97ffb1 1511
f291b4e3 1512 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 1513
cebdf5af 1514 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
1515 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1516 stderr=subprocess.PIPE);
08483580
SH
1517 self.gitOutput = importProcess.stdout
1518 self.gitStream = importProcess.stdin
1519 self.gitError = importProcess.stderr
b984733c 1520
1c49fc19 1521 if revision:
c208a243 1522 self.importHeadRevision(revision)
b984733c
SH
1523 else:
1524 changes = []
1525
0828ab14 1526 if len(self.changesFile) > 0:
b984733c
SH
1527 output = open(self.changesFile).readlines()
1528 changeSet = Set()
1529 for line in output:
1530 changeSet.add(int(line))
1531
1532 for change in changeSet:
1533 changes.append(change)
1534
1535 changes.sort()
1536 else:
29bdbac1 1537 if self.verbose:
86dff6b6 1538 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 1539 self.changeRange)
4f6432d8 1540 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 1541
01a9c9c5 1542 if len(self.maxChanges) > 0:
7fcff9de 1543 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 1544
b984733c 1545 if len(changes) == 0:
0828ab14 1546 if not self.silent:
341dc1c1 1547 print "No changes to import!"
1f52af6c 1548 return True
b984733c 1549
a9d1a27a
SH
1550 if not self.silent and not self.detectBranches:
1551 print "Import destination: %s" % self.branch
1552
341dc1c1
SH
1553 self.updatedBranches = set()
1554
e87f37ae 1555 self.importChanges(changes)
b984733c 1556
341dc1c1
SH
1557 if not self.silent:
1558 print ""
1559 if len(self.updatedBranches) > 0:
1560 sys.stdout.write("Updated branches: ")
1561 for b in self.updatedBranches:
1562 sys.stdout.write("%s " % b)
1563 sys.stdout.write("\n")
b984733c 1564
b984733c 1565 self.gitStream.close()
29bdbac1
SH
1566 if importProcess.wait() != 0:
1567 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
1568 self.gitOutput.close()
1569 self.gitError.close()
1570
b984733c
SH
1571 return True
1572
01ce1fe9
SH
1573class P4Rebase(Command):
1574 def __init__(self):
1575 Command.__init__(self)
01265103 1576 self.options = [ ]
cebdf5af
HWN
1577 self.description = ("Fetches the latest revision from perforce and "
1578 + "rebases the current work (branch) against it")
68c42153 1579 self.verbose = False
01ce1fe9
SH
1580
1581 def run(self, args):
1582 sync = P4Sync()
1583 sync.run([])
d7e3868c 1584
14594f4b
SH
1585 return self.rebase()
1586
1587 def rebase(self):
d7e3868c
SH
1588 [upstream, settings] = findUpstreamBranchPoint()
1589 if len(upstream) == 0:
1590 die("Cannot find upstream branchpoint for rebase")
1591
1592 # the branchpoint may be p4/foo~3, so strip off the parent
1593 upstream = re.sub("~[0-9]+$", "", upstream)
1594
1595 print "Rebasing the current branch onto %s" % upstream
b25b2065 1596 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 1597 system("git rebase %s" % upstream)
1f52af6c 1598 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
1599 return True
1600
f9a3a4f7
SH
1601class P4Clone(P4Sync):
1602 def __init__(self):
1603 P4Sync.__init__(self)
1604 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2
HWN
1605 self.usage = "usage: %prog [options] //depot/path[@revRange]"
1606 self.options.append(
1607 optparse.make_option("--destination", dest="cloneDestination",
1608 action='store', default=None,
1609 help="where to leave result of the clone"))
1610 self.cloneDestination = None
f9a3a4f7 1611 self.needsGit = False
f9a3a4f7 1612
6a49f8e2
HWN
1613 def defaultDestination(self, args):
1614 ## TODO: use common prefix of args?
1615 depotPath = args[0]
1616 depotDir = re.sub("(@[^@]*)$", "", depotPath)
1617 depotDir = re.sub("(#[^#]*)$", "", depotDir)
1618 depotDir = re.sub(r"\.\.\.$,", "", depotDir)
1619 depotDir = re.sub(r"/$", "", depotDir)
1620 return os.path.split(depotDir)[1]
1621
f9a3a4f7
SH
1622 def run(self, args):
1623 if len(args) < 1:
1624 return False
bb6e09b2
HWN
1625
1626 if self.keepRepoPath and not self.cloneDestination:
1627 sys.stderr.write("Must specify destination for --keep-path\n")
1628 sys.exit(1)
f9a3a4f7 1629
6326aa58 1630 depotPaths = args
5e100b5c
SH
1631
1632 if not self.cloneDestination and len(depotPaths) > 1:
1633 self.cloneDestination = depotPaths[-1]
1634 depotPaths = depotPaths[:-1]
1635
6326aa58
HWN
1636 for p in depotPaths:
1637 if not p.startswith("//"):
1638 return False
f9a3a4f7 1639
bb6e09b2 1640 if not self.cloneDestination:
98ad4faf 1641 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 1642
86dff6b6 1643 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
c3bf3f13
KG
1644 if not os.path.exists(self.cloneDestination):
1645 os.makedirs(self.cloneDestination)
bb6e09b2 1646 os.chdir(self.cloneDestination)
f9a3a4f7 1647 system("git init")
b86f7378 1648 self.gitdir = os.getcwd() + "/.git"
6326aa58 1649 if not P4Sync.run(self, depotPaths):
f9a3a4f7 1650 return False
f9a3a4f7 1651 if self.branch != "master":
8f9b2e08
SH
1652 if gitBranchExists("refs/remotes/p4/master"):
1653 system("git branch master refs/remotes/p4/master")
1654 system("git checkout -f")
1655 else:
1656 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 1657
f9a3a4f7
SH
1658 return True
1659
09d89de2
SH
1660class P4Branches(Command):
1661 def __init__(self):
1662 Command.__init__(self)
1663 self.options = [ ]
1664 self.description = ("Shows the git branches that hold imports and their "
1665 + "corresponding perforce depot paths")
1666 self.verbose = False
1667
1668 def run(self, args):
5ca44617
SH
1669 if originP4BranchesExist():
1670 createOrUpdateBranchesFromOrigin()
1671
09d89de2
SH
1672 cmdline = "git rev-parse --symbolic "
1673 cmdline += " --remotes"
1674
1675 for line in read_pipe_lines(cmdline):
1676 line = line.strip()
1677
1678 if not line.startswith('p4/') or line == "p4/HEAD":
1679 continue
1680 branch = line
1681
1682 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1683 settings = extractSettingsGitLog(log)
1684
1685 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1686 return True
1687
b984733c
SH
1688class HelpFormatter(optparse.IndentedHelpFormatter):
1689 def __init__(self):
1690 optparse.IndentedHelpFormatter.__init__(self)
1691
1692 def format_description(self, description):
1693 if description:
1694 return description + "\n"
1695 else:
1696 return ""
4f5cf76a 1697
86949eef
SH
1698def printUsage(commands):
1699 print "usage: %s <command> [options]" % sys.argv[0]
1700 print ""
1701 print "valid commands: %s" % ", ".join(commands)
1702 print ""
1703 print "Try %s <command> --help for command specific help." % sys.argv[0]
1704 print ""
1705
1706commands = {
b86f7378
HWN
1707 "debug" : P4Debug,
1708 "submit" : P4Submit,
a9834f58 1709 "commit" : P4Submit,
b86f7378
HWN
1710 "sync" : P4Sync,
1711 "rebase" : P4Rebase,
1712 "clone" : P4Clone,
09d89de2
SH
1713 "rollback" : P4RollBack,
1714 "branches" : P4Branches
86949eef
SH
1715}
1716
86949eef 1717
bb6e09b2
HWN
1718def main():
1719 if len(sys.argv[1:]) == 0:
1720 printUsage(commands.keys())
1721 sys.exit(2)
4f5cf76a 1722
bb6e09b2
HWN
1723 cmd = ""
1724 cmdName = sys.argv[1]
1725 try:
b86f7378
HWN
1726 klass = commands[cmdName]
1727 cmd = klass()
bb6e09b2
HWN
1728 except KeyError:
1729 print "unknown command %s" % cmdName
1730 print ""
1731 printUsage(commands.keys())
1732 sys.exit(2)
1733
1734 options = cmd.options
b86f7378 1735 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
1736
1737 args = sys.argv[2:]
1738
1739 if len(options) > 0:
1740 options.append(optparse.make_option("--git-dir", dest="gitdir"))
1741
1742 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1743 options,
1744 description = cmd.description,
1745 formatter = HelpFormatter())
1746
1747 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1748 global verbose
1749 verbose = cmd.verbose
1750 if cmd.needsGit:
b86f7378
HWN
1751 if cmd.gitdir == None:
1752 cmd.gitdir = os.path.abspath(".git")
1753 if not isValidGitDir(cmd.gitdir):
1754 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1755 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
1756 cdup = read_pipe("git rev-parse --show-cdup").strip()
1757 if len(cdup) > 0:
1758 os.chdir(cdup);
e20a9e53 1759
b86f7378
HWN
1760 if not isValidGitDir(cmd.gitdir):
1761 if isValidGitDir(cmd.gitdir + "/.git"):
1762 cmd.gitdir += "/.git"
bb6e09b2 1763 else:
b86f7378 1764 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 1765
b86f7378 1766 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 1767
bb6e09b2
HWN
1768 if not cmd.run(args):
1769 parser.print_help()
4f5cf76a 1770
4f5cf76a 1771
bb6e09b2
HWN
1772if __name__ == '__main__':
1773 main()