git-p4: git-p4 submit cleanups.
[git/git.git] / contrib / fast-import / git-p4
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
08483580 11import optparse, sys, os, marshal, popen2, subprocess, shelve
25df95cc 12import tempfile, getopt, sha, os.path, time, platform
ce6f33c8 13import re
8b41a97f 14
b984733c 15from sets import Set;
4f5cf76a 16
4addad22 17verbose = False
86949eef 18
86dff6b6
HWN
19def die(msg):
20 if verbose:
21 raise Exception(msg)
22 else:
23 sys.stderr.write(msg + "\n")
24 sys.exit(1)
25
bce4c5fc 26def write_pipe(c, str):
4addad22 27 if verbose:
86dff6b6 28 sys.stderr.write('Writing pipe: %s\n' % c)
b016d397 29
bce4c5fc 30 pipe = os.popen(c, 'w')
b016d397 31 val = pipe.write(str)
bce4c5fc 32 if pipe.close():
86dff6b6 33 die('Command failed: %s' % c)
b016d397
HWN
34
35 return val
36
4addad22
HWN
37def read_pipe(c, ignore_error=False):
38 if verbose:
86dff6b6 39 sys.stderr.write('Reading pipe: %s\n' % c)
8b41a97f 40
bce4c5fc 41 pipe = os.popen(c, 'rb')
b016d397 42 val = pipe.read()
4addad22 43 if pipe.close() and not ignore_error:
86dff6b6 44 die('Command failed: %s' % c)
b016d397
HWN
45
46 return val
47
48
bce4c5fc 49def read_pipe_lines(c):
4addad22 50 if verbose:
86dff6b6 51 sys.stderr.write('Reading pipe: %s\n' % c)
b016d397 52 ## todo: check return status
bce4c5fc 53 pipe = os.popen(c, 'rb')
b016d397 54 val = pipe.readlines()
bce4c5fc 55 if pipe.close():
86dff6b6 56 die('Command failed: %s' % c)
b016d397
HWN
57
58 return val
caace111 59
6754a299 60def system(cmd):
4addad22 61 if verbose:
bb6e09b2 62 sys.stderr.write("executing %s\n" % cmd)
6754a299
HWN
63 if os.system(cmd) != 0:
64 die("command failed: %s" % cmd)
65
b9fc6ea9
DB
66def isP4Exec(kind):
67 """Determine if a Perforce 'kind' should have execute permission
68
69 'p4 help filetypes' gives a list of the types. If it starts with 'x',
70 or x follows one of a few letters. Otherwise, if there is an 'x' after
71 a plus sign, it is also executable"""
72 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
73
c65b670e
CP
74def setP4ExecBit(file, mode):
75 # Reopens an already open file and changes the execute bit to match
76 # the execute bit setting in the passed in mode.
77
78 p4Type = "+x"
79
80 if not isModeExec(mode):
81 p4Type = getP4OpenedType(file)
82 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
83 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
84 if p4Type[-1] == "+":
85 p4Type = p4Type[0:-1]
86
87 system("p4 reopen -t %s %s" % (p4Type, file))
88
89def getP4OpenedType(file):
90 # Returns the perforce file type for the given file.
91
92 result = read_pipe("p4 opened %s" % file)
93 match = re.match(".*\((.+)\)$", result)
94 if match:
95 return match.group(1)
96 else:
97 die("Could not determine file type for %s" % file)
98
b43b0a3c
CP
99def diffTreePattern():
100 # This is a simple generator for the diff tree regex pattern. This could be
101 # a class variable if this and parseDiffTreeEntry were a part of a class.
102 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
103 while True:
104 yield pattern
105
106def parseDiffTreeEntry(entry):
107 """Parses a single diff tree entry into its component elements.
108
109 See git-diff-tree(1) manpage for details about the format of the diff
110 output. This method returns a dictionary with the following elements:
111
112 src_mode - The mode of the source file
113 dst_mode - The mode of the destination file
114 src_sha1 - The sha1 for the source file
115 dst_sha1 - The sha1 fr the destination file
116 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
117 status_score - The score for the status (applicable for 'C' and 'R'
118 statuses). This is None if there is no score.
119 src - The path for the source file.
120 dst - The path for the destination file. This is only present for
121 copy or renames. If it is not present, this is None.
122
123 If the pattern is not matched, None is returned."""
124
125 match = diffTreePattern().next().match(entry)
126 if match:
127 return {
128 'src_mode': match.group(1),
129 'dst_mode': match.group(2),
130 'src_sha1': match.group(3),
131 'dst_sha1': match.group(4),
132 'status': match.group(5),
133 'status_score': match.group(6),
134 'src': match.group(7),
135 'dst': match.group(10)
136 }
137 return None
138
c65b670e
CP
139def isModeExec(mode):
140 # Returns True if the given git mode represents an executable file,
141 # otherwise False.
142 return mode[-3:] == "755"
143
144def isModeExecChanged(src_mode, dst_mode):
145 return isModeExec(src_mode) != isModeExec(dst_mode)
146
9f90c733 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
86949eef 148 cmd = "p4 -G %s" % cmd
6a49f8e2
HWN
149 if verbose:
150 sys.stderr.write("Opening pipe: %s\n" % cmd)
9f90c733
SL
151
152 # Use a temporary file to avoid deadlocks without
153 # subprocess.communicate(), which would put another copy
154 # of stdout into memory.
155 stdin_file = None
156 if stdin is not None:
157 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
158 stdin_file.write(stdin)
159 stdin_file.flush()
160 stdin_file.seek(0)
161
162 p4 = subprocess.Popen(cmd, shell=True,
163 stdin=stdin_file,
164 stdout=subprocess.PIPE)
86949eef
SH
165
166 result = []
167 try:
168 while True:
9f90c733 169 entry = marshal.load(p4.stdout)
86949eef
SH
170 result.append(entry)
171 except EOFError:
172 pass
9f90c733
SL
173 exitCode = p4.wait()
174 if exitCode != 0:
ac3e0d79
SH
175 entry = {}
176 entry["p4ExitCode"] = exitCode
177 result.append(entry)
86949eef
SH
178
179 return result
180
181def p4Cmd(cmd):
182 list = p4CmdList(cmd)
183 result = {}
184 for entry in list:
185 result.update(entry)
186 return result;
187
cb2c9db5
SH
188def p4Where(depotPath):
189 if not depotPath.endswith("/"):
190 depotPath += "/"
191 output = p4Cmd("where %s..." % depotPath)
dc524036
SH
192 if output["code"] == "error":
193 return ""
cb2c9db5
SH
194 clientPath = ""
195 if "path" in output:
196 clientPath = output.get("path")
197 elif "data" in output:
198 data = output.get("data")
199 lastSpace = data.rfind(" ")
200 clientPath = data[lastSpace + 1:]
201
202 if clientPath.endswith("..."):
203 clientPath = clientPath[:-3]
204 return clientPath
205
86949eef 206def currentGitBranch():
b25b2065 207 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 208
4f5cf76a 209def isValidGitDir(path):
bb6e09b2
HWN
210 if (os.path.exists(path + "/HEAD")
211 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
212 return True;
213 return False
214
463e8af6 215def parseRevision(ref):
b25b2065 216 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 217
6ae8de88
SH
218def extractLogMessageFromGitCommit(commit):
219 logMessage = ""
b016d397
HWN
220
221 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 222 foundTitle = False
b016d397 223 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
224 if not foundTitle:
225 if len(log) == 1:
1c094184 226 foundTitle = True
6ae8de88
SH
227 continue
228
229 logMessage += log
230 return logMessage
231
bb6e09b2 232def extractSettingsGitLog(log):
6ae8de88
SH
233 values = {}
234 for line in log.split("\n"):
235 line = line.strip()
6326aa58
HWN
236 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
237 if not m:
238 continue
239
240 assignments = m.group(1).split (':')
241 for a in assignments:
242 vals = a.split ('=')
243 key = vals[0].strip()
244 val = ('='.join (vals[1:])).strip()
245 if val.endswith ('\"') and val.startswith('"'):
246 val = val[1:-1]
247
248 values[key] = val
249
845b42cb
SH
250 paths = values.get("depot-paths")
251 if not paths:
252 paths = values.get("depot-path")
a3fdd579
SH
253 if paths:
254 values['depot-paths'] = paths.split(',')
bb6e09b2 255 return values
6ae8de88 256
8136a639 257def gitBranchExists(branch):
bb6e09b2
HWN
258 proc = subprocess.Popen(["git", "rev-parse", branch],
259 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 260 return proc.wait() == 0;
8136a639 261
01265103 262def gitConfig(key):
4addad22 263 return read_pipe("git config %s" % key, ignore_error=True).strip()
01265103 264
062410bb
SH
265def p4BranchesInGit(branchesAreInRemotes = True):
266 branches = {}
267
268 cmdline = "git rev-parse --symbolic "
269 if branchesAreInRemotes:
270 cmdline += " --remotes"
271 else:
272 cmdline += " --branches"
273
274 for line in read_pipe_lines(cmdline):
275 line = line.strip()
276
277 ## only import to p4/
278 if not line.startswith('p4/') or line == "p4/HEAD":
279 continue
280 branch = line
281
282 # strip off p4
283 branch = re.sub ("^p4/", "", line)
284
285 branches[branch] = parseRevision(line)
286 return branches
287
9ceab363 288def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
289 branches = p4BranchesInGit()
290 # map from depot-path to branch name
291 branchByDepotPath = {}
292 for branch in branches.keys():
293 tip = branches[branch]
294 log = extractLogMessageFromGitCommit(tip)
295 settings = extractSettingsGitLog(log)
296 if settings.has_key("depot-paths"):
297 paths = ",".join(settings["depot-paths"])
298 branchByDepotPath[paths] = "remotes/p4/" + branch
299
27d2d811 300 settings = None
27d2d811
SH
301 parent = 0
302 while parent < 65535:
9ceab363 303 commit = head + "~%s" % parent
27d2d811
SH
304 log = extractLogMessageFromGitCommit(commit)
305 settings = extractSettingsGitLog(log)
86506fe5
SH
306 if settings.has_key("depot-paths"):
307 paths = ",".join(settings["depot-paths"])
308 if branchByDepotPath.has_key(paths):
309 return [branchByDepotPath[paths], settings]
27d2d811 310
86506fe5 311 parent = parent + 1
27d2d811 312
86506fe5 313 return ["", settings]
27d2d811 314
5ca44617
SH
315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
316 if not silent:
317 print ("Creating/updating branch(es) in %s based on origin branch(es)"
318 % localRefPrefix)
319
320 originPrefix = "origin/p4/"
321
322 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
323 line = line.strip()
324 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
325 continue
326
327 headName = line[len(originPrefix):]
328 remoteHead = localRefPrefix + headName
329 originHead = line
330
331 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
332 if (not original.has_key('depot-paths')
333 or not original.has_key('change')):
334 continue
335
336 update = False
337 if not gitBranchExists(remoteHead):
338 if verbose:
339 print "creating %s" % remoteHead
340 update = True
341 else:
342 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
343 if settings.has_key('change') > 0:
344 if settings['depot-paths'] == original['depot-paths']:
345 originP4Change = int(original['change'])
346 p4Change = int(settings['change'])
347 if originP4Change > p4Change:
348 print ("%s (%s) is newer than %s (%s). "
349 "Updating p4 branch from origin."
350 % (originHead, originP4Change,
351 remoteHead, p4Change))
352 update = True
353 else:
354 print ("Ignoring: %s was imported from %s while "
355 "%s was imported from %s"
356 % (originHead, ','.join(original['depot-paths']),
357 remoteHead, ','.join(settings['depot-paths'])))
358
359 if update:
360 system("git update-ref %s %s" % (remoteHead, originHead))
361
362def originP4BranchesExist():
363 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
364
4f6432d8
SH
365def p4ChangesForPaths(depotPaths, changeRange):
366 assert depotPaths
367 output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
368 for p in depotPaths]))
369
370 changes = []
371 for line in output:
372 changeNum = line.split(" ")[1]
373 changes.append(int(changeNum))
374
375 changes.sort()
376 return changes
377
b984733c
SH
378class Command:
379 def __init__(self):
380 self.usage = "usage: %prog [options]"
8910ac0e 381 self.needsGit = True
b984733c
SH
382
383class P4Debug(Command):
86949eef 384 def __init__(self):
6ae8de88 385 Command.__init__(self)
86949eef 386 self.options = [
b1ce9447
HWN
387 optparse.make_option("--verbose", dest="verbose", action="store_true",
388 default=False),
4addad22 389 ]
c8c39116 390 self.description = "A tool to debug the output of p4 -G."
8910ac0e 391 self.needsGit = False
b1ce9447 392 self.verbose = False
86949eef
SH
393
394 def run(self, args):
b1ce9447 395 j = 0
86949eef 396 for output in p4CmdList(" ".join(args)):
b1ce9447
HWN
397 print 'Element: %d' % j
398 j += 1
86949eef 399 print output
b984733c 400 return True
86949eef 401
5834684d
SH
402class P4RollBack(Command):
403 def __init__(self):
404 Command.__init__(self)
405 self.options = [
0c66a783
SH
406 optparse.make_option("--verbose", dest="verbose", action="store_true"),
407 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
408 ]
409 self.description = "A tool to debug the multi-branch import. Don't use :)"
52102d47 410 self.verbose = False
0c66a783 411 self.rollbackLocalBranches = False
5834684d
SH
412
413 def run(self, args):
414 if len(args) != 1:
415 return False
416 maxChange = int(args[0])
0c66a783 417
ad192f28 418 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
419 die("Problems executing p4");
420
0c66a783
SH
421 if self.rollbackLocalBranches:
422 refPrefix = "refs/heads/"
b016d397 423 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
424 else:
425 refPrefix = "refs/remotes/"
b016d397 426 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
427
428 for line in lines:
429 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
430 line = line.strip()
431 ref = refPrefix + line
5834684d 432 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
433 settings = extractSettingsGitLog(log)
434
435 depotPaths = settings['depot-paths']
436 change = settings['change']
437
5834684d 438 changed = False
52102d47 439
6326aa58
HWN
440 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
441 for p in depotPaths]))) == 0:
52102d47
SH
442 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
443 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
444 continue
445
bb6e09b2 446 while change and int(change) > maxChange:
5834684d 447 changed = True
52102d47
SH
448 if self.verbose:
449 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
450 system("git update-ref %s \"%s^\"" % (ref, ref))
451 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
452 settings = extractSettingsGitLog(log)
453
454
455 depotPaths = settings['depot-paths']
456 change = settings['change']
5834684d
SH
457
458 if changed:
52102d47 459 print "%s rewound to %s" % (ref, change)
5834684d
SH
460
461 return True
462
711544b0 463class P4Submit(Command):
4f5cf76a 464 def __init__(self):
b984733c 465 Command.__init__(self)
4f5cf76a 466 self.options = [
4addad22 467 optparse.make_option("--verbose", dest="verbose", action="store_true"),
4f5cf76a 468 optparse.make_option("--origin", dest="origin"),
d9a5f25b 469 optparse.make_option("-M", dest="detectRename", action="store_true"),
4f5cf76a
SH
470 ]
471 self.description = "Submit changes from git to the perforce depot."
c9b50e63 472 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 473 self.interactive = True
9512497b 474 self.origin = ""
d9a5f25b 475 self.detectRename = False
b0d10df7 476 self.verbose = False
f7baba8b 477 self.isWindows = (platform.system() == "Windows")
4f5cf76a 478
4f5cf76a
SH
479 def check(self):
480 if len(p4CmdList("opened ...")) > 0:
481 die("You have files opened with perforce! Close them before starting the sync.")
482
edae1e2f
SH
483 # replaces everything between 'Description:' and the next P4 submit template field with the
484 # commit message
4f5cf76a
SH
485 def prepareLogMessage(self, template, message):
486 result = ""
487
edae1e2f
SH
488 inDescriptionSection = False
489
4f5cf76a
SH
490 for line in template.split("\n"):
491 if line.startswith("#"):
492 result += line + "\n"
493 continue
494
edae1e2f
SH
495 if inDescriptionSection:
496 if line.startswith("Files:"):
497 inDescriptionSection = False
498 else:
499 continue
500 else:
501 if line.startswith("Description:"):
502 inDescriptionSection = True
503 line += "\n"
504 for messageLine in message.split("\n"):
505 line += "\t" + messageLine + "\n"
506
507 result += line + "\n"
4f5cf76a
SH
508
509 return result
510
ea99c3ae
SH
511 def prepareSubmitTemplate(self):
512 # remove lines in the Files section that show changes to files outside the depot path we're committing into
513 template = ""
514 inFilesSection = False
515 for line in read_pipe_lines("p4 change -o"):
516 if inFilesSection:
517 if line.startswith("\t"):
518 # path starts and ends with a tab
519 path = line[1:]
520 lastTab = path.rfind("\t")
521 if lastTab != -1:
522 path = path[:lastTab]
523 if not path.startswith(self.depotPath):
524 continue
525 else:
526 inFilesSection = False
527 else:
528 if line.startswith("Files:"):
529 inFilesSection = True
530
531 template += line
532
533 return template
534
7cb5cbef 535 def applyCommit(self, id):
0e36f2d7
SH
536 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
537 diffOpts = ("", "-M")[self.detectRename]
538 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
539 filesToAdd = set()
540 filesToDelete = set()
d336c158 541 editedFiles = set()
c65b670e 542 filesToChangeExecBit = {}
4f5cf76a 543 for line in diff:
b43b0a3c
CP
544 diff = parseDiffTreeEntry(line)
545 modifier = diff['status']
546 path = diff['src']
4f5cf76a 547 if modifier == "M":
d336c158 548 system("p4 edit \"%s\"" % path)
c65b670e
CP
549 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
550 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 551 editedFiles.add(path)
4f5cf76a
SH
552 elif modifier == "A":
553 filesToAdd.add(path)
c65b670e 554 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
555 if path in filesToDelete:
556 filesToDelete.remove(path)
557 elif modifier == "D":
558 filesToDelete.add(path)
559 if path in filesToAdd:
560 filesToAdd.remove(path)
d9a5f25b 561 elif modifier == "R":
b43b0a3c 562 src, dest = diff['src'], diff['dst']
d9a5f25b
CP
563 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
564 system("p4 edit \"%s\"" % (dest))
c65b670e
CP
565 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
566 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
567 os.unlink(dest)
568 editedFiles.add(dest)
569 filesToDelete.add(src)
4f5cf76a
SH
570 else:
571 die("unknown modifier %s for %s" % (modifier, path))
572
0e36f2d7 573 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 574 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
575 tryPatchCmd = patchcmd + "--check -"
576 applyPatchCmd = patchcmd + "--check --apply -"
51a2640a 577
47a130b7 578 if os.system(tryPatchCmd) != 0:
51a2640a
SH
579 print "Unfortunately applying the change failed!"
580 print "What do you want to do?"
581 response = "x"
582 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
583 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
584 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
585 if response == "s":
586 print "Skipping! Good luck with the next patches..."
20947149
SH
587 for f in editedFiles:
588 system("p4 revert \"%s\"" % f);
589 for f in filesToAdd:
590 system("rm %s" %f)
51a2640a
SH
591 return
592 elif response == "a":
47a130b7 593 os.system(applyPatchCmd)
51a2640a
SH
594 if len(filesToAdd) > 0:
595 print "You may also want to call p4 add on the following files:"
596 print " ".join(filesToAdd)
597 if len(filesToDelete):
598 print "The following files should be scheduled for deletion with p4 delete:"
599 print " ".join(filesToDelete)
cebdf5af
HWN
600 die("Please resolve and submit the conflict manually and "
601 + "continue afterwards with git-p4 submit --continue")
51a2640a
SH
602 elif response == "w":
603 system(diffcmd + " > patch.txt")
604 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af
HWN
605 die("Please resolve and submit the conflict manually and "
606 "continue afterwards with git-p4 submit --continue")
51a2640a 607
47a130b7 608 system(applyPatchCmd)
4f5cf76a
SH
609
610 for f in filesToAdd:
e6b711f0 611 system("p4 add \"%s\"" % f)
4f5cf76a 612 for f in filesToDelete:
e6b711f0
SH
613 system("p4 revert \"%s\"" % f)
614 system("p4 delete \"%s\"" % f)
4f5cf76a 615
c65b670e
CP
616 # Set/clear executable bits
617 for f in filesToChangeExecBit.keys():
618 mode = filesToChangeExecBit[f]
619 setP4ExecBit(f, mode)
620
0e36f2d7
SH
621 logMessage = extractLogMessageFromGitCommit(id)
622 if self.isWindows:
623 logMessage = logMessage.replace("\n", "\r\n")
624 logMessage = logMessage.strip()
4f5cf76a 625
ea99c3ae 626 template = self.prepareSubmitTemplate()
4f5cf76a
SH
627
628 if self.interactive:
629 submitTemplate = self.prepareLogMessage(template, logMessage)
b016d397 630 diff = read_pipe("p4 diff -du ...")
4f5cf76a
SH
631
632 for newFile in filesToAdd:
633 diff += "==== new file ====\n"
634 diff += "--- /dev/null\n"
635 diff += "+++ %s\n" % newFile
636 f = open(newFile, "r")
637 for line in f.readlines():
638 diff += "+" + line
639 f.close()
640
25df95cc
SH
641 separatorLine = "######## everything below this line is just the diff #######"
642 if platform.system() == "Windows":
643 separatorLine += "\r"
644 separatorLine += "\n"
4f5cf76a 645
e96e400f
SH
646 [handle, fileName] = tempfile.mkstemp()
647 tmpFile = os.fdopen(handle, "w+")
648 tmpFile.write(submitTemplate + separatorLine + diff)
649 tmpFile.close()
650 defaultEditor = "vi"
651 if platform.system() == "Windows":
652 defaultEditor = "notepad"
653 editor = os.environ.get("EDITOR", defaultEditor);
654 system(editor + " " + fileName)
655 tmpFile = open(fileName, "rb")
656 message = tmpFile.read()
657 tmpFile.close()
658 os.remove(fileName)
659 submitTemplate = message[:message.index(separatorLine)]
660 if self.isWindows:
661 submitTemplate = submitTemplate.replace("\r\n", "\n")
662
e96e400f 663 write_pipe("p4 submit -i", submitTemplate)
4f5cf76a
SH
664 else:
665 fileName = "submit.txt"
666 file = open(fileName, "w+")
667 file.write(self.prepareLogMessage(template, logMessage))
668 file.close()
cebdf5af
HWN
669 print ("Perforce submit template written as %s. "
670 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
671 % (fileName, fileName))
4f5cf76a
SH
672
673 def run(self, args):
c9b50e63
SH
674 if len(args) == 0:
675 self.master = currentGitBranch()
4280e533 676 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
677 die("Detecting current git branch failed!")
678 elif len(args) == 1:
679 self.master = args[0]
680 else:
681 return False
682
27d2d811 683 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 684 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
685 if len(self.origin) == 0:
686 self.origin = upstream
a3fdd579
SH
687
688 if self.verbose:
689 print "Origin branch is " + self.origin
9512497b 690
ea99c3ae 691 if len(self.depotPath) == 0:
9512497b
SH
692 print "Internal error: cannot locate perforce depot path from existing branches"
693 sys.exit(128)
694
ea99c3ae 695 self.clientPath = p4Where(self.depotPath)
9512497b 696
51a2640a 697 if len(self.clientPath) == 0:
ea99c3ae 698 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
9512497b
SH
699 sys.exit(128)
700
ea99c3ae 701 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 702 self.oldWorkingDirectory = os.getcwd()
c1b296b9 703
51a2640a 704 os.chdir(self.clientPath)
31f9ec12
SH
705 print "Syncronizing p4 checkout..."
706 system("p4 sync ...")
9512497b 707
4f5cf76a 708 self.check()
4f5cf76a 709
4c750c0d
SH
710 commits = []
711 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
712 commits.append(line.strip())
713 commits.reverse()
4f5cf76a
SH
714
715 while len(commits) > 0:
4f5cf76a
SH
716 commit = commits[0]
717 commits = commits[1:]
7cb5cbef 718 self.applyCommit(commit)
4f5cf76a
SH
719 if not self.interactive:
720 break
721
4f5cf76a 722 if len(commits) == 0:
4c750c0d
SH
723 print "All changes applied!"
724 os.chdir(self.oldWorkingDirectory)
14594f4b 725
4c750c0d
SH
726 sync = P4Sync()
727 sync.run([])
14594f4b 728
4c750c0d
SH
729 rebase = P4Rebase()
730 rebase.rebase()
4f5cf76a 731
b984733c
SH
732 return True
733
711544b0 734class P4Sync(Command):
b984733c
SH
735 def __init__(self):
736 Command.__init__(self)
737 self.options = [
738 optparse.make_option("--branch", dest="branch"),
739 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
740 optparse.make_option("--changesfile", dest="changesFile"),
741 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 742 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
a028a98e 743 optparse.make_option("--verbose", dest="verbose", action="store_true"),
d2c6dd30
HWN
744 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
745 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 746 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6
HWN
747 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
748 help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
b984733c
SH
749 ]
750 self.description = """Imports from Perforce into a git repository.\n
751 example:
752 //depot/my/project/ -- to import the current head
753 //depot/my/project/@all -- to import everything
754 //depot/my/project/@1,6 -- to import only from revision 1 to 6
755
756 (a ... is not needed in the path p4 specification, it's added implicitly)"""
757
758 self.usage += " //depot/path[@revRange]"
b984733c 759 self.silent = False
b984733c
SH
760 self.createdBranches = Set()
761 self.committedChanges = Set()
569d1bd4 762 self.branch = ""
b984733c 763 self.detectBranches = False
cb53e1f8 764 self.detectLabels = False
b984733c 765 self.changesFile = ""
01265103 766 self.syncWithOrigin = True
4b97ffb1 767 self.verbose = False
a028a98e 768 self.importIntoRemotes = True
01a9c9c5 769 self.maxChanges = ""
c1f9197f 770 self.isWindows = (platform.system() == "Windows")
8b41a97f 771 self.keepRepoPath = False
6326aa58 772 self.depotPaths = None
3c699645 773 self.p4BranchesInGit = []
354081d5 774 self.cloneExclude = []
b984733c 775
01265103
SH
776 if gitConfig("git-p4.syncFromOrigin") == "false":
777 self.syncWithOrigin = False
778
b984733c 779 def extractFilesFromCommit(self, commit):
354081d5
TT
780 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
781 for path in self.cloneExclude]
b984733c
SH
782 files = []
783 fnum = 0
784 while commit.has_key("depotFile%s" % fnum):
785 path = commit["depotFile%s" % fnum]
6326aa58 786
354081d5
TT
787 if [p for p in self.cloneExclude
788 if path.startswith (p)]:
789 found = False
790 else:
791 found = [p for p in self.depotPaths
792 if path.startswith (p)]
6326aa58 793 if not found:
b984733c
SH
794 fnum = fnum + 1
795 continue
796
797 file = {}
798 file["path"] = path
799 file["rev"] = commit["rev%s" % fnum]
800 file["action"] = commit["action%s" % fnum]
801 file["type"] = commit["type%s" % fnum]
802 files.append(file)
803 fnum = fnum + 1
804 return files
805
6326aa58 806 def stripRepoPath(self, path, prefixes):
8b41a97f 807 if self.keepRepoPath:
6326aa58
HWN
808 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
809
810 for p in prefixes:
811 if path.startswith(p):
812 path = path[len(p):]
8b41a97f 813
6326aa58 814 return path
6754a299 815
71b112d4 816 def splitFilesIntoBranches(self, commit):
d5904674 817 branches = {}
71b112d4
SH
818 fnum = 0
819 while commit.has_key("depotFile%s" % fnum):
820 path = commit["depotFile%s" % fnum]
6326aa58
HWN
821 found = [p for p in self.depotPaths
822 if path.startswith (p)]
823 if not found:
71b112d4
SH
824 fnum = fnum + 1
825 continue
826
827 file = {}
828 file["path"] = path
829 file["rev"] = commit["rev%s" % fnum]
830 file["action"] = commit["action%s" % fnum]
831 file["type"] = commit["type%s" % fnum]
832 fnum = fnum + 1
833
6326aa58 834 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 835
4b97ffb1 836 for branch in self.knownBranches.keys():
6754a299
HWN
837
838 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
839 if relPath.startswith(branch + "/"):
d5904674
SH
840 if branch not in branches:
841 branches[branch] = []
71b112d4 842 branches[branch].append(file)
6555b2cc 843 break
b984733c
SH
844
845 return branches
846
6a49f8e2
HWN
847 ## Should move this out, doesn't use SELF.
848 def readP4Files(self, files):
b1ce9447 849 files = [f for f in files
982bb8a3 850 if f['action'] != 'delete']
6a49f8e2 851
b1ce9447 852 if not files:
f2eda79f
HWN
853 return
854
78800190
SL
855 filedata = p4CmdList('-x - print',
856 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
857 for f in files]),
858 stdin_mode='w+')
859 if "p4ExitCode" in filedata[0]:
860 die("Problems executing p4. Error: [%d]."
861 % (filedata[0]['p4ExitCode']));
6a49f8e2 862
d2c6dd30
HWN
863 j = 0;
864 contents = {}
b1ce9447 865 while j < len(filedata):
d2c6dd30 866 stat = filedata[j]
b1ce9447
HWN
867 j += 1
868 text = ''
f3e9512b
JM
869 while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
870 tmp = filedata[j]['data']
871 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
872 tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
873 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
874 tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
875 text += tmp
b1ce9447 876 j += 1
6a49f8e2 877
1b9a4684
HWN
878
879 if not stat.has_key('depotFile'):
880 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
881 continue
882
b1ce9447 883 contents[stat['depotFile']] = text
6a49f8e2 884
d2c6dd30
HWN
885 for f in files:
886 assert not f.has_key('data')
887 f['data'] = contents[f['path']]
6a49f8e2 888
6326aa58 889 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
890 epoch = details["time"]
891 author = details["user"]
892
4b97ffb1
SH
893 if self.verbose:
894 print "commit into %s" % branch
895
96e07dd2
HWN
896 # start with reading files; if that fails, we should not
897 # create a commit.
898 new_files = []
899 for f in files:
900 if [p for p in branchPrefixes if f['path'].startswith(p)]:
901 new_files.append (f)
902 else:
903 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
904 files = new_files
905 self.readP4Files(files)
906
907
908
909
b984733c 910 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 911# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
912 self.committedChanges.add(int(details["change"]))
913 committer = ""
b607e71e
SH
914 if author not in self.users:
915 self.getUserMapFromPerforceServer()
b984733c 916 if author in self.users:
0828ab14 917 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
b984733c 918 else:
0828ab14 919 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
b984733c
SH
920
921 self.gitStream.write("committer %s\n" % committer)
922
923 self.gitStream.write("data <<EOT\n")
924 self.gitStream.write(details["desc"])
6581de09
SH
925 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
926 % (','.join (branchPrefixes), details["change"]))
927 if len(details['options']) > 0:
928 self.gitStream.write(": options = %s" % details['options'])
929 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
930
931 if len(parent) > 0:
4b97ffb1
SH
932 if self.verbose:
933 print "parent %s" % parent
b984733c
SH
934 self.gitStream.write("from %s\n" % parent)
935
6a49f8e2 936 for file in files:
b984733c 937 if file["type"] == "apple":
6a49f8e2 938 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
b984733c
SH
939 continue
940
6a49f8e2
HWN
941 relPath = self.stripRepoPath(file['path'], branchPrefixes)
942 if file["action"] == "delete":
b984733c
SH
943 self.gitStream.write("D %s\n" % relPath)
944 else:
6a49f8e2 945 data = file['data']
b984733c 946
74276ec6 947 mode = "644"
b9fc6ea9 948 if isP4Exec(file["type"]):
74276ec6
SH
949 mode = "755"
950 elif file["type"] == "symlink":
951 mode = "120000"
952 # p4 print on a symlink contains "target\n", so strip it off
953 data = data[:-1]
954
c1f9197f
MSO
955 if self.isWindows and file["type"].endswith("text"):
956 data = data.replace("\r\n", "\n")
957
74276ec6 958 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
b984733c
SH
959 self.gitStream.write("data %s\n" % len(data))
960 self.gitStream.write(data)
961 self.gitStream.write("\n")
962
963 self.gitStream.write("\n")
964
1f4ba1cb
SH
965 change = int(details["change"])
966
9bda3a85 967 if self.labels.has_key(change):
1f4ba1cb
SH
968 label = self.labels[change]
969 labelDetails = label[0]
970 labelRevisions = label[1]
71b112d4
SH
971 if self.verbose:
972 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 973
6326aa58
HWN
974 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
975 for p in branchPrefixes]))
1f4ba1cb
SH
976
977 if len(files) == len(labelRevisions):
978
979 cleanedFiles = {}
980 for info in files:
981 if info["action"] == "delete":
982 continue
983 cleanedFiles[info["depotFile"]] = info["rev"]
984
985 if cleanedFiles == labelRevisions:
986 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
987 self.gitStream.write("from %s\n" % branch)
988
989 owner = labelDetails["Owner"]
990 tagger = ""
991 if author in self.users:
992 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
993 else:
994 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
995 self.gitStream.write("tagger %s\n" % tagger)
996 self.gitStream.write("data <<EOT\n")
997 self.gitStream.write(labelDetails["Description"])
998 self.gitStream.write("EOT\n\n")
999
1000 else:
a46668fa 1001 if not self.silent:
cebdf5af
HWN
1002 print ("Tag %s does not match with change %s: files do not match."
1003 % (labelDetails["label"], change))
1f4ba1cb
SH
1004
1005 else:
a46668fa 1006 if not self.silent:
cebdf5af
HWN
1007 print ("Tag %s does not match with change %s: file count is different."
1008 % (labelDetails["label"], change))
b984733c 1009
183b8ef8 1010 def getUserCacheFilename(self):
b2d2d16a
SH
1011 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1012 return home + "/.gitp4-usercache.txt"
183b8ef8 1013
b607e71e 1014 def getUserMapFromPerforceServer(self):
ebd81168
SH
1015 if self.userMapFromPerforceServer:
1016 return
b984733c
SH
1017 self.users = {}
1018
1019 for output in p4CmdList("users"):
1020 if not output.has_key("User"):
1021 continue
1022 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1023
183b8ef8
HWN
1024
1025 s = ''
1026 for (key, val) in self.users.items():
1027 s += "%s\t%s\n" % (key, val)
1028
1029 open(self.getUserCacheFilename(), "wb").write(s)
ebd81168 1030 self.userMapFromPerforceServer = True
b607e71e
SH
1031
1032 def loadUserMapFromCache(self):
1033 self.users = {}
ebd81168 1034 self.userMapFromPerforceServer = False
b607e71e 1035 try:
183b8ef8 1036 cache = open(self.getUserCacheFilename(), "rb")
b607e71e
SH
1037 lines = cache.readlines()
1038 cache.close()
1039 for line in lines:
b25b2065 1040 entry = line.strip().split("\t")
b607e71e
SH
1041 self.users[entry[0]] = entry[1]
1042 except IOError:
1043 self.getUserMapFromPerforceServer()
1044
1f4ba1cb
SH
1045 def getLabels(self):
1046 self.labels = {}
1047
6326aa58 1048 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
10c3211b 1049 if len(l) > 0 and not self.silent:
183f8436 1050 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
1051
1052 for output in l:
1f4ba1cb
SH
1053 label = output["label"]
1054 revisions = {}
1055 newestChange = 0
71b112d4
SH
1056 if self.verbose:
1057 print "Querying files for label %s" % label
6326aa58
HWN
1058 for file in p4CmdList("files "
1059 + ' '.join (["%s...@%s" % (p, label)
1060 for p in self.depotPaths])):
1f4ba1cb
SH
1061 revisions[file["depotFile"]] = file["rev"]
1062 change = int(file["change"])
1063 if change > newestChange:
1064 newestChange = change
1065
9bda3a85
SH
1066 self.labels[newestChange] = [output, revisions]
1067
1068 if self.verbose:
1069 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 1070
86dff6b6
HWN
1071 def guessProjectName(self):
1072 for p in self.depotPaths:
6e5295c4
SH
1073 if p.endswith("/"):
1074 p = p[:-1]
1075 p = p[p.strip().rfind("/") + 1:]
1076 if not p.endswith("/"):
1077 p += "/"
1078 return p
86dff6b6 1079
4b97ffb1 1080 def getBranchMapping(self):
6555b2cc
SH
1081 lostAndFoundBranches = set()
1082
4b97ffb1
SH
1083 for info in p4CmdList("branches"):
1084 details = p4Cmd("branch -o %s" % info["branch"])
1085 viewIdx = 0
1086 while details.has_key("View%s" % viewIdx):
1087 paths = details["View%s" % viewIdx].split(" ")
1088 viewIdx = viewIdx + 1
1089 # require standard //depot/foo/... //depot/bar/... mapping
1090 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1091 continue
1092 source = paths[0]
1093 destination = paths[1]
6509e19c
SH
1094 ## HACK
1095 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1096 source = source[len(self.depotPaths[0]):-4]
1097 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 1098
1a2edf4e
SH
1099 if destination in self.knownBranches:
1100 if not self.silent:
1101 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1102 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1103 continue
1104
6555b2cc
SH
1105 self.knownBranches[destination] = source
1106
1107 lostAndFoundBranches.discard(destination)
1108
29bdbac1 1109 if source not in self.knownBranches:
6555b2cc
SH
1110 lostAndFoundBranches.add(source)
1111
1112
1113 for branch in lostAndFoundBranches:
1114 self.knownBranches[branch] = branch
29bdbac1 1115
38f9f5ec
SH
1116 def getBranchMappingFromGitBranches(self):
1117 branches = p4BranchesInGit(self.importIntoRemotes)
1118 for branch in branches.keys():
1119 if branch == "master":
1120 branch = "main"
1121 else:
1122 branch = branch[len(self.projectName):]
1123 self.knownBranches[branch] = branch
1124
29bdbac1 1125 def listExistingP4GitBranches(self):
144ff46b
SH
1126 # branches holds mapping from name to commit
1127 branches = p4BranchesInGit(self.importIntoRemotes)
1128 self.p4BranchesInGit = branches.keys()
1129 for branch in branches.keys():
1130 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 1131
bb6e09b2
HWN
1132 def updateOptionDict(self, d):
1133 option_keys = {}
1134 if self.keepRepoPath:
1135 option_keys['keepRepoPath'] = 1
1136
1137 d["options"] = ' '.join(sorted(option_keys.keys()))
1138
1139 def readOptions(self, d):
1140 self.keepRepoPath = (d.has_key('options')
1141 and ('keepRepoPath' in d['options']))
6326aa58 1142
8134f69c
SH
1143 def gitRefForBranch(self, branch):
1144 if branch == "main":
1145 return self.refPrefix + "master"
1146
1147 if len(branch) <= 0:
1148 return branch
1149
1150 return self.refPrefix + self.projectName + branch
1151
1ca3d710
SH
1152 def gitCommitByP4Change(self, ref, change):
1153 if self.verbose:
1154 print "looking in ref " + ref + " for change %s using bisect..." % change
1155
1156 earliestCommit = ""
1157 latestCommit = parseRevision(ref)
1158
1159 while True:
1160 if self.verbose:
1161 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1162 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1163 if len(next) == 0:
1164 if self.verbose:
1165 print "argh"
1166 return ""
1167 log = extractLogMessageFromGitCommit(next)
1168 settings = extractSettingsGitLog(log)
1169 currentChange = int(settings['change'])
1170 if self.verbose:
1171 print "current change %s" % currentChange
1172
1173 if currentChange == change:
1174 if self.verbose:
1175 print "found %s" % next
1176 return next
1177
1178 if currentChange < change:
1179 earliestCommit = "^%s" % next
1180 else:
1181 latestCommit = "%s" % next
1182
1183 return ""
1184
1185 def importNewBranch(self, branch, maxChange):
1186 # make fast-import flush all changes to disk and update the refs using the checkpoint
1187 # command so that we can try to find the branch parent in the git history
1188 self.gitStream.write("checkpoint\n\n");
1189 self.gitStream.flush();
1190 branchPrefix = self.depotPaths[0] + branch + "/"
1191 range = "@1,%s" % maxChange
1192 #print "prefix" + branchPrefix
1193 changes = p4ChangesForPaths([branchPrefix], range)
1194 if len(changes) <= 0:
1195 return False
1196 firstChange = changes[0]
1197 #print "first change in branch: %s" % firstChange
1198 sourceBranch = self.knownBranches[branch]
1199 sourceDepotPath = self.depotPaths[0] + sourceBranch
1200 sourceRef = self.gitRefForBranch(sourceBranch)
1201 #print "source " + sourceBranch
1202
1203 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1204 #print "branch parent: %s" % branchParentChange
1205 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1206 if len(gitParent) > 0:
1207 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1208 #print "parent git commit: %s" % gitParent
1209
1210 self.importChanges(changes)
1211 return True
1212
e87f37ae
SH
1213 def importChanges(self, changes):
1214 cnt = 1
1215 for change in changes:
1216 description = p4Cmd("describe %s" % change)
1217 self.updateOptionDict(description)
1218
1219 if not self.silent:
1220 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1221 sys.stdout.flush()
1222 cnt = cnt + 1
1223
1224 try:
1225 if self.detectBranches:
1226 branches = self.splitFilesIntoBranches(description)
1227 for branch in branches.keys():
1228 ## HACK --hwn
1229 branchPrefix = self.depotPaths[0] + branch + "/"
1230
1231 parent = ""
1232
1233 filesForCommit = branches[branch]
1234
1235 if self.verbose:
1236 print "branch is %s" % branch
1237
1238 self.updatedBranches.add(branch)
1239
1240 if branch not in self.createdBranches:
1241 self.createdBranches.add(branch)
1242 parent = self.knownBranches[branch]
1243 if parent == branch:
1244 parent = ""
1ca3d710
SH
1245 else:
1246 fullBranch = self.projectName + branch
1247 if fullBranch not in self.p4BranchesInGit:
1248 if not self.silent:
1249 print("\n Importing new branch %s" % fullBranch);
1250 if self.importNewBranch(branch, change - 1):
1251 parent = ""
1252 self.p4BranchesInGit.append(fullBranch)
1253 if not self.silent:
1254 print("\n Resuming with change %s" % change);
1255
1256 if self.verbose:
1257 print "parent determined through known branches: %s" % parent
e87f37ae 1258
8134f69c
SH
1259 branch = self.gitRefForBranch(branch)
1260 parent = self.gitRefForBranch(parent)
e87f37ae
SH
1261
1262 if self.verbose:
1263 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1264
1265 if len(parent) == 0 and branch in self.initialParents:
1266 parent = self.initialParents[branch]
1267 del self.initialParents[branch]
1268
1269 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1270 else:
1271 files = self.extractFilesFromCommit(description)
1272 self.commit(description, files, self.branch, self.depotPaths,
1273 self.initialParent)
1274 self.initialParent = ""
1275 except IOError:
1276 print self.gitError.read()
1277 sys.exit(1)
1278
c208a243
SH
1279 def importHeadRevision(self, revision):
1280 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1281
1282 details = { "user" : "git perforce import user", "time" : int(time.time()) }
1283 details["desc"] = ("Initial import of %s from the state at revision %s"
1284 % (' '.join(self.depotPaths), revision))
1285 details["change"] = revision
1286 newestRevision = 0
1287
1288 fileCnt = 0
1289 for info in p4CmdList("files "
1290 + ' '.join(["%s...%s"
1291 % (p, revision)
1292 for p in self.depotPaths])):
1293
1294 if info['code'] == 'error':
1295 sys.stderr.write("p4 returned an error: %s\n"
1296 % info['data'])
1297 sys.exit(1)
1298
1299
1300 change = int(info["change"])
1301 if change > newestRevision:
1302 newestRevision = change
1303
1304 if info["action"] == "delete":
1305 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1306 #fileCnt = fileCnt + 1
1307 continue
1308
1309 for prop in ["depotFile", "rev", "action", "type" ]:
1310 details["%s%s" % (prop, fileCnt)] = info[prop]
1311
1312 fileCnt = fileCnt + 1
1313
1314 details["change"] = newestRevision
1315 self.updateOptionDict(details)
1316 try:
1317 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1318 except IOError:
1319 print "IO error with git fast-import. Is your git version recent enough?"
1320 print self.gitError.read()
1321
1322
b984733c 1323 def run(self, args):
6326aa58 1324 self.depotPaths = []
179caebf
SH
1325 self.changeRange = ""
1326 self.initialParent = ""
6326aa58 1327 self.previousDepotPaths = []
ce6f33c8 1328
29bdbac1
SH
1329 # map from branch depot path to parent branch
1330 self.knownBranches = {}
1331 self.initialParents = {}
5ca44617 1332 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
1333 if not self.syncWithOrigin:
1334 self.hasOrigin = False
29bdbac1 1335
a028a98e
SH
1336 if self.importIntoRemotes:
1337 self.refPrefix = "refs/remotes/p4/"
1338 else:
db775559 1339 self.refPrefix = "refs/heads/p4/"
a028a98e 1340
cebdf5af
HWN
1341 if self.syncWithOrigin and self.hasOrigin:
1342 if not self.silent:
1343 print "Syncing with origin first by calling git fetch origin"
1344 system("git fetch origin")
10f880f8 1345
569d1bd4 1346 if len(self.branch) == 0:
db775559 1347 self.branch = self.refPrefix + "master"
a028a98e 1348 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 1349 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 1350 system("git branch -D p4");
faf1bd20 1351 # create it /after/ importing, when master exists
0058a33a 1352 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 1353 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 1354
6a49f8e2
HWN
1355 # TODO: should always look at previous commits,
1356 # merge with previous imports, if possible.
1357 if args == []:
d414c74a 1358 if self.hasOrigin:
5ca44617 1359 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
1360 self.listExistingP4GitBranches()
1361
1362 if len(self.p4BranchesInGit) > 1:
1363 if not self.silent:
1364 print "Importing from/into multiple branches"
1365 self.detectBranches = True
967f72e2 1366
29bdbac1
SH
1367 if self.verbose:
1368 print "branches: %s" % self.p4BranchesInGit
1369
1370 p4Change = 0
1371 for branch in self.p4BranchesInGit:
cebdf5af 1372 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
1373
1374 settings = extractSettingsGitLog(logMsg)
29bdbac1 1375
bb6e09b2
HWN
1376 self.readOptions(settings)
1377 if (settings.has_key('depot-paths')
1378 and settings.has_key ('change')):
1379 change = int(settings['change']) + 1
29bdbac1
SH
1380 p4Change = max(p4Change, change)
1381
bb6e09b2
HWN
1382 depotPaths = sorted(settings['depot-paths'])
1383 if self.previousDepotPaths == []:
6326aa58 1384 self.previousDepotPaths = depotPaths
29bdbac1 1385 else:
6326aa58
HWN
1386 paths = []
1387 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
583e1707 1388 for i in range(0, min(len(cur), len(prev))):
6326aa58 1389 if cur[i] <> prev[i]:
583e1707 1390 i = i - 1
6326aa58
HWN
1391 break
1392
583e1707 1393 paths.append (cur[:i + 1])
6326aa58
HWN
1394
1395 self.previousDepotPaths = paths
29bdbac1
SH
1396
1397 if p4Change > 0:
bb6e09b2 1398 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 1399 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
1400 if not self.detectBranches:
1401 self.initialParent = parseRevision(self.branch)
341dc1c1 1402 if not self.silent and not self.detectBranches:
967f72e2 1403 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 1404
f9162f6a
SH
1405 if not self.branch.startswith("refs/"):
1406 self.branch = "refs/heads/" + self.branch
179caebf 1407
6326aa58 1408 if len(args) == 0 and self.depotPaths:
b984733c 1409 if not self.silent:
6326aa58 1410 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 1411 else:
6326aa58 1412 if self.depotPaths and self.depotPaths != args:
cebdf5af 1413 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
1414 "This doesn't work!" % (' '.join (self.depotPaths),
1415 ' '.join (args)))
b984733c 1416 sys.exit(1)
6326aa58 1417
bb6e09b2 1418 self.depotPaths = sorted(args)
b984733c 1419
1c49fc19 1420 revision = ""
b984733c 1421 self.users = {}
b984733c 1422
6326aa58
HWN
1423 newPaths = []
1424 for p in self.depotPaths:
1425 if p.find("@") != -1:
1426 atIdx = p.index("@")
1427 self.changeRange = p[atIdx:]
1428 if self.changeRange == "@all":
1429 self.changeRange = ""
6a49f8e2 1430 elif ',' not in self.changeRange:
1c49fc19 1431 revision = self.changeRange
6326aa58 1432 self.changeRange = ""
7fcff9de 1433 p = p[:atIdx]
6326aa58
HWN
1434 elif p.find("#") != -1:
1435 hashIdx = p.index("#")
1c49fc19 1436 revision = p[hashIdx:]
7fcff9de 1437 p = p[:hashIdx]
6326aa58 1438 elif self.previousDepotPaths == []:
1c49fc19 1439 revision = "#head"
6326aa58
HWN
1440
1441 p = re.sub ("\.\.\.$", "", p)
1442 if not p.endswith("/"):
1443 p += "/"
1444
1445 newPaths.append(p)
1446
1447 self.depotPaths = newPaths
1448
b984733c 1449
b607e71e 1450 self.loadUserMapFromCache()
cb53e1f8
SH
1451 self.labels = {}
1452 if self.detectLabels:
1453 self.getLabels();
b984733c 1454
4b97ffb1 1455 if self.detectBranches:
df450923
SH
1456 ## FIXME - what's a P4 projectName ?
1457 self.projectName = self.guessProjectName()
1458
38f9f5ec
SH
1459 if self.hasOrigin:
1460 self.getBranchMappingFromGitBranches()
1461 else:
1462 self.getBranchMapping()
29bdbac1
SH
1463 if self.verbose:
1464 print "p4-git branches: %s" % self.p4BranchesInGit
1465 print "initial parents: %s" % self.initialParents
1466 for b in self.p4BranchesInGit:
1467 if b != "master":
6326aa58
HWN
1468
1469 ## FIXME
29bdbac1
SH
1470 b = b[len(self.projectName):]
1471 self.createdBranches.add(b)
4b97ffb1 1472
f291b4e3 1473 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 1474
cebdf5af 1475 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
1476 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1477 stderr=subprocess.PIPE);
08483580
SH
1478 self.gitOutput = importProcess.stdout
1479 self.gitStream = importProcess.stdin
1480 self.gitError = importProcess.stderr
b984733c 1481
1c49fc19 1482 if revision:
c208a243 1483 self.importHeadRevision(revision)
b984733c
SH
1484 else:
1485 changes = []
1486
0828ab14 1487 if len(self.changesFile) > 0:
b984733c
SH
1488 output = open(self.changesFile).readlines()
1489 changeSet = Set()
1490 for line in output:
1491 changeSet.add(int(line))
1492
1493 for change in changeSet:
1494 changes.append(change)
1495
1496 changes.sort()
1497 else:
29bdbac1 1498 if self.verbose:
86dff6b6 1499 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 1500 self.changeRange)
4f6432d8 1501 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 1502
01a9c9c5 1503 if len(self.maxChanges) > 0:
7fcff9de 1504 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 1505
b984733c 1506 if len(changes) == 0:
0828ab14 1507 if not self.silent:
341dc1c1 1508 print "No changes to import!"
1f52af6c 1509 return True
b984733c 1510
a9d1a27a
SH
1511 if not self.silent and not self.detectBranches:
1512 print "Import destination: %s" % self.branch
1513
341dc1c1
SH
1514 self.updatedBranches = set()
1515
e87f37ae 1516 self.importChanges(changes)
b984733c 1517
341dc1c1
SH
1518 if not self.silent:
1519 print ""
1520 if len(self.updatedBranches) > 0:
1521 sys.stdout.write("Updated branches: ")
1522 for b in self.updatedBranches:
1523 sys.stdout.write("%s " % b)
1524 sys.stdout.write("\n")
b984733c 1525
b984733c 1526 self.gitStream.close()
29bdbac1
SH
1527 if importProcess.wait() != 0:
1528 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
1529 self.gitOutput.close()
1530 self.gitError.close()
1531
b984733c
SH
1532 return True
1533
01ce1fe9
SH
1534class P4Rebase(Command):
1535 def __init__(self):
1536 Command.__init__(self)
01265103 1537 self.options = [ ]
cebdf5af
HWN
1538 self.description = ("Fetches the latest revision from perforce and "
1539 + "rebases the current work (branch) against it")
68c42153 1540 self.verbose = False
01ce1fe9
SH
1541
1542 def run(self, args):
1543 sync = P4Sync()
1544 sync.run([])
d7e3868c 1545
14594f4b
SH
1546 return self.rebase()
1547
1548 def rebase(self):
36ee4ee4
SH
1549 if os.system("git update-index --refresh") != 0:
1550 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1551 if len(read_pipe("git diff-index HEAD --")) > 0:
1552 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1553
d7e3868c
SH
1554 [upstream, settings] = findUpstreamBranchPoint()
1555 if len(upstream) == 0:
1556 die("Cannot find upstream branchpoint for rebase")
1557
1558 # the branchpoint may be p4/foo~3, so strip off the parent
1559 upstream = re.sub("~[0-9]+$", "", upstream)
1560
1561 print "Rebasing the current branch onto %s" % upstream
b25b2065 1562 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 1563 system("git rebase %s" % upstream)
1f52af6c 1564 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
1565 return True
1566
f9a3a4f7
SH
1567class P4Clone(P4Sync):
1568 def __init__(self):
1569 P4Sync.__init__(self)
1570 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 1571 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 1572 self.options += [
bb6e09b2
HWN
1573 optparse.make_option("--destination", dest="cloneDestination",
1574 action='store', default=None,
354081d5
TT
1575 help="where to leave result of the clone"),
1576 optparse.make_option("-/", dest="cloneExclude",
1577 action="append", type="string",
1578 help="exclude depot path")
1579 ]
bb6e09b2 1580 self.cloneDestination = None
f9a3a4f7 1581 self.needsGit = False
f9a3a4f7 1582
354081d5
TT
1583 # This is required for the "append" cloneExclude action
1584 def ensure_value(self, attr, value):
1585 if not hasattr(self, attr) or getattr(self, attr) is None:
1586 setattr(self, attr, value)
1587 return getattr(self, attr)
1588
6a49f8e2
HWN
1589 def defaultDestination(self, args):
1590 ## TODO: use common prefix of args?
1591 depotPath = args[0]
1592 depotDir = re.sub("(@[^@]*)$", "", depotPath)
1593 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 1594 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
1595 depotDir = re.sub(r"/$", "", depotDir)
1596 return os.path.split(depotDir)[1]
1597
f9a3a4f7
SH
1598 def run(self, args):
1599 if len(args) < 1:
1600 return False
bb6e09b2
HWN
1601
1602 if self.keepRepoPath and not self.cloneDestination:
1603 sys.stderr.write("Must specify destination for --keep-path\n")
1604 sys.exit(1)
f9a3a4f7 1605
6326aa58 1606 depotPaths = args
5e100b5c
SH
1607
1608 if not self.cloneDestination and len(depotPaths) > 1:
1609 self.cloneDestination = depotPaths[-1]
1610 depotPaths = depotPaths[:-1]
1611
354081d5 1612 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
1613 for p in depotPaths:
1614 if not p.startswith("//"):
1615 return False
f9a3a4f7 1616
bb6e09b2 1617 if not self.cloneDestination:
98ad4faf 1618 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 1619
86dff6b6 1620 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
c3bf3f13
KG
1621 if not os.path.exists(self.cloneDestination):
1622 os.makedirs(self.cloneDestination)
bb6e09b2 1623 os.chdir(self.cloneDestination)
f9a3a4f7 1624 system("git init")
b86f7378 1625 self.gitdir = os.getcwd() + "/.git"
6326aa58 1626 if not P4Sync.run(self, depotPaths):
f9a3a4f7 1627 return False
f9a3a4f7 1628 if self.branch != "master":
8f9b2e08
SH
1629 if gitBranchExists("refs/remotes/p4/master"):
1630 system("git branch master refs/remotes/p4/master")
1631 system("git checkout -f")
1632 else:
1633 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 1634
f9a3a4f7
SH
1635 return True
1636
09d89de2
SH
1637class P4Branches(Command):
1638 def __init__(self):
1639 Command.__init__(self)
1640 self.options = [ ]
1641 self.description = ("Shows the git branches that hold imports and their "
1642 + "corresponding perforce depot paths")
1643 self.verbose = False
1644
1645 def run(self, args):
5ca44617
SH
1646 if originP4BranchesExist():
1647 createOrUpdateBranchesFromOrigin()
1648
09d89de2
SH
1649 cmdline = "git rev-parse --symbolic "
1650 cmdline += " --remotes"
1651
1652 for line in read_pipe_lines(cmdline):
1653 line = line.strip()
1654
1655 if not line.startswith('p4/') or line == "p4/HEAD":
1656 continue
1657 branch = line
1658
1659 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1660 settings = extractSettingsGitLog(log)
1661
1662 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1663 return True
1664
b984733c
SH
1665class HelpFormatter(optparse.IndentedHelpFormatter):
1666 def __init__(self):
1667 optparse.IndentedHelpFormatter.__init__(self)
1668
1669 def format_description(self, description):
1670 if description:
1671 return description + "\n"
1672 else:
1673 return ""
4f5cf76a 1674
86949eef
SH
1675def printUsage(commands):
1676 print "usage: %s <command> [options]" % sys.argv[0]
1677 print ""
1678 print "valid commands: %s" % ", ".join(commands)
1679 print ""
1680 print "Try %s <command> --help for command specific help." % sys.argv[0]
1681 print ""
1682
1683commands = {
b86f7378
HWN
1684 "debug" : P4Debug,
1685 "submit" : P4Submit,
a9834f58 1686 "commit" : P4Submit,
b86f7378
HWN
1687 "sync" : P4Sync,
1688 "rebase" : P4Rebase,
1689 "clone" : P4Clone,
09d89de2
SH
1690 "rollback" : P4RollBack,
1691 "branches" : P4Branches
86949eef
SH
1692}
1693
86949eef 1694
bb6e09b2
HWN
1695def main():
1696 if len(sys.argv[1:]) == 0:
1697 printUsage(commands.keys())
1698 sys.exit(2)
4f5cf76a 1699
bb6e09b2
HWN
1700 cmd = ""
1701 cmdName = sys.argv[1]
1702 try:
b86f7378
HWN
1703 klass = commands[cmdName]
1704 cmd = klass()
bb6e09b2
HWN
1705 except KeyError:
1706 print "unknown command %s" % cmdName
1707 print ""
1708 printUsage(commands.keys())
1709 sys.exit(2)
1710
1711 options = cmd.options
b86f7378 1712 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
1713
1714 args = sys.argv[2:]
1715
1716 if len(options) > 0:
1717 options.append(optparse.make_option("--git-dir", dest="gitdir"))
1718
1719 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1720 options,
1721 description = cmd.description,
1722 formatter = HelpFormatter())
1723
1724 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1725 global verbose
1726 verbose = cmd.verbose
1727 if cmd.needsGit:
b86f7378
HWN
1728 if cmd.gitdir == None:
1729 cmd.gitdir = os.path.abspath(".git")
1730 if not isValidGitDir(cmd.gitdir):
1731 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1732 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
1733 cdup = read_pipe("git rev-parse --show-cdup").strip()
1734 if len(cdup) > 0:
1735 os.chdir(cdup);
e20a9e53 1736
b86f7378
HWN
1737 if not isValidGitDir(cmd.gitdir):
1738 if isValidGitDir(cmd.gitdir + "/.git"):
1739 cmd.gitdir += "/.git"
bb6e09b2 1740 else:
b86f7378 1741 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 1742
b86f7378 1743 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 1744
bb6e09b2
HWN
1745 if not cmd.run(args):
1746 parser.print_help()
4f5cf76a 1747
4f5cf76a 1748
bb6e09b2
HWN
1749if __name__ == '__main__':
1750 main()