git-p4: add option to preserve user names
[git/git.git] / contrib / fast-import / git-p4
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
1d7367dc
RG
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
ce6f33c8 13import re
8b41a97f 14
4addad22 15verbose = False
86949eef 16
21a50753
AK
17
18def p4_build_cmd(cmd):
19 """Build a suitable p4 command line.
20
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
24 """
abcaf073
AK
25 real_cmd = "%s " % "p4"
26
27 user = gitConfig("git-p4.user")
28 if len(user) > 0:
29 real_cmd += "-u %s " % user
30
31 password = gitConfig("git-p4.password")
32 if len(password) > 0:
33 real_cmd += "-P %s " % password
34
35 port = gitConfig("git-p4.port")
36 if len(port) > 0:
37 real_cmd += "-p %s " % port
38
39 host = gitConfig("git-p4.host")
40 if len(host) > 0:
41 real_cmd += "-h %s " % host
42
43 client = gitConfig("git-p4.client")
44 if len(client) > 0:
45 real_cmd += "-c %s " % client
46
47 real_cmd += "%s" % (cmd)
ee06427a
AK
48 if verbose:
49 print real_cmd
21a50753
AK
50 return real_cmd
51
053fd0c1
RB
52def chdir(dir):
53 if os.name == 'nt':
54 os.environ['PWD']=dir
55 os.chdir(dir)
56
86dff6b6
HWN
57def die(msg):
58 if verbose:
59 raise Exception(msg)
60 else:
61 sys.stderr.write(msg + "\n")
62 sys.exit(1)
63
bce4c5fc 64def write_pipe(c, str):
4addad22 65 if verbose:
86dff6b6 66 sys.stderr.write('Writing pipe: %s\n' % c)
b016d397 67
bce4c5fc 68 pipe = os.popen(c, 'w')
b016d397 69 val = pipe.write(str)
bce4c5fc 70 if pipe.close():
86dff6b6 71 die('Command failed: %s' % c)
b016d397
HWN
72
73 return val
74
d9429194
AK
75def p4_write_pipe(c, str):
76 real_cmd = p4_build_cmd(c)
893d340f 77 return write_pipe(real_cmd, str)
d9429194 78
4addad22
HWN
79def read_pipe(c, ignore_error=False):
80 if verbose:
86dff6b6 81 sys.stderr.write('Reading pipe: %s\n' % c)
8b41a97f 82
bce4c5fc 83 pipe = os.popen(c, 'rb')
b016d397 84 val = pipe.read()
4addad22 85 if pipe.close() and not ignore_error:
86dff6b6 86 die('Command failed: %s' % c)
b016d397
HWN
87
88 return val
89
d9429194
AK
90def p4_read_pipe(c, ignore_error=False):
91 real_cmd = p4_build_cmd(c)
92 return read_pipe(real_cmd, ignore_error)
b016d397 93
bce4c5fc 94def read_pipe_lines(c):
4addad22 95 if verbose:
86dff6b6 96 sys.stderr.write('Reading pipe: %s\n' % c)
b016d397 97 ## todo: check return status
bce4c5fc 98 pipe = os.popen(c, 'rb')
b016d397 99 val = pipe.readlines()
bce4c5fc 100 if pipe.close():
86dff6b6 101 die('Command failed: %s' % c)
b016d397
HWN
102
103 return val
caace111 104
2318121b
AK
105def p4_read_pipe_lines(c):
106 """Specifically invoke p4 on the command supplied. """
155af834 107 real_cmd = p4_build_cmd(c)
2318121b
AK
108 return read_pipe_lines(real_cmd)
109
6754a299 110def system(cmd):
4addad22 111 if verbose:
bb6e09b2 112 sys.stderr.write("executing %s\n" % cmd)
6754a299
HWN
113 if os.system(cmd) != 0:
114 die("command failed: %s" % cmd)
115
bf9320f1
AK
116def p4_system(cmd):
117 """Specifically invoke p4 as the system command. """
155af834 118 real_cmd = p4_build_cmd(cmd)
bf9320f1
AK
119 return system(real_cmd)
120
b9fc6ea9
DB
121def isP4Exec(kind):
122 """Determine if a Perforce 'kind' should have execute permission
123
124 'p4 help filetypes' gives a list of the types. If it starts with 'x',
125 or x follows one of a few letters. Otherwise, if there is an 'x' after
126 a plus sign, it is also executable"""
127 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
128
c65b670e
CP
129def setP4ExecBit(file, mode):
130 # Reopens an already open file and changes the execute bit to match
131 # the execute bit setting in the passed in mode.
132
133 p4Type = "+x"
134
135 if not isModeExec(mode):
136 p4Type = getP4OpenedType(file)
137 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
138 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
139 if p4Type[-1] == "+":
140 p4Type = p4Type[0:-1]
141
87b611d5 142 p4_system("reopen -t %s %s" % (p4Type, file))
c65b670e
CP
143
144def getP4OpenedType(file):
145 # Returns the perforce file type for the given file.
146
a7d3ef9d 147 result = p4_read_pipe("opened %s" % file)
f3e5ae4f 148 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
149 if match:
150 return match.group(1)
151 else:
f3e5ae4f 152 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 153
b43b0a3c
CP
154def diffTreePattern():
155 # This is a simple generator for the diff tree regex pattern. This could be
156 # a class variable if this and parseDiffTreeEntry were a part of a class.
157 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
158 while True:
159 yield pattern
160
161def parseDiffTreeEntry(entry):
162 """Parses a single diff tree entry into its component elements.
163
164 See git-diff-tree(1) manpage for details about the format of the diff
165 output. This method returns a dictionary with the following elements:
166
167 src_mode - The mode of the source file
168 dst_mode - The mode of the destination file
169 src_sha1 - The sha1 for the source file
170 dst_sha1 - The sha1 fr the destination file
171 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
172 status_score - The score for the status (applicable for 'C' and 'R'
173 statuses). This is None if there is no score.
174 src - The path for the source file.
175 dst - The path for the destination file. This is only present for
176 copy or renames. If it is not present, this is None.
177
178 If the pattern is not matched, None is returned."""
179
180 match = diffTreePattern().next().match(entry)
181 if match:
182 return {
183 'src_mode': match.group(1),
184 'dst_mode': match.group(2),
185 'src_sha1': match.group(3),
186 'dst_sha1': match.group(4),
187 'status': match.group(5),
188 'status_score': match.group(6),
189 'src': match.group(7),
190 'dst': match.group(10)
191 }
192 return None
193
c65b670e
CP
194def isModeExec(mode):
195 # Returns True if the given git mode represents an executable file,
196 # otherwise False.
197 return mode[-3:] == "755"
198
199def isModeExecChanged(src_mode, dst_mode):
200 return isModeExec(src_mode) != isModeExec(dst_mode)
201
b932705b 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
155af834 203 cmd = p4_build_cmd("-G %s" % (cmd))
6a49f8e2
HWN
204 if verbose:
205 sys.stderr.write("Opening pipe: %s\n" % cmd)
9f90c733
SL
206
207 # Use a temporary file to avoid deadlocks without
208 # subprocess.communicate(), which would put another copy
209 # of stdout into memory.
210 stdin_file = None
211 if stdin is not None:
212 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
213 stdin_file.write(stdin)
214 stdin_file.flush()
215 stdin_file.seek(0)
216
217 p4 = subprocess.Popen(cmd, shell=True,
218 stdin=stdin_file,
219 stdout=subprocess.PIPE)
86949eef
SH
220
221 result = []
222 try:
223 while True:
9f90c733 224 entry = marshal.load(p4.stdout)
c3f6163b
AG
225 if cb is not None:
226 cb(entry)
227 else:
228 result.append(entry)
86949eef
SH
229 except EOFError:
230 pass
9f90c733
SL
231 exitCode = p4.wait()
232 if exitCode != 0:
ac3e0d79
SH
233 entry = {}
234 entry["p4ExitCode"] = exitCode
235 result.append(entry)
86949eef
SH
236
237 return result
238
239def p4Cmd(cmd):
240 list = p4CmdList(cmd)
241 result = {}
242 for entry in list:
243 result.update(entry)
244 return result;
245
cb2c9db5
SH
246def p4Where(depotPath):
247 if not depotPath.endswith("/"):
248 depotPath += "/"
7f705dc3
TAL
249 depotPath = depotPath + "..."
250 outputList = p4CmdList("where %s" % depotPath)
251 output = None
252 for entry in outputList:
75bc9573
TAL
253 if "depotFile" in entry:
254 if entry["depotFile"] == depotPath:
255 output = entry
256 break
257 elif "data" in entry:
258 data = entry.get("data")
259 space = data.find(" ")
260 if data[:space] == depotPath:
261 output = entry
262 break
7f705dc3
TAL
263 if output == None:
264 return ""
dc524036
SH
265 if output["code"] == "error":
266 return ""
cb2c9db5
SH
267 clientPath = ""
268 if "path" in output:
269 clientPath = output.get("path")
270 elif "data" in output:
271 data = output.get("data")
272 lastSpace = data.rfind(" ")
273 clientPath = data[lastSpace + 1:]
274
275 if clientPath.endswith("..."):
276 clientPath = clientPath[:-3]
277 return clientPath
278
86949eef 279def currentGitBranch():
b25b2065 280 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 281
4f5cf76a 282def isValidGitDir(path):
bb6e09b2
HWN
283 if (os.path.exists(path + "/HEAD")
284 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
285 return True;
286 return False
287
463e8af6 288def parseRevision(ref):
b25b2065 289 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 290
6ae8de88
SH
291def extractLogMessageFromGitCommit(commit):
292 logMessage = ""
b016d397
HWN
293
294 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 295 foundTitle = False
b016d397 296 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
297 if not foundTitle:
298 if len(log) == 1:
1c094184 299 foundTitle = True
6ae8de88
SH
300 continue
301
302 logMessage += log
303 return logMessage
304
bb6e09b2 305def extractSettingsGitLog(log):
6ae8de88
SH
306 values = {}
307 for line in log.split("\n"):
308 line = line.strip()
6326aa58
HWN
309 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
310 if not m:
311 continue
312
313 assignments = m.group(1).split (':')
314 for a in assignments:
315 vals = a.split ('=')
316 key = vals[0].strip()
317 val = ('='.join (vals[1:])).strip()
318 if val.endswith ('\"') and val.startswith('"'):
319 val = val[1:-1]
320
321 values[key] = val
322
845b42cb
SH
323 paths = values.get("depot-paths")
324 if not paths:
325 paths = values.get("depot-path")
a3fdd579
SH
326 if paths:
327 values['depot-paths'] = paths.split(',')
bb6e09b2 328 return values
6ae8de88 329
8136a639 330def gitBranchExists(branch):
bb6e09b2
HWN
331 proc = subprocess.Popen(["git", "rev-parse", branch],
332 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 333 return proc.wait() == 0;
8136a639 334
36bd8446 335_gitConfig = {}
99f790f2 336def gitConfig(key, args = None): # set args to "--bool", for instance
36bd8446 337 if not _gitConfig.has_key(key):
99f790f2
TAL
338 argsFilter = ""
339 if args != None:
340 argsFilter = "%s " % args
341 cmd = "git config %s%s" % (argsFilter, key)
342 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
36bd8446 343 return _gitConfig[key]
01265103 344
062410bb
SH
345def p4BranchesInGit(branchesAreInRemotes = True):
346 branches = {}
347
348 cmdline = "git rev-parse --symbolic "
349 if branchesAreInRemotes:
350 cmdline += " --remotes"
351 else:
352 cmdline += " --branches"
353
354 for line in read_pipe_lines(cmdline):
355 line = line.strip()
356
357 ## only import to p4/
358 if not line.startswith('p4/') or line == "p4/HEAD":
359 continue
360 branch = line
361
362 # strip off p4
363 branch = re.sub ("^p4/", "", line)
364
365 branches[branch] = parseRevision(line)
366 return branches
367
9ceab363 368def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
369 branches = p4BranchesInGit()
370 # map from depot-path to branch name
371 branchByDepotPath = {}
372 for branch in branches.keys():
373 tip = branches[branch]
374 log = extractLogMessageFromGitCommit(tip)
375 settings = extractSettingsGitLog(log)
376 if settings.has_key("depot-paths"):
377 paths = ",".join(settings["depot-paths"])
378 branchByDepotPath[paths] = "remotes/p4/" + branch
379
27d2d811 380 settings = None
27d2d811
SH
381 parent = 0
382 while parent < 65535:
9ceab363 383 commit = head + "~%s" % parent
27d2d811
SH
384 log = extractLogMessageFromGitCommit(commit)
385 settings = extractSettingsGitLog(log)
86506fe5
SH
386 if settings.has_key("depot-paths"):
387 paths = ",".join(settings["depot-paths"])
388 if branchByDepotPath.has_key(paths):
389 return [branchByDepotPath[paths], settings]
27d2d811 390
86506fe5 391 parent = parent + 1
27d2d811 392
86506fe5 393 return ["", settings]
27d2d811 394
5ca44617
SH
395def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
396 if not silent:
397 print ("Creating/updating branch(es) in %s based on origin branch(es)"
398 % localRefPrefix)
399
400 originPrefix = "origin/p4/"
401
402 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
403 line = line.strip()
404 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
405 continue
406
407 headName = line[len(originPrefix):]
408 remoteHead = localRefPrefix + headName
409 originHead = line
410
411 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
412 if (not original.has_key('depot-paths')
413 or not original.has_key('change')):
414 continue
415
416 update = False
417 if not gitBranchExists(remoteHead):
418 if verbose:
419 print "creating %s" % remoteHead
420 update = True
421 else:
422 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
423 if settings.has_key('change') > 0:
424 if settings['depot-paths'] == original['depot-paths']:
425 originP4Change = int(original['change'])
426 p4Change = int(settings['change'])
427 if originP4Change > p4Change:
428 print ("%s (%s) is newer than %s (%s). "
429 "Updating p4 branch from origin."
430 % (originHead, originP4Change,
431 remoteHead, p4Change))
432 update = True
433 else:
434 print ("Ignoring: %s was imported from %s while "
435 "%s was imported from %s"
436 % (originHead, ','.join(original['depot-paths']),
437 remoteHead, ','.join(settings['depot-paths'])))
438
439 if update:
440 system("git update-ref %s %s" % (remoteHead, originHead))
441
442def originP4BranchesExist():
443 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
444
4f6432d8
SH
445def p4ChangesForPaths(depotPaths, changeRange):
446 assert depotPaths
b340fa43 447 output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
4f6432d8
SH
448 for p in depotPaths]))
449
b4b0ba06 450 changes = {}
4f6432d8 451 for line in output:
c3f6163b
AG
452 changeNum = int(line.split(" ")[1])
453 changes[changeNum] = True
4f6432d8 454
b4b0ba06
PW
455 changelist = changes.keys()
456 changelist.sort()
457 return changelist
4f6432d8 458
d53de8b9
TAL
459def p4PathStartsWith(path, prefix):
460 # This method tries to remedy a potential mixed-case issue:
461 #
462 # If UserA adds //depot/DirA/file1
463 # and UserB adds //depot/dira/file2
464 #
465 # we may or may not have a problem. If you have core.ignorecase=true,
466 # we treat DirA and dira as the same directory
467 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
468 if ignorecase:
469 return path.lower().startswith(prefix.lower())
470 return path.startswith(prefix)
471
b984733c
SH
472class Command:
473 def __init__(self):
474 self.usage = "usage: %prog [options]"
8910ac0e 475 self.needsGit = True
b984733c 476
3ea2cfd4
LD
477class P4UserMap:
478 def __init__(self):
479 self.userMapFromPerforceServer = False
480
481 def getUserCacheFilename(self):
482 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
483 return home + "/.gitp4-usercache.txt"
484
485 def getUserMapFromPerforceServer(self):
486 if self.userMapFromPerforceServer:
487 return
488 self.users = {}
489 self.emails = {}
490
491 for output in p4CmdList("users"):
492 if not output.has_key("User"):
493 continue
494 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
495 self.emails[output["Email"]] = output["User"]
496
497
498 s = ''
499 for (key, val) in self.users.items():
500 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
501
502 open(self.getUserCacheFilename(), "wb").write(s)
503 self.userMapFromPerforceServer = True
504
505 def loadUserMapFromCache(self):
506 self.users = {}
507 self.userMapFromPerforceServer = False
508 try:
509 cache = open(self.getUserCacheFilename(), "rb")
510 lines = cache.readlines()
511 cache.close()
512 for line in lines:
513 entry = line.strip().split("\t")
514 self.users[entry[0]] = entry[1]
515 except IOError:
516 self.getUserMapFromPerforceServer()
517
b984733c 518class P4Debug(Command):
86949eef 519 def __init__(self):
6ae8de88 520 Command.__init__(self)
86949eef 521 self.options = [
b1ce9447
HWN
522 optparse.make_option("--verbose", dest="verbose", action="store_true",
523 default=False),
4addad22 524 ]
c8c39116 525 self.description = "A tool to debug the output of p4 -G."
8910ac0e 526 self.needsGit = False
b1ce9447 527 self.verbose = False
86949eef
SH
528
529 def run(self, args):
b1ce9447 530 j = 0
86949eef 531 for output in p4CmdList(" ".join(args)):
b1ce9447
HWN
532 print 'Element: %d' % j
533 j += 1
86949eef 534 print output
b984733c 535 return True
86949eef 536
5834684d
SH
537class P4RollBack(Command):
538 def __init__(self):
539 Command.__init__(self)
540 self.options = [
0c66a783
SH
541 optparse.make_option("--verbose", dest="verbose", action="store_true"),
542 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
543 ]
544 self.description = "A tool to debug the multi-branch import. Don't use :)"
52102d47 545 self.verbose = False
0c66a783 546 self.rollbackLocalBranches = False
5834684d
SH
547
548 def run(self, args):
549 if len(args) != 1:
550 return False
551 maxChange = int(args[0])
0c66a783 552
ad192f28 553 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
554 die("Problems executing p4");
555
0c66a783
SH
556 if self.rollbackLocalBranches:
557 refPrefix = "refs/heads/"
b016d397 558 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
559 else:
560 refPrefix = "refs/remotes/"
b016d397 561 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
562
563 for line in lines:
564 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
565 line = line.strip()
566 ref = refPrefix + line
5834684d 567 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
568 settings = extractSettingsGitLog(log)
569
570 depotPaths = settings['depot-paths']
571 change = settings['change']
572
5834684d 573 changed = False
52102d47 574
6326aa58
HWN
575 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
576 for p in depotPaths]))) == 0:
52102d47
SH
577 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
578 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
579 continue
580
bb6e09b2 581 while change and int(change) > maxChange:
5834684d 582 changed = True
52102d47
SH
583 if self.verbose:
584 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
585 system("git update-ref %s \"%s^\"" % (ref, ref))
586 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
587 settings = extractSettingsGitLog(log)
588
589
590 depotPaths = settings['depot-paths']
591 change = settings['change']
5834684d
SH
592
593 if changed:
52102d47 594 print "%s rewound to %s" % (ref, change)
5834684d
SH
595
596 return True
597
3ea2cfd4 598class P4Submit(Command, P4UserMap):
4f5cf76a 599 def __init__(self):
b984733c 600 Command.__init__(self)
3ea2cfd4 601 P4UserMap.__init__(self)
4f5cf76a 602 self.options = [
4addad22 603 optparse.make_option("--verbose", dest="verbose", action="store_true"),
4f5cf76a 604 optparse.make_option("--origin", dest="origin"),
ae901090 605 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
606 # preserve the user, requires relevant p4 permissions
607 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
4f5cf76a
SH
608 ]
609 self.description = "Submit changes from git to the perforce depot."
c9b50e63 610 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 611 self.interactive = True
9512497b 612 self.origin = ""
ae901090 613 self.detectRenames = False
b0d10df7 614 self.verbose = False
3ea2cfd4 615 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
f7baba8b 616 self.isWindows = (platform.system() == "Windows")
4f5cf76a 617
4f5cf76a
SH
618 def check(self):
619 if len(p4CmdList("opened ...")) > 0:
620 die("You have files opened with perforce! Close them before starting the sync.")
621
edae1e2f
SH
622 # replaces everything between 'Description:' and the next P4 submit template field with the
623 # commit message
4f5cf76a
SH
624 def prepareLogMessage(self, template, message):
625 result = ""
626
edae1e2f
SH
627 inDescriptionSection = False
628
4f5cf76a
SH
629 for line in template.split("\n"):
630 if line.startswith("#"):
631 result += line + "\n"
632 continue
633
edae1e2f 634 if inDescriptionSection:
c9dbab04 635 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f
SH
636 inDescriptionSection = False
637 else:
638 continue
639 else:
640 if line.startswith("Description:"):
641 inDescriptionSection = True
642 line += "\n"
643 for messageLine in message.split("\n"):
644 line += "\t" + messageLine + "\n"
645
646 result += line + "\n"
4f5cf76a
SH
647
648 return result
649
3ea2cfd4
LD
650 def p4UserForCommit(self,id):
651 # Return the tuple (perforce user,git email) for a given git commit id
652 self.getUserMapFromPerforceServer()
653 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
654 gitEmail = gitEmail.strip()
655 if not self.emails.has_key(gitEmail):
656 return (None,gitEmail)
657 else:
658 return (self.emails[gitEmail],gitEmail)
659
660 def checkValidP4Users(self,commits):
661 # check if any git authors cannot be mapped to p4 users
662 for id in commits:
663 (user,email) = self.p4UserForCommit(id)
664 if not user:
665 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
666 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
667 print "%s" % msg
668 else:
669 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
670
671 def lastP4Changelist(self):
672 # Get back the last changelist number submitted in this client spec. This
673 # then gets used to patch up the username in the change. If the same
674 # client spec is being used by multiple processes then this might go
675 # wrong.
676 results = p4CmdList("client -o") # find the current client
677 client = None
678 for r in results:
679 if r.has_key('Client'):
680 client = r['Client']
681 break
682 if not client:
683 die("could not get client spec")
684 results = p4CmdList("changes -c %s -m 1" % client)
685 for r in results:
686 if r.has_key('change'):
687 return r['change']
688 die("Could not get changelist number for last submit - cannot patch up user details")
689
690 def modifyChangelistUser(self, changelist, newUser):
691 # fixup the user field of a changelist after it has been submitted.
692 changes = p4CmdList("change -o %s" % changelist)
693 for c in changes:
694 if c.has_key('User'):
695 c['User'] = newUser
696 input = marshal.dumps(changes[0])
697 result = p4CmdList("change -f -i", stdin=input)
698 for r in result:
699 if r.has_key('code'):
700 if r['code'] == 'error':
701 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
702 if r.has_key('data'):
703 print("Updated user field for changelist %s to %s" % (changelist, newUser))
704 return
705 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
706
707 def canChangeChangelists(self):
708 # check to see if we have p4 admin or super-user permissions, either of
709 # which are required to modify changelists.
710 results = p4CmdList("-G protects %s" % self.depotPath)
711 for r in results:
712 if r.has_key('perm'):
713 if r['perm'] == 'admin':
714 return 1
715 if r['perm'] == 'super':
716 return 1
717 return 0
718
ea99c3ae
SH
719 def prepareSubmitTemplate(self):
720 # remove lines in the Files section that show changes to files outside the depot path we're committing into
721 template = ""
722 inFilesSection = False
b340fa43 723 for line in p4_read_pipe_lines("change -o"):
f3e5ae4f
MSO
724 if line.endswith("\r\n"):
725 line = line[:-2] + "\n"
ea99c3ae
SH
726 if inFilesSection:
727 if line.startswith("\t"):
728 # path starts and ends with a tab
729 path = line[1:]
730 lastTab = path.rfind("\t")
731 if lastTab != -1:
732 path = path[:lastTab]
d53de8b9 733 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
734 continue
735 else:
736 inFilesSection = False
737 else:
738 if line.startswith("Files:"):
739 inFilesSection = True
740
741 template += line
742
743 return template
744
7cb5cbef 745 def applyCommit(self, id):
0e36f2d7 746 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
ae901090 747
3ea2cfd4
LD
748 if self.preserveUser:
749 (p4User, gitEmail) = self.p4UserForCommit(id)
750
ae901090
VA
751 if not self.detectRenames:
752 # If not explicitly set check the config variable
753 self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true"
754
755 if self.detectRenames:
756 diffOpts = "-M"
757 else:
758 diffOpts = ""
759
4fddb41b
VA
760 if gitConfig("git-p4.detectCopies").lower() == "true":
761 diffOpts += " -C"
762
763 if gitConfig("git-p4.detectCopiesHarder").lower() == "true":
764 diffOpts += " --find-copies-harder"
765
0e36f2d7 766 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
767 filesToAdd = set()
768 filesToDelete = set()
d336c158 769 editedFiles = set()
c65b670e 770 filesToChangeExecBit = {}
4f5cf76a 771 for line in diff:
b43b0a3c
CP
772 diff = parseDiffTreeEntry(line)
773 modifier = diff['status']
774 path = diff['src']
4f5cf76a 775 if modifier == "M":
87b611d5 776 p4_system("edit \"%s\"" % path)
c65b670e
CP
777 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
778 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 779 editedFiles.add(path)
4f5cf76a
SH
780 elif modifier == "A":
781 filesToAdd.add(path)
c65b670e 782 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
783 if path in filesToDelete:
784 filesToDelete.remove(path)
785 elif modifier == "D":
786 filesToDelete.add(path)
787 if path in filesToAdd:
788 filesToAdd.remove(path)
4fddb41b
VA
789 elif modifier == "C":
790 src, dest = diff['src'], diff['dst']
791 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
792 if diff['src_sha1'] != diff['dst_sha1']:
793 p4_system("edit \"%s\"" % (dest))
794 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
795 p4_system("edit \"%s\"" % (dest))
796 filesToChangeExecBit[dest] = diff['dst_mode']
797 os.unlink(dest)
798 editedFiles.add(dest)
d9a5f25b 799 elif modifier == "R":
b43b0a3c 800 src, dest = diff['src'], diff['dst']
87b611d5 801 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
ae901090
VA
802 if diff['src_sha1'] != diff['dst_sha1']:
803 p4_system("edit \"%s\"" % (dest))
c65b670e 804 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
ae901090 805 p4_system("edit \"%s\"" % (dest))
c65b670e 806 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
807 os.unlink(dest)
808 editedFiles.add(dest)
809 filesToDelete.add(src)
4f5cf76a
SH
810 else:
811 die("unknown modifier %s for %s" % (modifier, path))
812
0e36f2d7 813 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 814 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
815 tryPatchCmd = patchcmd + "--check -"
816 applyPatchCmd = patchcmd + "--check --apply -"
51a2640a 817
47a130b7 818 if os.system(tryPatchCmd) != 0:
51a2640a
SH
819 print "Unfortunately applying the change failed!"
820 print "What do you want to do?"
821 response = "x"
822 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
823 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
824 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
825 if response == "s":
826 print "Skipping! Good luck with the next patches..."
20947149 827 for f in editedFiles:
87b611d5 828 p4_system("revert \"%s\"" % f);
20947149
SH
829 for f in filesToAdd:
830 system("rm %s" %f)
51a2640a
SH
831 return
832 elif response == "a":
47a130b7 833 os.system(applyPatchCmd)
51a2640a
SH
834 if len(filesToAdd) > 0:
835 print "You may also want to call p4 add on the following files:"
836 print " ".join(filesToAdd)
837 if len(filesToDelete):
838 print "The following files should be scheduled for deletion with p4 delete:"
839 print " ".join(filesToDelete)
cebdf5af
HWN
840 die("Please resolve and submit the conflict manually and "
841 + "continue afterwards with git-p4 submit --continue")
51a2640a
SH
842 elif response == "w":
843 system(diffcmd + " > patch.txt")
844 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af
HWN
845 die("Please resolve and submit the conflict manually and "
846 "continue afterwards with git-p4 submit --continue")
51a2640a 847
47a130b7 848 system(applyPatchCmd)
4f5cf76a
SH
849
850 for f in filesToAdd:
87b611d5 851 p4_system("add \"%s\"" % f)
4f5cf76a 852 for f in filesToDelete:
87b611d5
AK
853 p4_system("revert \"%s\"" % f)
854 p4_system("delete \"%s\"" % f)
4f5cf76a 855
c65b670e
CP
856 # Set/clear executable bits
857 for f in filesToChangeExecBit.keys():
858 mode = filesToChangeExecBit[f]
859 setP4ExecBit(f, mode)
860
0e36f2d7 861 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 862 logMessage = logMessage.strip()
4f5cf76a 863
ea99c3ae 864 template = self.prepareSubmitTemplate()
4f5cf76a
SH
865
866 if self.interactive:
867 submitTemplate = self.prepareLogMessage(template, logMessage)
67abd417
SB
868 if os.environ.has_key("P4DIFF"):
869 del(os.environ["P4DIFF"])
8b130262
AW
870 diff = ""
871 for editedFile in editedFiles:
872 diff += p4_read_pipe("diff -du %r" % editedFile)
4f5cf76a 873
f3e5ae4f 874 newdiff = ""
4f5cf76a 875 for newFile in filesToAdd:
f3e5ae4f
MSO
876 newdiff += "==== new file ====\n"
877 newdiff += "--- /dev/null\n"
878 newdiff += "+++ %s\n" % newFile
4f5cf76a
SH
879 f = open(newFile, "r")
880 for line in f.readlines():
f3e5ae4f 881 newdiff += "+" + line
4f5cf76a
SH
882 f.close()
883
f3e5ae4f 884 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a 885
e96e400f
SH
886 [handle, fileName] = tempfile.mkstemp()
887 tmpFile = os.fdopen(handle, "w+")
f3e5ae4f
MSO
888 if self.isWindows:
889 submitTemplate = submitTemplate.replace("\n", "\r\n")
890 separatorLine = separatorLine.replace("\n", "\r\n")
891 newdiff = newdiff.replace("\n", "\r\n")
892 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
e96e400f 893 tmpFile.close()
cdc7e388 894 mtime = os.stat(fileName).st_mtime
82cea9ff
SB
895 if os.environ.has_key("P4EDITOR"):
896 editor = os.environ.get("P4EDITOR")
897 else:
8b187e6b 898 editor = read_pipe("git var GIT_EDITOR").strip()
e96e400f 899 system(editor + " " + fileName)
e96e400f 900
3ea2cfd4
LD
901 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
902 checkModTime = False
903 else:
904 checkModTime = True
905
cdc7e388 906 response = "y"
3ea2cfd4 907 if checkModTime and (os.stat(fileName).st_mtime <= mtime):
cdc7e388
SH
908 response = "x"
909 while response != "y" and response != "n":
910 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
911
912 if response == "y":
913 tmpFile = open(fileName, "rb")
914 message = tmpFile.read()
915 tmpFile.close()
916 submitTemplate = message[:message.index(separatorLine)]
917 if self.isWindows:
918 submitTemplate = submitTemplate.replace("\r\n", "\n")
919 p4_write_pipe("submit -i", submitTemplate)
3ea2cfd4
LD
920
921 if self.preserveUser:
922 if p4User:
923 # Get last changelist number. Cannot easily get it from
924 # the submit command output as the output is unmarshalled.
925 changelist = self.lastP4Changelist()
926 self.modifyChangelistUser(changelist, p4User)
927
cdc7e388
SH
928 else:
929 for f in editedFiles:
930 p4_system("revert \"%s\"" % f);
931 for f in filesToAdd:
932 p4_system("revert \"%s\"" % f);
933 system("rm %s" %f)
934
935 os.remove(fileName)
4f5cf76a
SH
936 else:
937 fileName = "submit.txt"
938 file = open(fileName, "w+")
939 file.write(self.prepareLogMessage(template, logMessage))
940 file.close()
cebdf5af
HWN
941 print ("Perforce submit template written as %s. "
942 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
943 % (fileName, fileName))
4f5cf76a
SH
944
945 def run(self, args):
c9b50e63
SH
946 if len(args) == 0:
947 self.master = currentGitBranch()
4280e533 948 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
949 die("Detecting current git branch failed!")
950 elif len(args) == 1:
951 self.master = args[0]
952 else:
953 return False
954
4c2d5d72
JX
955 allowSubmit = gitConfig("git-p4.allowSubmit")
956 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
957 die("%s is not in git-p4.allowSubmit" % self.master)
958
27d2d811 959 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 960 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
961 if len(self.origin) == 0:
962 self.origin = upstream
a3fdd579 963
3ea2cfd4
LD
964 if self.preserveUser:
965 if not self.canChangeChangelists():
966 die("Cannot preserve user names without p4 super-user or admin permissions")
967
a3fdd579
SH
968 if self.verbose:
969 print "Origin branch is " + self.origin
9512497b 970
ea99c3ae 971 if len(self.depotPath) == 0:
9512497b
SH
972 print "Internal error: cannot locate perforce depot path from existing branches"
973 sys.exit(128)
974
ea99c3ae 975 self.clientPath = p4Where(self.depotPath)
9512497b 976
51a2640a 977 if len(self.clientPath) == 0:
ea99c3ae 978 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
9512497b
SH
979 sys.exit(128)
980
ea99c3ae 981 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 982 self.oldWorkingDirectory = os.getcwd()
c1b296b9 983
053fd0c1 984 chdir(self.clientPath)
6a01298a 985 print "Synchronizing p4 checkout..."
87b611d5 986 p4_system("sync ...")
9512497b 987
4f5cf76a 988 self.check()
4f5cf76a 989
4c750c0d
SH
990 commits = []
991 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
992 commits.append(line.strip())
993 commits.reverse()
4f5cf76a 994
3ea2cfd4
LD
995 if self.preserveUser:
996 self.checkValidP4Users(commits)
997
4f5cf76a 998 while len(commits) > 0:
4f5cf76a
SH
999 commit = commits[0]
1000 commits = commits[1:]
7cb5cbef 1001 self.applyCommit(commit)
4f5cf76a
SH
1002 if not self.interactive:
1003 break
1004
4f5cf76a 1005 if len(commits) == 0:
4c750c0d 1006 print "All changes applied!"
053fd0c1 1007 chdir(self.oldWorkingDirectory)
14594f4b 1008
4c750c0d
SH
1009 sync = P4Sync()
1010 sync.run([])
14594f4b 1011
4c750c0d
SH
1012 rebase = P4Rebase()
1013 rebase.rebase()
4f5cf76a 1014
b984733c
SH
1015 return True
1016
3ea2cfd4 1017class P4Sync(Command, P4UserMap):
56c09345
PW
1018 delete_actions = ( "delete", "move/delete", "purge" )
1019
b984733c
SH
1020 def __init__(self):
1021 Command.__init__(self)
3ea2cfd4 1022 P4UserMap.__init__(self)
b984733c
SH
1023 self.options = [
1024 optparse.make_option("--branch", dest="branch"),
1025 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1026 optparse.make_option("--changesfile", dest="changesFile"),
1027 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1028 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
a028a98e 1029 optparse.make_option("--verbose", dest="verbose", action="store_true"),
d2c6dd30
HWN
1030 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1031 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 1032 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 1033 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1034 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1035 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1036 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
1037 ]
1038 self.description = """Imports from Perforce into a git repository.\n
1039 example:
1040 //depot/my/project/ -- to import the current head
1041 //depot/my/project/@all -- to import everything
1042 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1043
1044 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1045
1046 self.usage += " //depot/path[@revRange]"
b984733c 1047 self.silent = False
1d7367dc
RG
1048 self.createdBranches = set()
1049 self.committedChanges = set()
569d1bd4 1050 self.branch = ""
b984733c 1051 self.detectBranches = False
cb53e1f8 1052 self.detectLabels = False
b984733c 1053 self.changesFile = ""
01265103 1054 self.syncWithOrigin = True
4b97ffb1 1055 self.verbose = False
a028a98e 1056 self.importIntoRemotes = True
01a9c9c5 1057 self.maxChanges = ""
c1f9197f 1058 self.isWindows = (platform.system() == "Windows")
8b41a97f 1059 self.keepRepoPath = False
6326aa58 1060 self.depotPaths = None
3c699645 1061 self.p4BranchesInGit = []
354081d5 1062 self.cloneExclude = []
3a70cdfa
TAL
1063 self.useClientSpec = False
1064 self.clientSpecDirs = []
b984733c 1065
01265103
SH
1066 if gitConfig("git-p4.syncFromOrigin") == "false":
1067 self.syncWithOrigin = False
1068
084f6306
PW
1069 #
1070 # P4 wildcards are not allowed in filenames. P4 complains
1071 # if you simply add them, but you can force it with "-f", in
1072 # which case it translates them into %xx encoding internally.
1073 # Search for and fix just these four characters. Do % last so
1074 # that fixing it does not inadvertently create new %-escapes.
1075 #
1076 def wildcard_decode(self, path):
1077 # Cannot have * in a filename in windows; untested as to
1078 # what p4 would do in such a case.
1079 if not self.isWindows:
1080 path = path.replace("%2A", "*")
1081 path = path.replace("%23", "#") \
1082 .replace("%40", "@") \
1083 .replace("%25", "%")
1084 return path
1085
b984733c 1086 def extractFilesFromCommit(self, commit):
354081d5
TT
1087 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1088 for path in self.cloneExclude]
b984733c
SH
1089 files = []
1090 fnum = 0
1091 while commit.has_key("depotFile%s" % fnum):
1092 path = commit["depotFile%s" % fnum]
6326aa58 1093
354081d5 1094 if [p for p in self.cloneExclude
d53de8b9 1095 if p4PathStartsWith(path, p)]:
354081d5
TT
1096 found = False
1097 else:
1098 found = [p for p in self.depotPaths
d53de8b9 1099 if p4PathStartsWith(path, p)]
6326aa58 1100 if not found:
b984733c
SH
1101 fnum = fnum + 1
1102 continue
1103
1104 file = {}
1105 file["path"] = path
1106 file["rev"] = commit["rev%s" % fnum]
1107 file["action"] = commit["action%s" % fnum]
1108 file["type"] = commit["type%s" % fnum]
1109 files.append(file)
1110 fnum = fnum + 1
1111 return files
1112
6326aa58 1113 def stripRepoPath(self, path, prefixes):
3952710b
IW
1114 if self.useClientSpec:
1115
1116 # if using the client spec, we use the output directory
1117 # specified in the client. For example, a view
1118 # //depot/foo/branch/... //client/branch/foo/...
1119 # will end up putting all foo/branch files into
1120 # branch/foo/
1121 for val in self.clientSpecDirs:
1122 if path.startswith(val[0]):
1123 # replace the depot path with the client path
1124 path = path.replace(val[0], val[1][1])
1125 # now strip out the client (//client/...)
1126 path = re.sub("^(//[^/]+/)", '', path)
1127 # the rest is all path
1128 return path
1129
8b41a97f 1130 if self.keepRepoPath:
6326aa58
HWN
1131 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1132
1133 for p in prefixes:
d53de8b9 1134 if p4PathStartsWith(path, p):
6326aa58 1135 path = path[len(p):]
8b41a97f 1136
6326aa58 1137 return path
6754a299 1138
71b112d4 1139 def splitFilesIntoBranches(self, commit):
d5904674 1140 branches = {}
71b112d4
SH
1141 fnum = 0
1142 while commit.has_key("depotFile%s" % fnum):
1143 path = commit["depotFile%s" % fnum]
6326aa58 1144 found = [p for p in self.depotPaths
d53de8b9 1145 if p4PathStartsWith(path, p)]
6326aa58 1146 if not found:
71b112d4
SH
1147 fnum = fnum + 1
1148 continue
1149
1150 file = {}
1151 file["path"] = path
1152 file["rev"] = commit["rev%s" % fnum]
1153 file["action"] = commit["action%s" % fnum]
1154 file["type"] = commit["type%s" % fnum]
1155 fnum = fnum + 1
1156
6326aa58 1157 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 1158
4b97ffb1 1159 for branch in self.knownBranches.keys():
6754a299
HWN
1160
1161 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1162 if relPath.startswith(branch + "/"):
d5904674
SH
1163 if branch not in branches:
1164 branches[branch] = []
71b112d4 1165 branches[branch].append(file)
6555b2cc 1166 break
b984733c
SH
1167
1168 return branches
1169
b932705b
LD
1170 # output one file from the P4 stream
1171 # - helper for streamP4Files
1172
1173 def streamOneP4File(self, file, contents):
c3f6163b
AG
1174 if file["type"] == "apple":
1175 print "\nfile %s is a strange apple file that forks. Ignoring" % \
1176 file['depotFile']
1177 return
b932705b
LD
1178
1179 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
084f6306 1180 relPath = self.wildcard_decode(relPath)
b932705b
LD
1181 if verbose:
1182 sys.stderr.write("%s\n" % relPath)
1183
1184 mode = "644"
1185 if isP4Exec(file["type"]):
1186 mode = "755"
1187 elif file["type"] == "symlink":
1188 mode = "120000"
1189 # p4 print on a symlink contains "target\n", so strip it off
b39c3612
EP
1190 data = ''.join(contents)
1191 contents = [data[:-1]]
b932705b
LD
1192
1193 if self.isWindows and file["type"].endswith("text"):
1194 mangled = []
1195 for data in contents:
1196 data = data.replace("\r\n", "\n")
1197 mangled.append(data)
1198 contents = mangled
1199
1200 if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1201 contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1202 elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1203 contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1204
1205 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1206
1207 # total length...
1208 length = 0
1209 for d in contents:
1210 length = length + len(d)
1211
1212 self.gitStream.write("data %d\n" % length)
1213 for d in contents:
1214 self.gitStream.write(d)
1215 self.gitStream.write("\n")
1216
1217 def streamOneP4Deletion(self, file):
1218 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1219 if verbose:
1220 sys.stderr.write("delete %s\n" % relPath)
1221 self.gitStream.write("D %s\n" % relPath)
1222
1223 # handle another chunk of streaming data
1224 def streamP4FilesCb(self, marshalled):
1225
c3f6163b
AG
1226 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1227 # start of a new file - output the old one first
1228 self.streamOneP4File(self.stream_file, self.stream_contents)
1229 self.stream_file = {}
1230 self.stream_contents = []
1231 self.stream_have_file_info = False
b932705b 1232
c3f6163b
AG
1233 # pick up the new file information... for the
1234 # 'data' field we need to append to our array
1235 for k in marshalled.keys():
1236 if k == 'data':
1237 self.stream_contents.append(marshalled['data'])
1238 else:
1239 self.stream_file[k] = marshalled[k]
b932705b 1240
c3f6163b 1241 self.stream_have_file_info = True
b932705b
LD
1242
1243 # Stream directly from "p4 files" into "git fast-import"
1244 def streamP4Files(self, files):
30b5940b
SH
1245 filesForCommit = []
1246 filesToRead = []
b932705b 1247 filesToDelete = []
30b5940b 1248
3a70cdfa 1249 for f in files:
30b5940b 1250 includeFile = True
3a70cdfa
TAL
1251 for val in self.clientSpecDirs:
1252 if f['path'].startswith(val[0]):
3952710b 1253 if val[1][0] <= 0:
30b5940b 1254 includeFile = False
3a70cdfa
TAL
1255 break
1256
30b5940b
SH
1257 if includeFile:
1258 filesForCommit.append(f)
56c09345 1259 if f['action'] in self.delete_actions:
b932705b 1260 filesToDelete.append(f)
56c09345
PW
1261 else:
1262 filesToRead.append(f)
6a49f8e2 1263
b932705b
LD
1264 # deleted files...
1265 for f in filesToDelete:
1266 self.streamOneP4Deletion(f)
1b9a4684 1267
b932705b
LD
1268 if len(filesToRead) > 0:
1269 self.stream_file = {}
1270 self.stream_contents = []
1271 self.stream_have_file_info = False
8ff45f2a 1272
c3f6163b
AG
1273 # curry self argument
1274 def streamP4FilesCbSelf(entry):
1275 self.streamP4FilesCb(entry)
6a49f8e2 1276
c3f6163b
AG
1277 p4CmdList("-x - print",
1278 '\n'.join(['%s#%s' % (f['path'], f['rev'])
b932705b 1279 for f in filesToRead]),
c3f6163b 1280 cb=streamP4FilesCbSelf)
30b5940b 1281
b932705b
LD
1282 # do the last chunk
1283 if self.stream_file.has_key('depotFile'):
1284 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 1285
6326aa58 1286 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
1287 epoch = details["time"]
1288 author = details["user"]
c3f6163b 1289 self.branchPrefixes = branchPrefixes
b984733c 1290
4b97ffb1
SH
1291 if self.verbose:
1292 print "commit into %s" % branch
1293
96e07dd2
HWN
1294 # start with reading files; if that fails, we should not
1295 # create a commit.
1296 new_files = []
1297 for f in files:
d53de8b9 1298 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
1299 new_files.append (f)
1300 else:
afa1dd9a 1301 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 1302
b984733c 1303 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 1304# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
1305 self.committedChanges.add(int(details["change"]))
1306 committer = ""
b607e71e
SH
1307 if author not in self.users:
1308 self.getUserMapFromPerforceServer()
b984733c 1309 if author in self.users:
0828ab14 1310 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
b984733c 1311 else:
0828ab14 1312 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
b984733c
SH
1313
1314 self.gitStream.write("committer %s\n" % committer)
1315
1316 self.gitStream.write("data <<EOT\n")
1317 self.gitStream.write(details["desc"])
6581de09
SH
1318 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1319 % (','.join (branchPrefixes), details["change"]))
1320 if len(details['options']) > 0:
1321 self.gitStream.write(": options = %s" % details['options'])
1322 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
1323
1324 if len(parent) > 0:
4b97ffb1
SH
1325 if self.verbose:
1326 print "parent %s" % parent
b984733c
SH
1327 self.gitStream.write("from %s\n" % parent)
1328
b932705b 1329 self.streamP4Files(new_files)
b984733c
SH
1330 self.gitStream.write("\n")
1331
1f4ba1cb
SH
1332 change = int(details["change"])
1333
9bda3a85 1334 if self.labels.has_key(change):
1f4ba1cb
SH
1335 label = self.labels[change]
1336 labelDetails = label[0]
1337 labelRevisions = label[1]
71b112d4
SH
1338 if self.verbose:
1339 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 1340
6326aa58
HWN
1341 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1342 for p in branchPrefixes]))
1f4ba1cb
SH
1343
1344 if len(files) == len(labelRevisions):
1345
1346 cleanedFiles = {}
1347 for info in files:
56c09345 1348 if info["action"] in self.delete_actions:
1f4ba1cb
SH
1349 continue
1350 cleanedFiles[info["depotFile"]] = info["rev"]
1351
1352 if cleanedFiles == labelRevisions:
1353 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1354 self.gitStream.write("from %s\n" % branch)
1355
1356 owner = labelDetails["Owner"]
1357 tagger = ""
1358 if author in self.users:
1359 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1360 else:
1361 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1362 self.gitStream.write("tagger %s\n" % tagger)
1363 self.gitStream.write("data <<EOT\n")
1364 self.gitStream.write(labelDetails["Description"])
1365 self.gitStream.write("EOT\n\n")
1366
1367 else:
a46668fa 1368 if not self.silent:
cebdf5af
HWN
1369 print ("Tag %s does not match with change %s: files do not match."
1370 % (labelDetails["label"], change))
1f4ba1cb
SH
1371
1372 else:
a46668fa 1373 if not self.silent:
cebdf5af
HWN
1374 print ("Tag %s does not match with change %s: file count is different."
1375 % (labelDetails["label"], change))
b984733c 1376
1f4ba1cb
SH
1377 def getLabels(self):
1378 self.labels = {}
1379
6326aa58 1380 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
10c3211b 1381 if len(l) > 0 and not self.silent:
183f8436 1382 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
1383
1384 for output in l:
1f4ba1cb
SH
1385 label = output["label"]
1386 revisions = {}
1387 newestChange = 0
71b112d4
SH
1388 if self.verbose:
1389 print "Querying files for label %s" % label
6326aa58
HWN
1390 for file in p4CmdList("files "
1391 + ' '.join (["%s...@%s" % (p, label)
1392 for p in self.depotPaths])):
1f4ba1cb
SH
1393 revisions[file["depotFile"]] = file["rev"]
1394 change = int(file["change"])
1395 if change > newestChange:
1396 newestChange = change
1397
9bda3a85
SH
1398 self.labels[newestChange] = [output, revisions]
1399
1400 if self.verbose:
1401 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 1402
86dff6b6
HWN
1403 def guessProjectName(self):
1404 for p in self.depotPaths:
6e5295c4
SH
1405 if p.endswith("/"):
1406 p = p[:-1]
1407 p = p[p.strip().rfind("/") + 1:]
1408 if not p.endswith("/"):
1409 p += "/"
1410 return p
86dff6b6 1411
4b97ffb1 1412 def getBranchMapping(self):
6555b2cc
SH
1413 lostAndFoundBranches = set()
1414
4b97ffb1
SH
1415 for info in p4CmdList("branches"):
1416 details = p4Cmd("branch -o %s" % info["branch"])
1417 viewIdx = 0
1418 while details.has_key("View%s" % viewIdx):
1419 paths = details["View%s" % viewIdx].split(" ")
1420 viewIdx = viewIdx + 1
1421 # require standard //depot/foo/... //depot/bar/... mapping
1422 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1423 continue
1424 source = paths[0]
1425 destination = paths[1]
6509e19c 1426 ## HACK
d53de8b9 1427 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
1428 source = source[len(self.depotPaths[0]):-4]
1429 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 1430
1a2edf4e
SH
1431 if destination in self.knownBranches:
1432 if not self.silent:
1433 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1434 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1435 continue
1436
6555b2cc
SH
1437 self.knownBranches[destination] = source
1438
1439 lostAndFoundBranches.discard(destination)
1440
29bdbac1 1441 if source not in self.knownBranches:
6555b2cc
SH
1442 lostAndFoundBranches.add(source)
1443
1444
1445 for branch in lostAndFoundBranches:
1446 self.knownBranches[branch] = branch
29bdbac1 1447
38f9f5ec
SH
1448 def getBranchMappingFromGitBranches(self):
1449 branches = p4BranchesInGit(self.importIntoRemotes)
1450 for branch in branches.keys():
1451 if branch == "master":
1452 branch = "main"
1453 else:
1454 branch = branch[len(self.projectName):]
1455 self.knownBranches[branch] = branch
1456
29bdbac1 1457 def listExistingP4GitBranches(self):
144ff46b
SH
1458 # branches holds mapping from name to commit
1459 branches = p4BranchesInGit(self.importIntoRemotes)
1460 self.p4BranchesInGit = branches.keys()
1461 for branch in branches.keys():
1462 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 1463
bb6e09b2
HWN
1464 def updateOptionDict(self, d):
1465 option_keys = {}
1466 if self.keepRepoPath:
1467 option_keys['keepRepoPath'] = 1
1468
1469 d["options"] = ' '.join(sorted(option_keys.keys()))
1470
1471 def readOptions(self, d):
1472 self.keepRepoPath = (d.has_key('options')
1473 and ('keepRepoPath' in d['options']))
6326aa58 1474
8134f69c
SH
1475 def gitRefForBranch(self, branch):
1476 if branch == "main":
1477 return self.refPrefix + "master"
1478
1479 if len(branch) <= 0:
1480 return branch
1481
1482 return self.refPrefix + self.projectName + branch
1483
1ca3d710
SH
1484 def gitCommitByP4Change(self, ref, change):
1485 if self.verbose:
1486 print "looking in ref " + ref + " for change %s using bisect..." % change
1487
1488 earliestCommit = ""
1489 latestCommit = parseRevision(ref)
1490
1491 while True:
1492 if self.verbose:
1493 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1494 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1495 if len(next) == 0:
1496 if self.verbose:
1497 print "argh"
1498 return ""
1499 log = extractLogMessageFromGitCommit(next)
1500 settings = extractSettingsGitLog(log)
1501 currentChange = int(settings['change'])
1502 if self.verbose:
1503 print "current change %s" % currentChange
1504
1505 if currentChange == change:
1506 if self.verbose:
1507 print "found %s" % next
1508 return next
1509
1510 if currentChange < change:
1511 earliestCommit = "^%s" % next
1512 else:
1513 latestCommit = "%s" % next
1514
1515 return ""
1516
1517 def importNewBranch(self, branch, maxChange):
1518 # make fast-import flush all changes to disk and update the refs using the checkpoint
1519 # command so that we can try to find the branch parent in the git history
1520 self.gitStream.write("checkpoint\n\n");
1521 self.gitStream.flush();
1522 branchPrefix = self.depotPaths[0] + branch + "/"
1523 range = "@1,%s" % maxChange
1524 #print "prefix" + branchPrefix
1525 changes = p4ChangesForPaths([branchPrefix], range)
1526 if len(changes) <= 0:
1527 return False
1528 firstChange = changes[0]
1529 #print "first change in branch: %s" % firstChange
1530 sourceBranch = self.knownBranches[branch]
1531 sourceDepotPath = self.depotPaths[0] + sourceBranch
1532 sourceRef = self.gitRefForBranch(sourceBranch)
1533 #print "source " + sourceBranch
1534
1535 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1536 #print "branch parent: %s" % branchParentChange
1537 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1538 if len(gitParent) > 0:
1539 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1540 #print "parent git commit: %s" % gitParent
1541
1542 self.importChanges(changes)
1543 return True
1544
e87f37ae
SH
1545 def importChanges(self, changes):
1546 cnt = 1
1547 for change in changes:
1548 description = p4Cmd("describe %s" % change)
1549 self.updateOptionDict(description)
1550
1551 if not self.silent:
1552 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1553 sys.stdout.flush()
1554 cnt = cnt + 1
1555
1556 try:
1557 if self.detectBranches:
1558 branches = self.splitFilesIntoBranches(description)
1559 for branch in branches.keys():
1560 ## HACK --hwn
1561 branchPrefix = self.depotPaths[0] + branch + "/"
1562
1563 parent = ""
1564
1565 filesForCommit = branches[branch]
1566
1567 if self.verbose:
1568 print "branch is %s" % branch
1569
1570 self.updatedBranches.add(branch)
1571
1572 if branch not in self.createdBranches:
1573 self.createdBranches.add(branch)
1574 parent = self.knownBranches[branch]
1575 if parent == branch:
1576 parent = ""
1ca3d710
SH
1577 else:
1578 fullBranch = self.projectName + branch
1579 if fullBranch not in self.p4BranchesInGit:
1580 if not self.silent:
1581 print("\n Importing new branch %s" % fullBranch);
1582 if self.importNewBranch(branch, change - 1):
1583 parent = ""
1584 self.p4BranchesInGit.append(fullBranch)
1585 if not self.silent:
1586 print("\n Resuming with change %s" % change);
1587
1588 if self.verbose:
1589 print "parent determined through known branches: %s" % parent
e87f37ae 1590
8134f69c
SH
1591 branch = self.gitRefForBranch(branch)
1592 parent = self.gitRefForBranch(parent)
e87f37ae
SH
1593
1594 if self.verbose:
1595 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1596
1597 if len(parent) == 0 and branch in self.initialParents:
1598 parent = self.initialParents[branch]
1599 del self.initialParents[branch]
1600
1601 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1602 else:
1603 files = self.extractFilesFromCommit(description)
1604 self.commit(description, files, self.branch, self.depotPaths,
1605 self.initialParent)
1606 self.initialParent = ""
1607 except IOError:
1608 print self.gitError.read()
1609 sys.exit(1)
1610
c208a243
SH
1611 def importHeadRevision(self, revision):
1612 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1613
1614 details = { "user" : "git perforce import user", "time" : int(time.time()) }
1494fcbb 1615 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
1616 % (' '.join(self.depotPaths), revision))
1617 details["change"] = revision
1618 newestRevision = 0
1619
1620 fileCnt = 0
1621 for info in p4CmdList("files "
1622 + ' '.join(["%s...%s"
1623 % (p, revision)
1624 for p in self.depotPaths])):
1625
68b28593 1626 if 'code' in info and info['code'] == 'error':
c208a243
SH
1627 sys.stderr.write("p4 returned an error: %s\n"
1628 % info['data'])
d88e707f
PW
1629 if info['data'].find("must refer to client") >= 0:
1630 sys.stderr.write("This particular p4 error is misleading.\n")
1631 sys.stderr.write("Perhaps the depot path was misspelled.\n");
1632 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 1633 sys.exit(1)
68b28593
PW
1634 if 'p4ExitCode' in info:
1635 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
1636 sys.exit(1)
1637
1638
1639 change = int(info["change"])
1640 if change > newestRevision:
1641 newestRevision = change
1642
56c09345 1643 if info["action"] in self.delete_actions:
c208a243
SH
1644 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1645 #fileCnt = fileCnt + 1
1646 continue
1647
1648 for prop in ["depotFile", "rev", "action", "type" ]:
1649 details["%s%s" % (prop, fileCnt)] = info[prop]
1650
1651 fileCnt = fileCnt + 1
1652
1653 details["change"] = newestRevision
1654 self.updateOptionDict(details)
1655 try:
1656 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1657 except IOError:
1658 print "IO error with git fast-import. Is your git version recent enough?"
1659 print self.gitError.read()
1660
1661
3a70cdfa
TAL
1662 def getClientSpec(self):
1663 specList = p4CmdList( "client -o" )
1664 temp = {}
1665 for entry in specList:
1666 for k,v in entry.iteritems():
1667 if k.startswith("View"):
3952710b
IW
1668
1669 # p4 has these %%1 to %%9 arguments in specs to
1670 # reorder paths; which we can't handle (yet :)
1671 if re.match('%%\d', v) != None:
1672 print "Sorry, can't handle %%n arguments in client specs"
1673 sys.exit(1)
1674
3a70cdfa
TAL
1675 if v.startswith('"'):
1676 start = 1
1677 else:
1678 start = 0
1679 index = v.find("...")
3952710b
IW
1680
1681 # save the "client view"; i.e the RHS of the view
1682 # line that tells the client where to put the
1683 # files for this view.
1684 cv = v[index+3:].strip() # +3 to remove previous '...'
1685
1686 # if the client view doesn't end with a
1687 # ... wildcard, then we're going to mess up the
1688 # output directory, so fail gracefully.
1689 if not cv.endswith('...'):
1690 print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1691 sys.exit(1)
1692 cv=cv[:-3]
1693
1694 # now save the view; +index means included, -index
1695 # means it should be filtered out.
3a70cdfa
TAL
1696 v = v[start:index]
1697 if v.startswith("-"):
1698 v = v[1:]
3952710b 1699 include = -len(v)
3a70cdfa 1700 else:
3952710b
IW
1701 include = len(v)
1702
1703 temp[v] = (include, cv)
1704
3a70cdfa 1705 self.clientSpecDirs = temp.items()
3952710b 1706 self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
3a70cdfa 1707
b984733c 1708 def run(self, args):
6326aa58 1709 self.depotPaths = []
179caebf
SH
1710 self.changeRange = ""
1711 self.initialParent = ""
6326aa58 1712 self.previousDepotPaths = []
ce6f33c8 1713
29bdbac1
SH
1714 # map from branch depot path to parent branch
1715 self.knownBranches = {}
1716 self.initialParents = {}
5ca44617 1717 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
1718 if not self.syncWithOrigin:
1719 self.hasOrigin = False
29bdbac1 1720
a028a98e
SH
1721 if self.importIntoRemotes:
1722 self.refPrefix = "refs/remotes/p4/"
1723 else:
db775559 1724 self.refPrefix = "refs/heads/p4/"
a028a98e 1725
cebdf5af
HWN
1726 if self.syncWithOrigin and self.hasOrigin:
1727 if not self.silent:
1728 print "Syncing with origin first by calling git fetch origin"
1729 system("git fetch origin")
10f880f8 1730
569d1bd4 1731 if len(self.branch) == 0:
db775559 1732 self.branch = self.refPrefix + "master"
a028a98e 1733 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 1734 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 1735 system("git branch -D p4");
faf1bd20 1736 # create it /after/ importing, when master exists
0058a33a 1737 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 1738 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 1739
3cafb7d8 1740 if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
3a70cdfa
TAL
1741 self.getClientSpec()
1742
6a49f8e2
HWN
1743 # TODO: should always look at previous commits,
1744 # merge with previous imports, if possible.
1745 if args == []:
d414c74a 1746 if self.hasOrigin:
5ca44617 1747 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
1748 self.listExistingP4GitBranches()
1749
1750 if len(self.p4BranchesInGit) > 1:
1751 if not self.silent:
1752 print "Importing from/into multiple branches"
1753 self.detectBranches = True
967f72e2 1754
29bdbac1
SH
1755 if self.verbose:
1756 print "branches: %s" % self.p4BranchesInGit
1757
1758 p4Change = 0
1759 for branch in self.p4BranchesInGit:
cebdf5af 1760 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
1761
1762 settings = extractSettingsGitLog(logMsg)
29bdbac1 1763
bb6e09b2
HWN
1764 self.readOptions(settings)
1765 if (settings.has_key('depot-paths')
1766 and settings.has_key ('change')):
1767 change = int(settings['change']) + 1
29bdbac1
SH
1768 p4Change = max(p4Change, change)
1769
bb6e09b2
HWN
1770 depotPaths = sorted(settings['depot-paths'])
1771 if self.previousDepotPaths == []:
6326aa58 1772 self.previousDepotPaths = depotPaths
29bdbac1 1773 else:
6326aa58
HWN
1774 paths = []
1775 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
583e1707 1776 for i in range(0, min(len(cur), len(prev))):
6326aa58 1777 if cur[i] <> prev[i]:
583e1707 1778 i = i - 1
6326aa58
HWN
1779 break
1780
583e1707 1781 paths.append (cur[:i + 1])
6326aa58
HWN
1782
1783 self.previousDepotPaths = paths
29bdbac1
SH
1784
1785 if p4Change > 0:
bb6e09b2 1786 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 1787 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
1788 if not self.detectBranches:
1789 self.initialParent = parseRevision(self.branch)
341dc1c1 1790 if not self.silent and not self.detectBranches:
967f72e2 1791 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 1792
f9162f6a
SH
1793 if not self.branch.startswith("refs/"):
1794 self.branch = "refs/heads/" + self.branch
179caebf 1795
6326aa58 1796 if len(args) == 0 and self.depotPaths:
b984733c 1797 if not self.silent:
6326aa58 1798 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 1799 else:
6326aa58 1800 if self.depotPaths and self.depotPaths != args:
cebdf5af 1801 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
1802 "This doesn't work!" % (' '.join (self.depotPaths),
1803 ' '.join (args)))
b984733c 1804 sys.exit(1)
6326aa58 1805
bb6e09b2 1806 self.depotPaths = sorted(args)
b984733c 1807
1c49fc19 1808 revision = ""
b984733c 1809 self.users = {}
b984733c 1810
6326aa58
HWN
1811 newPaths = []
1812 for p in self.depotPaths:
1813 if p.find("@") != -1:
1814 atIdx = p.index("@")
1815 self.changeRange = p[atIdx:]
1816 if self.changeRange == "@all":
1817 self.changeRange = ""
6a49f8e2 1818 elif ',' not in self.changeRange:
1c49fc19 1819 revision = self.changeRange
6326aa58 1820 self.changeRange = ""
7fcff9de 1821 p = p[:atIdx]
6326aa58
HWN
1822 elif p.find("#") != -1:
1823 hashIdx = p.index("#")
1c49fc19 1824 revision = p[hashIdx:]
7fcff9de 1825 p = p[:hashIdx]
6326aa58 1826 elif self.previousDepotPaths == []:
1c49fc19 1827 revision = "#head"
6326aa58
HWN
1828
1829 p = re.sub ("\.\.\.$", "", p)
1830 if not p.endswith("/"):
1831 p += "/"
1832
1833 newPaths.append(p)
1834
1835 self.depotPaths = newPaths
1836
b984733c 1837
b607e71e 1838 self.loadUserMapFromCache()
cb53e1f8
SH
1839 self.labels = {}
1840 if self.detectLabels:
1841 self.getLabels();
b984733c 1842
4b97ffb1 1843 if self.detectBranches:
df450923
SH
1844 ## FIXME - what's a P4 projectName ?
1845 self.projectName = self.guessProjectName()
1846
38f9f5ec
SH
1847 if self.hasOrigin:
1848 self.getBranchMappingFromGitBranches()
1849 else:
1850 self.getBranchMapping()
29bdbac1
SH
1851 if self.verbose:
1852 print "p4-git branches: %s" % self.p4BranchesInGit
1853 print "initial parents: %s" % self.initialParents
1854 for b in self.p4BranchesInGit:
1855 if b != "master":
6326aa58
HWN
1856
1857 ## FIXME
29bdbac1
SH
1858 b = b[len(self.projectName):]
1859 self.createdBranches.add(b)
4b97ffb1 1860
f291b4e3 1861 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 1862
cebdf5af 1863 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
1864 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1865 stderr=subprocess.PIPE);
08483580
SH
1866 self.gitOutput = importProcess.stdout
1867 self.gitStream = importProcess.stdin
1868 self.gitError = importProcess.stderr
b984733c 1869
1c49fc19 1870 if revision:
c208a243 1871 self.importHeadRevision(revision)
b984733c
SH
1872 else:
1873 changes = []
1874
0828ab14 1875 if len(self.changesFile) > 0:
b984733c 1876 output = open(self.changesFile).readlines()
1d7367dc 1877 changeSet = set()
b984733c
SH
1878 for line in output:
1879 changeSet.add(int(line))
1880
1881 for change in changeSet:
1882 changes.append(change)
1883
1884 changes.sort()
1885 else:
accad8e0
PW
1886 # catch "git-p4 sync" with no new branches, in a repo that
1887 # does not have any existing git-p4 branches
1888 if len(args) == 0 and not self.p4BranchesInGit:
e32e00dc 1889 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
29bdbac1 1890 if self.verbose:
86dff6b6 1891 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 1892 self.changeRange)
4f6432d8 1893 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 1894
01a9c9c5 1895 if len(self.maxChanges) > 0:
7fcff9de 1896 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 1897
b984733c 1898 if len(changes) == 0:
0828ab14 1899 if not self.silent:
341dc1c1 1900 print "No changes to import!"
1f52af6c 1901 return True
b984733c 1902
a9d1a27a
SH
1903 if not self.silent and not self.detectBranches:
1904 print "Import destination: %s" % self.branch
1905
341dc1c1
SH
1906 self.updatedBranches = set()
1907
e87f37ae 1908 self.importChanges(changes)
b984733c 1909
341dc1c1
SH
1910 if not self.silent:
1911 print ""
1912 if len(self.updatedBranches) > 0:
1913 sys.stdout.write("Updated branches: ")
1914 for b in self.updatedBranches:
1915 sys.stdout.write("%s " % b)
1916 sys.stdout.write("\n")
b984733c 1917
b984733c 1918 self.gitStream.close()
29bdbac1
SH
1919 if importProcess.wait() != 0:
1920 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
1921 self.gitOutput.close()
1922 self.gitError.close()
1923
b984733c
SH
1924 return True
1925
01ce1fe9
SH
1926class P4Rebase(Command):
1927 def __init__(self):
1928 Command.__init__(self)
01265103 1929 self.options = [ ]
cebdf5af
HWN
1930 self.description = ("Fetches the latest revision from perforce and "
1931 + "rebases the current work (branch) against it")
68c42153 1932 self.verbose = False
01ce1fe9
SH
1933
1934 def run(self, args):
1935 sync = P4Sync()
1936 sync.run([])
d7e3868c 1937
14594f4b
SH
1938 return self.rebase()
1939
1940 def rebase(self):
36ee4ee4
SH
1941 if os.system("git update-index --refresh") != 0:
1942 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1943 if len(read_pipe("git diff-index HEAD --")) > 0:
1944 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1945
d7e3868c
SH
1946 [upstream, settings] = findUpstreamBranchPoint()
1947 if len(upstream) == 0:
1948 die("Cannot find upstream branchpoint for rebase")
1949
1950 # the branchpoint may be p4/foo~3, so strip off the parent
1951 upstream = re.sub("~[0-9]+$", "", upstream)
1952
1953 print "Rebasing the current branch onto %s" % upstream
b25b2065 1954 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 1955 system("git rebase %s" % upstream)
1f52af6c 1956 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
1957 return True
1958
f9a3a4f7
SH
1959class P4Clone(P4Sync):
1960 def __init__(self):
1961 P4Sync.__init__(self)
1962 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 1963 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 1964 self.options += [
bb6e09b2
HWN
1965 optparse.make_option("--destination", dest="cloneDestination",
1966 action='store', default=None,
354081d5
TT
1967 help="where to leave result of the clone"),
1968 optparse.make_option("-/", dest="cloneExclude",
1969 action="append", type="string",
38200076
PW
1970 help="exclude depot path"),
1971 optparse.make_option("--bare", dest="cloneBare",
1972 action="store_true", default=False),
354081d5 1973 ]
bb6e09b2 1974 self.cloneDestination = None
f9a3a4f7 1975 self.needsGit = False
38200076 1976 self.cloneBare = False
f9a3a4f7 1977
354081d5
TT
1978 # This is required for the "append" cloneExclude action
1979 def ensure_value(self, attr, value):
1980 if not hasattr(self, attr) or getattr(self, attr) is None:
1981 setattr(self, attr, value)
1982 return getattr(self, attr)
1983
6a49f8e2
HWN
1984 def defaultDestination(self, args):
1985 ## TODO: use common prefix of args?
1986 depotPath = args[0]
1987 depotDir = re.sub("(@[^@]*)$", "", depotPath)
1988 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 1989 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
1990 depotDir = re.sub(r"/$", "", depotDir)
1991 return os.path.split(depotDir)[1]
1992
f9a3a4f7
SH
1993 def run(self, args):
1994 if len(args) < 1:
1995 return False
bb6e09b2
HWN
1996
1997 if self.keepRepoPath and not self.cloneDestination:
1998 sys.stderr.write("Must specify destination for --keep-path\n")
1999 sys.exit(1)
f9a3a4f7 2000
6326aa58 2001 depotPaths = args
5e100b5c
SH
2002
2003 if not self.cloneDestination and len(depotPaths) > 1:
2004 self.cloneDestination = depotPaths[-1]
2005 depotPaths = depotPaths[:-1]
2006
354081d5 2007 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
2008 for p in depotPaths:
2009 if not p.startswith("//"):
2010 return False
f9a3a4f7 2011
bb6e09b2 2012 if not self.cloneDestination:
98ad4faf 2013 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 2014
86dff6b6 2015 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 2016
c3bf3f13
KG
2017 if not os.path.exists(self.cloneDestination):
2018 os.makedirs(self.cloneDestination)
053fd0c1 2019 chdir(self.cloneDestination)
38200076
PW
2020
2021 init_cmd = [ "git", "init" ]
2022 if self.cloneBare:
2023 init_cmd.append("--bare")
2024 subprocess.check_call(init_cmd)
2025
6326aa58 2026 if not P4Sync.run(self, depotPaths):
f9a3a4f7 2027 return False
f9a3a4f7 2028 if self.branch != "master":
e9905013
TAL
2029 if self.importIntoRemotes:
2030 masterbranch = "refs/remotes/p4/master"
2031 else:
2032 masterbranch = "refs/heads/p4/master"
2033 if gitBranchExists(masterbranch):
2034 system("git branch master %s" % masterbranch)
38200076
PW
2035 if not self.cloneBare:
2036 system("git checkout -f")
8f9b2e08
SH
2037 else:
2038 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 2039
f9a3a4f7
SH
2040 return True
2041
09d89de2
SH
2042class P4Branches(Command):
2043 def __init__(self):
2044 Command.__init__(self)
2045 self.options = [ ]
2046 self.description = ("Shows the git branches that hold imports and their "
2047 + "corresponding perforce depot paths")
2048 self.verbose = False
2049
2050 def run(self, args):
5ca44617
SH
2051 if originP4BranchesExist():
2052 createOrUpdateBranchesFromOrigin()
2053
09d89de2
SH
2054 cmdline = "git rev-parse --symbolic "
2055 cmdline += " --remotes"
2056
2057 for line in read_pipe_lines(cmdline):
2058 line = line.strip()
2059
2060 if not line.startswith('p4/') or line == "p4/HEAD":
2061 continue
2062 branch = line
2063
2064 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2065 settings = extractSettingsGitLog(log)
2066
2067 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2068 return True
2069
b984733c
SH
2070class HelpFormatter(optparse.IndentedHelpFormatter):
2071 def __init__(self):
2072 optparse.IndentedHelpFormatter.__init__(self)
2073
2074 def format_description(self, description):
2075 if description:
2076 return description + "\n"
2077 else:
2078 return ""
4f5cf76a 2079
86949eef
SH
2080def printUsage(commands):
2081 print "usage: %s <command> [options]" % sys.argv[0]
2082 print ""
2083 print "valid commands: %s" % ", ".join(commands)
2084 print ""
2085 print "Try %s <command> --help for command specific help." % sys.argv[0]
2086 print ""
2087
2088commands = {
b86f7378
HWN
2089 "debug" : P4Debug,
2090 "submit" : P4Submit,
a9834f58 2091 "commit" : P4Submit,
b86f7378
HWN
2092 "sync" : P4Sync,
2093 "rebase" : P4Rebase,
2094 "clone" : P4Clone,
09d89de2
SH
2095 "rollback" : P4RollBack,
2096 "branches" : P4Branches
86949eef
SH
2097}
2098
86949eef 2099
bb6e09b2
HWN
2100def main():
2101 if len(sys.argv[1:]) == 0:
2102 printUsage(commands.keys())
2103 sys.exit(2)
4f5cf76a 2104
bb6e09b2
HWN
2105 cmd = ""
2106 cmdName = sys.argv[1]
2107 try:
b86f7378
HWN
2108 klass = commands[cmdName]
2109 cmd = klass()
bb6e09b2
HWN
2110 except KeyError:
2111 print "unknown command %s" % cmdName
2112 print ""
2113 printUsage(commands.keys())
2114 sys.exit(2)
2115
2116 options = cmd.options
b86f7378 2117 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
2118
2119 args = sys.argv[2:]
2120
2121 if len(options) > 0:
2122 options.append(optparse.make_option("--git-dir", dest="gitdir"))
2123
2124 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2125 options,
2126 description = cmd.description,
2127 formatter = HelpFormatter())
2128
2129 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2130 global verbose
2131 verbose = cmd.verbose
2132 if cmd.needsGit:
b86f7378
HWN
2133 if cmd.gitdir == None:
2134 cmd.gitdir = os.path.abspath(".git")
2135 if not isValidGitDir(cmd.gitdir):
2136 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2137 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
2138 cdup = read_pipe("git rev-parse --show-cdup").strip()
2139 if len(cdup) > 0:
053fd0c1 2140 chdir(cdup);
e20a9e53 2141
b86f7378
HWN
2142 if not isValidGitDir(cmd.gitdir):
2143 if isValidGitDir(cmd.gitdir + "/.git"):
2144 cmd.gitdir += "/.git"
bb6e09b2 2145 else:
b86f7378 2146 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 2147
b86f7378 2148 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 2149
bb6e09b2
HWN
2150 if not cmd.run(args):
2151 parser.print_help()
4f5cf76a 2152
4f5cf76a 2153
bb6e09b2
HWN
2154if __name__ == '__main__':
2155 main()