git-p4: Process detectCopiesHarder with --bool
[git/git.git] / contrib / fast-import / git-p4
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
1d7367dc
RG
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
ce6f33c8 13import re
8b41a97f 14
4addad22 15verbose = False
86949eef 16
21a50753
AK
17
18def p4_build_cmd(cmd):
19 """Build a suitable p4 command line.
20
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
24 """
abcaf073
AK
25 real_cmd = "%s " % "p4"
26
27 user = gitConfig("git-p4.user")
28 if len(user) > 0:
29 real_cmd += "-u %s " % user
30
31 password = gitConfig("git-p4.password")
32 if len(password) > 0:
33 real_cmd += "-P %s " % password
34
35 port = gitConfig("git-p4.port")
36 if len(port) > 0:
37 real_cmd += "-p %s " % port
38
39 host = gitConfig("git-p4.host")
40 if len(host) > 0:
41 real_cmd += "-h %s " % host
42
43 client = gitConfig("git-p4.client")
44 if len(client) > 0:
45 real_cmd += "-c %s " % client
46
47 real_cmd += "%s" % (cmd)
ee06427a
AK
48 if verbose:
49 print real_cmd
21a50753
AK
50 return real_cmd
51
053fd0c1
RB
52def chdir(dir):
53 if os.name == 'nt':
54 os.environ['PWD']=dir
55 os.chdir(dir)
56
86dff6b6
HWN
57def die(msg):
58 if verbose:
59 raise Exception(msg)
60 else:
61 sys.stderr.write(msg + "\n")
62 sys.exit(1)
63
bce4c5fc 64def write_pipe(c, str):
4addad22 65 if verbose:
86dff6b6 66 sys.stderr.write('Writing pipe: %s\n' % c)
b016d397 67
bce4c5fc 68 pipe = os.popen(c, 'w')
b016d397 69 val = pipe.write(str)
bce4c5fc 70 if pipe.close():
86dff6b6 71 die('Command failed: %s' % c)
b016d397
HWN
72
73 return val
74
d9429194
AK
75def p4_write_pipe(c, str):
76 real_cmd = p4_build_cmd(c)
893d340f 77 return write_pipe(real_cmd, str)
d9429194 78
4addad22
HWN
79def read_pipe(c, ignore_error=False):
80 if verbose:
86dff6b6 81 sys.stderr.write('Reading pipe: %s\n' % c)
8b41a97f 82
bce4c5fc 83 pipe = os.popen(c, 'rb')
b016d397 84 val = pipe.read()
4addad22 85 if pipe.close() and not ignore_error:
86dff6b6 86 die('Command failed: %s' % c)
b016d397
HWN
87
88 return val
89
d9429194
AK
90def p4_read_pipe(c, ignore_error=False):
91 real_cmd = p4_build_cmd(c)
92 return read_pipe(real_cmd, ignore_error)
b016d397 93
bce4c5fc 94def read_pipe_lines(c):
4addad22 95 if verbose:
86dff6b6 96 sys.stderr.write('Reading pipe: %s\n' % c)
b016d397 97 ## todo: check return status
bce4c5fc 98 pipe = os.popen(c, 'rb')
b016d397 99 val = pipe.readlines()
bce4c5fc 100 if pipe.close():
86dff6b6 101 die('Command failed: %s' % c)
b016d397
HWN
102
103 return val
caace111 104
2318121b
AK
105def p4_read_pipe_lines(c):
106 """Specifically invoke p4 on the command supplied. """
155af834 107 real_cmd = p4_build_cmd(c)
2318121b
AK
108 return read_pipe_lines(real_cmd)
109
6754a299 110def system(cmd):
4addad22 111 if verbose:
bb6e09b2 112 sys.stderr.write("executing %s\n" % cmd)
6754a299
HWN
113 if os.system(cmd) != 0:
114 die("command failed: %s" % cmd)
115
bf9320f1
AK
116def p4_system(cmd):
117 """Specifically invoke p4 as the system command. """
155af834 118 real_cmd = p4_build_cmd(cmd)
bf9320f1
AK
119 return system(real_cmd)
120
b9fc6ea9
DB
121def isP4Exec(kind):
122 """Determine if a Perforce 'kind' should have execute permission
123
124 'p4 help filetypes' gives a list of the types. If it starts with 'x',
125 or x follows one of a few letters. Otherwise, if there is an 'x' after
126 a plus sign, it is also executable"""
127 return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
128
c65b670e
CP
129def setP4ExecBit(file, mode):
130 # Reopens an already open file and changes the execute bit to match
131 # the execute bit setting in the passed in mode.
132
133 p4Type = "+x"
134
135 if not isModeExec(mode):
136 p4Type = getP4OpenedType(file)
137 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
138 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
139 if p4Type[-1] == "+":
140 p4Type = p4Type[0:-1]
141
87b611d5 142 p4_system("reopen -t %s %s" % (p4Type, file))
c65b670e
CP
143
144def getP4OpenedType(file):
145 # Returns the perforce file type for the given file.
146
a7d3ef9d 147 result = p4_read_pipe("opened %s" % file)
f3e5ae4f 148 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
149 if match:
150 return match.group(1)
151 else:
f3e5ae4f 152 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 153
b43b0a3c
CP
154def diffTreePattern():
155 # This is a simple generator for the diff tree regex pattern. This could be
156 # a class variable if this and parseDiffTreeEntry were a part of a class.
157 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
158 while True:
159 yield pattern
160
161def parseDiffTreeEntry(entry):
162 """Parses a single diff tree entry into its component elements.
163
164 See git-diff-tree(1) manpage for details about the format of the diff
165 output. This method returns a dictionary with the following elements:
166
167 src_mode - The mode of the source file
168 dst_mode - The mode of the destination file
169 src_sha1 - The sha1 for the source file
170 dst_sha1 - The sha1 fr the destination file
171 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
172 status_score - The score for the status (applicable for 'C' and 'R'
173 statuses). This is None if there is no score.
174 src - The path for the source file.
175 dst - The path for the destination file. This is only present for
176 copy or renames. If it is not present, this is None.
177
178 If the pattern is not matched, None is returned."""
179
180 match = diffTreePattern().next().match(entry)
181 if match:
182 return {
183 'src_mode': match.group(1),
184 'dst_mode': match.group(2),
185 'src_sha1': match.group(3),
186 'dst_sha1': match.group(4),
187 'status': match.group(5),
188 'status_score': match.group(6),
189 'src': match.group(7),
190 'dst': match.group(10)
191 }
192 return None
193
c65b670e
CP
194def isModeExec(mode):
195 # Returns True if the given git mode represents an executable file,
196 # otherwise False.
197 return mode[-3:] == "755"
198
199def isModeExecChanged(src_mode, dst_mode):
200 return isModeExec(src_mode) != isModeExec(dst_mode)
201
b932705b 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
155af834 203 cmd = p4_build_cmd("-G %s" % (cmd))
6a49f8e2
HWN
204 if verbose:
205 sys.stderr.write("Opening pipe: %s\n" % cmd)
9f90c733
SL
206
207 # Use a temporary file to avoid deadlocks without
208 # subprocess.communicate(), which would put another copy
209 # of stdout into memory.
210 stdin_file = None
211 if stdin is not None:
212 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
213 stdin_file.write(stdin)
214 stdin_file.flush()
215 stdin_file.seek(0)
216
217 p4 = subprocess.Popen(cmd, shell=True,
218 stdin=stdin_file,
219 stdout=subprocess.PIPE)
86949eef
SH
220
221 result = []
222 try:
223 while True:
9f90c733 224 entry = marshal.load(p4.stdout)
c3f6163b
AG
225 if cb is not None:
226 cb(entry)
227 else:
228 result.append(entry)
86949eef
SH
229 except EOFError:
230 pass
9f90c733
SL
231 exitCode = p4.wait()
232 if exitCode != 0:
ac3e0d79
SH
233 entry = {}
234 entry["p4ExitCode"] = exitCode
235 result.append(entry)
86949eef
SH
236
237 return result
238
239def p4Cmd(cmd):
240 list = p4CmdList(cmd)
241 result = {}
242 for entry in list:
243 result.update(entry)
244 return result;
245
cb2c9db5
SH
246def p4Where(depotPath):
247 if not depotPath.endswith("/"):
248 depotPath += "/"
7f705dc3
TAL
249 depotPath = depotPath + "..."
250 outputList = p4CmdList("where %s" % depotPath)
251 output = None
252 for entry in outputList:
75bc9573
TAL
253 if "depotFile" in entry:
254 if entry["depotFile"] == depotPath:
255 output = entry
256 break
257 elif "data" in entry:
258 data = entry.get("data")
259 space = data.find(" ")
260 if data[:space] == depotPath:
261 output = entry
262 break
7f705dc3
TAL
263 if output == None:
264 return ""
dc524036
SH
265 if output["code"] == "error":
266 return ""
cb2c9db5
SH
267 clientPath = ""
268 if "path" in output:
269 clientPath = output.get("path")
270 elif "data" in output:
271 data = output.get("data")
272 lastSpace = data.rfind(" ")
273 clientPath = data[lastSpace + 1:]
274
275 if clientPath.endswith("..."):
276 clientPath = clientPath[:-3]
277 return clientPath
278
86949eef 279def currentGitBranch():
b25b2065 280 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 281
4f5cf76a 282def isValidGitDir(path):
bb6e09b2
HWN
283 if (os.path.exists(path + "/HEAD")
284 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
285 return True;
286 return False
287
463e8af6 288def parseRevision(ref):
b25b2065 289 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 290
6ae8de88
SH
291def extractLogMessageFromGitCommit(commit):
292 logMessage = ""
b016d397
HWN
293
294 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 295 foundTitle = False
b016d397 296 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
297 if not foundTitle:
298 if len(log) == 1:
1c094184 299 foundTitle = True
6ae8de88
SH
300 continue
301
302 logMessage += log
303 return logMessage
304
bb6e09b2 305def extractSettingsGitLog(log):
6ae8de88
SH
306 values = {}
307 for line in log.split("\n"):
308 line = line.strip()
6326aa58
HWN
309 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
310 if not m:
311 continue
312
313 assignments = m.group(1).split (':')
314 for a in assignments:
315 vals = a.split ('=')
316 key = vals[0].strip()
317 val = ('='.join (vals[1:])).strip()
318 if val.endswith ('\"') and val.startswith('"'):
319 val = val[1:-1]
320
321 values[key] = val
322
845b42cb
SH
323 paths = values.get("depot-paths")
324 if not paths:
325 paths = values.get("depot-path")
a3fdd579
SH
326 if paths:
327 values['depot-paths'] = paths.split(',')
bb6e09b2 328 return values
6ae8de88 329
8136a639 330def gitBranchExists(branch):
bb6e09b2
HWN
331 proc = subprocess.Popen(["git", "rev-parse", branch],
332 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 333 return proc.wait() == 0;
8136a639 334
36bd8446 335_gitConfig = {}
99f790f2 336def gitConfig(key, args = None): # set args to "--bool", for instance
36bd8446 337 if not _gitConfig.has_key(key):
99f790f2
TAL
338 argsFilter = ""
339 if args != None:
340 argsFilter = "%s " % args
341 cmd = "git config %s%s" % (argsFilter, key)
342 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
36bd8446 343 return _gitConfig[key]
01265103 344
062410bb
SH
345def p4BranchesInGit(branchesAreInRemotes = True):
346 branches = {}
347
348 cmdline = "git rev-parse --symbolic "
349 if branchesAreInRemotes:
350 cmdline += " --remotes"
351 else:
352 cmdline += " --branches"
353
354 for line in read_pipe_lines(cmdline):
355 line = line.strip()
356
357 ## only import to p4/
358 if not line.startswith('p4/') or line == "p4/HEAD":
359 continue
360 branch = line
361
362 # strip off p4
363 branch = re.sub ("^p4/", "", line)
364
365 branches[branch] = parseRevision(line)
366 return branches
367
9ceab363 368def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
369 branches = p4BranchesInGit()
370 # map from depot-path to branch name
371 branchByDepotPath = {}
372 for branch in branches.keys():
373 tip = branches[branch]
374 log = extractLogMessageFromGitCommit(tip)
375 settings = extractSettingsGitLog(log)
376 if settings.has_key("depot-paths"):
377 paths = ",".join(settings["depot-paths"])
378 branchByDepotPath[paths] = "remotes/p4/" + branch
379
27d2d811 380 settings = None
27d2d811
SH
381 parent = 0
382 while parent < 65535:
9ceab363 383 commit = head + "~%s" % parent
27d2d811
SH
384 log = extractLogMessageFromGitCommit(commit)
385 settings = extractSettingsGitLog(log)
86506fe5
SH
386 if settings.has_key("depot-paths"):
387 paths = ",".join(settings["depot-paths"])
388 if branchByDepotPath.has_key(paths):
389 return [branchByDepotPath[paths], settings]
27d2d811 390
86506fe5 391 parent = parent + 1
27d2d811 392
86506fe5 393 return ["", settings]
27d2d811 394
5ca44617
SH
395def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
396 if not silent:
397 print ("Creating/updating branch(es) in %s based on origin branch(es)"
398 % localRefPrefix)
399
400 originPrefix = "origin/p4/"
401
402 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
403 line = line.strip()
404 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
405 continue
406
407 headName = line[len(originPrefix):]
408 remoteHead = localRefPrefix + headName
409 originHead = line
410
411 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
412 if (not original.has_key('depot-paths')
413 or not original.has_key('change')):
414 continue
415
416 update = False
417 if not gitBranchExists(remoteHead):
418 if verbose:
419 print "creating %s" % remoteHead
420 update = True
421 else:
422 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
423 if settings.has_key('change') > 0:
424 if settings['depot-paths'] == original['depot-paths']:
425 originP4Change = int(original['change'])
426 p4Change = int(settings['change'])
427 if originP4Change > p4Change:
428 print ("%s (%s) is newer than %s (%s). "
429 "Updating p4 branch from origin."
430 % (originHead, originP4Change,
431 remoteHead, p4Change))
432 update = True
433 else:
434 print ("Ignoring: %s was imported from %s while "
435 "%s was imported from %s"
436 % (originHead, ','.join(original['depot-paths']),
437 remoteHead, ','.join(settings['depot-paths'])))
438
439 if update:
440 system("git update-ref %s %s" % (remoteHead, originHead))
441
442def originP4BranchesExist():
443 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
444
4f6432d8
SH
445def p4ChangesForPaths(depotPaths, changeRange):
446 assert depotPaths
b340fa43 447 output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
4f6432d8
SH
448 for p in depotPaths]))
449
b4b0ba06 450 changes = {}
4f6432d8 451 for line in output:
c3f6163b
AG
452 changeNum = int(line.split(" ")[1])
453 changes[changeNum] = True
4f6432d8 454
b4b0ba06
PW
455 changelist = changes.keys()
456 changelist.sort()
457 return changelist
4f6432d8 458
d53de8b9
TAL
459def p4PathStartsWith(path, prefix):
460 # This method tries to remedy a potential mixed-case issue:
461 #
462 # If UserA adds //depot/DirA/file1
463 # and UserB adds //depot/dira/file2
464 #
465 # we may or may not have a problem. If you have core.ignorecase=true,
466 # we treat DirA and dira as the same directory
467 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
468 if ignorecase:
469 return path.lower().startswith(prefix.lower())
470 return path.startswith(prefix)
471
b984733c
SH
472class Command:
473 def __init__(self):
474 self.usage = "usage: %prog [options]"
8910ac0e 475 self.needsGit = True
b984733c 476
3ea2cfd4
LD
477class P4UserMap:
478 def __init__(self):
479 self.userMapFromPerforceServer = False
480
481 def getUserCacheFilename(self):
482 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
483 return home + "/.gitp4-usercache.txt"
484
485 def getUserMapFromPerforceServer(self):
486 if self.userMapFromPerforceServer:
487 return
488 self.users = {}
489 self.emails = {}
490
491 for output in p4CmdList("users"):
492 if not output.has_key("User"):
493 continue
494 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
495 self.emails[output["Email"]] = output["User"]
496
497
498 s = ''
499 for (key, val) in self.users.items():
500 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
501
502 open(self.getUserCacheFilename(), "wb").write(s)
503 self.userMapFromPerforceServer = True
504
505 def loadUserMapFromCache(self):
506 self.users = {}
507 self.userMapFromPerforceServer = False
508 try:
509 cache = open(self.getUserCacheFilename(), "rb")
510 lines = cache.readlines()
511 cache.close()
512 for line in lines:
513 entry = line.strip().split("\t")
514 self.users[entry[0]] = entry[1]
515 except IOError:
516 self.getUserMapFromPerforceServer()
517
b984733c 518class P4Debug(Command):
86949eef 519 def __init__(self):
6ae8de88 520 Command.__init__(self)
86949eef 521 self.options = [
b1ce9447
HWN
522 optparse.make_option("--verbose", dest="verbose", action="store_true",
523 default=False),
4addad22 524 ]
c8c39116 525 self.description = "A tool to debug the output of p4 -G."
8910ac0e 526 self.needsGit = False
b1ce9447 527 self.verbose = False
86949eef
SH
528
529 def run(self, args):
b1ce9447 530 j = 0
86949eef 531 for output in p4CmdList(" ".join(args)):
b1ce9447
HWN
532 print 'Element: %d' % j
533 j += 1
86949eef 534 print output
b984733c 535 return True
86949eef 536
5834684d
SH
537class P4RollBack(Command):
538 def __init__(self):
539 Command.__init__(self)
540 self.options = [
0c66a783
SH
541 optparse.make_option("--verbose", dest="verbose", action="store_true"),
542 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
543 ]
544 self.description = "A tool to debug the multi-branch import. Don't use :)"
52102d47 545 self.verbose = False
0c66a783 546 self.rollbackLocalBranches = False
5834684d
SH
547
548 def run(self, args):
549 if len(args) != 1:
550 return False
551 maxChange = int(args[0])
0c66a783 552
ad192f28 553 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
554 die("Problems executing p4");
555
0c66a783
SH
556 if self.rollbackLocalBranches:
557 refPrefix = "refs/heads/"
b016d397 558 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
559 else:
560 refPrefix = "refs/remotes/"
b016d397 561 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
562
563 for line in lines:
564 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
565 line = line.strip()
566 ref = refPrefix + line
5834684d 567 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
568 settings = extractSettingsGitLog(log)
569
570 depotPaths = settings['depot-paths']
571 change = settings['change']
572
5834684d 573 changed = False
52102d47 574
6326aa58
HWN
575 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
576 for p in depotPaths]))) == 0:
52102d47
SH
577 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
578 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
579 continue
580
bb6e09b2 581 while change and int(change) > maxChange:
5834684d 582 changed = True
52102d47
SH
583 if self.verbose:
584 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
585 system("git update-ref %s \"%s^\"" % (ref, ref))
586 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
587 settings = extractSettingsGitLog(log)
588
589
590 depotPaths = settings['depot-paths']
591 change = settings['change']
5834684d
SH
592
593 if changed:
52102d47 594 print "%s rewound to %s" % (ref, change)
5834684d
SH
595
596 return True
597
3ea2cfd4 598class P4Submit(Command, P4UserMap):
4f5cf76a 599 def __init__(self):
b984733c 600 Command.__init__(self)
3ea2cfd4 601 P4UserMap.__init__(self)
4f5cf76a 602 self.options = [
4addad22 603 optparse.make_option("--verbose", dest="verbose", action="store_true"),
4f5cf76a 604 optparse.make_option("--origin", dest="origin"),
ae901090 605 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
606 # preserve the user, requires relevant p4 permissions
607 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
4f5cf76a
SH
608 ]
609 self.description = "Submit changes from git to the perforce depot."
c9b50e63 610 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 611 self.interactive = True
9512497b 612 self.origin = ""
ae901090 613 self.detectRenames = False
b0d10df7 614 self.verbose = False
3ea2cfd4 615 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
f7baba8b 616 self.isWindows = (platform.system() == "Windows")
848de9c3 617 self.myP4UserId = None
4f5cf76a 618
4f5cf76a
SH
619 def check(self):
620 if len(p4CmdList("opened ...")) > 0:
621 die("You have files opened with perforce! Close them before starting the sync.")
622
edae1e2f
SH
623 # replaces everything between 'Description:' and the next P4 submit template field with the
624 # commit message
4f5cf76a
SH
625 def prepareLogMessage(self, template, message):
626 result = ""
627
edae1e2f
SH
628 inDescriptionSection = False
629
4f5cf76a
SH
630 for line in template.split("\n"):
631 if line.startswith("#"):
632 result += line + "\n"
633 continue
634
edae1e2f 635 if inDescriptionSection:
c9dbab04 636 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f
SH
637 inDescriptionSection = False
638 else:
639 continue
640 else:
641 if line.startswith("Description:"):
642 inDescriptionSection = True
643 line += "\n"
644 for messageLine in message.split("\n"):
645 line += "\t" + messageLine + "\n"
646
647 result += line + "\n"
4f5cf76a
SH
648
649 return result
650
3ea2cfd4
LD
651 def p4UserForCommit(self,id):
652 # Return the tuple (perforce user,git email) for a given git commit id
653 self.getUserMapFromPerforceServer()
654 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
655 gitEmail = gitEmail.strip()
656 if not self.emails.has_key(gitEmail):
657 return (None,gitEmail)
658 else:
659 return (self.emails[gitEmail],gitEmail)
660
661 def checkValidP4Users(self,commits):
662 # check if any git authors cannot be mapped to p4 users
663 for id in commits:
664 (user,email) = self.p4UserForCommit(id)
665 if not user:
666 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
667 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
668 print "%s" % msg
669 else:
670 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
671
672 def lastP4Changelist(self):
673 # Get back the last changelist number submitted in this client spec. This
674 # then gets used to patch up the username in the change. If the same
675 # client spec is being used by multiple processes then this might go
676 # wrong.
677 results = p4CmdList("client -o") # find the current client
678 client = None
679 for r in results:
680 if r.has_key('Client'):
681 client = r['Client']
682 break
683 if not client:
684 die("could not get client spec")
685 results = p4CmdList("changes -c %s -m 1" % client)
686 for r in results:
687 if r.has_key('change'):
688 return r['change']
689 die("Could not get changelist number for last submit - cannot patch up user details")
690
691 def modifyChangelistUser(self, changelist, newUser):
692 # fixup the user field of a changelist after it has been submitted.
693 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
694 if len(changes) != 1:
695 die("Bad output from p4 change modifying %s to user %s" %
696 (changelist, newUser))
697
698 c = changes[0]
699 if c['User'] == newUser: return # nothing to do
700 c['User'] = newUser
701 input = marshal.dumps(c)
702
3ea2cfd4
LD
703 result = p4CmdList("change -f -i", stdin=input)
704 for r in result:
705 if r.has_key('code'):
706 if r['code'] == 'error':
707 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
708 if r.has_key('data'):
709 print("Updated user field for changelist %s to %s" % (changelist, newUser))
710 return
711 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
712
713 def canChangeChangelists(self):
714 # check to see if we have p4 admin or super-user permissions, either of
715 # which are required to modify changelists.
ecdba36d 716 results = p4CmdList("protects %s" % self.depotPath)
3ea2cfd4
LD
717 for r in results:
718 if r.has_key('perm'):
719 if r['perm'] == 'admin':
720 return 1
721 if r['perm'] == 'super':
722 return 1
723 return 0
724
848de9c3
LD
725 def p4UserId(self):
726 if self.myP4UserId:
727 return self.myP4UserId
728
729 results = p4CmdList("user -o")
730 for r in results:
731 if r.has_key('User'):
732 self.myP4UserId = r['User']
733 return r['User']
734 die("Could not find your p4 user id")
735
736 def p4UserIsMe(self, p4User):
737 # return True if the given p4 user is actually me
738 me = self.p4UserId()
739 if not p4User or p4User != me:
740 return False
741 else:
742 return True
743
ea99c3ae
SH
744 def prepareSubmitTemplate(self):
745 # remove lines in the Files section that show changes to files outside the depot path we're committing into
746 template = ""
747 inFilesSection = False
b340fa43 748 for line in p4_read_pipe_lines("change -o"):
f3e5ae4f
MSO
749 if line.endswith("\r\n"):
750 line = line[:-2] + "\n"
ea99c3ae
SH
751 if inFilesSection:
752 if line.startswith("\t"):
753 # path starts and ends with a tab
754 path = line[1:]
755 lastTab = path.rfind("\t")
756 if lastTab != -1:
757 path = path[:lastTab]
d53de8b9 758 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
759 continue
760 else:
761 inFilesSection = False
762 else:
763 if line.startswith("Files:"):
764 inFilesSection = True
765
766 template += line
767
768 return template
769
7cb5cbef 770 def applyCommit(self, id):
0e36f2d7 771 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
ae901090 772
848de9c3 773 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 774
ae901090
VA
775 if not self.detectRenames:
776 # If not explicitly set check the config variable
0a9feffc 777 self.detectRenames = gitConfig("git-p4.detectRenames")
ae901090 778
0a9feffc
VA
779 if self.detectRenames.lower() == "false" or self.detectRenames == "":
780 diffOpts = ""
781 elif self.detectRenames.lower() == "true":
ae901090
VA
782 diffOpts = "-M"
783 else:
0a9feffc 784 diffOpts = "-M%s" % self.detectRenames
ae901090 785
0a9feffc
VA
786 detectCopies = gitConfig("git-p4.detectCopies")
787 if detectCopies.lower() == "true":
4fddb41b 788 diffOpts += " -C"
0a9feffc
VA
789 elif detectCopies != "" and detectCopies.lower() != "false":
790 diffOpts += " -C%s" % detectCopies
4fddb41b 791
68cbcf1b 792 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
4fddb41b
VA
793 diffOpts += " --find-copies-harder"
794
0e36f2d7 795 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
796 filesToAdd = set()
797 filesToDelete = set()
d336c158 798 editedFiles = set()
c65b670e 799 filesToChangeExecBit = {}
4f5cf76a 800 for line in diff:
b43b0a3c
CP
801 diff = parseDiffTreeEntry(line)
802 modifier = diff['status']
803 path = diff['src']
4f5cf76a 804 if modifier == "M":
87b611d5 805 p4_system("edit \"%s\"" % path)
c65b670e
CP
806 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
807 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 808 editedFiles.add(path)
4f5cf76a
SH
809 elif modifier == "A":
810 filesToAdd.add(path)
c65b670e 811 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
812 if path in filesToDelete:
813 filesToDelete.remove(path)
814 elif modifier == "D":
815 filesToDelete.add(path)
816 if path in filesToAdd:
817 filesToAdd.remove(path)
4fddb41b
VA
818 elif modifier == "C":
819 src, dest = diff['src'], diff['dst']
820 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
821 if diff['src_sha1'] != diff['dst_sha1']:
822 p4_system("edit \"%s\"" % (dest))
823 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
824 p4_system("edit \"%s\"" % (dest))
825 filesToChangeExecBit[dest] = diff['dst_mode']
826 os.unlink(dest)
827 editedFiles.add(dest)
d9a5f25b 828 elif modifier == "R":
b43b0a3c 829 src, dest = diff['src'], diff['dst']
87b611d5 830 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
ae901090
VA
831 if diff['src_sha1'] != diff['dst_sha1']:
832 p4_system("edit \"%s\"" % (dest))
c65b670e 833 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
ae901090 834 p4_system("edit \"%s\"" % (dest))
c65b670e 835 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
836 os.unlink(dest)
837 editedFiles.add(dest)
838 filesToDelete.add(src)
4f5cf76a
SH
839 else:
840 die("unknown modifier %s for %s" % (modifier, path))
841
0e36f2d7 842 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 843 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
844 tryPatchCmd = patchcmd + "--check -"
845 applyPatchCmd = patchcmd + "--check --apply -"
51a2640a 846
47a130b7 847 if os.system(tryPatchCmd) != 0:
51a2640a
SH
848 print "Unfortunately applying the change failed!"
849 print "What do you want to do?"
850 response = "x"
851 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
852 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
853 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
854 if response == "s":
855 print "Skipping! Good luck with the next patches..."
20947149 856 for f in editedFiles:
87b611d5 857 p4_system("revert \"%s\"" % f);
20947149
SH
858 for f in filesToAdd:
859 system("rm %s" %f)
51a2640a
SH
860 return
861 elif response == "a":
47a130b7 862 os.system(applyPatchCmd)
51a2640a
SH
863 if len(filesToAdd) > 0:
864 print "You may also want to call p4 add on the following files:"
865 print " ".join(filesToAdd)
866 if len(filesToDelete):
867 print "The following files should be scheduled for deletion with p4 delete:"
868 print " ".join(filesToDelete)
cebdf5af
HWN
869 die("Please resolve and submit the conflict manually and "
870 + "continue afterwards with git-p4 submit --continue")
51a2640a
SH
871 elif response == "w":
872 system(diffcmd + " > patch.txt")
873 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af
HWN
874 die("Please resolve and submit the conflict manually and "
875 "continue afterwards with git-p4 submit --continue")
51a2640a 876
47a130b7 877 system(applyPatchCmd)
4f5cf76a
SH
878
879 for f in filesToAdd:
87b611d5 880 p4_system("add \"%s\"" % f)
4f5cf76a 881 for f in filesToDelete:
87b611d5
AK
882 p4_system("revert \"%s\"" % f)
883 p4_system("delete \"%s\"" % f)
4f5cf76a 884
c65b670e
CP
885 # Set/clear executable bits
886 for f in filesToChangeExecBit.keys():
887 mode = filesToChangeExecBit[f]
888 setP4ExecBit(f, mode)
889
0e36f2d7 890 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 891 logMessage = logMessage.strip()
4f5cf76a 892
ea99c3ae 893 template = self.prepareSubmitTemplate()
4f5cf76a
SH
894
895 if self.interactive:
896 submitTemplate = self.prepareLogMessage(template, logMessage)
ecdba36d
LD
897
898 if self.preserveUser:
899 submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
900
67abd417
SB
901 if os.environ.has_key("P4DIFF"):
902 del(os.environ["P4DIFF"])
8b130262
AW
903 diff = ""
904 for editedFile in editedFiles:
905 diff += p4_read_pipe("diff -du %r" % editedFile)
4f5cf76a 906
f3e5ae4f 907 newdiff = ""
4f5cf76a 908 for newFile in filesToAdd:
f3e5ae4f
MSO
909 newdiff += "==== new file ====\n"
910 newdiff += "--- /dev/null\n"
911 newdiff += "+++ %s\n" % newFile
4f5cf76a
SH
912 f = open(newFile, "r")
913 for line in f.readlines():
f3e5ae4f 914 newdiff += "+" + line
4f5cf76a
SH
915 f.close()
916
848de9c3
LD
917 if self.checkAuthorship and not self.p4UserIsMe(p4User):
918 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
919 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
920 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
921
f3e5ae4f 922 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a 923
e96e400f
SH
924 [handle, fileName] = tempfile.mkstemp()
925 tmpFile = os.fdopen(handle, "w+")
f3e5ae4f
MSO
926 if self.isWindows:
927 submitTemplate = submitTemplate.replace("\n", "\r\n")
928 separatorLine = separatorLine.replace("\n", "\r\n")
929 newdiff = newdiff.replace("\n", "\r\n")
930 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
e96e400f 931 tmpFile.close()
cdc7e388 932 mtime = os.stat(fileName).st_mtime
82cea9ff
SB
933 if os.environ.has_key("P4EDITOR"):
934 editor = os.environ.get("P4EDITOR")
935 else:
8b187e6b 936 editor = read_pipe("git var GIT_EDITOR").strip()
e96e400f 937 system(editor + " " + fileName)
e96e400f 938
3ea2cfd4
LD
939 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
940 checkModTime = False
941 else:
942 checkModTime = True
943
cdc7e388 944 response = "y"
3ea2cfd4 945 if checkModTime and (os.stat(fileName).st_mtime <= mtime):
cdc7e388
SH
946 response = "x"
947 while response != "y" and response != "n":
948 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
949
950 if response == "y":
951 tmpFile = open(fileName, "rb")
952 message = tmpFile.read()
953 tmpFile.close()
954 submitTemplate = message[:message.index(separatorLine)]
955 if self.isWindows:
956 submitTemplate = submitTemplate.replace("\r\n", "\n")
957 p4_write_pipe("submit -i", submitTemplate)
3ea2cfd4
LD
958
959 if self.preserveUser:
960 if p4User:
961 # Get last changelist number. Cannot easily get it from
962 # the submit command output as the output is unmarshalled.
963 changelist = self.lastP4Changelist()
964 self.modifyChangelistUser(changelist, p4User)
965
cdc7e388
SH
966 else:
967 for f in editedFiles:
968 p4_system("revert \"%s\"" % f);
969 for f in filesToAdd:
970 p4_system("revert \"%s\"" % f);
971 system("rm %s" %f)
972
973 os.remove(fileName)
4f5cf76a
SH
974 else:
975 fileName = "submit.txt"
976 file = open(fileName, "w+")
977 file.write(self.prepareLogMessage(template, logMessage))
978 file.close()
cebdf5af
HWN
979 print ("Perforce submit template written as %s. "
980 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
981 % (fileName, fileName))
4f5cf76a
SH
982
983 def run(self, args):
c9b50e63
SH
984 if len(args) == 0:
985 self.master = currentGitBranch()
4280e533 986 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
987 die("Detecting current git branch failed!")
988 elif len(args) == 1:
989 self.master = args[0]
990 else:
991 return False
992
4c2d5d72
JX
993 allowSubmit = gitConfig("git-p4.allowSubmit")
994 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
995 die("%s is not in git-p4.allowSubmit" % self.master)
996
27d2d811 997 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 998 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
999 if len(self.origin) == 0:
1000 self.origin = upstream
a3fdd579 1001
3ea2cfd4
LD
1002 if self.preserveUser:
1003 if not self.canChangeChangelists():
1004 die("Cannot preserve user names without p4 super-user or admin permissions")
1005
a3fdd579
SH
1006 if self.verbose:
1007 print "Origin branch is " + self.origin
9512497b 1008
ea99c3ae 1009 if len(self.depotPath) == 0:
9512497b
SH
1010 print "Internal error: cannot locate perforce depot path from existing branches"
1011 sys.exit(128)
1012
ea99c3ae 1013 self.clientPath = p4Where(self.depotPath)
9512497b 1014
51a2640a 1015 if len(self.clientPath) == 0:
ea99c3ae 1016 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
9512497b
SH
1017 sys.exit(128)
1018
ea99c3ae 1019 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1020 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1021
053fd0c1 1022 chdir(self.clientPath)
6a01298a 1023 print "Synchronizing p4 checkout..."
87b611d5 1024 p4_system("sync ...")
9512497b 1025
4f5cf76a 1026 self.check()
4f5cf76a 1027
4c750c0d
SH
1028 commits = []
1029 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1030 commits.append(line.strip())
1031 commits.reverse()
4f5cf76a 1032
848de9c3
LD
1033 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1034 self.checkAuthorship = False
1035 else:
1036 self.checkAuthorship = True
1037
3ea2cfd4
LD
1038 if self.preserveUser:
1039 self.checkValidP4Users(commits)
1040
4f5cf76a 1041 while len(commits) > 0:
4f5cf76a
SH
1042 commit = commits[0]
1043 commits = commits[1:]
7cb5cbef 1044 self.applyCommit(commit)
4f5cf76a
SH
1045 if not self.interactive:
1046 break
1047
4f5cf76a 1048 if len(commits) == 0:
4c750c0d 1049 print "All changes applied!"
053fd0c1 1050 chdir(self.oldWorkingDirectory)
14594f4b 1051
4c750c0d
SH
1052 sync = P4Sync()
1053 sync.run([])
14594f4b 1054
4c750c0d
SH
1055 rebase = P4Rebase()
1056 rebase.rebase()
4f5cf76a 1057
b984733c
SH
1058 return True
1059
3ea2cfd4 1060class P4Sync(Command, P4UserMap):
56c09345
PW
1061 delete_actions = ( "delete", "move/delete", "purge" )
1062
b984733c
SH
1063 def __init__(self):
1064 Command.__init__(self)
3ea2cfd4 1065 P4UserMap.__init__(self)
b984733c
SH
1066 self.options = [
1067 optparse.make_option("--branch", dest="branch"),
1068 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1069 optparse.make_option("--changesfile", dest="changesFile"),
1070 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1071 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
a028a98e 1072 optparse.make_option("--verbose", dest="verbose", action="store_true"),
d2c6dd30
HWN
1073 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1074 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 1075 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 1076 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1077 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1078 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1079 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
1080 ]
1081 self.description = """Imports from Perforce into a git repository.\n
1082 example:
1083 //depot/my/project/ -- to import the current head
1084 //depot/my/project/@all -- to import everything
1085 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1086
1087 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1088
1089 self.usage += " //depot/path[@revRange]"
b984733c 1090 self.silent = False
1d7367dc
RG
1091 self.createdBranches = set()
1092 self.committedChanges = set()
569d1bd4 1093 self.branch = ""
b984733c 1094 self.detectBranches = False
cb53e1f8 1095 self.detectLabels = False
b984733c 1096 self.changesFile = ""
01265103 1097 self.syncWithOrigin = True
4b97ffb1 1098 self.verbose = False
a028a98e 1099 self.importIntoRemotes = True
01a9c9c5 1100 self.maxChanges = ""
c1f9197f 1101 self.isWindows = (platform.system() == "Windows")
8b41a97f 1102 self.keepRepoPath = False
6326aa58 1103 self.depotPaths = None
3c699645 1104 self.p4BranchesInGit = []
354081d5 1105 self.cloneExclude = []
3a70cdfa
TAL
1106 self.useClientSpec = False
1107 self.clientSpecDirs = []
b984733c 1108
01265103
SH
1109 if gitConfig("git-p4.syncFromOrigin") == "false":
1110 self.syncWithOrigin = False
1111
084f6306
PW
1112 #
1113 # P4 wildcards are not allowed in filenames. P4 complains
1114 # if you simply add them, but you can force it with "-f", in
1115 # which case it translates them into %xx encoding internally.
1116 # Search for and fix just these four characters. Do % last so
1117 # that fixing it does not inadvertently create new %-escapes.
1118 #
1119 def wildcard_decode(self, path):
1120 # Cannot have * in a filename in windows; untested as to
1121 # what p4 would do in such a case.
1122 if not self.isWindows:
1123 path = path.replace("%2A", "*")
1124 path = path.replace("%23", "#") \
1125 .replace("%40", "@") \
1126 .replace("%25", "%")
1127 return path
1128
b984733c 1129 def extractFilesFromCommit(self, commit):
354081d5
TT
1130 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1131 for path in self.cloneExclude]
b984733c
SH
1132 files = []
1133 fnum = 0
1134 while commit.has_key("depotFile%s" % fnum):
1135 path = commit["depotFile%s" % fnum]
6326aa58 1136
354081d5 1137 if [p for p in self.cloneExclude
d53de8b9 1138 if p4PathStartsWith(path, p)]:
354081d5
TT
1139 found = False
1140 else:
1141 found = [p for p in self.depotPaths
d53de8b9 1142 if p4PathStartsWith(path, p)]
6326aa58 1143 if not found:
b984733c
SH
1144 fnum = fnum + 1
1145 continue
1146
1147 file = {}
1148 file["path"] = path
1149 file["rev"] = commit["rev%s" % fnum]
1150 file["action"] = commit["action%s" % fnum]
1151 file["type"] = commit["type%s" % fnum]
1152 files.append(file)
1153 fnum = fnum + 1
1154 return files
1155
6326aa58 1156 def stripRepoPath(self, path, prefixes):
3952710b
IW
1157 if self.useClientSpec:
1158
1159 # if using the client spec, we use the output directory
1160 # specified in the client. For example, a view
1161 # //depot/foo/branch/... //client/branch/foo/...
1162 # will end up putting all foo/branch files into
1163 # branch/foo/
1164 for val in self.clientSpecDirs:
1165 if path.startswith(val[0]):
1166 # replace the depot path with the client path
1167 path = path.replace(val[0], val[1][1])
1168 # now strip out the client (//client/...)
1169 path = re.sub("^(//[^/]+/)", '', path)
1170 # the rest is all path
1171 return path
1172
8b41a97f 1173 if self.keepRepoPath:
6326aa58
HWN
1174 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1175
1176 for p in prefixes:
d53de8b9 1177 if p4PathStartsWith(path, p):
6326aa58 1178 path = path[len(p):]
8b41a97f 1179
6326aa58 1180 return path
6754a299 1181
71b112d4 1182 def splitFilesIntoBranches(self, commit):
d5904674 1183 branches = {}
71b112d4
SH
1184 fnum = 0
1185 while commit.has_key("depotFile%s" % fnum):
1186 path = commit["depotFile%s" % fnum]
6326aa58 1187 found = [p for p in self.depotPaths
d53de8b9 1188 if p4PathStartsWith(path, p)]
6326aa58 1189 if not found:
71b112d4
SH
1190 fnum = fnum + 1
1191 continue
1192
1193 file = {}
1194 file["path"] = path
1195 file["rev"] = commit["rev%s" % fnum]
1196 file["action"] = commit["action%s" % fnum]
1197 file["type"] = commit["type%s" % fnum]
1198 fnum = fnum + 1
1199
6326aa58 1200 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 1201
4b97ffb1 1202 for branch in self.knownBranches.keys():
6754a299
HWN
1203
1204 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1205 if relPath.startswith(branch + "/"):
d5904674
SH
1206 if branch not in branches:
1207 branches[branch] = []
71b112d4 1208 branches[branch].append(file)
6555b2cc 1209 break
b984733c
SH
1210
1211 return branches
1212
b932705b
LD
1213 # output one file from the P4 stream
1214 # - helper for streamP4Files
1215
1216 def streamOneP4File(self, file, contents):
c3f6163b
AG
1217 if file["type"] == "apple":
1218 print "\nfile %s is a strange apple file that forks. Ignoring" % \
1219 file['depotFile']
1220 return
b932705b
LD
1221
1222 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
084f6306 1223 relPath = self.wildcard_decode(relPath)
b932705b
LD
1224 if verbose:
1225 sys.stderr.write("%s\n" % relPath)
1226
1227 mode = "644"
1228 if isP4Exec(file["type"]):
1229 mode = "755"
1230 elif file["type"] == "symlink":
1231 mode = "120000"
1232 # p4 print on a symlink contains "target\n", so strip it off
b39c3612
EP
1233 data = ''.join(contents)
1234 contents = [data[:-1]]
b932705b
LD
1235
1236 if self.isWindows and file["type"].endswith("text"):
1237 mangled = []
1238 for data in contents:
1239 data = data.replace("\r\n", "\n")
1240 mangled.append(data)
1241 contents = mangled
1242
1243 if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1244 contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1245 elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1246 contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1247
1248 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1249
1250 # total length...
1251 length = 0
1252 for d in contents:
1253 length = length + len(d)
1254
1255 self.gitStream.write("data %d\n" % length)
1256 for d in contents:
1257 self.gitStream.write(d)
1258 self.gitStream.write("\n")
1259
1260 def streamOneP4Deletion(self, file):
1261 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1262 if verbose:
1263 sys.stderr.write("delete %s\n" % relPath)
1264 self.gitStream.write("D %s\n" % relPath)
1265
1266 # handle another chunk of streaming data
1267 def streamP4FilesCb(self, marshalled):
1268
c3f6163b
AG
1269 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1270 # start of a new file - output the old one first
1271 self.streamOneP4File(self.stream_file, self.stream_contents)
1272 self.stream_file = {}
1273 self.stream_contents = []
1274 self.stream_have_file_info = False
b932705b 1275
c3f6163b
AG
1276 # pick up the new file information... for the
1277 # 'data' field we need to append to our array
1278 for k in marshalled.keys():
1279 if k == 'data':
1280 self.stream_contents.append(marshalled['data'])
1281 else:
1282 self.stream_file[k] = marshalled[k]
b932705b 1283
c3f6163b 1284 self.stream_have_file_info = True
b932705b
LD
1285
1286 # Stream directly from "p4 files" into "git fast-import"
1287 def streamP4Files(self, files):
30b5940b
SH
1288 filesForCommit = []
1289 filesToRead = []
b932705b 1290 filesToDelete = []
30b5940b 1291
3a70cdfa 1292 for f in files:
30b5940b 1293 includeFile = True
3a70cdfa
TAL
1294 for val in self.clientSpecDirs:
1295 if f['path'].startswith(val[0]):
3952710b 1296 if val[1][0] <= 0:
30b5940b 1297 includeFile = False
3a70cdfa
TAL
1298 break
1299
30b5940b
SH
1300 if includeFile:
1301 filesForCommit.append(f)
56c09345 1302 if f['action'] in self.delete_actions:
b932705b 1303 filesToDelete.append(f)
56c09345
PW
1304 else:
1305 filesToRead.append(f)
6a49f8e2 1306
b932705b
LD
1307 # deleted files...
1308 for f in filesToDelete:
1309 self.streamOneP4Deletion(f)
1b9a4684 1310
b932705b
LD
1311 if len(filesToRead) > 0:
1312 self.stream_file = {}
1313 self.stream_contents = []
1314 self.stream_have_file_info = False
8ff45f2a 1315
c3f6163b
AG
1316 # curry self argument
1317 def streamP4FilesCbSelf(entry):
1318 self.streamP4FilesCb(entry)
6a49f8e2 1319
c3f6163b
AG
1320 p4CmdList("-x - print",
1321 '\n'.join(['%s#%s' % (f['path'], f['rev'])
b932705b 1322 for f in filesToRead]),
c3f6163b 1323 cb=streamP4FilesCbSelf)
30b5940b 1324
b932705b
LD
1325 # do the last chunk
1326 if self.stream_file.has_key('depotFile'):
1327 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 1328
6326aa58 1329 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
1330 epoch = details["time"]
1331 author = details["user"]
c3f6163b 1332 self.branchPrefixes = branchPrefixes
b984733c 1333
4b97ffb1
SH
1334 if self.verbose:
1335 print "commit into %s" % branch
1336
96e07dd2
HWN
1337 # start with reading files; if that fails, we should not
1338 # create a commit.
1339 new_files = []
1340 for f in files:
d53de8b9 1341 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
1342 new_files.append (f)
1343 else:
afa1dd9a 1344 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 1345
b984733c 1346 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 1347# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
1348 self.committedChanges.add(int(details["change"]))
1349 committer = ""
b607e71e
SH
1350 if author not in self.users:
1351 self.getUserMapFromPerforceServer()
b984733c 1352 if author in self.users:
0828ab14 1353 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
b984733c 1354 else:
0828ab14 1355 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
b984733c
SH
1356
1357 self.gitStream.write("committer %s\n" % committer)
1358
1359 self.gitStream.write("data <<EOT\n")
1360 self.gitStream.write(details["desc"])
6581de09
SH
1361 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1362 % (','.join (branchPrefixes), details["change"]))
1363 if len(details['options']) > 0:
1364 self.gitStream.write(": options = %s" % details['options'])
1365 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
1366
1367 if len(parent) > 0:
4b97ffb1
SH
1368 if self.verbose:
1369 print "parent %s" % parent
b984733c
SH
1370 self.gitStream.write("from %s\n" % parent)
1371
b932705b 1372 self.streamP4Files(new_files)
b984733c
SH
1373 self.gitStream.write("\n")
1374
1f4ba1cb
SH
1375 change = int(details["change"])
1376
9bda3a85 1377 if self.labels.has_key(change):
1f4ba1cb
SH
1378 label = self.labels[change]
1379 labelDetails = label[0]
1380 labelRevisions = label[1]
71b112d4
SH
1381 if self.verbose:
1382 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 1383
6326aa58
HWN
1384 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1385 for p in branchPrefixes]))
1f4ba1cb
SH
1386
1387 if len(files) == len(labelRevisions):
1388
1389 cleanedFiles = {}
1390 for info in files:
56c09345 1391 if info["action"] in self.delete_actions:
1f4ba1cb
SH
1392 continue
1393 cleanedFiles[info["depotFile"]] = info["rev"]
1394
1395 if cleanedFiles == labelRevisions:
1396 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1397 self.gitStream.write("from %s\n" % branch)
1398
1399 owner = labelDetails["Owner"]
1400 tagger = ""
1401 if author in self.users:
1402 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1403 else:
1404 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1405 self.gitStream.write("tagger %s\n" % tagger)
1406 self.gitStream.write("data <<EOT\n")
1407 self.gitStream.write(labelDetails["Description"])
1408 self.gitStream.write("EOT\n\n")
1409
1410 else:
a46668fa 1411 if not self.silent:
cebdf5af
HWN
1412 print ("Tag %s does not match with change %s: files do not match."
1413 % (labelDetails["label"], change))
1f4ba1cb
SH
1414
1415 else:
a46668fa 1416 if not self.silent:
cebdf5af
HWN
1417 print ("Tag %s does not match with change %s: file count is different."
1418 % (labelDetails["label"], change))
b984733c 1419
1f4ba1cb
SH
1420 def getLabels(self):
1421 self.labels = {}
1422
6326aa58 1423 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
10c3211b 1424 if len(l) > 0 and not self.silent:
183f8436 1425 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
1426
1427 for output in l:
1f4ba1cb
SH
1428 label = output["label"]
1429 revisions = {}
1430 newestChange = 0
71b112d4
SH
1431 if self.verbose:
1432 print "Querying files for label %s" % label
6326aa58
HWN
1433 for file in p4CmdList("files "
1434 + ' '.join (["%s...@%s" % (p, label)
1435 for p in self.depotPaths])):
1f4ba1cb
SH
1436 revisions[file["depotFile"]] = file["rev"]
1437 change = int(file["change"])
1438 if change > newestChange:
1439 newestChange = change
1440
9bda3a85
SH
1441 self.labels[newestChange] = [output, revisions]
1442
1443 if self.verbose:
1444 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 1445
86dff6b6
HWN
1446 def guessProjectName(self):
1447 for p in self.depotPaths:
6e5295c4
SH
1448 if p.endswith("/"):
1449 p = p[:-1]
1450 p = p[p.strip().rfind("/") + 1:]
1451 if not p.endswith("/"):
1452 p += "/"
1453 return p
86dff6b6 1454
4b97ffb1 1455 def getBranchMapping(self):
6555b2cc
SH
1456 lostAndFoundBranches = set()
1457
4b97ffb1
SH
1458 for info in p4CmdList("branches"):
1459 details = p4Cmd("branch -o %s" % info["branch"])
1460 viewIdx = 0
1461 while details.has_key("View%s" % viewIdx):
1462 paths = details["View%s" % viewIdx].split(" ")
1463 viewIdx = viewIdx + 1
1464 # require standard //depot/foo/... //depot/bar/... mapping
1465 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1466 continue
1467 source = paths[0]
1468 destination = paths[1]
6509e19c 1469 ## HACK
d53de8b9 1470 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
1471 source = source[len(self.depotPaths[0]):-4]
1472 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 1473
1a2edf4e
SH
1474 if destination in self.knownBranches:
1475 if not self.silent:
1476 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1477 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1478 continue
1479
6555b2cc
SH
1480 self.knownBranches[destination] = source
1481
1482 lostAndFoundBranches.discard(destination)
1483
29bdbac1 1484 if source not in self.knownBranches:
6555b2cc
SH
1485 lostAndFoundBranches.add(source)
1486
1487
1488 for branch in lostAndFoundBranches:
1489 self.knownBranches[branch] = branch
29bdbac1 1490
38f9f5ec
SH
1491 def getBranchMappingFromGitBranches(self):
1492 branches = p4BranchesInGit(self.importIntoRemotes)
1493 for branch in branches.keys():
1494 if branch == "master":
1495 branch = "main"
1496 else:
1497 branch = branch[len(self.projectName):]
1498 self.knownBranches[branch] = branch
1499
29bdbac1 1500 def listExistingP4GitBranches(self):
144ff46b
SH
1501 # branches holds mapping from name to commit
1502 branches = p4BranchesInGit(self.importIntoRemotes)
1503 self.p4BranchesInGit = branches.keys()
1504 for branch in branches.keys():
1505 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 1506
bb6e09b2
HWN
1507 def updateOptionDict(self, d):
1508 option_keys = {}
1509 if self.keepRepoPath:
1510 option_keys['keepRepoPath'] = 1
1511
1512 d["options"] = ' '.join(sorted(option_keys.keys()))
1513
1514 def readOptions(self, d):
1515 self.keepRepoPath = (d.has_key('options')
1516 and ('keepRepoPath' in d['options']))
6326aa58 1517
8134f69c
SH
1518 def gitRefForBranch(self, branch):
1519 if branch == "main":
1520 return self.refPrefix + "master"
1521
1522 if len(branch) <= 0:
1523 return branch
1524
1525 return self.refPrefix + self.projectName + branch
1526
1ca3d710
SH
1527 def gitCommitByP4Change(self, ref, change):
1528 if self.verbose:
1529 print "looking in ref " + ref + " for change %s using bisect..." % change
1530
1531 earliestCommit = ""
1532 latestCommit = parseRevision(ref)
1533
1534 while True:
1535 if self.verbose:
1536 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1537 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1538 if len(next) == 0:
1539 if self.verbose:
1540 print "argh"
1541 return ""
1542 log = extractLogMessageFromGitCommit(next)
1543 settings = extractSettingsGitLog(log)
1544 currentChange = int(settings['change'])
1545 if self.verbose:
1546 print "current change %s" % currentChange
1547
1548 if currentChange == change:
1549 if self.verbose:
1550 print "found %s" % next
1551 return next
1552
1553 if currentChange < change:
1554 earliestCommit = "^%s" % next
1555 else:
1556 latestCommit = "%s" % next
1557
1558 return ""
1559
1560 def importNewBranch(self, branch, maxChange):
1561 # make fast-import flush all changes to disk and update the refs using the checkpoint
1562 # command so that we can try to find the branch parent in the git history
1563 self.gitStream.write("checkpoint\n\n");
1564 self.gitStream.flush();
1565 branchPrefix = self.depotPaths[0] + branch + "/"
1566 range = "@1,%s" % maxChange
1567 #print "prefix" + branchPrefix
1568 changes = p4ChangesForPaths([branchPrefix], range)
1569 if len(changes) <= 0:
1570 return False
1571 firstChange = changes[0]
1572 #print "first change in branch: %s" % firstChange
1573 sourceBranch = self.knownBranches[branch]
1574 sourceDepotPath = self.depotPaths[0] + sourceBranch
1575 sourceRef = self.gitRefForBranch(sourceBranch)
1576 #print "source " + sourceBranch
1577
1578 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1579 #print "branch parent: %s" % branchParentChange
1580 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1581 if len(gitParent) > 0:
1582 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1583 #print "parent git commit: %s" % gitParent
1584
1585 self.importChanges(changes)
1586 return True
1587
e87f37ae
SH
1588 def importChanges(self, changes):
1589 cnt = 1
1590 for change in changes:
1591 description = p4Cmd("describe %s" % change)
1592 self.updateOptionDict(description)
1593
1594 if not self.silent:
1595 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1596 sys.stdout.flush()
1597 cnt = cnt + 1
1598
1599 try:
1600 if self.detectBranches:
1601 branches = self.splitFilesIntoBranches(description)
1602 for branch in branches.keys():
1603 ## HACK --hwn
1604 branchPrefix = self.depotPaths[0] + branch + "/"
1605
1606 parent = ""
1607
1608 filesForCommit = branches[branch]
1609
1610 if self.verbose:
1611 print "branch is %s" % branch
1612
1613 self.updatedBranches.add(branch)
1614
1615 if branch not in self.createdBranches:
1616 self.createdBranches.add(branch)
1617 parent = self.knownBranches[branch]
1618 if parent == branch:
1619 parent = ""
1ca3d710
SH
1620 else:
1621 fullBranch = self.projectName + branch
1622 if fullBranch not in self.p4BranchesInGit:
1623 if not self.silent:
1624 print("\n Importing new branch %s" % fullBranch);
1625 if self.importNewBranch(branch, change - 1):
1626 parent = ""
1627 self.p4BranchesInGit.append(fullBranch)
1628 if not self.silent:
1629 print("\n Resuming with change %s" % change);
1630
1631 if self.verbose:
1632 print "parent determined through known branches: %s" % parent
e87f37ae 1633
8134f69c
SH
1634 branch = self.gitRefForBranch(branch)
1635 parent = self.gitRefForBranch(parent)
e87f37ae
SH
1636
1637 if self.verbose:
1638 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1639
1640 if len(parent) == 0 and branch in self.initialParents:
1641 parent = self.initialParents[branch]
1642 del self.initialParents[branch]
1643
1644 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1645 else:
1646 files = self.extractFilesFromCommit(description)
1647 self.commit(description, files, self.branch, self.depotPaths,
1648 self.initialParent)
1649 self.initialParent = ""
1650 except IOError:
1651 print self.gitError.read()
1652 sys.exit(1)
1653
c208a243
SH
1654 def importHeadRevision(self, revision):
1655 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1656
4e2e6ce4
PW
1657 details = {}
1658 details["user"] = "git perforce import user"
1494fcbb 1659 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
1660 % (' '.join(self.depotPaths), revision))
1661 details["change"] = revision
1662 newestRevision = 0
1663
1664 fileCnt = 0
1665 for info in p4CmdList("files "
1666 + ' '.join(["%s...%s"
1667 % (p, revision)
1668 for p in self.depotPaths])):
1669
68b28593 1670 if 'code' in info and info['code'] == 'error':
c208a243
SH
1671 sys.stderr.write("p4 returned an error: %s\n"
1672 % info['data'])
d88e707f
PW
1673 if info['data'].find("must refer to client") >= 0:
1674 sys.stderr.write("This particular p4 error is misleading.\n")
1675 sys.stderr.write("Perhaps the depot path was misspelled.\n");
1676 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 1677 sys.exit(1)
68b28593
PW
1678 if 'p4ExitCode' in info:
1679 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
1680 sys.exit(1)
1681
1682
1683 change = int(info["change"])
1684 if change > newestRevision:
1685 newestRevision = change
1686
56c09345 1687 if info["action"] in self.delete_actions:
c208a243
SH
1688 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1689 #fileCnt = fileCnt + 1
1690 continue
1691
1692 for prop in ["depotFile", "rev", "action", "type" ]:
1693 details["%s%s" % (prop, fileCnt)] = info[prop]
1694
1695 fileCnt = fileCnt + 1
1696
1697 details["change"] = newestRevision
4e2e6ce4
PW
1698
1699 # Use time from top-most change so that all git-p4 clones of
1700 # the same p4 repo have the same commit SHA1s.
1701 res = p4CmdList("describe -s %d" % newestRevision)
1702 newestTime = None
1703 for r in res:
1704 if r.has_key('time'):
1705 newestTime = int(r['time'])
1706 if newestTime is None:
1707 die("\"describe -s\" on newest change %d did not give a time")
1708 details["time"] = newestTime
1709
c208a243
SH
1710 self.updateOptionDict(details)
1711 try:
1712 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1713 except IOError:
1714 print "IO error with git fast-import. Is your git version recent enough?"
1715 print self.gitError.read()
1716
1717
3a70cdfa
TAL
1718 def getClientSpec(self):
1719 specList = p4CmdList( "client -o" )
1720 temp = {}
1721 for entry in specList:
1722 for k,v in entry.iteritems():
1723 if k.startswith("View"):
3952710b
IW
1724
1725 # p4 has these %%1 to %%9 arguments in specs to
1726 # reorder paths; which we can't handle (yet :)
1727 if re.match('%%\d', v) != None:
1728 print "Sorry, can't handle %%n arguments in client specs"
1729 sys.exit(1)
1730
3a70cdfa
TAL
1731 if v.startswith('"'):
1732 start = 1
1733 else:
1734 start = 0
1735 index = v.find("...")
3952710b
IW
1736
1737 # save the "client view"; i.e the RHS of the view
1738 # line that tells the client where to put the
1739 # files for this view.
1740 cv = v[index+3:].strip() # +3 to remove previous '...'
1741
1742 # if the client view doesn't end with a
1743 # ... wildcard, then we're going to mess up the
1744 # output directory, so fail gracefully.
1745 if not cv.endswith('...'):
1746 print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1747 sys.exit(1)
1748 cv=cv[:-3]
1749
1750 # now save the view; +index means included, -index
1751 # means it should be filtered out.
3a70cdfa
TAL
1752 v = v[start:index]
1753 if v.startswith("-"):
1754 v = v[1:]
3952710b 1755 include = -len(v)
3a70cdfa 1756 else:
3952710b
IW
1757 include = len(v)
1758
1759 temp[v] = (include, cv)
1760
3a70cdfa 1761 self.clientSpecDirs = temp.items()
3952710b 1762 self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
3a70cdfa 1763
b984733c 1764 def run(self, args):
6326aa58 1765 self.depotPaths = []
179caebf
SH
1766 self.changeRange = ""
1767 self.initialParent = ""
6326aa58 1768 self.previousDepotPaths = []
ce6f33c8 1769
29bdbac1
SH
1770 # map from branch depot path to parent branch
1771 self.knownBranches = {}
1772 self.initialParents = {}
5ca44617 1773 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
1774 if not self.syncWithOrigin:
1775 self.hasOrigin = False
29bdbac1 1776
a028a98e
SH
1777 if self.importIntoRemotes:
1778 self.refPrefix = "refs/remotes/p4/"
1779 else:
db775559 1780 self.refPrefix = "refs/heads/p4/"
a028a98e 1781
cebdf5af
HWN
1782 if self.syncWithOrigin and self.hasOrigin:
1783 if not self.silent:
1784 print "Syncing with origin first by calling git fetch origin"
1785 system("git fetch origin")
10f880f8 1786
569d1bd4 1787 if len(self.branch) == 0:
db775559 1788 self.branch = self.refPrefix + "master"
a028a98e 1789 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 1790 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 1791 system("git branch -D p4");
faf1bd20 1792 # create it /after/ importing, when master exists
0058a33a 1793 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 1794 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 1795
3cafb7d8 1796 if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
3a70cdfa
TAL
1797 self.getClientSpec()
1798
6a49f8e2
HWN
1799 # TODO: should always look at previous commits,
1800 # merge with previous imports, if possible.
1801 if args == []:
d414c74a 1802 if self.hasOrigin:
5ca44617 1803 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
1804 self.listExistingP4GitBranches()
1805
1806 if len(self.p4BranchesInGit) > 1:
1807 if not self.silent:
1808 print "Importing from/into multiple branches"
1809 self.detectBranches = True
967f72e2 1810
29bdbac1
SH
1811 if self.verbose:
1812 print "branches: %s" % self.p4BranchesInGit
1813
1814 p4Change = 0
1815 for branch in self.p4BranchesInGit:
cebdf5af 1816 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
1817
1818 settings = extractSettingsGitLog(logMsg)
29bdbac1 1819
bb6e09b2
HWN
1820 self.readOptions(settings)
1821 if (settings.has_key('depot-paths')
1822 and settings.has_key ('change')):
1823 change = int(settings['change']) + 1
29bdbac1
SH
1824 p4Change = max(p4Change, change)
1825
bb6e09b2
HWN
1826 depotPaths = sorted(settings['depot-paths'])
1827 if self.previousDepotPaths == []:
6326aa58 1828 self.previousDepotPaths = depotPaths
29bdbac1 1829 else:
6326aa58
HWN
1830 paths = []
1831 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
583e1707 1832 for i in range(0, min(len(cur), len(prev))):
6326aa58 1833 if cur[i] <> prev[i]:
583e1707 1834 i = i - 1
6326aa58
HWN
1835 break
1836
583e1707 1837 paths.append (cur[:i + 1])
6326aa58
HWN
1838
1839 self.previousDepotPaths = paths
29bdbac1
SH
1840
1841 if p4Change > 0:
bb6e09b2 1842 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 1843 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
1844 if not self.detectBranches:
1845 self.initialParent = parseRevision(self.branch)
341dc1c1 1846 if not self.silent and not self.detectBranches:
967f72e2 1847 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 1848
f9162f6a
SH
1849 if not self.branch.startswith("refs/"):
1850 self.branch = "refs/heads/" + self.branch
179caebf 1851
6326aa58 1852 if len(args) == 0 and self.depotPaths:
b984733c 1853 if not self.silent:
6326aa58 1854 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 1855 else:
6326aa58 1856 if self.depotPaths and self.depotPaths != args:
cebdf5af 1857 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
1858 "This doesn't work!" % (' '.join (self.depotPaths),
1859 ' '.join (args)))
b984733c 1860 sys.exit(1)
6326aa58 1861
bb6e09b2 1862 self.depotPaths = sorted(args)
b984733c 1863
1c49fc19 1864 revision = ""
b984733c 1865 self.users = {}
b984733c 1866
6326aa58
HWN
1867 newPaths = []
1868 for p in self.depotPaths:
1869 if p.find("@") != -1:
1870 atIdx = p.index("@")
1871 self.changeRange = p[atIdx:]
1872 if self.changeRange == "@all":
1873 self.changeRange = ""
6a49f8e2 1874 elif ',' not in self.changeRange:
1c49fc19 1875 revision = self.changeRange
6326aa58 1876 self.changeRange = ""
7fcff9de 1877 p = p[:atIdx]
6326aa58
HWN
1878 elif p.find("#") != -1:
1879 hashIdx = p.index("#")
1c49fc19 1880 revision = p[hashIdx:]
7fcff9de 1881 p = p[:hashIdx]
6326aa58 1882 elif self.previousDepotPaths == []:
1c49fc19 1883 revision = "#head"
6326aa58
HWN
1884
1885 p = re.sub ("\.\.\.$", "", p)
1886 if not p.endswith("/"):
1887 p += "/"
1888
1889 newPaths.append(p)
1890
1891 self.depotPaths = newPaths
1892
b984733c 1893
b607e71e 1894 self.loadUserMapFromCache()
cb53e1f8
SH
1895 self.labels = {}
1896 if self.detectLabels:
1897 self.getLabels();
b984733c 1898
4b97ffb1 1899 if self.detectBranches:
df450923
SH
1900 ## FIXME - what's a P4 projectName ?
1901 self.projectName = self.guessProjectName()
1902
38f9f5ec
SH
1903 if self.hasOrigin:
1904 self.getBranchMappingFromGitBranches()
1905 else:
1906 self.getBranchMapping()
29bdbac1
SH
1907 if self.verbose:
1908 print "p4-git branches: %s" % self.p4BranchesInGit
1909 print "initial parents: %s" % self.initialParents
1910 for b in self.p4BranchesInGit:
1911 if b != "master":
6326aa58
HWN
1912
1913 ## FIXME
29bdbac1
SH
1914 b = b[len(self.projectName):]
1915 self.createdBranches.add(b)
4b97ffb1 1916
f291b4e3 1917 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 1918
cebdf5af 1919 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
1920 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1921 stderr=subprocess.PIPE);
08483580
SH
1922 self.gitOutput = importProcess.stdout
1923 self.gitStream = importProcess.stdin
1924 self.gitError = importProcess.stderr
b984733c 1925
1c49fc19 1926 if revision:
c208a243 1927 self.importHeadRevision(revision)
b984733c
SH
1928 else:
1929 changes = []
1930
0828ab14 1931 if len(self.changesFile) > 0:
b984733c 1932 output = open(self.changesFile).readlines()
1d7367dc 1933 changeSet = set()
b984733c
SH
1934 for line in output:
1935 changeSet.add(int(line))
1936
1937 for change in changeSet:
1938 changes.append(change)
1939
1940 changes.sort()
1941 else:
accad8e0
PW
1942 # catch "git-p4 sync" with no new branches, in a repo that
1943 # does not have any existing git-p4 branches
1944 if len(args) == 0 and not self.p4BranchesInGit:
e32e00dc 1945 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
29bdbac1 1946 if self.verbose:
86dff6b6 1947 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 1948 self.changeRange)
4f6432d8 1949 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 1950
01a9c9c5 1951 if len(self.maxChanges) > 0:
7fcff9de 1952 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 1953
b984733c 1954 if len(changes) == 0:
0828ab14 1955 if not self.silent:
341dc1c1 1956 print "No changes to import!"
1f52af6c 1957 return True
b984733c 1958
a9d1a27a
SH
1959 if not self.silent and not self.detectBranches:
1960 print "Import destination: %s" % self.branch
1961
341dc1c1
SH
1962 self.updatedBranches = set()
1963
e87f37ae 1964 self.importChanges(changes)
b984733c 1965
341dc1c1
SH
1966 if not self.silent:
1967 print ""
1968 if len(self.updatedBranches) > 0:
1969 sys.stdout.write("Updated branches: ")
1970 for b in self.updatedBranches:
1971 sys.stdout.write("%s " % b)
1972 sys.stdout.write("\n")
b984733c 1973
b984733c 1974 self.gitStream.close()
29bdbac1
SH
1975 if importProcess.wait() != 0:
1976 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
1977 self.gitOutput.close()
1978 self.gitError.close()
1979
b984733c
SH
1980 return True
1981
01ce1fe9
SH
1982class P4Rebase(Command):
1983 def __init__(self):
1984 Command.__init__(self)
01265103 1985 self.options = [ ]
cebdf5af
HWN
1986 self.description = ("Fetches the latest revision from perforce and "
1987 + "rebases the current work (branch) against it")
68c42153 1988 self.verbose = False
01ce1fe9
SH
1989
1990 def run(self, args):
1991 sync = P4Sync()
1992 sync.run([])
d7e3868c 1993
14594f4b
SH
1994 return self.rebase()
1995
1996 def rebase(self):
36ee4ee4
SH
1997 if os.system("git update-index --refresh") != 0:
1998 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1999 if len(read_pipe("git diff-index HEAD --")) > 0:
2000 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2001
d7e3868c
SH
2002 [upstream, settings] = findUpstreamBranchPoint()
2003 if len(upstream) == 0:
2004 die("Cannot find upstream branchpoint for rebase")
2005
2006 # the branchpoint may be p4/foo~3, so strip off the parent
2007 upstream = re.sub("~[0-9]+$", "", upstream)
2008
2009 print "Rebasing the current branch onto %s" % upstream
b25b2065 2010 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 2011 system("git rebase %s" % upstream)
1f52af6c 2012 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
2013 return True
2014
f9a3a4f7
SH
2015class P4Clone(P4Sync):
2016 def __init__(self):
2017 P4Sync.__init__(self)
2018 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 2019 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 2020 self.options += [
bb6e09b2
HWN
2021 optparse.make_option("--destination", dest="cloneDestination",
2022 action='store', default=None,
354081d5
TT
2023 help="where to leave result of the clone"),
2024 optparse.make_option("-/", dest="cloneExclude",
2025 action="append", type="string",
38200076
PW
2026 help="exclude depot path"),
2027 optparse.make_option("--bare", dest="cloneBare",
2028 action="store_true", default=False),
354081d5 2029 ]
bb6e09b2 2030 self.cloneDestination = None
f9a3a4f7 2031 self.needsGit = False
38200076 2032 self.cloneBare = False
f9a3a4f7 2033
354081d5
TT
2034 # This is required for the "append" cloneExclude action
2035 def ensure_value(self, attr, value):
2036 if not hasattr(self, attr) or getattr(self, attr) is None:
2037 setattr(self, attr, value)
2038 return getattr(self, attr)
2039
6a49f8e2
HWN
2040 def defaultDestination(self, args):
2041 ## TODO: use common prefix of args?
2042 depotPath = args[0]
2043 depotDir = re.sub("(@[^@]*)$", "", depotPath)
2044 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 2045 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
2046 depotDir = re.sub(r"/$", "", depotDir)
2047 return os.path.split(depotDir)[1]
2048
f9a3a4f7
SH
2049 def run(self, args):
2050 if len(args) < 1:
2051 return False
bb6e09b2
HWN
2052
2053 if self.keepRepoPath and not self.cloneDestination:
2054 sys.stderr.write("Must specify destination for --keep-path\n")
2055 sys.exit(1)
f9a3a4f7 2056
6326aa58 2057 depotPaths = args
5e100b5c
SH
2058
2059 if not self.cloneDestination and len(depotPaths) > 1:
2060 self.cloneDestination = depotPaths[-1]
2061 depotPaths = depotPaths[:-1]
2062
354081d5 2063 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
2064 for p in depotPaths:
2065 if not p.startswith("//"):
2066 return False
f9a3a4f7 2067
bb6e09b2 2068 if not self.cloneDestination:
98ad4faf 2069 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 2070
86dff6b6 2071 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 2072
c3bf3f13
KG
2073 if not os.path.exists(self.cloneDestination):
2074 os.makedirs(self.cloneDestination)
053fd0c1 2075 chdir(self.cloneDestination)
38200076
PW
2076
2077 init_cmd = [ "git", "init" ]
2078 if self.cloneBare:
2079 init_cmd.append("--bare")
2080 subprocess.check_call(init_cmd)
2081
6326aa58 2082 if not P4Sync.run(self, depotPaths):
f9a3a4f7 2083 return False
f9a3a4f7 2084 if self.branch != "master":
e9905013
TAL
2085 if self.importIntoRemotes:
2086 masterbranch = "refs/remotes/p4/master"
2087 else:
2088 masterbranch = "refs/heads/p4/master"
2089 if gitBranchExists(masterbranch):
2090 system("git branch master %s" % masterbranch)
38200076
PW
2091 if not self.cloneBare:
2092 system("git checkout -f")
8f9b2e08
SH
2093 else:
2094 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 2095
f9a3a4f7
SH
2096 return True
2097
09d89de2
SH
2098class P4Branches(Command):
2099 def __init__(self):
2100 Command.__init__(self)
2101 self.options = [ ]
2102 self.description = ("Shows the git branches that hold imports and their "
2103 + "corresponding perforce depot paths")
2104 self.verbose = False
2105
2106 def run(self, args):
5ca44617
SH
2107 if originP4BranchesExist():
2108 createOrUpdateBranchesFromOrigin()
2109
09d89de2
SH
2110 cmdline = "git rev-parse --symbolic "
2111 cmdline += " --remotes"
2112
2113 for line in read_pipe_lines(cmdline):
2114 line = line.strip()
2115
2116 if not line.startswith('p4/') or line == "p4/HEAD":
2117 continue
2118 branch = line
2119
2120 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2121 settings = extractSettingsGitLog(log)
2122
2123 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2124 return True
2125
b984733c
SH
2126class HelpFormatter(optparse.IndentedHelpFormatter):
2127 def __init__(self):
2128 optparse.IndentedHelpFormatter.__init__(self)
2129
2130 def format_description(self, description):
2131 if description:
2132 return description + "\n"
2133 else:
2134 return ""
4f5cf76a 2135
86949eef
SH
2136def printUsage(commands):
2137 print "usage: %s <command> [options]" % sys.argv[0]
2138 print ""
2139 print "valid commands: %s" % ", ".join(commands)
2140 print ""
2141 print "Try %s <command> --help for command specific help." % sys.argv[0]
2142 print ""
2143
2144commands = {
b86f7378
HWN
2145 "debug" : P4Debug,
2146 "submit" : P4Submit,
a9834f58 2147 "commit" : P4Submit,
b86f7378
HWN
2148 "sync" : P4Sync,
2149 "rebase" : P4Rebase,
2150 "clone" : P4Clone,
09d89de2
SH
2151 "rollback" : P4RollBack,
2152 "branches" : P4Branches
86949eef
SH
2153}
2154
86949eef 2155
bb6e09b2
HWN
2156def main():
2157 if len(sys.argv[1:]) == 0:
2158 printUsage(commands.keys())
2159 sys.exit(2)
4f5cf76a 2160
bb6e09b2
HWN
2161 cmd = ""
2162 cmdName = sys.argv[1]
2163 try:
b86f7378
HWN
2164 klass = commands[cmdName]
2165 cmd = klass()
bb6e09b2
HWN
2166 except KeyError:
2167 print "unknown command %s" % cmdName
2168 print ""
2169 printUsage(commands.keys())
2170 sys.exit(2)
2171
2172 options = cmd.options
b86f7378 2173 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
2174
2175 args = sys.argv[2:]
2176
2177 if len(options) > 0:
2178 options.append(optparse.make_option("--git-dir", dest="gitdir"))
2179
2180 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2181 options,
2182 description = cmd.description,
2183 formatter = HelpFormatter())
2184
2185 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2186 global verbose
2187 verbose = cmd.verbose
2188 if cmd.needsGit:
b86f7378
HWN
2189 if cmd.gitdir == None:
2190 cmd.gitdir = os.path.abspath(".git")
2191 if not isValidGitDir(cmd.gitdir):
2192 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2193 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
2194 cdup = read_pipe("git rev-parse --show-cdup").strip()
2195 if len(cdup) > 0:
053fd0c1 2196 chdir(cdup);
e20a9e53 2197
b86f7378
HWN
2198 if not isValidGitDir(cmd.gitdir):
2199 if isValidGitDir(cmd.gitdir + "/.git"):
2200 cmd.gitdir += "/.git"
bb6e09b2 2201 else:
b86f7378 2202 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 2203
b86f7378 2204 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 2205
bb6e09b2
HWN
2206 if not cmd.run(args):
2207 parser.print_help()
4f5cf76a 2208
4f5cf76a 2209
bb6e09b2
HWN
2210if __name__ == '__main__':
2211 main()