git-p4: keyword flattening fixes
[git/git.git] / contrib / fast-import / git-p4
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
1d7367dc
RG
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
ce6f33c8 13import re
8b41a97f 14
4addad22 15verbose = False
86949eef 16
21a50753
AK
17
18def p4_build_cmd(cmd):
19 """Build a suitable p4 command line.
20
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
24 """
abcaf073
AK
25 real_cmd = "%s " % "p4"
26
27 user = gitConfig("git-p4.user")
28 if len(user) > 0:
29 real_cmd += "-u %s " % user
30
31 password = gitConfig("git-p4.password")
32 if len(password) > 0:
33 real_cmd += "-P %s " % password
34
35 port = gitConfig("git-p4.port")
36 if len(port) > 0:
37 real_cmd += "-p %s " % port
38
39 host = gitConfig("git-p4.host")
40 if len(host) > 0:
41 real_cmd += "-h %s " % host
42
43 client = gitConfig("git-p4.client")
44 if len(client) > 0:
45 real_cmd += "-c %s " % client
46
47 real_cmd += "%s" % (cmd)
ee06427a
AK
48 if verbose:
49 print real_cmd
21a50753
AK
50 return real_cmd
51
053fd0c1
RB
52def chdir(dir):
53 if os.name == 'nt':
54 os.environ['PWD']=dir
55 os.chdir(dir)
56
86dff6b6
HWN
57def die(msg):
58 if verbose:
59 raise Exception(msg)
60 else:
61 sys.stderr.write(msg + "\n")
62 sys.exit(1)
63
bce4c5fc 64def write_pipe(c, str):
4addad22 65 if verbose:
86dff6b6 66 sys.stderr.write('Writing pipe: %s\n' % c)
b016d397 67
bce4c5fc 68 pipe = os.popen(c, 'w')
b016d397 69 val = pipe.write(str)
bce4c5fc 70 if pipe.close():
86dff6b6 71 die('Command failed: %s' % c)
b016d397
HWN
72
73 return val
74
d9429194
AK
75def p4_write_pipe(c, str):
76 real_cmd = p4_build_cmd(c)
893d340f 77 return write_pipe(real_cmd, str)
d9429194 78
4addad22
HWN
79def read_pipe(c, ignore_error=False):
80 if verbose:
86dff6b6 81 sys.stderr.write('Reading pipe: %s\n' % c)
8b41a97f 82
bce4c5fc 83 pipe = os.popen(c, 'rb')
b016d397 84 val = pipe.read()
4addad22 85 if pipe.close() and not ignore_error:
86dff6b6 86 die('Command failed: %s' % c)
b016d397
HWN
87
88 return val
89
d9429194
AK
90def p4_read_pipe(c, ignore_error=False):
91 real_cmd = p4_build_cmd(c)
92 return read_pipe(real_cmd, ignore_error)
b016d397 93
bce4c5fc 94def read_pipe_lines(c):
4addad22 95 if verbose:
86dff6b6 96 sys.stderr.write('Reading pipe: %s\n' % c)
b016d397 97 ## todo: check return status
bce4c5fc 98 pipe = os.popen(c, 'rb')
b016d397 99 val = pipe.readlines()
bce4c5fc 100 if pipe.close():
86dff6b6 101 die('Command failed: %s' % c)
b016d397
HWN
102
103 return val
caace111 104
2318121b
AK
105def p4_read_pipe_lines(c):
106 """Specifically invoke p4 on the command supplied. """
155af834 107 real_cmd = p4_build_cmd(c)
2318121b
AK
108 return read_pipe_lines(real_cmd)
109
6754a299 110def system(cmd):
4addad22 111 if verbose:
bb6e09b2 112 sys.stderr.write("executing %s\n" % cmd)
6754a299
HWN
113 if os.system(cmd) != 0:
114 die("command failed: %s" % cmd)
115
bf9320f1
AK
116def p4_system(cmd):
117 """Specifically invoke p4 as the system command. """
155af834 118 real_cmd = p4_build_cmd(cmd)
bf9320f1
AK
119 return system(real_cmd)
120
9cffb8c8
PW
121#
122# Canonicalize the p4 type and return a tuple of the
123# base type, plus any modifiers. See "p4 help filetypes"
124# for a list and explanation.
125#
126def split_p4_type(p4type):
127
128 p4_filetypes_historical = {
129 "ctempobj": "binary+Sw",
130 "ctext": "text+C",
131 "cxtext": "text+Cx",
132 "ktext": "text+k",
133 "kxtext": "text+kx",
134 "ltext": "text+F",
135 "tempobj": "binary+FSw",
136 "ubinary": "binary+F",
137 "uresource": "resource+F",
138 "uxbinary": "binary+Fx",
139 "xbinary": "binary+x",
140 "xltext": "text+Fx",
141 "xtempobj": "binary+Swx",
142 "xtext": "text+x",
143 "xunicode": "unicode+x",
144 "xutf16": "utf16+x",
145 }
146 if p4type in p4_filetypes_historical:
147 p4type = p4_filetypes_historical[p4type]
148 mods = ""
149 s = p4type.split("+")
150 base = s[0]
151 mods = ""
152 if len(s) > 1:
153 mods = s[1]
154 return (base, mods)
b9fc6ea9 155
b9fc6ea9 156
c65b670e
CP
157def setP4ExecBit(file, mode):
158 # Reopens an already open file and changes the execute bit to match
159 # the execute bit setting in the passed in mode.
160
161 p4Type = "+x"
162
163 if not isModeExec(mode):
164 p4Type = getP4OpenedType(file)
165 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
166 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
167 if p4Type[-1] == "+":
168 p4Type = p4Type[0:-1]
169
87b611d5 170 p4_system("reopen -t %s %s" % (p4Type, file))
c65b670e
CP
171
172def getP4OpenedType(file):
173 # Returns the perforce file type for the given file.
174
a7d3ef9d 175 result = p4_read_pipe("opened %s" % file)
f3e5ae4f 176 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
177 if match:
178 return match.group(1)
179 else:
f3e5ae4f 180 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 181
b43b0a3c
CP
182def diffTreePattern():
183 # This is a simple generator for the diff tree regex pattern. This could be
184 # a class variable if this and parseDiffTreeEntry were a part of a class.
185 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
186 while True:
187 yield pattern
188
189def parseDiffTreeEntry(entry):
190 """Parses a single diff tree entry into its component elements.
191
192 See git-diff-tree(1) manpage for details about the format of the diff
193 output. This method returns a dictionary with the following elements:
194
195 src_mode - The mode of the source file
196 dst_mode - The mode of the destination file
197 src_sha1 - The sha1 for the source file
198 dst_sha1 - The sha1 fr the destination file
199 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
200 status_score - The score for the status (applicable for 'C' and 'R'
201 statuses). This is None if there is no score.
202 src - The path for the source file.
203 dst - The path for the destination file. This is only present for
204 copy or renames. If it is not present, this is None.
205
206 If the pattern is not matched, None is returned."""
207
208 match = diffTreePattern().next().match(entry)
209 if match:
210 return {
211 'src_mode': match.group(1),
212 'dst_mode': match.group(2),
213 'src_sha1': match.group(3),
214 'dst_sha1': match.group(4),
215 'status': match.group(5),
216 'status_score': match.group(6),
217 'src': match.group(7),
218 'dst': match.group(10)
219 }
220 return None
221
c65b670e
CP
222def isModeExec(mode):
223 # Returns True if the given git mode represents an executable file,
224 # otherwise False.
225 return mode[-3:] == "755"
226
227def isModeExecChanged(src_mode, dst_mode):
228 return isModeExec(src_mode) != isModeExec(dst_mode)
229
b932705b 230def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
155af834 231 cmd = p4_build_cmd("-G %s" % (cmd))
6a49f8e2
HWN
232 if verbose:
233 sys.stderr.write("Opening pipe: %s\n" % cmd)
9f90c733
SL
234
235 # Use a temporary file to avoid deadlocks without
236 # subprocess.communicate(), which would put another copy
237 # of stdout into memory.
238 stdin_file = None
239 if stdin is not None:
240 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
241 stdin_file.write(stdin)
242 stdin_file.flush()
243 stdin_file.seek(0)
244
245 p4 = subprocess.Popen(cmd, shell=True,
246 stdin=stdin_file,
247 stdout=subprocess.PIPE)
86949eef
SH
248
249 result = []
250 try:
251 while True:
9f90c733 252 entry = marshal.load(p4.stdout)
c3f6163b
AG
253 if cb is not None:
254 cb(entry)
255 else:
256 result.append(entry)
86949eef
SH
257 except EOFError:
258 pass
9f90c733
SL
259 exitCode = p4.wait()
260 if exitCode != 0:
ac3e0d79
SH
261 entry = {}
262 entry["p4ExitCode"] = exitCode
263 result.append(entry)
86949eef
SH
264
265 return result
266
267def p4Cmd(cmd):
268 list = p4CmdList(cmd)
269 result = {}
270 for entry in list:
271 result.update(entry)
272 return result;
273
cb2c9db5
SH
274def p4Where(depotPath):
275 if not depotPath.endswith("/"):
276 depotPath += "/"
7f705dc3
TAL
277 depotPath = depotPath + "..."
278 outputList = p4CmdList("where %s" % depotPath)
279 output = None
280 for entry in outputList:
75bc9573
TAL
281 if "depotFile" in entry:
282 if entry["depotFile"] == depotPath:
283 output = entry
284 break
285 elif "data" in entry:
286 data = entry.get("data")
287 space = data.find(" ")
288 if data[:space] == depotPath:
289 output = entry
290 break
7f705dc3
TAL
291 if output == None:
292 return ""
dc524036
SH
293 if output["code"] == "error":
294 return ""
cb2c9db5
SH
295 clientPath = ""
296 if "path" in output:
297 clientPath = output.get("path")
298 elif "data" in output:
299 data = output.get("data")
300 lastSpace = data.rfind(" ")
301 clientPath = data[lastSpace + 1:]
302
303 if clientPath.endswith("..."):
304 clientPath = clientPath[:-3]
305 return clientPath
306
86949eef 307def currentGitBranch():
b25b2065 308 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 309
4f5cf76a 310def isValidGitDir(path):
bb6e09b2
HWN
311 if (os.path.exists(path + "/HEAD")
312 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
313 return True;
314 return False
315
463e8af6 316def parseRevision(ref):
b25b2065 317 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 318
6ae8de88
SH
319def extractLogMessageFromGitCommit(commit):
320 logMessage = ""
b016d397
HWN
321
322 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 323 foundTitle = False
b016d397 324 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
325 if not foundTitle:
326 if len(log) == 1:
1c094184 327 foundTitle = True
6ae8de88
SH
328 continue
329
330 logMessage += log
331 return logMessage
332
bb6e09b2 333def extractSettingsGitLog(log):
6ae8de88
SH
334 values = {}
335 for line in log.split("\n"):
336 line = line.strip()
6326aa58
HWN
337 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
338 if not m:
339 continue
340
341 assignments = m.group(1).split (':')
342 for a in assignments:
343 vals = a.split ('=')
344 key = vals[0].strip()
345 val = ('='.join (vals[1:])).strip()
346 if val.endswith ('\"') and val.startswith('"'):
347 val = val[1:-1]
348
349 values[key] = val
350
845b42cb
SH
351 paths = values.get("depot-paths")
352 if not paths:
353 paths = values.get("depot-path")
a3fdd579
SH
354 if paths:
355 values['depot-paths'] = paths.split(',')
bb6e09b2 356 return values
6ae8de88 357
8136a639 358def gitBranchExists(branch):
bb6e09b2
HWN
359 proc = subprocess.Popen(["git", "rev-parse", branch],
360 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 361 return proc.wait() == 0;
8136a639 362
36bd8446 363_gitConfig = {}
99f790f2 364def gitConfig(key, args = None): # set args to "--bool", for instance
36bd8446 365 if not _gitConfig.has_key(key):
99f790f2
TAL
366 argsFilter = ""
367 if args != None:
368 argsFilter = "%s " % args
369 cmd = "git config %s%s" % (argsFilter, key)
370 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
36bd8446 371 return _gitConfig[key]
01265103 372
7199cf13
VA
373def gitConfigList(key):
374 if not _gitConfig.has_key(key):
375 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
376 return _gitConfig[key]
377
062410bb
SH
378def p4BranchesInGit(branchesAreInRemotes = True):
379 branches = {}
380
381 cmdline = "git rev-parse --symbolic "
382 if branchesAreInRemotes:
383 cmdline += " --remotes"
384 else:
385 cmdline += " --branches"
386
387 for line in read_pipe_lines(cmdline):
388 line = line.strip()
389
390 ## only import to p4/
391 if not line.startswith('p4/') or line == "p4/HEAD":
392 continue
393 branch = line
394
395 # strip off p4
396 branch = re.sub ("^p4/", "", line)
397
398 branches[branch] = parseRevision(line)
399 return branches
400
9ceab363 401def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
402 branches = p4BranchesInGit()
403 # map from depot-path to branch name
404 branchByDepotPath = {}
405 for branch in branches.keys():
406 tip = branches[branch]
407 log = extractLogMessageFromGitCommit(tip)
408 settings = extractSettingsGitLog(log)
409 if settings.has_key("depot-paths"):
410 paths = ",".join(settings["depot-paths"])
411 branchByDepotPath[paths] = "remotes/p4/" + branch
412
27d2d811 413 settings = None
27d2d811
SH
414 parent = 0
415 while parent < 65535:
9ceab363 416 commit = head + "~%s" % parent
27d2d811
SH
417 log = extractLogMessageFromGitCommit(commit)
418 settings = extractSettingsGitLog(log)
86506fe5
SH
419 if settings.has_key("depot-paths"):
420 paths = ",".join(settings["depot-paths"])
421 if branchByDepotPath.has_key(paths):
422 return [branchByDepotPath[paths], settings]
27d2d811 423
86506fe5 424 parent = parent + 1
27d2d811 425
86506fe5 426 return ["", settings]
27d2d811 427
5ca44617
SH
428def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
429 if not silent:
430 print ("Creating/updating branch(es) in %s based on origin branch(es)"
431 % localRefPrefix)
432
433 originPrefix = "origin/p4/"
434
435 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
436 line = line.strip()
437 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
438 continue
439
440 headName = line[len(originPrefix):]
441 remoteHead = localRefPrefix + headName
442 originHead = line
443
444 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
445 if (not original.has_key('depot-paths')
446 or not original.has_key('change')):
447 continue
448
449 update = False
450 if not gitBranchExists(remoteHead):
451 if verbose:
452 print "creating %s" % remoteHead
453 update = True
454 else:
455 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
456 if settings.has_key('change') > 0:
457 if settings['depot-paths'] == original['depot-paths']:
458 originP4Change = int(original['change'])
459 p4Change = int(settings['change'])
460 if originP4Change > p4Change:
461 print ("%s (%s) is newer than %s (%s). "
462 "Updating p4 branch from origin."
463 % (originHead, originP4Change,
464 remoteHead, p4Change))
465 update = True
466 else:
467 print ("Ignoring: %s was imported from %s while "
468 "%s was imported from %s"
469 % (originHead, ','.join(original['depot-paths']),
470 remoteHead, ','.join(settings['depot-paths'])))
471
472 if update:
473 system("git update-ref %s %s" % (remoteHead, originHead))
474
475def originP4BranchesExist():
476 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
477
4f6432d8
SH
478def p4ChangesForPaths(depotPaths, changeRange):
479 assert depotPaths
b340fa43 480 output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
4f6432d8
SH
481 for p in depotPaths]))
482
b4b0ba06 483 changes = {}
4f6432d8 484 for line in output:
c3f6163b
AG
485 changeNum = int(line.split(" ")[1])
486 changes[changeNum] = True
4f6432d8 487
b4b0ba06
PW
488 changelist = changes.keys()
489 changelist.sort()
490 return changelist
4f6432d8 491
d53de8b9
TAL
492def p4PathStartsWith(path, prefix):
493 # This method tries to remedy a potential mixed-case issue:
494 #
495 # If UserA adds //depot/DirA/file1
496 # and UserB adds //depot/dira/file2
497 #
498 # we may or may not have a problem. If you have core.ignorecase=true,
499 # we treat DirA and dira as the same directory
500 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
501 if ignorecase:
502 return path.lower().startswith(prefix.lower())
503 return path.startswith(prefix)
504
b984733c
SH
505class Command:
506 def __init__(self):
507 self.usage = "usage: %prog [options]"
8910ac0e 508 self.needsGit = True
b984733c 509
3ea2cfd4
LD
510class P4UserMap:
511 def __init__(self):
512 self.userMapFromPerforceServer = False
513
514 def getUserCacheFilename(self):
515 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
516 return home + "/.gitp4-usercache.txt"
517
518 def getUserMapFromPerforceServer(self):
519 if self.userMapFromPerforceServer:
520 return
521 self.users = {}
522 self.emails = {}
523
524 for output in p4CmdList("users"):
525 if not output.has_key("User"):
526 continue
527 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
528 self.emails[output["Email"]] = output["User"]
529
530
531 s = ''
532 for (key, val) in self.users.items():
533 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
534
535 open(self.getUserCacheFilename(), "wb").write(s)
536 self.userMapFromPerforceServer = True
537
538 def loadUserMapFromCache(self):
539 self.users = {}
540 self.userMapFromPerforceServer = False
541 try:
542 cache = open(self.getUserCacheFilename(), "rb")
543 lines = cache.readlines()
544 cache.close()
545 for line in lines:
546 entry = line.strip().split("\t")
547 self.users[entry[0]] = entry[1]
548 except IOError:
549 self.getUserMapFromPerforceServer()
550
b984733c 551class P4Debug(Command):
86949eef 552 def __init__(self):
6ae8de88 553 Command.__init__(self)
86949eef 554 self.options = [
b1ce9447
HWN
555 optparse.make_option("--verbose", dest="verbose", action="store_true",
556 default=False),
4addad22 557 ]
c8c39116 558 self.description = "A tool to debug the output of p4 -G."
8910ac0e 559 self.needsGit = False
b1ce9447 560 self.verbose = False
86949eef
SH
561
562 def run(self, args):
b1ce9447 563 j = 0
86949eef 564 for output in p4CmdList(" ".join(args)):
b1ce9447
HWN
565 print 'Element: %d' % j
566 j += 1
86949eef 567 print output
b984733c 568 return True
86949eef 569
5834684d
SH
570class P4RollBack(Command):
571 def __init__(self):
572 Command.__init__(self)
573 self.options = [
0c66a783
SH
574 optparse.make_option("--verbose", dest="verbose", action="store_true"),
575 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
576 ]
577 self.description = "A tool to debug the multi-branch import. Don't use :)"
52102d47 578 self.verbose = False
0c66a783 579 self.rollbackLocalBranches = False
5834684d
SH
580
581 def run(self, args):
582 if len(args) != 1:
583 return False
584 maxChange = int(args[0])
0c66a783 585
ad192f28 586 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
587 die("Problems executing p4");
588
0c66a783
SH
589 if self.rollbackLocalBranches:
590 refPrefix = "refs/heads/"
b016d397 591 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
592 else:
593 refPrefix = "refs/remotes/"
b016d397 594 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
595
596 for line in lines:
597 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
598 line = line.strip()
599 ref = refPrefix + line
5834684d 600 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
601 settings = extractSettingsGitLog(log)
602
603 depotPaths = settings['depot-paths']
604 change = settings['change']
605
5834684d 606 changed = False
52102d47 607
6326aa58
HWN
608 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
609 for p in depotPaths]))) == 0:
52102d47
SH
610 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
611 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
612 continue
613
bb6e09b2 614 while change and int(change) > maxChange:
5834684d 615 changed = True
52102d47
SH
616 if self.verbose:
617 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
618 system("git update-ref %s \"%s^\"" % (ref, ref))
619 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
620 settings = extractSettingsGitLog(log)
621
622
623 depotPaths = settings['depot-paths']
624 change = settings['change']
5834684d
SH
625
626 if changed:
52102d47 627 print "%s rewound to %s" % (ref, change)
5834684d
SH
628
629 return True
630
3ea2cfd4 631class P4Submit(Command, P4UserMap):
4f5cf76a 632 def __init__(self):
b984733c 633 Command.__init__(self)
3ea2cfd4 634 P4UserMap.__init__(self)
4f5cf76a 635 self.options = [
4addad22 636 optparse.make_option("--verbose", dest="verbose", action="store_true"),
4f5cf76a 637 optparse.make_option("--origin", dest="origin"),
ae901090 638 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
639 # preserve the user, requires relevant p4 permissions
640 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
4f5cf76a
SH
641 ]
642 self.description = "Submit changes from git to the perforce depot."
c9b50e63 643 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 644 self.interactive = True
9512497b 645 self.origin = ""
ae901090 646 self.detectRenames = False
b0d10df7 647 self.verbose = False
3ea2cfd4 648 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
f7baba8b 649 self.isWindows = (platform.system() == "Windows")
848de9c3 650 self.myP4UserId = None
4f5cf76a 651
4f5cf76a
SH
652 def check(self):
653 if len(p4CmdList("opened ...")) > 0:
654 die("You have files opened with perforce! Close them before starting the sync.")
655
edae1e2f
SH
656 # replaces everything between 'Description:' and the next P4 submit template field with the
657 # commit message
4f5cf76a
SH
658 def prepareLogMessage(self, template, message):
659 result = ""
660
edae1e2f
SH
661 inDescriptionSection = False
662
4f5cf76a
SH
663 for line in template.split("\n"):
664 if line.startswith("#"):
665 result += line + "\n"
666 continue
667
edae1e2f 668 if inDescriptionSection:
c9dbab04 669 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f
SH
670 inDescriptionSection = False
671 else:
672 continue
673 else:
674 if line.startswith("Description:"):
675 inDescriptionSection = True
676 line += "\n"
677 for messageLine in message.split("\n"):
678 line += "\t" + messageLine + "\n"
679
680 result += line + "\n"
4f5cf76a
SH
681
682 return result
683
3ea2cfd4
LD
684 def p4UserForCommit(self,id):
685 # Return the tuple (perforce user,git email) for a given git commit id
686 self.getUserMapFromPerforceServer()
687 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
688 gitEmail = gitEmail.strip()
689 if not self.emails.has_key(gitEmail):
690 return (None,gitEmail)
691 else:
692 return (self.emails[gitEmail],gitEmail)
693
694 def checkValidP4Users(self,commits):
695 # check if any git authors cannot be mapped to p4 users
696 for id in commits:
697 (user,email) = self.p4UserForCommit(id)
698 if not user:
699 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
700 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
701 print "%s" % msg
702 else:
703 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
704
705 def lastP4Changelist(self):
706 # Get back the last changelist number submitted in this client spec. This
707 # then gets used to patch up the username in the change. If the same
708 # client spec is being used by multiple processes then this might go
709 # wrong.
710 results = p4CmdList("client -o") # find the current client
711 client = None
712 for r in results:
713 if r.has_key('Client'):
714 client = r['Client']
715 break
716 if not client:
717 die("could not get client spec")
718 results = p4CmdList("changes -c %s -m 1" % client)
719 for r in results:
720 if r.has_key('change'):
721 return r['change']
722 die("Could not get changelist number for last submit - cannot patch up user details")
723
724 def modifyChangelistUser(self, changelist, newUser):
725 # fixup the user field of a changelist after it has been submitted.
726 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
727 if len(changes) != 1:
728 die("Bad output from p4 change modifying %s to user %s" %
729 (changelist, newUser))
730
731 c = changes[0]
732 if c['User'] == newUser: return # nothing to do
733 c['User'] = newUser
734 input = marshal.dumps(c)
735
3ea2cfd4
LD
736 result = p4CmdList("change -f -i", stdin=input)
737 for r in result:
738 if r.has_key('code'):
739 if r['code'] == 'error':
740 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
741 if r.has_key('data'):
742 print("Updated user field for changelist %s to %s" % (changelist, newUser))
743 return
744 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
745
746 def canChangeChangelists(self):
747 # check to see if we have p4 admin or super-user permissions, either of
748 # which are required to modify changelists.
ecdba36d 749 results = p4CmdList("protects %s" % self.depotPath)
3ea2cfd4
LD
750 for r in results:
751 if r.has_key('perm'):
752 if r['perm'] == 'admin':
753 return 1
754 if r['perm'] == 'super':
755 return 1
756 return 0
757
848de9c3
LD
758 def p4UserId(self):
759 if self.myP4UserId:
760 return self.myP4UserId
761
762 results = p4CmdList("user -o")
763 for r in results:
764 if r.has_key('User'):
765 self.myP4UserId = r['User']
766 return r['User']
767 die("Could not find your p4 user id")
768
769 def p4UserIsMe(self, p4User):
770 # return True if the given p4 user is actually me
771 me = self.p4UserId()
772 if not p4User or p4User != me:
773 return False
774 else:
775 return True
776
ea99c3ae
SH
777 def prepareSubmitTemplate(self):
778 # remove lines in the Files section that show changes to files outside the depot path we're committing into
779 template = ""
780 inFilesSection = False
b340fa43 781 for line in p4_read_pipe_lines("change -o"):
f3e5ae4f
MSO
782 if line.endswith("\r\n"):
783 line = line[:-2] + "\n"
ea99c3ae
SH
784 if inFilesSection:
785 if line.startswith("\t"):
786 # path starts and ends with a tab
787 path = line[1:]
788 lastTab = path.rfind("\t")
789 if lastTab != -1:
790 path = path[:lastTab]
d53de8b9 791 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
792 continue
793 else:
794 inFilesSection = False
795 else:
796 if line.startswith("Files:"):
797 inFilesSection = True
798
799 template += line
800
801 return template
802
7cb5cbef 803 def applyCommit(self, id):
0e36f2d7 804 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
ae901090 805
848de9c3 806 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 807
ae901090
VA
808 if not self.detectRenames:
809 # If not explicitly set check the config variable
0a9feffc 810 self.detectRenames = gitConfig("git-p4.detectRenames")
ae901090 811
0a9feffc
VA
812 if self.detectRenames.lower() == "false" or self.detectRenames == "":
813 diffOpts = ""
814 elif self.detectRenames.lower() == "true":
ae901090
VA
815 diffOpts = "-M"
816 else:
0a9feffc 817 diffOpts = "-M%s" % self.detectRenames
ae901090 818
0a9feffc
VA
819 detectCopies = gitConfig("git-p4.detectCopies")
820 if detectCopies.lower() == "true":
4fddb41b 821 diffOpts += " -C"
0a9feffc
VA
822 elif detectCopies != "" and detectCopies.lower() != "false":
823 diffOpts += " -C%s" % detectCopies
4fddb41b 824
68cbcf1b 825 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
4fddb41b
VA
826 diffOpts += " --find-copies-harder"
827
0e36f2d7 828 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
829 filesToAdd = set()
830 filesToDelete = set()
d336c158 831 editedFiles = set()
c65b670e 832 filesToChangeExecBit = {}
4f5cf76a 833 for line in diff:
b43b0a3c
CP
834 diff = parseDiffTreeEntry(line)
835 modifier = diff['status']
836 path = diff['src']
4f5cf76a 837 if modifier == "M":
87b611d5 838 p4_system("edit \"%s\"" % path)
c65b670e
CP
839 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
840 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 841 editedFiles.add(path)
4f5cf76a
SH
842 elif modifier == "A":
843 filesToAdd.add(path)
c65b670e 844 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
845 if path in filesToDelete:
846 filesToDelete.remove(path)
847 elif modifier == "D":
848 filesToDelete.add(path)
849 if path in filesToAdd:
850 filesToAdd.remove(path)
4fddb41b
VA
851 elif modifier == "C":
852 src, dest = diff['src'], diff['dst']
853 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
854 if diff['src_sha1'] != diff['dst_sha1']:
855 p4_system("edit \"%s\"" % (dest))
856 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
857 p4_system("edit \"%s\"" % (dest))
858 filesToChangeExecBit[dest] = diff['dst_mode']
859 os.unlink(dest)
860 editedFiles.add(dest)
d9a5f25b 861 elif modifier == "R":
b43b0a3c 862 src, dest = diff['src'], diff['dst']
87b611d5 863 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
ae901090
VA
864 if diff['src_sha1'] != diff['dst_sha1']:
865 p4_system("edit \"%s\"" % (dest))
c65b670e 866 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
ae901090 867 p4_system("edit \"%s\"" % (dest))
c65b670e 868 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
869 os.unlink(dest)
870 editedFiles.add(dest)
871 filesToDelete.add(src)
4f5cf76a
SH
872 else:
873 die("unknown modifier %s for %s" % (modifier, path))
874
0e36f2d7 875 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 876 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
877 tryPatchCmd = patchcmd + "--check -"
878 applyPatchCmd = patchcmd + "--check --apply -"
51a2640a 879
47a130b7 880 if os.system(tryPatchCmd) != 0:
51a2640a
SH
881 print "Unfortunately applying the change failed!"
882 print "What do you want to do?"
883 response = "x"
884 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
885 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
886 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
887 if response == "s":
888 print "Skipping! Good luck with the next patches..."
20947149 889 for f in editedFiles:
87b611d5 890 p4_system("revert \"%s\"" % f);
20947149
SH
891 for f in filesToAdd:
892 system("rm %s" %f)
51a2640a
SH
893 return
894 elif response == "a":
47a130b7 895 os.system(applyPatchCmd)
51a2640a
SH
896 if len(filesToAdd) > 0:
897 print "You may also want to call p4 add on the following files:"
898 print " ".join(filesToAdd)
899 if len(filesToDelete):
900 print "The following files should be scheduled for deletion with p4 delete:"
901 print " ".join(filesToDelete)
cebdf5af
HWN
902 die("Please resolve and submit the conflict manually and "
903 + "continue afterwards with git-p4 submit --continue")
51a2640a
SH
904 elif response == "w":
905 system(diffcmd + " > patch.txt")
906 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af
HWN
907 die("Please resolve and submit the conflict manually and "
908 "continue afterwards with git-p4 submit --continue")
51a2640a 909
47a130b7 910 system(applyPatchCmd)
4f5cf76a
SH
911
912 for f in filesToAdd:
87b611d5 913 p4_system("add \"%s\"" % f)
4f5cf76a 914 for f in filesToDelete:
87b611d5
AK
915 p4_system("revert \"%s\"" % f)
916 p4_system("delete \"%s\"" % f)
4f5cf76a 917
c65b670e
CP
918 # Set/clear executable bits
919 for f in filesToChangeExecBit.keys():
920 mode = filesToChangeExecBit[f]
921 setP4ExecBit(f, mode)
922
0e36f2d7 923 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 924 logMessage = logMessage.strip()
4f5cf76a 925
ea99c3ae 926 template = self.prepareSubmitTemplate()
4f5cf76a
SH
927
928 if self.interactive:
929 submitTemplate = self.prepareLogMessage(template, logMessage)
ecdba36d
LD
930
931 if self.preserveUser:
932 submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
933
67abd417
SB
934 if os.environ.has_key("P4DIFF"):
935 del(os.environ["P4DIFF"])
8b130262
AW
936 diff = ""
937 for editedFile in editedFiles:
938 diff += p4_read_pipe("diff -du %r" % editedFile)
4f5cf76a 939
f3e5ae4f 940 newdiff = ""
4f5cf76a 941 for newFile in filesToAdd:
f3e5ae4f
MSO
942 newdiff += "==== new file ====\n"
943 newdiff += "--- /dev/null\n"
944 newdiff += "+++ %s\n" % newFile
4f5cf76a
SH
945 f = open(newFile, "r")
946 for line in f.readlines():
f3e5ae4f 947 newdiff += "+" + line
4f5cf76a
SH
948 f.close()
949
848de9c3
LD
950 if self.checkAuthorship and not self.p4UserIsMe(p4User):
951 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
952 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
953 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
954
f3e5ae4f 955 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a 956
e96e400f
SH
957 [handle, fileName] = tempfile.mkstemp()
958 tmpFile = os.fdopen(handle, "w+")
f3e5ae4f
MSO
959 if self.isWindows:
960 submitTemplate = submitTemplate.replace("\n", "\r\n")
961 separatorLine = separatorLine.replace("\n", "\r\n")
962 newdiff = newdiff.replace("\n", "\r\n")
963 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
e96e400f 964 tmpFile.close()
cdc7e388 965 mtime = os.stat(fileName).st_mtime
82cea9ff
SB
966 if os.environ.has_key("P4EDITOR"):
967 editor = os.environ.get("P4EDITOR")
968 else:
8b187e6b 969 editor = read_pipe("git var GIT_EDITOR").strip()
e96e400f 970 system(editor + " " + fileName)
e96e400f 971
3ea2cfd4
LD
972 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
973 checkModTime = False
974 else:
975 checkModTime = True
976
cdc7e388 977 response = "y"
3ea2cfd4 978 if checkModTime and (os.stat(fileName).st_mtime <= mtime):
cdc7e388
SH
979 response = "x"
980 while response != "y" and response != "n":
981 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
982
983 if response == "y":
984 tmpFile = open(fileName, "rb")
985 message = tmpFile.read()
986 tmpFile.close()
987 submitTemplate = message[:message.index(separatorLine)]
988 if self.isWindows:
989 submitTemplate = submitTemplate.replace("\r\n", "\n")
990 p4_write_pipe("submit -i", submitTemplate)
3ea2cfd4
LD
991
992 if self.preserveUser:
993 if p4User:
994 # Get last changelist number. Cannot easily get it from
995 # the submit command output as the output is unmarshalled.
996 changelist = self.lastP4Changelist()
997 self.modifyChangelistUser(changelist, p4User)
998
cdc7e388
SH
999 else:
1000 for f in editedFiles:
1001 p4_system("revert \"%s\"" % f);
1002 for f in filesToAdd:
1003 p4_system("revert \"%s\"" % f);
1004 system("rm %s" %f)
1005
1006 os.remove(fileName)
4f5cf76a
SH
1007 else:
1008 fileName = "submit.txt"
1009 file = open(fileName, "w+")
1010 file.write(self.prepareLogMessage(template, logMessage))
1011 file.close()
cebdf5af
HWN
1012 print ("Perforce submit template written as %s. "
1013 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1014 % (fileName, fileName))
4f5cf76a
SH
1015
1016 def run(self, args):
c9b50e63
SH
1017 if len(args) == 0:
1018 self.master = currentGitBranch()
4280e533 1019 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1020 die("Detecting current git branch failed!")
1021 elif len(args) == 1:
1022 self.master = args[0]
1023 else:
1024 return False
1025
4c2d5d72
JX
1026 allowSubmit = gitConfig("git-p4.allowSubmit")
1027 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1028 die("%s is not in git-p4.allowSubmit" % self.master)
1029
27d2d811 1030 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1031 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1032 if len(self.origin) == 0:
1033 self.origin = upstream
a3fdd579 1034
3ea2cfd4
LD
1035 if self.preserveUser:
1036 if not self.canChangeChangelists():
1037 die("Cannot preserve user names without p4 super-user or admin permissions")
1038
a3fdd579
SH
1039 if self.verbose:
1040 print "Origin branch is " + self.origin
9512497b 1041
ea99c3ae 1042 if len(self.depotPath) == 0:
9512497b
SH
1043 print "Internal error: cannot locate perforce depot path from existing branches"
1044 sys.exit(128)
1045
ea99c3ae 1046 self.clientPath = p4Where(self.depotPath)
9512497b 1047
51a2640a 1048 if len(self.clientPath) == 0:
ea99c3ae 1049 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
9512497b
SH
1050 sys.exit(128)
1051
ea99c3ae 1052 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1053 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1054
053fd0c1 1055 chdir(self.clientPath)
6a01298a 1056 print "Synchronizing p4 checkout..."
87b611d5 1057 p4_system("sync ...")
9512497b 1058
4f5cf76a 1059 self.check()
4f5cf76a 1060
4c750c0d
SH
1061 commits = []
1062 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1063 commits.append(line.strip())
1064 commits.reverse()
4f5cf76a 1065
848de9c3
LD
1066 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1067 self.checkAuthorship = False
1068 else:
1069 self.checkAuthorship = True
1070
3ea2cfd4
LD
1071 if self.preserveUser:
1072 self.checkValidP4Users(commits)
1073
4f5cf76a 1074 while len(commits) > 0:
4f5cf76a
SH
1075 commit = commits[0]
1076 commits = commits[1:]
7cb5cbef 1077 self.applyCommit(commit)
4f5cf76a
SH
1078 if not self.interactive:
1079 break
1080
4f5cf76a 1081 if len(commits) == 0:
4c750c0d 1082 print "All changes applied!"
053fd0c1 1083 chdir(self.oldWorkingDirectory)
14594f4b 1084
4c750c0d
SH
1085 sync = P4Sync()
1086 sync.run([])
14594f4b 1087
4c750c0d
SH
1088 rebase = P4Rebase()
1089 rebase.rebase()
4f5cf76a 1090
b984733c
SH
1091 return True
1092
3ea2cfd4 1093class P4Sync(Command, P4UserMap):
56c09345
PW
1094 delete_actions = ( "delete", "move/delete", "purge" )
1095
b984733c
SH
1096 def __init__(self):
1097 Command.__init__(self)
3ea2cfd4 1098 P4UserMap.__init__(self)
b984733c
SH
1099 self.options = [
1100 optparse.make_option("--branch", dest="branch"),
1101 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1102 optparse.make_option("--changesfile", dest="changesFile"),
1103 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1104 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
a028a98e 1105 optparse.make_option("--verbose", dest="verbose", action="store_true"),
d2c6dd30
HWN
1106 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1107 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 1108 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 1109 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1110 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1111 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1112 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
1113 ]
1114 self.description = """Imports from Perforce into a git repository.\n
1115 example:
1116 //depot/my/project/ -- to import the current head
1117 //depot/my/project/@all -- to import everything
1118 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1119
1120 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1121
1122 self.usage += " //depot/path[@revRange]"
b984733c 1123 self.silent = False
1d7367dc
RG
1124 self.createdBranches = set()
1125 self.committedChanges = set()
569d1bd4 1126 self.branch = ""
b984733c 1127 self.detectBranches = False
cb53e1f8 1128 self.detectLabels = False
b984733c 1129 self.changesFile = ""
01265103 1130 self.syncWithOrigin = True
4b97ffb1 1131 self.verbose = False
a028a98e 1132 self.importIntoRemotes = True
01a9c9c5 1133 self.maxChanges = ""
c1f9197f 1134 self.isWindows = (platform.system() == "Windows")
8b41a97f 1135 self.keepRepoPath = False
6326aa58 1136 self.depotPaths = None
3c699645 1137 self.p4BranchesInGit = []
354081d5 1138 self.cloneExclude = []
3a70cdfa
TAL
1139 self.useClientSpec = False
1140 self.clientSpecDirs = []
b984733c 1141
01265103
SH
1142 if gitConfig("git-p4.syncFromOrigin") == "false":
1143 self.syncWithOrigin = False
1144
084f6306
PW
1145 #
1146 # P4 wildcards are not allowed in filenames. P4 complains
1147 # if you simply add them, but you can force it with "-f", in
1148 # which case it translates them into %xx encoding internally.
1149 # Search for and fix just these four characters. Do % last so
1150 # that fixing it does not inadvertently create new %-escapes.
1151 #
1152 def wildcard_decode(self, path):
1153 # Cannot have * in a filename in windows; untested as to
1154 # what p4 would do in such a case.
1155 if not self.isWindows:
1156 path = path.replace("%2A", "*")
1157 path = path.replace("%23", "#") \
1158 .replace("%40", "@") \
1159 .replace("%25", "%")
1160 return path
1161
b984733c 1162 def extractFilesFromCommit(self, commit):
354081d5
TT
1163 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1164 for path in self.cloneExclude]
b984733c
SH
1165 files = []
1166 fnum = 0
1167 while commit.has_key("depotFile%s" % fnum):
1168 path = commit["depotFile%s" % fnum]
6326aa58 1169
354081d5 1170 if [p for p in self.cloneExclude
d53de8b9 1171 if p4PathStartsWith(path, p)]:
354081d5
TT
1172 found = False
1173 else:
1174 found = [p for p in self.depotPaths
d53de8b9 1175 if p4PathStartsWith(path, p)]
6326aa58 1176 if not found:
b984733c
SH
1177 fnum = fnum + 1
1178 continue
1179
1180 file = {}
1181 file["path"] = path
1182 file["rev"] = commit["rev%s" % fnum]
1183 file["action"] = commit["action%s" % fnum]
1184 file["type"] = commit["type%s" % fnum]
1185 files.append(file)
1186 fnum = fnum + 1
1187 return files
1188
6326aa58 1189 def stripRepoPath(self, path, prefixes):
3952710b
IW
1190 if self.useClientSpec:
1191
1192 # if using the client spec, we use the output directory
1193 # specified in the client. For example, a view
1194 # //depot/foo/branch/... //client/branch/foo/...
1195 # will end up putting all foo/branch files into
1196 # branch/foo/
1197 for val in self.clientSpecDirs:
1198 if path.startswith(val[0]):
1199 # replace the depot path with the client path
1200 path = path.replace(val[0], val[1][1])
1201 # now strip out the client (//client/...)
1202 path = re.sub("^(//[^/]+/)", '', path)
1203 # the rest is all path
1204 return path
1205
8b41a97f 1206 if self.keepRepoPath:
6326aa58
HWN
1207 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1208
1209 for p in prefixes:
d53de8b9 1210 if p4PathStartsWith(path, p):
6326aa58 1211 path = path[len(p):]
8b41a97f 1212
6326aa58 1213 return path
6754a299 1214
71b112d4 1215 def splitFilesIntoBranches(self, commit):
d5904674 1216 branches = {}
71b112d4
SH
1217 fnum = 0
1218 while commit.has_key("depotFile%s" % fnum):
1219 path = commit["depotFile%s" % fnum]
6326aa58 1220 found = [p for p in self.depotPaths
d53de8b9 1221 if p4PathStartsWith(path, p)]
6326aa58 1222 if not found:
71b112d4
SH
1223 fnum = fnum + 1
1224 continue
1225
1226 file = {}
1227 file["path"] = path
1228 file["rev"] = commit["rev%s" % fnum]
1229 file["action"] = commit["action%s" % fnum]
1230 file["type"] = commit["type%s" % fnum]
1231 fnum = fnum + 1
1232
6326aa58 1233 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 1234
4b97ffb1 1235 for branch in self.knownBranches.keys():
6754a299
HWN
1236
1237 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1238 if relPath.startswith(branch + "/"):
d5904674
SH
1239 if branch not in branches:
1240 branches[branch] = []
71b112d4 1241 branches[branch].append(file)
6555b2cc 1242 break
b984733c
SH
1243
1244 return branches
1245
b932705b
LD
1246 # output one file from the P4 stream
1247 # - helper for streamP4Files
1248
1249 def streamOneP4File(self, file, contents):
b932705b 1250 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
084f6306 1251 relPath = self.wildcard_decode(relPath)
b932705b
LD
1252 if verbose:
1253 sys.stderr.write("%s\n" % relPath)
1254
9cffb8c8
PW
1255 (type_base, type_mods) = split_p4_type(file["type"])
1256
1257 git_mode = "100644"
1258 if "x" in type_mods:
1259 git_mode = "100755"
1260 if type_base == "symlink":
1261 git_mode = "120000"
1262 # p4 print on a symlink contains "target\n"; remove the newline
b39c3612
EP
1263 data = ''.join(contents)
1264 contents = [data[:-1]]
b932705b 1265
9cffb8c8 1266 if type_base == "utf16":
55aa5714
PW
1267 # p4 delivers different text in the python output to -G
1268 # than it does when using "print -o", or normal p4 client
1269 # operations. utf16 is converted to ascii or utf8, perhaps.
1270 # But ascii text saved as -t utf16 is completely mangled.
1271 # Invoke print -o to get the real contents.
1272 text = p4_read_pipe('print -q -o - "%s"' % file['depotFile'])
1273 contents = [ text ]
1274
9cffb8c8
PW
1275 # Perhaps windows wants unicode, utf16 newlines translated too;
1276 # but this is not doing it.
1277 if self.isWindows and type_base == "text":
b932705b
LD
1278 mangled = []
1279 for data in contents:
1280 data = data.replace("\r\n", "\n")
1281 mangled.append(data)
1282 contents = mangled
1283
55aa5714
PW
1284 # Note that we do not try to de-mangle keywords on utf16 files,
1285 # even though in theory somebody may want that.
9cffb8c8
PW
1286 if type_base in ("text", "unicode", "binary"):
1287 if "ko" in type_mods:
cb585a9c
PW
1288 text = ''.join(contents)
1289 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1290 contents = [ text ]
9cffb8c8 1291 elif "k" in type_mods:
cb585a9c
PW
1292 text = ''.join(contents)
1293 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1294 contents = [ text ]
b932705b 1295
9cffb8c8 1296 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
1297
1298 # total length...
1299 length = 0
1300 for d in contents:
1301 length = length + len(d)
1302
1303 self.gitStream.write("data %d\n" % length)
1304 for d in contents:
1305 self.gitStream.write(d)
1306 self.gitStream.write("\n")
1307
1308 def streamOneP4Deletion(self, file):
1309 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1310 if verbose:
1311 sys.stderr.write("delete %s\n" % relPath)
1312 self.gitStream.write("D %s\n" % relPath)
1313
1314 # handle another chunk of streaming data
1315 def streamP4FilesCb(self, marshalled):
1316
c3f6163b
AG
1317 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1318 # start of a new file - output the old one first
1319 self.streamOneP4File(self.stream_file, self.stream_contents)
1320 self.stream_file = {}
1321 self.stream_contents = []
1322 self.stream_have_file_info = False
b932705b 1323
c3f6163b
AG
1324 # pick up the new file information... for the
1325 # 'data' field we need to append to our array
1326 for k in marshalled.keys():
1327 if k == 'data':
1328 self.stream_contents.append(marshalled['data'])
1329 else:
1330 self.stream_file[k] = marshalled[k]
b932705b 1331
c3f6163b 1332 self.stream_have_file_info = True
b932705b
LD
1333
1334 # Stream directly from "p4 files" into "git fast-import"
1335 def streamP4Files(self, files):
30b5940b
SH
1336 filesForCommit = []
1337 filesToRead = []
b932705b 1338 filesToDelete = []
30b5940b 1339
3a70cdfa 1340 for f in files:
30b5940b 1341 includeFile = True
3a70cdfa
TAL
1342 for val in self.clientSpecDirs:
1343 if f['path'].startswith(val[0]):
3952710b 1344 if val[1][0] <= 0:
30b5940b 1345 includeFile = False
3a70cdfa
TAL
1346 break
1347
30b5940b
SH
1348 if includeFile:
1349 filesForCommit.append(f)
56c09345 1350 if f['action'] in self.delete_actions:
b932705b 1351 filesToDelete.append(f)
56c09345
PW
1352 else:
1353 filesToRead.append(f)
6a49f8e2 1354
b932705b
LD
1355 # deleted files...
1356 for f in filesToDelete:
1357 self.streamOneP4Deletion(f)
1b9a4684 1358
b932705b
LD
1359 if len(filesToRead) > 0:
1360 self.stream_file = {}
1361 self.stream_contents = []
1362 self.stream_have_file_info = False
8ff45f2a 1363
c3f6163b
AG
1364 # curry self argument
1365 def streamP4FilesCbSelf(entry):
1366 self.streamP4FilesCb(entry)
6a49f8e2 1367
c3f6163b
AG
1368 p4CmdList("-x - print",
1369 '\n'.join(['%s#%s' % (f['path'], f['rev'])
b932705b 1370 for f in filesToRead]),
c3f6163b 1371 cb=streamP4FilesCbSelf)
30b5940b 1372
b932705b
LD
1373 # do the last chunk
1374 if self.stream_file.has_key('depotFile'):
1375 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 1376
6326aa58 1377 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
1378 epoch = details["time"]
1379 author = details["user"]
c3f6163b 1380 self.branchPrefixes = branchPrefixes
b984733c 1381
4b97ffb1
SH
1382 if self.verbose:
1383 print "commit into %s" % branch
1384
96e07dd2
HWN
1385 # start with reading files; if that fails, we should not
1386 # create a commit.
1387 new_files = []
1388 for f in files:
d53de8b9 1389 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
1390 new_files.append (f)
1391 else:
afa1dd9a 1392 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 1393
b984733c 1394 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 1395# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
1396 self.committedChanges.add(int(details["change"]))
1397 committer = ""
b607e71e
SH
1398 if author not in self.users:
1399 self.getUserMapFromPerforceServer()
b984733c 1400 if author in self.users:
0828ab14 1401 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
b984733c 1402 else:
0828ab14 1403 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
b984733c
SH
1404
1405 self.gitStream.write("committer %s\n" % committer)
1406
1407 self.gitStream.write("data <<EOT\n")
1408 self.gitStream.write(details["desc"])
6581de09
SH
1409 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1410 % (','.join (branchPrefixes), details["change"]))
1411 if len(details['options']) > 0:
1412 self.gitStream.write(": options = %s" % details['options'])
1413 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
1414
1415 if len(parent) > 0:
4b97ffb1
SH
1416 if self.verbose:
1417 print "parent %s" % parent
b984733c
SH
1418 self.gitStream.write("from %s\n" % parent)
1419
b932705b 1420 self.streamP4Files(new_files)
b984733c
SH
1421 self.gitStream.write("\n")
1422
1f4ba1cb
SH
1423 change = int(details["change"])
1424
9bda3a85 1425 if self.labels.has_key(change):
1f4ba1cb
SH
1426 label = self.labels[change]
1427 labelDetails = label[0]
1428 labelRevisions = label[1]
71b112d4
SH
1429 if self.verbose:
1430 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 1431
6326aa58
HWN
1432 files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1433 for p in branchPrefixes]))
1f4ba1cb
SH
1434
1435 if len(files) == len(labelRevisions):
1436
1437 cleanedFiles = {}
1438 for info in files:
56c09345 1439 if info["action"] in self.delete_actions:
1f4ba1cb
SH
1440 continue
1441 cleanedFiles[info["depotFile"]] = info["rev"]
1442
1443 if cleanedFiles == labelRevisions:
1444 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1445 self.gitStream.write("from %s\n" % branch)
1446
1447 owner = labelDetails["Owner"]
1448 tagger = ""
1449 if author in self.users:
1450 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1451 else:
1452 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1453 self.gitStream.write("tagger %s\n" % tagger)
1454 self.gitStream.write("data <<EOT\n")
1455 self.gitStream.write(labelDetails["Description"])
1456 self.gitStream.write("EOT\n\n")
1457
1458 else:
a46668fa 1459 if not self.silent:
cebdf5af
HWN
1460 print ("Tag %s does not match with change %s: files do not match."
1461 % (labelDetails["label"], change))
1f4ba1cb
SH
1462
1463 else:
a46668fa 1464 if not self.silent:
cebdf5af
HWN
1465 print ("Tag %s does not match with change %s: file count is different."
1466 % (labelDetails["label"], change))
b984733c 1467
1f4ba1cb
SH
1468 def getLabels(self):
1469 self.labels = {}
1470
6326aa58 1471 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
10c3211b 1472 if len(l) > 0 and not self.silent:
183f8436 1473 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
1474
1475 for output in l:
1f4ba1cb
SH
1476 label = output["label"]
1477 revisions = {}
1478 newestChange = 0
71b112d4
SH
1479 if self.verbose:
1480 print "Querying files for label %s" % label
6326aa58
HWN
1481 for file in p4CmdList("files "
1482 + ' '.join (["%s...@%s" % (p, label)
1483 for p in self.depotPaths])):
1f4ba1cb
SH
1484 revisions[file["depotFile"]] = file["rev"]
1485 change = int(file["change"])
1486 if change > newestChange:
1487 newestChange = change
1488
9bda3a85
SH
1489 self.labels[newestChange] = [output, revisions]
1490
1491 if self.verbose:
1492 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 1493
86dff6b6
HWN
1494 def guessProjectName(self):
1495 for p in self.depotPaths:
6e5295c4
SH
1496 if p.endswith("/"):
1497 p = p[:-1]
1498 p = p[p.strip().rfind("/") + 1:]
1499 if not p.endswith("/"):
1500 p += "/"
1501 return p
86dff6b6 1502
4b97ffb1 1503 def getBranchMapping(self):
6555b2cc
SH
1504 lostAndFoundBranches = set()
1505
8ace74c0
VA
1506 user = gitConfig("git-p4.branchUser")
1507 if len(user) > 0:
1508 command = "branches -u %s" % user
1509 else:
1510 command = "branches"
1511
1512 for info in p4CmdList(command):
4b97ffb1
SH
1513 details = p4Cmd("branch -o %s" % info["branch"])
1514 viewIdx = 0
1515 while details.has_key("View%s" % viewIdx):
1516 paths = details["View%s" % viewIdx].split(" ")
1517 viewIdx = viewIdx + 1
1518 # require standard //depot/foo/... //depot/bar/... mapping
1519 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1520 continue
1521 source = paths[0]
1522 destination = paths[1]
6509e19c 1523 ## HACK
d53de8b9 1524 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
1525 source = source[len(self.depotPaths[0]):-4]
1526 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 1527
1a2edf4e
SH
1528 if destination in self.knownBranches:
1529 if not self.silent:
1530 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1531 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1532 continue
1533
6555b2cc
SH
1534 self.knownBranches[destination] = source
1535
1536 lostAndFoundBranches.discard(destination)
1537
29bdbac1 1538 if source not in self.knownBranches:
6555b2cc
SH
1539 lostAndFoundBranches.add(source)
1540
7199cf13
VA
1541 # Perforce does not strictly require branches to be defined, so we also
1542 # check git config for a branch list.
1543 #
1544 # Example of branch definition in git config file:
1545 # [git-p4]
1546 # branchList=main:branchA
1547 # branchList=main:branchB
1548 # branchList=branchA:branchC
1549 configBranches = gitConfigList("git-p4.branchList")
1550 for branch in configBranches:
1551 if branch:
1552 (source, destination) = branch.split(":")
1553 self.knownBranches[destination] = source
1554
1555 lostAndFoundBranches.discard(destination)
1556
1557 if source not in self.knownBranches:
1558 lostAndFoundBranches.add(source)
1559
6555b2cc
SH
1560
1561 for branch in lostAndFoundBranches:
1562 self.knownBranches[branch] = branch
29bdbac1 1563
38f9f5ec
SH
1564 def getBranchMappingFromGitBranches(self):
1565 branches = p4BranchesInGit(self.importIntoRemotes)
1566 for branch in branches.keys():
1567 if branch == "master":
1568 branch = "main"
1569 else:
1570 branch = branch[len(self.projectName):]
1571 self.knownBranches[branch] = branch
1572
29bdbac1 1573 def listExistingP4GitBranches(self):
144ff46b
SH
1574 # branches holds mapping from name to commit
1575 branches = p4BranchesInGit(self.importIntoRemotes)
1576 self.p4BranchesInGit = branches.keys()
1577 for branch in branches.keys():
1578 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 1579
bb6e09b2
HWN
1580 def updateOptionDict(self, d):
1581 option_keys = {}
1582 if self.keepRepoPath:
1583 option_keys['keepRepoPath'] = 1
1584
1585 d["options"] = ' '.join(sorted(option_keys.keys()))
1586
1587 def readOptions(self, d):
1588 self.keepRepoPath = (d.has_key('options')
1589 and ('keepRepoPath' in d['options']))
6326aa58 1590
8134f69c
SH
1591 def gitRefForBranch(self, branch):
1592 if branch == "main":
1593 return self.refPrefix + "master"
1594
1595 if len(branch) <= 0:
1596 return branch
1597
1598 return self.refPrefix + self.projectName + branch
1599
1ca3d710
SH
1600 def gitCommitByP4Change(self, ref, change):
1601 if self.verbose:
1602 print "looking in ref " + ref + " for change %s using bisect..." % change
1603
1604 earliestCommit = ""
1605 latestCommit = parseRevision(ref)
1606
1607 while True:
1608 if self.verbose:
1609 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1610 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1611 if len(next) == 0:
1612 if self.verbose:
1613 print "argh"
1614 return ""
1615 log = extractLogMessageFromGitCommit(next)
1616 settings = extractSettingsGitLog(log)
1617 currentChange = int(settings['change'])
1618 if self.verbose:
1619 print "current change %s" % currentChange
1620
1621 if currentChange == change:
1622 if self.verbose:
1623 print "found %s" % next
1624 return next
1625
1626 if currentChange < change:
1627 earliestCommit = "^%s" % next
1628 else:
1629 latestCommit = "%s" % next
1630
1631 return ""
1632
1633 def importNewBranch(self, branch, maxChange):
1634 # make fast-import flush all changes to disk and update the refs using the checkpoint
1635 # command so that we can try to find the branch parent in the git history
1636 self.gitStream.write("checkpoint\n\n");
1637 self.gitStream.flush();
1638 branchPrefix = self.depotPaths[0] + branch + "/"
1639 range = "@1,%s" % maxChange
1640 #print "prefix" + branchPrefix
1641 changes = p4ChangesForPaths([branchPrefix], range)
1642 if len(changes) <= 0:
1643 return False
1644 firstChange = changes[0]
1645 #print "first change in branch: %s" % firstChange
1646 sourceBranch = self.knownBranches[branch]
1647 sourceDepotPath = self.depotPaths[0] + sourceBranch
1648 sourceRef = self.gitRefForBranch(sourceBranch)
1649 #print "source " + sourceBranch
1650
1651 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1652 #print "branch parent: %s" % branchParentChange
1653 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1654 if len(gitParent) > 0:
1655 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1656 #print "parent git commit: %s" % gitParent
1657
1658 self.importChanges(changes)
1659 return True
1660
e87f37ae
SH
1661 def importChanges(self, changes):
1662 cnt = 1
1663 for change in changes:
1664 description = p4Cmd("describe %s" % change)
1665 self.updateOptionDict(description)
1666
1667 if not self.silent:
1668 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1669 sys.stdout.flush()
1670 cnt = cnt + 1
1671
1672 try:
1673 if self.detectBranches:
1674 branches = self.splitFilesIntoBranches(description)
1675 for branch in branches.keys():
1676 ## HACK --hwn
1677 branchPrefix = self.depotPaths[0] + branch + "/"
1678
1679 parent = ""
1680
1681 filesForCommit = branches[branch]
1682
1683 if self.verbose:
1684 print "branch is %s" % branch
1685
1686 self.updatedBranches.add(branch)
1687
1688 if branch not in self.createdBranches:
1689 self.createdBranches.add(branch)
1690 parent = self.knownBranches[branch]
1691 if parent == branch:
1692 parent = ""
1ca3d710
SH
1693 else:
1694 fullBranch = self.projectName + branch
1695 if fullBranch not in self.p4BranchesInGit:
1696 if not self.silent:
1697 print("\n Importing new branch %s" % fullBranch);
1698 if self.importNewBranch(branch, change - 1):
1699 parent = ""
1700 self.p4BranchesInGit.append(fullBranch)
1701 if not self.silent:
1702 print("\n Resuming with change %s" % change);
1703
1704 if self.verbose:
1705 print "parent determined through known branches: %s" % parent
e87f37ae 1706
8134f69c
SH
1707 branch = self.gitRefForBranch(branch)
1708 parent = self.gitRefForBranch(parent)
e87f37ae
SH
1709
1710 if self.verbose:
1711 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1712
1713 if len(parent) == 0 and branch in self.initialParents:
1714 parent = self.initialParents[branch]
1715 del self.initialParents[branch]
1716
1717 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1718 else:
1719 files = self.extractFilesFromCommit(description)
1720 self.commit(description, files, self.branch, self.depotPaths,
1721 self.initialParent)
1722 self.initialParent = ""
1723 except IOError:
1724 print self.gitError.read()
1725 sys.exit(1)
1726
c208a243
SH
1727 def importHeadRevision(self, revision):
1728 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1729
4e2e6ce4
PW
1730 details = {}
1731 details["user"] = "git perforce import user"
1494fcbb 1732 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
1733 % (' '.join(self.depotPaths), revision))
1734 details["change"] = revision
1735 newestRevision = 0
1736
1737 fileCnt = 0
1738 for info in p4CmdList("files "
1739 + ' '.join(["%s...%s"
1740 % (p, revision)
1741 for p in self.depotPaths])):
1742
68b28593 1743 if 'code' in info and info['code'] == 'error':
c208a243
SH
1744 sys.stderr.write("p4 returned an error: %s\n"
1745 % info['data'])
d88e707f
PW
1746 if info['data'].find("must refer to client") >= 0:
1747 sys.stderr.write("This particular p4 error is misleading.\n")
1748 sys.stderr.write("Perhaps the depot path was misspelled.\n");
1749 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 1750 sys.exit(1)
68b28593
PW
1751 if 'p4ExitCode' in info:
1752 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
1753 sys.exit(1)
1754
1755
1756 change = int(info["change"])
1757 if change > newestRevision:
1758 newestRevision = change
1759
56c09345 1760 if info["action"] in self.delete_actions:
c208a243
SH
1761 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1762 #fileCnt = fileCnt + 1
1763 continue
1764
1765 for prop in ["depotFile", "rev", "action", "type" ]:
1766 details["%s%s" % (prop, fileCnt)] = info[prop]
1767
1768 fileCnt = fileCnt + 1
1769
1770 details["change"] = newestRevision
4e2e6ce4
PW
1771
1772 # Use time from top-most change so that all git-p4 clones of
1773 # the same p4 repo have the same commit SHA1s.
1774 res = p4CmdList("describe -s %d" % newestRevision)
1775 newestTime = None
1776 for r in res:
1777 if r.has_key('time'):
1778 newestTime = int(r['time'])
1779 if newestTime is None:
1780 die("\"describe -s\" on newest change %d did not give a time")
1781 details["time"] = newestTime
1782
c208a243
SH
1783 self.updateOptionDict(details)
1784 try:
1785 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1786 except IOError:
1787 print "IO error with git fast-import. Is your git version recent enough?"
1788 print self.gitError.read()
1789
1790
3a70cdfa
TAL
1791 def getClientSpec(self):
1792 specList = p4CmdList( "client -o" )
1793 temp = {}
1794 for entry in specList:
1795 for k,v in entry.iteritems():
1796 if k.startswith("View"):
3952710b
IW
1797
1798 # p4 has these %%1 to %%9 arguments in specs to
1799 # reorder paths; which we can't handle (yet :)
1800 if re.match('%%\d', v) != None:
1801 print "Sorry, can't handle %%n arguments in client specs"
1802 sys.exit(1)
1803
3a70cdfa
TAL
1804 if v.startswith('"'):
1805 start = 1
1806 else:
1807 start = 0
1808 index = v.find("...")
3952710b
IW
1809
1810 # save the "client view"; i.e the RHS of the view
1811 # line that tells the client where to put the
1812 # files for this view.
1813 cv = v[index+3:].strip() # +3 to remove previous '...'
1814
1815 # if the client view doesn't end with a
1816 # ... wildcard, then we're going to mess up the
1817 # output directory, so fail gracefully.
1818 if not cv.endswith('...'):
1819 print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1820 sys.exit(1)
1821 cv=cv[:-3]
1822
1823 # now save the view; +index means included, -index
1824 # means it should be filtered out.
3a70cdfa
TAL
1825 v = v[start:index]
1826 if v.startswith("-"):
1827 v = v[1:]
3952710b 1828 include = -len(v)
3a70cdfa 1829 else:
3952710b
IW
1830 include = len(v)
1831
1832 temp[v] = (include, cv)
1833
3a70cdfa 1834 self.clientSpecDirs = temp.items()
3952710b 1835 self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
3a70cdfa 1836
b984733c 1837 def run(self, args):
6326aa58 1838 self.depotPaths = []
179caebf
SH
1839 self.changeRange = ""
1840 self.initialParent = ""
6326aa58 1841 self.previousDepotPaths = []
ce6f33c8 1842
29bdbac1
SH
1843 # map from branch depot path to parent branch
1844 self.knownBranches = {}
1845 self.initialParents = {}
5ca44617 1846 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
1847 if not self.syncWithOrigin:
1848 self.hasOrigin = False
29bdbac1 1849
a028a98e
SH
1850 if self.importIntoRemotes:
1851 self.refPrefix = "refs/remotes/p4/"
1852 else:
db775559 1853 self.refPrefix = "refs/heads/p4/"
a028a98e 1854
cebdf5af
HWN
1855 if self.syncWithOrigin and self.hasOrigin:
1856 if not self.silent:
1857 print "Syncing with origin first by calling git fetch origin"
1858 system("git fetch origin")
10f880f8 1859
569d1bd4 1860 if len(self.branch) == 0:
db775559 1861 self.branch = self.refPrefix + "master"
a028a98e 1862 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 1863 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 1864 system("git branch -D p4");
faf1bd20 1865 # create it /after/ importing, when master exists
0058a33a 1866 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 1867 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 1868
3cafb7d8 1869 if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
3a70cdfa
TAL
1870 self.getClientSpec()
1871
6a49f8e2
HWN
1872 # TODO: should always look at previous commits,
1873 # merge with previous imports, if possible.
1874 if args == []:
d414c74a 1875 if self.hasOrigin:
5ca44617 1876 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
1877 self.listExistingP4GitBranches()
1878
1879 if len(self.p4BranchesInGit) > 1:
1880 if not self.silent:
1881 print "Importing from/into multiple branches"
1882 self.detectBranches = True
967f72e2 1883
29bdbac1
SH
1884 if self.verbose:
1885 print "branches: %s" % self.p4BranchesInGit
1886
1887 p4Change = 0
1888 for branch in self.p4BranchesInGit:
cebdf5af 1889 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
1890
1891 settings = extractSettingsGitLog(logMsg)
29bdbac1 1892
bb6e09b2
HWN
1893 self.readOptions(settings)
1894 if (settings.has_key('depot-paths')
1895 and settings.has_key ('change')):
1896 change = int(settings['change']) + 1
29bdbac1
SH
1897 p4Change = max(p4Change, change)
1898
bb6e09b2
HWN
1899 depotPaths = sorted(settings['depot-paths'])
1900 if self.previousDepotPaths == []:
6326aa58 1901 self.previousDepotPaths = depotPaths
29bdbac1 1902 else:
6326aa58
HWN
1903 paths = []
1904 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
1905 prev_list = prev.split("/")
1906 cur_list = cur.split("/")
1907 for i in range(0, min(len(cur_list), len(prev_list))):
1908 if cur_list[i] <> prev_list[i]:
583e1707 1909 i = i - 1
6326aa58
HWN
1910 break
1911
04d277b3 1912 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
1913
1914 self.previousDepotPaths = paths
29bdbac1
SH
1915
1916 if p4Change > 0:
bb6e09b2 1917 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 1918 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
1919 if not self.detectBranches:
1920 self.initialParent = parseRevision(self.branch)
341dc1c1 1921 if not self.silent and not self.detectBranches:
967f72e2 1922 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 1923
f9162f6a
SH
1924 if not self.branch.startswith("refs/"):
1925 self.branch = "refs/heads/" + self.branch
179caebf 1926
6326aa58 1927 if len(args) == 0 and self.depotPaths:
b984733c 1928 if not self.silent:
6326aa58 1929 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 1930 else:
6326aa58 1931 if self.depotPaths and self.depotPaths != args:
cebdf5af 1932 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
1933 "This doesn't work!" % (' '.join (self.depotPaths),
1934 ' '.join (args)))
b984733c 1935 sys.exit(1)
6326aa58 1936
bb6e09b2 1937 self.depotPaths = sorted(args)
b984733c 1938
1c49fc19 1939 revision = ""
b984733c 1940 self.users = {}
b984733c 1941
6326aa58
HWN
1942 newPaths = []
1943 for p in self.depotPaths:
1944 if p.find("@") != -1:
1945 atIdx = p.index("@")
1946 self.changeRange = p[atIdx:]
1947 if self.changeRange == "@all":
1948 self.changeRange = ""
6a49f8e2 1949 elif ',' not in self.changeRange:
1c49fc19 1950 revision = self.changeRange
6326aa58 1951 self.changeRange = ""
7fcff9de 1952 p = p[:atIdx]
6326aa58
HWN
1953 elif p.find("#") != -1:
1954 hashIdx = p.index("#")
1c49fc19 1955 revision = p[hashIdx:]
7fcff9de 1956 p = p[:hashIdx]
6326aa58 1957 elif self.previousDepotPaths == []:
1c49fc19 1958 revision = "#head"
6326aa58
HWN
1959
1960 p = re.sub ("\.\.\.$", "", p)
1961 if not p.endswith("/"):
1962 p += "/"
1963
1964 newPaths.append(p)
1965
1966 self.depotPaths = newPaths
1967
b984733c 1968
b607e71e 1969 self.loadUserMapFromCache()
cb53e1f8
SH
1970 self.labels = {}
1971 if self.detectLabels:
1972 self.getLabels();
b984733c 1973
4b97ffb1 1974 if self.detectBranches:
df450923
SH
1975 ## FIXME - what's a P4 projectName ?
1976 self.projectName = self.guessProjectName()
1977
38f9f5ec
SH
1978 if self.hasOrigin:
1979 self.getBranchMappingFromGitBranches()
1980 else:
1981 self.getBranchMapping()
29bdbac1
SH
1982 if self.verbose:
1983 print "p4-git branches: %s" % self.p4BranchesInGit
1984 print "initial parents: %s" % self.initialParents
1985 for b in self.p4BranchesInGit:
1986 if b != "master":
6326aa58
HWN
1987
1988 ## FIXME
29bdbac1
SH
1989 b = b[len(self.projectName):]
1990 self.createdBranches.add(b)
4b97ffb1 1991
f291b4e3 1992 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 1993
cebdf5af 1994 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
1995 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1996 stderr=subprocess.PIPE);
08483580
SH
1997 self.gitOutput = importProcess.stdout
1998 self.gitStream = importProcess.stdin
1999 self.gitError = importProcess.stderr
b984733c 2000
1c49fc19 2001 if revision:
c208a243 2002 self.importHeadRevision(revision)
b984733c
SH
2003 else:
2004 changes = []
2005
0828ab14 2006 if len(self.changesFile) > 0:
b984733c 2007 output = open(self.changesFile).readlines()
1d7367dc 2008 changeSet = set()
b984733c
SH
2009 for line in output:
2010 changeSet.add(int(line))
2011
2012 for change in changeSet:
2013 changes.append(change)
2014
2015 changes.sort()
2016 else:
accad8e0
PW
2017 # catch "git-p4 sync" with no new branches, in a repo that
2018 # does not have any existing git-p4 branches
2019 if len(args) == 0 and not self.p4BranchesInGit:
e32e00dc 2020 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
29bdbac1 2021 if self.verbose:
86dff6b6 2022 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 2023 self.changeRange)
4f6432d8 2024 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 2025
01a9c9c5 2026 if len(self.maxChanges) > 0:
7fcff9de 2027 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 2028
b984733c 2029 if len(changes) == 0:
0828ab14 2030 if not self.silent:
341dc1c1 2031 print "No changes to import!"
1f52af6c 2032 return True
b984733c 2033
a9d1a27a
SH
2034 if not self.silent and not self.detectBranches:
2035 print "Import destination: %s" % self.branch
2036
341dc1c1
SH
2037 self.updatedBranches = set()
2038
e87f37ae 2039 self.importChanges(changes)
b984733c 2040
341dc1c1
SH
2041 if not self.silent:
2042 print ""
2043 if len(self.updatedBranches) > 0:
2044 sys.stdout.write("Updated branches: ")
2045 for b in self.updatedBranches:
2046 sys.stdout.write("%s " % b)
2047 sys.stdout.write("\n")
b984733c 2048
b984733c 2049 self.gitStream.close()
29bdbac1
SH
2050 if importProcess.wait() != 0:
2051 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
2052 self.gitOutput.close()
2053 self.gitError.close()
2054
b984733c
SH
2055 return True
2056
01ce1fe9
SH
2057class P4Rebase(Command):
2058 def __init__(self):
2059 Command.__init__(self)
01265103 2060 self.options = [ ]
cebdf5af
HWN
2061 self.description = ("Fetches the latest revision from perforce and "
2062 + "rebases the current work (branch) against it")
68c42153 2063 self.verbose = False
01ce1fe9
SH
2064
2065 def run(self, args):
2066 sync = P4Sync()
2067 sync.run([])
d7e3868c 2068
14594f4b
SH
2069 return self.rebase()
2070
2071 def rebase(self):
36ee4ee4
SH
2072 if os.system("git update-index --refresh") != 0:
2073 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2074 if len(read_pipe("git diff-index HEAD --")) > 0:
2075 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2076
d7e3868c
SH
2077 [upstream, settings] = findUpstreamBranchPoint()
2078 if len(upstream) == 0:
2079 die("Cannot find upstream branchpoint for rebase")
2080
2081 # the branchpoint may be p4/foo~3, so strip off the parent
2082 upstream = re.sub("~[0-9]+$", "", upstream)
2083
2084 print "Rebasing the current branch onto %s" % upstream
b25b2065 2085 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 2086 system("git rebase %s" % upstream)
1f52af6c 2087 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
2088 return True
2089
f9a3a4f7
SH
2090class P4Clone(P4Sync):
2091 def __init__(self):
2092 P4Sync.__init__(self)
2093 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 2094 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 2095 self.options += [
bb6e09b2
HWN
2096 optparse.make_option("--destination", dest="cloneDestination",
2097 action='store', default=None,
354081d5
TT
2098 help="where to leave result of the clone"),
2099 optparse.make_option("-/", dest="cloneExclude",
2100 action="append", type="string",
38200076
PW
2101 help="exclude depot path"),
2102 optparse.make_option("--bare", dest="cloneBare",
2103 action="store_true", default=False),
354081d5 2104 ]
bb6e09b2 2105 self.cloneDestination = None
f9a3a4f7 2106 self.needsGit = False
38200076 2107 self.cloneBare = False
f9a3a4f7 2108
354081d5
TT
2109 # This is required for the "append" cloneExclude action
2110 def ensure_value(self, attr, value):
2111 if not hasattr(self, attr) or getattr(self, attr) is None:
2112 setattr(self, attr, value)
2113 return getattr(self, attr)
2114
6a49f8e2
HWN
2115 def defaultDestination(self, args):
2116 ## TODO: use common prefix of args?
2117 depotPath = args[0]
2118 depotDir = re.sub("(@[^@]*)$", "", depotPath)
2119 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 2120 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
2121 depotDir = re.sub(r"/$", "", depotDir)
2122 return os.path.split(depotDir)[1]
2123
f9a3a4f7
SH
2124 def run(self, args):
2125 if len(args) < 1:
2126 return False
bb6e09b2
HWN
2127
2128 if self.keepRepoPath and not self.cloneDestination:
2129 sys.stderr.write("Must specify destination for --keep-path\n")
2130 sys.exit(1)
f9a3a4f7 2131
6326aa58 2132 depotPaths = args
5e100b5c
SH
2133
2134 if not self.cloneDestination and len(depotPaths) > 1:
2135 self.cloneDestination = depotPaths[-1]
2136 depotPaths = depotPaths[:-1]
2137
354081d5 2138 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
2139 for p in depotPaths:
2140 if not p.startswith("//"):
2141 return False
f9a3a4f7 2142
bb6e09b2 2143 if not self.cloneDestination:
98ad4faf 2144 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 2145
86dff6b6 2146 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 2147
c3bf3f13
KG
2148 if not os.path.exists(self.cloneDestination):
2149 os.makedirs(self.cloneDestination)
053fd0c1 2150 chdir(self.cloneDestination)
38200076
PW
2151
2152 init_cmd = [ "git", "init" ]
2153 if self.cloneBare:
2154 init_cmd.append("--bare")
2155 subprocess.check_call(init_cmd)
2156
6326aa58 2157 if not P4Sync.run(self, depotPaths):
f9a3a4f7 2158 return False
f9a3a4f7 2159 if self.branch != "master":
e9905013
TAL
2160 if self.importIntoRemotes:
2161 masterbranch = "refs/remotes/p4/master"
2162 else:
2163 masterbranch = "refs/heads/p4/master"
2164 if gitBranchExists(masterbranch):
2165 system("git branch master %s" % masterbranch)
38200076
PW
2166 if not self.cloneBare:
2167 system("git checkout -f")
8f9b2e08
SH
2168 else:
2169 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 2170
f9a3a4f7
SH
2171 return True
2172
09d89de2
SH
2173class P4Branches(Command):
2174 def __init__(self):
2175 Command.__init__(self)
2176 self.options = [ ]
2177 self.description = ("Shows the git branches that hold imports and their "
2178 + "corresponding perforce depot paths")
2179 self.verbose = False
2180
2181 def run(self, args):
5ca44617
SH
2182 if originP4BranchesExist():
2183 createOrUpdateBranchesFromOrigin()
2184
09d89de2
SH
2185 cmdline = "git rev-parse --symbolic "
2186 cmdline += " --remotes"
2187
2188 for line in read_pipe_lines(cmdline):
2189 line = line.strip()
2190
2191 if not line.startswith('p4/') or line == "p4/HEAD":
2192 continue
2193 branch = line
2194
2195 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2196 settings = extractSettingsGitLog(log)
2197
2198 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2199 return True
2200
b984733c
SH
2201class HelpFormatter(optparse.IndentedHelpFormatter):
2202 def __init__(self):
2203 optparse.IndentedHelpFormatter.__init__(self)
2204
2205 def format_description(self, description):
2206 if description:
2207 return description + "\n"
2208 else:
2209 return ""
4f5cf76a 2210
86949eef
SH
2211def printUsage(commands):
2212 print "usage: %s <command> [options]" % sys.argv[0]
2213 print ""
2214 print "valid commands: %s" % ", ".join(commands)
2215 print ""
2216 print "Try %s <command> --help for command specific help." % sys.argv[0]
2217 print ""
2218
2219commands = {
b86f7378
HWN
2220 "debug" : P4Debug,
2221 "submit" : P4Submit,
a9834f58 2222 "commit" : P4Submit,
b86f7378
HWN
2223 "sync" : P4Sync,
2224 "rebase" : P4Rebase,
2225 "clone" : P4Clone,
09d89de2
SH
2226 "rollback" : P4RollBack,
2227 "branches" : P4Branches
86949eef
SH
2228}
2229
86949eef 2230
bb6e09b2
HWN
2231def main():
2232 if len(sys.argv[1:]) == 0:
2233 printUsage(commands.keys())
2234 sys.exit(2)
4f5cf76a 2235
bb6e09b2
HWN
2236 cmd = ""
2237 cmdName = sys.argv[1]
2238 try:
b86f7378
HWN
2239 klass = commands[cmdName]
2240 cmd = klass()
bb6e09b2
HWN
2241 except KeyError:
2242 print "unknown command %s" % cmdName
2243 print ""
2244 printUsage(commands.keys())
2245 sys.exit(2)
2246
2247 options = cmd.options
b86f7378 2248 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
2249
2250 args = sys.argv[2:]
2251
2252 if len(options) > 0:
2253 options.append(optparse.make_option("--git-dir", dest="gitdir"))
2254
2255 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2256 options,
2257 description = cmd.description,
2258 formatter = HelpFormatter())
2259
2260 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2261 global verbose
2262 verbose = cmd.verbose
2263 if cmd.needsGit:
b86f7378
HWN
2264 if cmd.gitdir == None:
2265 cmd.gitdir = os.path.abspath(".git")
2266 if not isValidGitDir(cmd.gitdir):
2267 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2268 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
2269 cdup = read_pipe("git rev-parse --show-cdup").strip()
2270 if len(cdup) > 0:
053fd0c1 2271 chdir(cdup);
e20a9e53 2272
b86f7378
HWN
2273 if not isValidGitDir(cmd.gitdir):
2274 if isValidGitDir(cmd.gitdir + "/.git"):
2275 cmd.gitdir += "/.git"
bb6e09b2 2276 else:
b86f7378 2277 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 2278
b86f7378 2279 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 2280
bb6e09b2
HWN
2281 if not cmd.run(args):
2282 parser.print_help()
4f5cf76a 2283
4f5cf76a 2284
bb6e09b2
HWN
2285if __name__ == '__main__':
2286 main()