Merge branch 'js/merge-edit-option'
[git/git.git] / contrib / fast-import / git-p4
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
1d7367dc
RG
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
ce6f33c8 13import re
8b41a97f 14
4addad22 15verbose = False
86949eef 16
21a50753
AK
17
18def p4_build_cmd(cmd):
19 """Build a suitable p4 command line.
20
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
24 """
6de040df 25 real_cmd = ["p4"]
abcaf073
AK
26
27 user = gitConfig("git-p4.user")
28 if len(user) > 0:
6de040df 29 real_cmd += ["-u",user]
abcaf073
AK
30
31 password = gitConfig("git-p4.password")
32 if len(password) > 0:
6de040df 33 real_cmd += ["-P", password]
abcaf073
AK
34
35 port = gitConfig("git-p4.port")
36 if len(port) > 0:
6de040df 37 real_cmd += ["-p", port]
abcaf073
AK
38
39 host = gitConfig("git-p4.host")
40 if len(host) > 0:
6de040df 41 real_cmd += ["-h", host]
abcaf073
AK
42
43 client = gitConfig("git-p4.client")
44 if len(client) > 0:
6de040df 45 real_cmd += ["-c", client]
abcaf073 46
6de040df
LD
47
48 if isinstance(cmd,basestring):
49 real_cmd = ' '.join(real_cmd) + ' ' + cmd
50 else:
51 real_cmd += cmd
21a50753
AK
52 return real_cmd
53
053fd0c1 54def chdir(dir):
6de040df
LD
55 # P4 uses the PWD environment variable rather than getcwd(). Since we're
56 # not using the shell, we have to set it ourselves.
57 os.environ['PWD']=dir
053fd0c1
RB
58 os.chdir(dir)
59
86dff6b6
HWN
60def die(msg):
61 if verbose:
62 raise Exception(msg)
63 else:
64 sys.stderr.write(msg + "\n")
65 sys.exit(1)
66
6de040df 67def write_pipe(c, stdin):
4addad22 68 if verbose:
6de040df 69 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 70
6de040df
LD
71 expand = isinstance(c,basestring)
72 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
73 pipe = p.stdin
74 val = pipe.write(stdin)
75 pipe.close()
76 if p.wait():
77 die('Command failed: %s' % str(c))
b016d397
HWN
78
79 return val
80
6de040df 81def p4_write_pipe(c, stdin):
d9429194 82 real_cmd = p4_build_cmd(c)
6de040df 83 return write_pipe(real_cmd, stdin)
d9429194 84
4addad22
HWN
85def read_pipe(c, ignore_error=False):
86 if verbose:
6de040df 87 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 88
6de040df
LD
89 expand = isinstance(c,basestring)
90 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
91 pipe = p.stdout
b016d397 92 val = pipe.read()
6de040df
LD
93 if p.wait() and not ignore_error:
94 die('Command failed: %s' % str(c))
b016d397
HWN
95
96 return val
97
d9429194
AK
98def p4_read_pipe(c, ignore_error=False):
99 real_cmd = p4_build_cmd(c)
100 return read_pipe(real_cmd, ignore_error)
b016d397 101
bce4c5fc 102def read_pipe_lines(c):
4addad22 103 if verbose:
6de040df
LD
104 sys.stderr.write('Reading pipe: %s\n' % str(c))
105
106 expand = isinstance(c, basestring)
107 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
108 pipe = p.stdout
b016d397 109 val = pipe.readlines()
6de040df
LD
110 if pipe.close() or p.wait():
111 die('Command failed: %s' % str(c))
b016d397
HWN
112
113 return val
caace111 114
2318121b
AK
115def p4_read_pipe_lines(c):
116 """Specifically invoke p4 on the command supplied. """
155af834 117 real_cmd = p4_build_cmd(c)
2318121b
AK
118 return read_pipe_lines(real_cmd)
119
6754a299 120def system(cmd):
6de040df 121 expand = isinstance(cmd,basestring)
4addad22 122 if verbose:
6de040df
LD
123 sys.stderr.write("executing %s\n" % str(cmd))
124 subprocess.check_call(cmd, shell=expand)
6754a299 125
bf9320f1
AK
126def p4_system(cmd):
127 """Specifically invoke p4 as the system command. """
155af834 128 real_cmd = p4_build_cmd(cmd)
6de040df
LD
129 expand = isinstance(real_cmd, basestring)
130 subprocess.check_call(real_cmd, shell=expand)
131
132def p4_integrate(src, dest):
133 p4_system(["integrate", "-Dt", src, dest])
134
135def p4_sync(path):
136 p4_system(["sync", path])
137
138def p4_add(f):
139 p4_system(["add", f])
140
141def p4_delete(f):
142 p4_system(["delete", f])
143
144def p4_edit(f):
145 p4_system(["edit", f])
146
147def p4_revert(f):
148 p4_system(["revert", f])
149
150def p4_reopen(type, file):
151 p4_system(["reopen", "-t", type, file])
bf9320f1 152
9cffb8c8
PW
153#
154# Canonicalize the p4 type and return a tuple of the
155# base type, plus any modifiers. See "p4 help filetypes"
156# for a list and explanation.
157#
158def split_p4_type(p4type):
159
160 p4_filetypes_historical = {
161 "ctempobj": "binary+Sw",
162 "ctext": "text+C",
163 "cxtext": "text+Cx",
164 "ktext": "text+k",
165 "kxtext": "text+kx",
166 "ltext": "text+F",
167 "tempobj": "binary+FSw",
168 "ubinary": "binary+F",
169 "uresource": "resource+F",
170 "uxbinary": "binary+Fx",
171 "xbinary": "binary+x",
172 "xltext": "text+Fx",
173 "xtempobj": "binary+Swx",
174 "xtext": "text+x",
175 "xunicode": "unicode+x",
176 "xutf16": "utf16+x",
177 }
178 if p4type in p4_filetypes_historical:
179 p4type = p4_filetypes_historical[p4type]
180 mods = ""
181 s = p4type.split("+")
182 base = s[0]
183 mods = ""
184 if len(s) > 1:
185 mods = s[1]
186 return (base, mods)
b9fc6ea9 187
b9fc6ea9 188
c65b670e
CP
189def setP4ExecBit(file, mode):
190 # Reopens an already open file and changes the execute bit to match
191 # the execute bit setting in the passed in mode.
192
193 p4Type = "+x"
194
195 if not isModeExec(mode):
196 p4Type = getP4OpenedType(file)
197 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
198 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
199 if p4Type[-1] == "+":
200 p4Type = p4Type[0:-1]
201
6de040df 202 p4_reopen(p4Type, file)
c65b670e
CP
203
204def getP4OpenedType(file):
205 # Returns the perforce file type for the given file.
206
6de040df 207 result = p4_read_pipe(["opened", file])
f3e5ae4f 208 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
209 if match:
210 return match.group(1)
211 else:
f3e5ae4f 212 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 213
b43b0a3c
CP
214def diffTreePattern():
215 # This is a simple generator for the diff tree regex pattern. This could be
216 # a class variable if this and parseDiffTreeEntry were a part of a class.
217 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
218 while True:
219 yield pattern
220
221def parseDiffTreeEntry(entry):
222 """Parses a single diff tree entry into its component elements.
223
224 See git-diff-tree(1) manpage for details about the format of the diff
225 output. This method returns a dictionary with the following elements:
226
227 src_mode - The mode of the source file
228 dst_mode - The mode of the destination file
229 src_sha1 - The sha1 for the source file
230 dst_sha1 - The sha1 fr the destination file
231 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
232 status_score - The score for the status (applicable for 'C' and 'R'
233 statuses). This is None if there is no score.
234 src - The path for the source file.
235 dst - The path for the destination file. This is only present for
236 copy or renames. If it is not present, this is None.
237
238 If the pattern is not matched, None is returned."""
239
240 match = diffTreePattern().next().match(entry)
241 if match:
242 return {
243 'src_mode': match.group(1),
244 'dst_mode': match.group(2),
245 'src_sha1': match.group(3),
246 'dst_sha1': match.group(4),
247 'status': match.group(5),
248 'status_score': match.group(6),
249 'src': match.group(7),
250 'dst': match.group(10)
251 }
252 return None
253
c65b670e
CP
254def isModeExec(mode):
255 # Returns True if the given git mode represents an executable file,
256 # otherwise False.
257 return mode[-3:] == "755"
258
259def isModeExecChanged(src_mode, dst_mode):
260 return isModeExec(src_mode) != isModeExec(dst_mode)
261
b932705b 262def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
263
264 if isinstance(cmd,basestring):
265 cmd = "-G " + cmd
266 expand = True
267 else:
268 cmd = ["-G"] + cmd
269 expand = False
270
271 cmd = p4_build_cmd(cmd)
6a49f8e2 272 if verbose:
6de040df 273 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
274
275 # Use a temporary file to avoid deadlocks without
276 # subprocess.communicate(), which would put another copy
277 # of stdout into memory.
278 stdin_file = None
279 if stdin is not None:
280 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
281 if isinstance(stdin,basestring):
282 stdin_file.write(stdin)
283 else:
284 for i in stdin:
285 stdin_file.write(i + '\n')
9f90c733
SL
286 stdin_file.flush()
287 stdin_file.seek(0)
288
6de040df
LD
289 p4 = subprocess.Popen(cmd,
290 shell=expand,
9f90c733
SL
291 stdin=stdin_file,
292 stdout=subprocess.PIPE)
86949eef
SH
293
294 result = []
295 try:
296 while True:
9f90c733 297 entry = marshal.load(p4.stdout)
c3f6163b
AG
298 if cb is not None:
299 cb(entry)
300 else:
301 result.append(entry)
86949eef
SH
302 except EOFError:
303 pass
9f90c733
SL
304 exitCode = p4.wait()
305 if exitCode != 0:
ac3e0d79
SH
306 entry = {}
307 entry["p4ExitCode"] = exitCode
308 result.append(entry)
86949eef
SH
309
310 return result
311
312def p4Cmd(cmd):
313 list = p4CmdList(cmd)
314 result = {}
315 for entry in list:
316 result.update(entry)
317 return result;
318
cb2c9db5
SH
319def p4Where(depotPath):
320 if not depotPath.endswith("/"):
321 depotPath += "/"
7f705dc3 322 depotPath = depotPath + "..."
6de040df 323 outputList = p4CmdList(["where", depotPath])
7f705dc3
TAL
324 output = None
325 for entry in outputList:
75bc9573
TAL
326 if "depotFile" in entry:
327 if entry["depotFile"] == depotPath:
328 output = entry
329 break
330 elif "data" in entry:
331 data = entry.get("data")
332 space = data.find(" ")
333 if data[:space] == depotPath:
334 output = entry
335 break
7f705dc3
TAL
336 if output == None:
337 return ""
dc524036
SH
338 if output["code"] == "error":
339 return ""
cb2c9db5
SH
340 clientPath = ""
341 if "path" in output:
342 clientPath = output.get("path")
343 elif "data" in output:
344 data = output.get("data")
345 lastSpace = data.rfind(" ")
346 clientPath = data[lastSpace + 1:]
347
348 if clientPath.endswith("..."):
349 clientPath = clientPath[:-3]
350 return clientPath
351
86949eef 352def currentGitBranch():
b25b2065 353 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 354
4f5cf76a 355def isValidGitDir(path):
bb6e09b2
HWN
356 if (os.path.exists(path + "/HEAD")
357 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
358 return True;
359 return False
360
463e8af6 361def parseRevision(ref):
b25b2065 362 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 363
6ae8de88
SH
364def extractLogMessageFromGitCommit(commit):
365 logMessage = ""
b016d397
HWN
366
367 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 368 foundTitle = False
b016d397 369 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
370 if not foundTitle:
371 if len(log) == 1:
1c094184 372 foundTitle = True
6ae8de88
SH
373 continue
374
375 logMessage += log
376 return logMessage
377
bb6e09b2 378def extractSettingsGitLog(log):
6ae8de88
SH
379 values = {}
380 for line in log.split("\n"):
381 line = line.strip()
6326aa58
HWN
382 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
383 if not m:
384 continue
385
386 assignments = m.group(1).split (':')
387 for a in assignments:
388 vals = a.split ('=')
389 key = vals[0].strip()
390 val = ('='.join (vals[1:])).strip()
391 if val.endswith ('\"') and val.startswith('"'):
392 val = val[1:-1]
393
394 values[key] = val
395
845b42cb
SH
396 paths = values.get("depot-paths")
397 if not paths:
398 paths = values.get("depot-path")
a3fdd579
SH
399 if paths:
400 values['depot-paths'] = paths.split(',')
bb6e09b2 401 return values
6ae8de88 402
8136a639 403def gitBranchExists(branch):
bb6e09b2
HWN
404 proc = subprocess.Popen(["git", "rev-parse", branch],
405 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 406 return proc.wait() == 0;
8136a639 407
36bd8446 408_gitConfig = {}
99f790f2 409def gitConfig(key, args = None): # set args to "--bool", for instance
36bd8446 410 if not _gitConfig.has_key(key):
99f790f2
TAL
411 argsFilter = ""
412 if args != None:
413 argsFilter = "%s " % args
414 cmd = "git config %s%s" % (argsFilter, key)
415 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
36bd8446 416 return _gitConfig[key]
01265103 417
7199cf13
VA
418def gitConfigList(key):
419 if not _gitConfig.has_key(key):
420 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
421 return _gitConfig[key]
422
062410bb
SH
423def p4BranchesInGit(branchesAreInRemotes = True):
424 branches = {}
425
426 cmdline = "git rev-parse --symbolic "
427 if branchesAreInRemotes:
428 cmdline += " --remotes"
429 else:
430 cmdline += " --branches"
431
432 for line in read_pipe_lines(cmdline):
433 line = line.strip()
434
435 ## only import to p4/
436 if not line.startswith('p4/') or line == "p4/HEAD":
437 continue
438 branch = line
439
440 # strip off p4
441 branch = re.sub ("^p4/", "", line)
442
443 branches[branch] = parseRevision(line)
444 return branches
445
9ceab363 446def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
447 branches = p4BranchesInGit()
448 # map from depot-path to branch name
449 branchByDepotPath = {}
450 for branch in branches.keys():
451 tip = branches[branch]
452 log = extractLogMessageFromGitCommit(tip)
453 settings = extractSettingsGitLog(log)
454 if settings.has_key("depot-paths"):
455 paths = ",".join(settings["depot-paths"])
456 branchByDepotPath[paths] = "remotes/p4/" + branch
457
27d2d811 458 settings = None
27d2d811
SH
459 parent = 0
460 while parent < 65535:
9ceab363 461 commit = head + "~%s" % parent
27d2d811
SH
462 log = extractLogMessageFromGitCommit(commit)
463 settings = extractSettingsGitLog(log)
86506fe5
SH
464 if settings.has_key("depot-paths"):
465 paths = ",".join(settings["depot-paths"])
466 if branchByDepotPath.has_key(paths):
467 return [branchByDepotPath[paths], settings]
27d2d811 468
86506fe5 469 parent = parent + 1
27d2d811 470
86506fe5 471 return ["", settings]
27d2d811 472
5ca44617
SH
473def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
474 if not silent:
475 print ("Creating/updating branch(es) in %s based on origin branch(es)"
476 % localRefPrefix)
477
478 originPrefix = "origin/p4/"
479
480 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
481 line = line.strip()
482 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
483 continue
484
485 headName = line[len(originPrefix):]
486 remoteHead = localRefPrefix + headName
487 originHead = line
488
489 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
490 if (not original.has_key('depot-paths')
491 or not original.has_key('change')):
492 continue
493
494 update = False
495 if not gitBranchExists(remoteHead):
496 if verbose:
497 print "creating %s" % remoteHead
498 update = True
499 else:
500 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
501 if settings.has_key('change') > 0:
502 if settings['depot-paths'] == original['depot-paths']:
503 originP4Change = int(original['change'])
504 p4Change = int(settings['change'])
505 if originP4Change > p4Change:
506 print ("%s (%s) is newer than %s (%s). "
507 "Updating p4 branch from origin."
508 % (originHead, originP4Change,
509 remoteHead, p4Change))
510 update = True
511 else:
512 print ("Ignoring: %s was imported from %s while "
513 "%s was imported from %s"
514 % (originHead, ','.join(original['depot-paths']),
515 remoteHead, ','.join(settings['depot-paths'])))
516
517 if update:
518 system("git update-ref %s %s" % (remoteHead, originHead))
519
520def originP4BranchesExist():
521 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
522
4f6432d8
SH
523def p4ChangesForPaths(depotPaths, changeRange):
524 assert depotPaths
6de040df
LD
525 cmd = ['changes']
526 for p in depotPaths:
527 cmd += ["%s...%s" % (p, changeRange)]
528 output = p4_read_pipe_lines(cmd)
4f6432d8 529
b4b0ba06 530 changes = {}
4f6432d8 531 for line in output:
c3f6163b
AG
532 changeNum = int(line.split(" ")[1])
533 changes[changeNum] = True
4f6432d8 534
b4b0ba06
PW
535 changelist = changes.keys()
536 changelist.sort()
537 return changelist
4f6432d8 538
d53de8b9
TAL
539def p4PathStartsWith(path, prefix):
540 # This method tries to remedy a potential mixed-case issue:
541 #
542 # If UserA adds //depot/DirA/file1
543 # and UserB adds //depot/dira/file2
544 #
545 # we may or may not have a problem. If you have core.ignorecase=true,
546 # we treat DirA and dira as the same directory
547 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
548 if ignorecase:
549 return path.lower().startswith(prefix.lower())
550 return path.startswith(prefix)
551
b984733c
SH
552class Command:
553 def __init__(self):
554 self.usage = "usage: %prog [options]"
8910ac0e 555 self.needsGit = True
b984733c 556
3ea2cfd4
LD
557class P4UserMap:
558 def __init__(self):
559 self.userMapFromPerforceServer = False
560
561 def getUserCacheFilename(self):
562 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
563 return home + "/.gitp4-usercache.txt"
564
565 def getUserMapFromPerforceServer(self):
566 if self.userMapFromPerforceServer:
567 return
568 self.users = {}
569 self.emails = {}
570
571 for output in p4CmdList("users"):
572 if not output.has_key("User"):
573 continue
574 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
575 self.emails[output["Email"]] = output["User"]
576
577
578 s = ''
579 for (key, val) in self.users.items():
580 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
581
582 open(self.getUserCacheFilename(), "wb").write(s)
583 self.userMapFromPerforceServer = True
584
585 def loadUserMapFromCache(self):
586 self.users = {}
587 self.userMapFromPerforceServer = False
588 try:
589 cache = open(self.getUserCacheFilename(), "rb")
590 lines = cache.readlines()
591 cache.close()
592 for line in lines:
593 entry = line.strip().split("\t")
594 self.users[entry[0]] = entry[1]
595 except IOError:
596 self.getUserMapFromPerforceServer()
597
b984733c 598class P4Debug(Command):
86949eef 599 def __init__(self):
6ae8de88 600 Command.__init__(self)
86949eef 601 self.options = [
b1ce9447
HWN
602 optparse.make_option("--verbose", dest="verbose", action="store_true",
603 default=False),
4addad22 604 ]
c8c39116 605 self.description = "A tool to debug the output of p4 -G."
8910ac0e 606 self.needsGit = False
b1ce9447 607 self.verbose = False
86949eef
SH
608
609 def run(self, args):
b1ce9447 610 j = 0
6de040df 611 for output in p4CmdList(args):
b1ce9447
HWN
612 print 'Element: %d' % j
613 j += 1
86949eef 614 print output
b984733c 615 return True
86949eef 616
5834684d
SH
617class P4RollBack(Command):
618 def __init__(self):
619 Command.__init__(self)
620 self.options = [
0c66a783
SH
621 optparse.make_option("--verbose", dest="verbose", action="store_true"),
622 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
623 ]
624 self.description = "A tool to debug the multi-branch import. Don't use :)"
52102d47 625 self.verbose = False
0c66a783 626 self.rollbackLocalBranches = False
5834684d
SH
627
628 def run(self, args):
629 if len(args) != 1:
630 return False
631 maxChange = int(args[0])
0c66a783 632
ad192f28 633 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
634 die("Problems executing p4");
635
0c66a783
SH
636 if self.rollbackLocalBranches:
637 refPrefix = "refs/heads/"
b016d397 638 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
639 else:
640 refPrefix = "refs/remotes/"
b016d397 641 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
642
643 for line in lines:
644 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
645 line = line.strip()
646 ref = refPrefix + line
5834684d 647 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
648 settings = extractSettingsGitLog(log)
649
650 depotPaths = settings['depot-paths']
651 change = settings['change']
652
5834684d 653 changed = False
52102d47 654
6326aa58
HWN
655 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
656 for p in depotPaths]))) == 0:
52102d47
SH
657 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
658 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
659 continue
660
bb6e09b2 661 while change and int(change) > maxChange:
5834684d 662 changed = True
52102d47
SH
663 if self.verbose:
664 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
665 system("git update-ref %s \"%s^\"" % (ref, ref))
666 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
667 settings = extractSettingsGitLog(log)
668
669
670 depotPaths = settings['depot-paths']
671 change = settings['change']
5834684d
SH
672
673 if changed:
52102d47 674 print "%s rewound to %s" % (ref, change)
5834684d
SH
675
676 return True
677
3ea2cfd4 678class P4Submit(Command, P4UserMap):
4f5cf76a 679 def __init__(self):
b984733c 680 Command.__init__(self)
3ea2cfd4 681 P4UserMap.__init__(self)
4f5cf76a 682 self.options = [
4addad22 683 optparse.make_option("--verbose", dest="verbose", action="store_true"),
4f5cf76a 684 optparse.make_option("--origin", dest="origin"),
ae901090 685 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
686 # preserve the user, requires relevant p4 permissions
687 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
4f5cf76a
SH
688 ]
689 self.description = "Submit changes from git to the perforce depot."
c9b50e63 690 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 691 self.interactive = True
9512497b 692 self.origin = ""
ae901090 693 self.detectRenames = False
b0d10df7 694 self.verbose = False
3ea2cfd4 695 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
f7baba8b 696 self.isWindows = (platform.system() == "Windows")
848de9c3 697 self.myP4UserId = None
4f5cf76a 698
4f5cf76a
SH
699 def check(self):
700 if len(p4CmdList("opened ...")) > 0:
701 die("You have files opened with perforce! Close them before starting the sync.")
702
edae1e2f
SH
703 # replaces everything between 'Description:' and the next P4 submit template field with the
704 # commit message
4f5cf76a
SH
705 def prepareLogMessage(self, template, message):
706 result = ""
707
edae1e2f
SH
708 inDescriptionSection = False
709
4f5cf76a
SH
710 for line in template.split("\n"):
711 if line.startswith("#"):
712 result += line + "\n"
713 continue
714
edae1e2f 715 if inDescriptionSection:
c9dbab04 716 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f
SH
717 inDescriptionSection = False
718 else:
719 continue
720 else:
721 if line.startswith("Description:"):
722 inDescriptionSection = True
723 line += "\n"
724 for messageLine in message.split("\n"):
725 line += "\t" + messageLine + "\n"
726
727 result += line + "\n"
4f5cf76a
SH
728
729 return result
730
3ea2cfd4
LD
731 def p4UserForCommit(self,id):
732 # Return the tuple (perforce user,git email) for a given git commit id
733 self.getUserMapFromPerforceServer()
734 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
735 gitEmail = gitEmail.strip()
736 if not self.emails.has_key(gitEmail):
737 return (None,gitEmail)
738 else:
739 return (self.emails[gitEmail],gitEmail)
740
741 def checkValidP4Users(self,commits):
742 # check if any git authors cannot be mapped to p4 users
743 for id in commits:
744 (user,email) = self.p4UserForCommit(id)
745 if not user:
746 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
747 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
748 print "%s" % msg
749 else:
750 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
751
752 def lastP4Changelist(self):
753 # Get back the last changelist number submitted in this client spec. This
754 # then gets used to patch up the username in the change. If the same
755 # client spec is being used by multiple processes then this might go
756 # wrong.
757 results = p4CmdList("client -o") # find the current client
758 client = None
759 for r in results:
760 if r.has_key('Client'):
761 client = r['Client']
762 break
763 if not client:
764 die("could not get client spec")
6de040df 765 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
766 for r in results:
767 if r.has_key('change'):
768 return r['change']
769 die("Could not get changelist number for last submit - cannot patch up user details")
770
771 def modifyChangelistUser(self, changelist, newUser):
772 # fixup the user field of a changelist after it has been submitted.
773 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
774 if len(changes) != 1:
775 die("Bad output from p4 change modifying %s to user %s" %
776 (changelist, newUser))
777
778 c = changes[0]
779 if c['User'] == newUser: return # nothing to do
780 c['User'] = newUser
781 input = marshal.dumps(c)
782
3ea2cfd4
LD
783 result = p4CmdList("change -f -i", stdin=input)
784 for r in result:
785 if r.has_key('code'):
786 if r['code'] == 'error':
787 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
788 if r.has_key('data'):
789 print("Updated user field for changelist %s to %s" % (changelist, newUser))
790 return
791 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
792
793 def canChangeChangelists(self):
794 # check to see if we have p4 admin or super-user permissions, either of
795 # which are required to modify changelists.
ecdba36d 796 results = p4CmdList("protects %s" % self.depotPath)
3ea2cfd4
LD
797 for r in results:
798 if r.has_key('perm'):
799 if r['perm'] == 'admin':
800 return 1
801 if r['perm'] == 'super':
802 return 1
803 return 0
804
848de9c3
LD
805 def p4UserId(self):
806 if self.myP4UserId:
807 return self.myP4UserId
808
809 results = p4CmdList("user -o")
810 for r in results:
811 if r.has_key('User'):
812 self.myP4UserId = r['User']
813 return r['User']
814 die("Could not find your p4 user id")
815
816 def p4UserIsMe(self, p4User):
817 # return True if the given p4 user is actually me
818 me = self.p4UserId()
819 if not p4User or p4User != me:
820 return False
821 else:
822 return True
823
ea99c3ae
SH
824 def prepareSubmitTemplate(self):
825 # remove lines in the Files section that show changes to files outside the depot path we're committing into
826 template = ""
827 inFilesSection = False
6de040df 828 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
829 if line.endswith("\r\n"):
830 line = line[:-2] + "\n"
ea99c3ae
SH
831 if inFilesSection:
832 if line.startswith("\t"):
833 # path starts and ends with a tab
834 path = line[1:]
835 lastTab = path.rfind("\t")
836 if lastTab != -1:
837 path = path[:lastTab]
d53de8b9 838 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
839 continue
840 else:
841 inFilesSection = False
842 else:
843 if line.startswith("Files:"):
844 inFilesSection = True
845
846 template += line
847
848 return template
849
7cb5cbef 850 def applyCommit(self, id):
0e36f2d7 851 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
ae901090 852
848de9c3 853 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 854
ae901090
VA
855 if not self.detectRenames:
856 # If not explicitly set check the config variable
0a9feffc 857 self.detectRenames = gitConfig("git-p4.detectRenames")
ae901090 858
0a9feffc
VA
859 if self.detectRenames.lower() == "false" or self.detectRenames == "":
860 diffOpts = ""
861 elif self.detectRenames.lower() == "true":
ae901090
VA
862 diffOpts = "-M"
863 else:
0a9feffc 864 diffOpts = "-M%s" % self.detectRenames
ae901090 865
0a9feffc
VA
866 detectCopies = gitConfig("git-p4.detectCopies")
867 if detectCopies.lower() == "true":
4fddb41b 868 diffOpts += " -C"
0a9feffc
VA
869 elif detectCopies != "" and detectCopies.lower() != "false":
870 diffOpts += " -C%s" % detectCopies
4fddb41b 871
68cbcf1b 872 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
4fddb41b
VA
873 diffOpts += " --find-copies-harder"
874
0e36f2d7 875 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
876 filesToAdd = set()
877 filesToDelete = set()
d336c158 878 editedFiles = set()
c65b670e 879 filesToChangeExecBit = {}
4f5cf76a 880 for line in diff:
b43b0a3c
CP
881 diff = parseDiffTreeEntry(line)
882 modifier = diff['status']
883 path = diff['src']
4f5cf76a 884 if modifier == "M":
6de040df 885 p4_edit(path)
c65b670e
CP
886 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
887 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 888 editedFiles.add(path)
4f5cf76a
SH
889 elif modifier == "A":
890 filesToAdd.add(path)
c65b670e 891 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
892 if path in filesToDelete:
893 filesToDelete.remove(path)
894 elif modifier == "D":
895 filesToDelete.add(path)
896 if path in filesToAdd:
897 filesToAdd.remove(path)
4fddb41b
VA
898 elif modifier == "C":
899 src, dest = diff['src'], diff['dst']
6de040df 900 p4_integrate(src, dest)
4fddb41b 901 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 902 p4_edit(dest)
4fddb41b 903 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 904 p4_edit(dest)
4fddb41b
VA
905 filesToChangeExecBit[dest] = diff['dst_mode']
906 os.unlink(dest)
907 editedFiles.add(dest)
d9a5f25b 908 elif modifier == "R":
b43b0a3c 909 src, dest = diff['src'], diff['dst']
6de040df 910 p4_integrate(src, dest)
ae901090 911 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 912 p4_edit(dest)
c65b670e 913 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 914 p4_edit(dest)
c65b670e 915 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
916 os.unlink(dest)
917 editedFiles.add(dest)
918 filesToDelete.add(src)
4f5cf76a
SH
919 else:
920 die("unknown modifier %s for %s" % (modifier, path))
921
0e36f2d7 922 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 923 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
924 tryPatchCmd = patchcmd + "--check -"
925 applyPatchCmd = patchcmd + "--check --apply -"
51a2640a 926
47a130b7 927 if os.system(tryPatchCmd) != 0:
51a2640a
SH
928 print "Unfortunately applying the change failed!"
929 print "What do you want to do?"
930 response = "x"
931 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
932 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
933 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
934 if response == "s":
935 print "Skipping! Good luck with the next patches..."
20947149 936 for f in editedFiles:
6de040df 937 p4_revert(f)
20947149 938 for f in filesToAdd:
6de040df 939 os.remove(f)
51a2640a
SH
940 return
941 elif response == "a":
47a130b7 942 os.system(applyPatchCmd)
51a2640a
SH
943 if len(filesToAdd) > 0:
944 print "You may also want to call p4 add on the following files:"
945 print " ".join(filesToAdd)
946 if len(filesToDelete):
947 print "The following files should be scheduled for deletion with p4 delete:"
948 print " ".join(filesToDelete)
cebdf5af
HWN
949 die("Please resolve and submit the conflict manually and "
950 + "continue afterwards with git-p4 submit --continue")
51a2640a
SH
951 elif response == "w":
952 system(diffcmd + " > patch.txt")
953 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af
HWN
954 die("Please resolve and submit the conflict manually and "
955 "continue afterwards with git-p4 submit --continue")
51a2640a 956
47a130b7 957 system(applyPatchCmd)
4f5cf76a
SH
958
959 for f in filesToAdd:
6de040df 960 p4_add(f)
4f5cf76a 961 for f in filesToDelete:
6de040df
LD
962 p4_revert(f)
963 p4_delete(f)
4f5cf76a 964
c65b670e
CP
965 # Set/clear executable bits
966 for f in filesToChangeExecBit.keys():
967 mode = filesToChangeExecBit[f]
968 setP4ExecBit(f, mode)
969
0e36f2d7 970 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 971 logMessage = logMessage.strip()
4f5cf76a 972
ea99c3ae 973 template = self.prepareSubmitTemplate()
4f5cf76a
SH
974
975 if self.interactive:
976 submitTemplate = self.prepareLogMessage(template, logMessage)
ecdba36d
LD
977
978 if self.preserveUser:
979 submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
980
67abd417
SB
981 if os.environ.has_key("P4DIFF"):
982 del(os.environ["P4DIFF"])
8b130262
AW
983 diff = ""
984 for editedFile in editedFiles:
6de040df 985 diff += p4_read_pipe(['diff', '-du', editedFile])
4f5cf76a 986
f3e5ae4f 987 newdiff = ""
4f5cf76a 988 for newFile in filesToAdd:
f3e5ae4f
MSO
989 newdiff += "==== new file ====\n"
990 newdiff += "--- /dev/null\n"
991 newdiff += "+++ %s\n" % newFile
4f5cf76a
SH
992 f = open(newFile, "r")
993 for line in f.readlines():
f3e5ae4f 994 newdiff += "+" + line
4f5cf76a
SH
995 f.close()
996
848de9c3
LD
997 if self.checkAuthorship and not self.p4UserIsMe(p4User):
998 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
999 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1000 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1001
f3e5ae4f 1002 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a 1003
e96e400f
SH
1004 [handle, fileName] = tempfile.mkstemp()
1005 tmpFile = os.fdopen(handle, "w+")
f3e5ae4f
MSO
1006 if self.isWindows:
1007 submitTemplate = submitTemplate.replace("\n", "\r\n")
1008 separatorLine = separatorLine.replace("\n", "\r\n")
1009 newdiff = newdiff.replace("\n", "\r\n")
1010 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
e96e400f 1011 tmpFile.close()
cdc7e388 1012 mtime = os.stat(fileName).st_mtime
82cea9ff
SB
1013 if os.environ.has_key("P4EDITOR"):
1014 editor = os.environ.get("P4EDITOR")
1015 else:
8b187e6b 1016 editor = read_pipe("git var GIT_EDITOR").strip()
e96e400f 1017 system(editor + " " + fileName)
e96e400f 1018
3ea2cfd4
LD
1019 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1020 checkModTime = False
1021 else:
1022 checkModTime = True
1023
cdc7e388 1024 response = "y"
3ea2cfd4 1025 if checkModTime and (os.stat(fileName).st_mtime <= mtime):
cdc7e388
SH
1026 response = "x"
1027 while response != "y" and response != "n":
1028 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1029
1030 if response == "y":
1031 tmpFile = open(fileName, "rb")
1032 message = tmpFile.read()
1033 tmpFile.close()
1034 submitTemplate = message[:message.index(separatorLine)]
1035 if self.isWindows:
1036 submitTemplate = submitTemplate.replace("\r\n", "\n")
6de040df 1037 p4_write_pipe(['submit', '-i'], submitTemplate)
3ea2cfd4
LD
1038
1039 if self.preserveUser:
1040 if p4User:
1041 # Get last changelist number. Cannot easily get it from
1042 # the submit command output as the output is unmarshalled.
1043 changelist = self.lastP4Changelist()
1044 self.modifyChangelistUser(changelist, p4User)
1045
cdc7e388
SH
1046 else:
1047 for f in editedFiles:
6de040df 1048 p4_revert(f)
cdc7e388 1049 for f in filesToAdd:
6de040df
LD
1050 p4_revert(f)
1051 os.remove(f)
cdc7e388
SH
1052
1053 os.remove(fileName)
4f5cf76a
SH
1054 else:
1055 fileName = "submit.txt"
1056 file = open(fileName, "w+")
1057 file.write(self.prepareLogMessage(template, logMessage))
1058 file.close()
cebdf5af
HWN
1059 print ("Perforce submit template written as %s. "
1060 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1061 % (fileName, fileName))
4f5cf76a
SH
1062
1063 def run(self, args):
c9b50e63
SH
1064 if len(args) == 0:
1065 self.master = currentGitBranch()
4280e533 1066 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1067 die("Detecting current git branch failed!")
1068 elif len(args) == 1:
1069 self.master = args[0]
1070 else:
1071 return False
1072
4c2d5d72
JX
1073 allowSubmit = gitConfig("git-p4.allowSubmit")
1074 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1075 die("%s is not in git-p4.allowSubmit" % self.master)
1076
27d2d811 1077 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1078 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1079 if len(self.origin) == 0:
1080 self.origin = upstream
a3fdd579 1081
3ea2cfd4
LD
1082 if self.preserveUser:
1083 if not self.canChangeChangelists():
1084 die("Cannot preserve user names without p4 super-user or admin permissions")
1085
a3fdd579
SH
1086 if self.verbose:
1087 print "Origin branch is " + self.origin
9512497b 1088
ea99c3ae 1089 if len(self.depotPath) == 0:
9512497b
SH
1090 print "Internal error: cannot locate perforce depot path from existing branches"
1091 sys.exit(128)
1092
ea99c3ae 1093 self.clientPath = p4Where(self.depotPath)
9512497b 1094
51a2640a 1095 if len(self.clientPath) == 0:
ea99c3ae 1096 print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
9512497b
SH
1097 sys.exit(128)
1098
ea99c3ae 1099 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1100 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1101
053fd0c1 1102 chdir(self.clientPath)
6a01298a 1103 print "Synchronizing p4 checkout..."
6de040df 1104 p4_sync("...")
4f5cf76a 1105 self.check()
4f5cf76a 1106
4c750c0d
SH
1107 commits = []
1108 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1109 commits.append(line.strip())
1110 commits.reverse()
4f5cf76a 1111
848de9c3
LD
1112 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1113 self.checkAuthorship = False
1114 else:
1115 self.checkAuthorship = True
1116
3ea2cfd4
LD
1117 if self.preserveUser:
1118 self.checkValidP4Users(commits)
1119
4f5cf76a 1120 while len(commits) > 0:
4f5cf76a
SH
1121 commit = commits[0]
1122 commits = commits[1:]
7cb5cbef 1123 self.applyCommit(commit)
4f5cf76a
SH
1124 if not self.interactive:
1125 break
1126
4f5cf76a 1127 if len(commits) == 0:
4c750c0d 1128 print "All changes applied!"
053fd0c1 1129 chdir(self.oldWorkingDirectory)
14594f4b 1130
4c750c0d
SH
1131 sync = P4Sync()
1132 sync.run([])
14594f4b 1133
4c750c0d
SH
1134 rebase = P4Rebase()
1135 rebase.rebase()
4f5cf76a 1136
b984733c
SH
1137 return True
1138
3ea2cfd4 1139class P4Sync(Command, P4UserMap):
56c09345
PW
1140 delete_actions = ( "delete", "move/delete", "purge" )
1141
b984733c
SH
1142 def __init__(self):
1143 Command.__init__(self)
3ea2cfd4 1144 P4UserMap.__init__(self)
b984733c
SH
1145 self.options = [
1146 optparse.make_option("--branch", dest="branch"),
1147 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1148 optparse.make_option("--changesfile", dest="changesFile"),
1149 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1150 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
a028a98e 1151 optparse.make_option("--verbose", dest="verbose", action="store_true"),
d2c6dd30
HWN
1152 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1153 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 1154 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 1155 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1156 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1157 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1158 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
1159 ]
1160 self.description = """Imports from Perforce into a git repository.\n
1161 example:
1162 //depot/my/project/ -- to import the current head
1163 //depot/my/project/@all -- to import everything
1164 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1165
1166 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1167
1168 self.usage += " //depot/path[@revRange]"
b984733c 1169 self.silent = False
1d7367dc
RG
1170 self.createdBranches = set()
1171 self.committedChanges = set()
569d1bd4 1172 self.branch = ""
b984733c 1173 self.detectBranches = False
cb53e1f8 1174 self.detectLabels = False
b984733c 1175 self.changesFile = ""
01265103 1176 self.syncWithOrigin = True
4b97ffb1 1177 self.verbose = False
a028a98e 1178 self.importIntoRemotes = True
01a9c9c5 1179 self.maxChanges = ""
c1f9197f 1180 self.isWindows = (platform.system() == "Windows")
8b41a97f 1181 self.keepRepoPath = False
6326aa58 1182 self.depotPaths = None
3c699645 1183 self.p4BranchesInGit = []
354081d5 1184 self.cloneExclude = []
3a70cdfa
TAL
1185 self.useClientSpec = False
1186 self.clientSpecDirs = []
b984733c 1187
01265103
SH
1188 if gitConfig("git-p4.syncFromOrigin") == "false":
1189 self.syncWithOrigin = False
1190
084f6306
PW
1191 #
1192 # P4 wildcards are not allowed in filenames. P4 complains
1193 # if you simply add them, but you can force it with "-f", in
1194 # which case it translates them into %xx encoding internally.
1195 # Search for and fix just these four characters. Do % last so
1196 # that fixing it does not inadvertently create new %-escapes.
1197 #
1198 def wildcard_decode(self, path):
1199 # Cannot have * in a filename in windows; untested as to
1200 # what p4 would do in such a case.
1201 if not self.isWindows:
1202 path = path.replace("%2A", "*")
1203 path = path.replace("%23", "#") \
1204 .replace("%40", "@") \
1205 .replace("%25", "%")
1206 return path
1207
b984733c 1208 def extractFilesFromCommit(self, commit):
354081d5
TT
1209 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1210 for path in self.cloneExclude]
b984733c
SH
1211 files = []
1212 fnum = 0
1213 while commit.has_key("depotFile%s" % fnum):
1214 path = commit["depotFile%s" % fnum]
6326aa58 1215
354081d5 1216 if [p for p in self.cloneExclude
d53de8b9 1217 if p4PathStartsWith(path, p)]:
354081d5
TT
1218 found = False
1219 else:
1220 found = [p for p in self.depotPaths
d53de8b9 1221 if p4PathStartsWith(path, p)]
6326aa58 1222 if not found:
b984733c
SH
1223 fnum = fnum + 1
1224 continue
1225
1226 file = {}
1227 file["path"] = path
1228 file["rev"] = commit["rev%s" % fnum]
1229 file["action"] = commit["action%s" % fnum]
1230 file["type"] = commit["type%s" % fnum]
1231 files.append(file)
1232 fnum = fnum + 1
1233 return files
1234
6326aa58 1235 def stripRepoPath(self, path, prefixes):
3952710b
IW
1236 if self.useClientSpec:
1237
1238 # if using the client spec, we use the output directory
1239 # specified in the client. For example, a view
1240 # //depot/foo/branch/... //client/branch/foo/...
1241 # will end up putting all foo/branch files into
1242 # branch/foo/
1243 for val in self.clientSpecDirs:
1244 if path.startswith(val[0]):
1245 # replace the depot path with the client path
1246 path = path.replace(val[0], val[1][1])
1247 # now strip out the client (//client/...)
1248 path = re.sub("^(//[^/]+/)", '', path)
1249 # the rest is all path
1250 return path
1251
8b41a97f 1252 if self.keepRepoPath:
6326aa58
HWN
1253 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1254
1255 for p in prefixes:
d53de8b9 1256 if p4PathStartsWith(path, p):
6326aa58 1257 path = path[len(p):]
8b41a97f 1258
6326aa58 1259 return path
6754a299 1260
71b112d4 1261 def splitFilesIntoBranches(self, commit):
d5904674 1262 branches = {}
71b112d4
SH
1263 fnum = 0
1264 while commit.has_key("depotFile%s" % fnum):
1265 path = commit["depotFile%s" % fnum]
6326aa58 1266 found = [p for p in self.depotPaths
d53de8b9 1267 if p4PathStartsWith(path, p)]
6326aa58 1268 if not found:
71b112d4
SH
1269 fnum = fnum + 1
1270 continue
1271
1272 file = {}
1273 file["path"] = path
1274 file["rev"] = commit["rev%s" % fnum]
1275 file["action"] = commit["action%s" % fnum]
1276 file["type"] = commit["type%s" % fnum]
1277 fnum = fnum + 1
1278
6326aa58 1279 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 1280
4b97ffb1 1281 for branch in self.knownBranches.keys():
6754a299
HWN
1282
1283 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1284 if relPath.startswith(branch + "/"):
d5904674
SH
1285 if branch not in branches:
1286 branches[branch] = []
71b112d4 1287 branches[branch].append(file)
6555b2cc 1288 break
b984733c
SH
1289
1290 return branches
1291
b932705b
LD
1292 # output one file from the P4 stream
1293 # - helper for streamP4Files
1294
1295 def streamOneP4File(self, file, contents):
b932705b 1296 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
084f6306 1297 relPath = self.wildcard_decode(relPath)
b932705b
LD
1298 if verbose:
1299 sys.stderr.write("%s\n" % relPath)
1300
9cffb8c8
PW
1301 (type_base, type_mods) = split_p4_type(file["type"])
1302
1303 git_mode = "100644"
1304 if "x" in type_mods:
1305 git_mode = "100755"
1306 if type_base == "symlink":
1307 git_mode = "120000"
1308 # p4 print on a symlink contains "target\n"; remove the newline
b39c3612
EP
1309 data = ''.join(contents)
1310 contents = [data[:-1]]
b932705b 1311
9cffb8c8 1312 if type_base == "utf16":
55aa5714
PW
1313 # p4 delivers different text in the python output to -G
1314 # than it does when using "print -o", or normal p4 client
1315 # operations. utf16 is converted to ascii or utf8, perhaps.
1316 # But ascii text saved as -t utf16 is completely mangled.
1317 # Invoke print -o to get the real contents.
6de040df 1318 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
55aa5714
PW
1319 contents = [ text ]
1320
9f7ef0ea
PW
1321 if type_base == "apple":
1322 # Apple filetype files will be streamed as a concatenation of
1323 # its appledouble header and the contents. This is useless
1324 # on both macs and non-macs. If using "print -q -o xx", it
1325 # will create "xx" with the data, and "%xx" with the header.
1326 # This is also not very useful.
1327 #
1328 # Ideally, someday, this script can learn how to generate
1329 # appledouble files directly and import those to git, but
1330 # non-mac machines can never find a use for apple filetype.
1331 print "\nIgnoring apple filetype file %s" % file['depotFile']
1332 return
1333
9cffb8c8
PW
1334 # Perhaps windows wants unicode, utf16 newlines translated too;
1335 # but this is not doing it.
1336 if self.isWindows and type_base == "text":
b932705b
LD
1337 mangled = []
1338 for data in contents:
1339 data = data.replace("\r\n", "\n")
1340 mangled.append(data)
1341 contents = mangled
1342
55aa5714
PW
1343 # Note that we do not try to de-mangle keywords on utf16 files,
1344 # even though in theory somebody may want that.
9cffb8c8
PW
1345 if type_base in ("text", "unicode", "binary"):
1346 if "ko" in type_mods:
cb585a9c
PW
1347 text = ''.join(contents)
1348 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1349 contents = [ text ]
9cffb8c8 1350 elif "k" in type_mods:
cb585a9c
PW
1351 text = ''.join(contents)
1352 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1353 contents = [ text ]
b932705b 1354
9cffb8c8 1355 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
1356
1357 # total length...
1358 length = 0
1359 for d in contents:
1360 length = length + len(d)
1361
1362 self.gitStream.write("data %d\n" % length)
1363 for d in contents:
1364 self.gitStream.write(d)
1365 self.gitStream.write("\n")
1366
1367 def streamOneP4Deletion(self, file):
1368 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1369 if verbose:
1370 sys.stderr.write("delete %s\n" % relPath)
1371 self.gitStream.write("D %s\n" % relPath)
1372
1373 # handle another chunk of streaming data
1374 def streamP4FilesCb(self, marshalled):
1375
c3f6163b
AG
1376 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1377 # start of a new file - output the old one first
1378 self.streamOneP4File(self.stream_file, self.stream_contents)
1379 self.stream_file = {}
1380 self.stream_contents = []
1381 self.stream_have_file_info = False
b932705b 1382
c3f6163b
AG
1383 # pick up the new file information... for the
1384 # 'data' field we need to append to our array
1385 for k in marshalled.keys():
1386 if k == 'data':
1387 self.stream_contents.append(marshalled['data'])
1388 else:
1389 self.stream_file[k] = marshalled[k]
b932705b 1390
c3f6163b 1391 self.stream_have_file_info = True
b932705b
LD
1392
1393 # Stream directly from "p4 files" into "git fast-import"
1394 def streamP4Files(self, files):
30b5940b
SH
1395 filesForCommit = []
1396 filesToRead = []
b932705b 1397 filesToDelete = []
30b5940b 1398
3a70cdfa 1399 for f in files:
30b5940b 1400 includeFile = True
3a70cdfa
TAL
1401 for val in self.clientSpecDirs:
1402 if f['path'].startswith(val[0]):
3952710b 1403 if val[1][0] <= 0:
30b5940b 1404 includeFile = False
3a70cdfa
TAL
1405 break
1406
30b5940b
SH
1407 if includeFile:
1408 filesForCommit.append(f)
56c09345 1409 if f['action'] in self.delete_actions:
b932705b 1410 filesToDelete.append(f)
56c09345
PW
1411 else:
1412 filesToRead.append(f)
6a49f8e2 1413
b932705b
LD
1414 # deleted files...
1415 for f in filesToDelete:
1416 self.streamOneP4Deletion(f)
1b9a4684 1417
b932705b
LD
1418 if len(filesToRead) > 0:
1419 self.stream_file = {}
1420 self.stream_contents = []
1421 self.stream_have_file_info = False
8ff45f2a 1422
c3f6163b
AG
1423 # curry self argument
1424 def streamP4FilesCbSelf(entry):
1425 self.streamP4FilesCb(entry)
6a49f8e2 1426
6de040df
LD
1427 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1428
1429 p4CmdList(["-x", "-", "print"],
1430 stdin=fileArgs,
1431 cb=streamP4FilesCbSelf)
30b5940b 1432
b932705b
LD
1433 # do the last chunk
1434 if self.stream_file.has_key('depotFile'):
1435 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 1436
6326aa58 1437 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
1438 epoch = details["time"]
1439 author = details["user"]
c3f6163b 1440 self.branchPrefixes = branchPrefixes
b984733c 1441
4b97ffb1
SH
1442 if self.verbose:
1443 print "commit into %s" % branch
1444
96e07dd2
HWN
1445 # start with reading files; if that fails, we should not
1446 # create a commit.
1447 new_files = []
1448 for f in files:
d53de8b9 1449 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
1450 new_files.append (f)
1451 else:
afa1dd9a 1452 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 1453
b984733c 1454 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 1455# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
1456 self.committedChanges.add(int(details["change"]))
1457 committer = ""
b607e71e
SH
1458 if author not in self.users:
1459 self.getUserMapFromPerforceServer()
b984733c 1460 if author in self.users:
0828ab14 1461 committer = "%s %s %s" % (self.users[author], epoch, self.tz)
b984733c 1462 else:
0828ab14 1463 committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
b984733c
SH
1464
1465 self.gitStream.write("committer %s\n" % committer)
1466
1467 self.gitStream.write("data <<EOT\n")
1468 self.gitStream.write(details["desc"])
6581de09
SH
1469 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1470 % (','.join (branchPrefixes), details["change"]))
1471 if len(details['options']) > 0:
1472 self.gitStream.write(": options = %s" % details['options'])
1473 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
1474
1475 if len(parent) > 0:
4b97ffb1
SH
1476 if self.verbose:
1477 print "parent %s" % parent
b984733c
SH
1478 self.gitStream.write("from %s\n" % parent)
1479
b932705b 1480 self.streamP4Files(new_files)
b984733c
SH
1481 self.gitStream.write("\n")
1482
1f4ba1cb
SH
1483 change = int(details["change"])
1484
9bda3a85 1485 if self.labels.has_key(change):
1f4ba1cb
SH
1486 label = self.labels[change]
1487 labelDetails = label[0]
1488 labelRevisions = label[1]
71b112d4
SH
1489 if self.verbose:
1490 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 1491
6de040df
LD
1492 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1493 for p in branchPrefixes])
1f4ba1cb
SH
1494
1495 if len(files) == len(labelRevisions):
1496
1497 cleanedFiles = {}
1498 for info in files:
56c09345 1499 if info["action"] in self.delete_actions:
1f4ba1cb
SH
1500 continue
1501 cleanedFiles[info["depotFile"]] = info["rev"]
1502
1503 if cleanedFiles == labelRevisions:
1504 self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1505 self.gitStream.write("from %s\n" % branch)
1506
1507 owner = labelDetails["Owner"]
1508 tagger = ""
1509 if author in self.users:
1510 tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1511 else:
1512 tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1513 self.gitStream.write("tagger %s\n" % tagger)
1514 self.gitStream.write("data <<EOT\n")
1515 self.gitStream.write(labelDetails["Description"])
1516 self.gitStream.write("EOT\n\n")
1517
1518 else:
a46668fa 1519 if not self.silent:
cebdf5af
HWN
1520 print ("Tag %s does not match with change %s: files do not match."
1521 % (labelDetails["label"], change))
1f4ba1cb
SH
1522
1523 else:
a46668fa 1524 if not self.silent:
cebdf5af
HWN
1525 print ("Tag %s does not match with change %s: file count is different."
1526 % (labelDetails["label"], change))
b984733c 1527
1f4ba1cb
SH
1528 def getLabels(self):
1529 self.labels = {}
1530
6326aa58 1531 l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
10c3211b 1532 if len(l) > 0 and not self.silent:
183f8436 1533 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
1534
1535 for output in l:
1f4ba1cb
SH
1536 label = output["label"]
1537 revisions = {}
1538 newestChange = 0
71b112d4
SH
1539 if self.verbose:
1540 print "Querying files for label %s" % label
6de040df
LD
1541 for file in p4CmdList(["files"] +
1542 ["%s...@%s" % (p, label)
1543 for p in self.depotPaths]):
1f4ba1cb
SH
1544 revisions[file["depotFile"]] = file["rev"]
1545 change = int(file["change"])
1546 if change > newestChange:
1547 newestChange = change
1548
9bda3a85
SH
1549 self.labels[newestChange] = [output, revisions]
1550
1551 if self.verbose:
1552 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 1553
86dff6b6
HWN
1554 def guessProjectName(self):
1555 for p in self.depotPaths:
6e5295c4
SH
1556 if p.endswith("/"):
1557 p = p[:-1]
1558 p = p[p.strip().rfind("/") + 1:]
1559 if not p.endswith("/"):
1560 p += "/"
1561 return p
86dff6b6 1562
4b97ffb1 1563 def getBranchMapping(self):
6555b2cc
SH
1564 lostAndFoundBranches = set()
1565
8ace74c0
VA
1566 user = gitConfig("git-p4.branchUser")
1567 if len(user) > 0:
1568 command = "branches -u %s" % user
1569 else:
1570 command = "branches"
1571
1572 for info in p4CmdList(command):
4b97ffb1
SH
1573 details = p4Cmd("branch -o %s" % info["branch"])
1574 viewIdx = 0
1575 while details.has_key("View%s" % viewIdx):
1576 paths = details["View%s" % viewIdx].split(" ")
1577 viewIdx = viewIdx + 1
1578 # require standard //depot/foo/... //depot/bar/... mapping
1579 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1580 continue
1581 source = paths[0]
1582 destination = paths[1]
6509e19c 1583 ## HACK
d53de8b9 1584 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
1585 source = source[len(self.depotPaths[0]):-4]
1586 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 1587
1a2edf4e
SH
1588 if destination in self.knownBranches:
1589 if not self.silent:
1590 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1591 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1592 continue
1593
6555b2cc
SH
1594 self.knownBranches[destination] = source
1595
1596 lostAndFoundBranches.discard(destination)
1597
29bdbac1 1598 if source not in self.knownBranches:
6555b2cc
SH
1599 lostAndFoundBranches.add(source)
1600
7199cf13
VA
1601 # Perforce does not strictly require branches to be defined, so we also
1602 # check git config for a branch list.
1603 #
1604 # Example of branch definition in git config file:
1605 # [git-p4]
1606 # branchList=main:branchA
1607 # branchList=main:branchB
1608 # branchList=branchA:branchC
1609 configBranches = gitConfigList("git-p4.branchList")
1610 for branch in configBranches:
1611 if branch:
1612 (source, destination) = branch.split(":")
1613 self.knownBranches[destination] = source
1614
1615 lostAndFoundBranches.discard(destination)
1616
1617 if source not in self.knownBranches:
1618 lostAndFoundBranches.add(source)
1619
6555b2cc
SH
1620
1621 for branch in lostAndFoundBranches:
1622 self.knownBranches[branch] = branch
29bdbac1 1623
38f9f5ec
SH
1624 def getBranchMappingFromGitBranches(self):
1625 branches = p4BranchesInGit(self.importIntoRemotes)
1626 for branch in branches.keys():
1627 if branch == "master":
1628 branch = "main"
1629 else:
1630 branch = branch[len(self.projectName):]
1631 self.knownBranches[branch] = branch
1632
29bdbac1 1633 def listExistingP4GitBranches(self):
144ff46b
SH
1634 # branches holds mapping from name to commit
1635 branches = p4BranchesInGit(self.importIntoRemotes)
1636 self.p4BranchesInGit = branches.keys()
1637 for branch in branches.keys():
1638 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 1639
bb6e09b2
HWN
1640 def updateOptionDict(self, d):
1641 option_keys = {}
1642 if self.keepRepoPath:
1643 option_keys['keepRepoPath'] = 1
1644
1645 d["options"] = ' '.join(sorted(option_keys.keys()))
1646
1647 def readOptions(self, d):
1648 self.keepRepoPath = (d.has_key('options')
1649 and ('keepRepoPath' in d['options']))
6326aa58 1650
8134f69c
SH
1651 def gitRefForBranch(self, branch):
1652 if branch == "main":
1653 return self.refPrefix + "master"
1654
1655 if len(branch) <= 0:
1656 return branch
1657
1658 return self.refPrefix + self.projectName + branch
1659
1ca3d710
SH
1660 def gitCommitByP4Change(self, ref, change):
1661 if self.verbose:
1662 print "looking in ref " + ref + " for change %s using bisect..." % change
1663
1664 earliestCommit = ""
1665 latestCommit = parseRevision(ref)
1666
1667 while True:
1668 if self.verbose:
1669 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1670 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1671 if len(next) == 0:
1672 if self.verbose:
1673 print "argh"
1674 return ""
1675 log = extractLogMessageFromGitCommit(next)
1676 settings = extractSettingsGitLog(log)
1677 currentChange = int(settings['change'])
1678 if self.verbose:
1679 print "current change %s" % currentChange
1680
1681 if currentChange == change:
1682 if self.verbose:
1683 print "found %s" % next
1684 return next
1685
1686 if currentChange < change:
1687 earliestCommit = "^%s" % next
1688 else:
1689 latestCommit = "%s" % next
1690
1691 return ""
1692
1693 def importNewBranch(self, branch, maxChange):
1694 # make fast-import flush all changes to disk and update the refs using the checkpoint
1695 # command so that we can try to find the branch parent in the git history
1696 self.gitStream.write("checkpoint\n\n");
1697 self.gitStream.flush();
1698 branchPrefix = self.depotPaths[0] + branch + "/"
1699 range = "@1,%s" % maxChange
1700 #print "prefix" + branchPrefix
1701 changes = p4ChangesForPaths([branchPrefix], range)
1702 if len(changes) <= 0:
1703 return False
1704 firstChange = changes[0]
1705 #print "first change in branch: %s" % firstChange
1706 sourceBranch = self.knownBranches[branch]
1707 sourceDepotPath = self.depotPaths[0] + sourceBranch
1708 sourceRef = self.gitRefForBranch(sourceBranch)
1709 #print "source " + sourceBranch
1710
1711 branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1712 #print "branch parent: %s" % branchParentChange
1713 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1714 if len(gitParent) > 0:
1715 self.initialParents[self.gitRefForBranch(branch)] = gitParent
1716 #print "parent git commit: %s" % gitParent
1717
1718 self.importChanges(changes)
1719 return True
1720
e87f37ae
SH
1721 def importChanges(self, changes):
1722 cnt = 1
1723 for change in changes:
1724 description = p4Cmd("describe %s" % change)
1725 self.updateOptionDict(description)
1726
1727 if not self.silent:
1728 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1729 sys.stdout.flush()
1730 cnt = cnt + 1
1731
1732 try:
1733 if self.detectBranches:
1734 branches = self.splitFilesIntoBranches(description)
1735 for branch in branches.keys():
1736 ## HACK --hwn
1737 branchPrefix = self.depotPaths[0] + branch + "/"
1738
1739 parent = ""
1740
1741 filesForCommit = branches[branch]
1742
1743 if self.verbose:
1744 print "branch is %s" % branch
1745
1746 self.updatedBranches.add(branch)
1747
1748 if branch not in self.createdBranches:
1749 self.createdBranches.add(branch)
1750 parent = self.knownBranches[branch]
1751 if parent == branch:
1752 parent = ""
1ca3d710
SH
1753 else:
1754 fullBranch = self.projectName + branch
1755 if fullBranch not in self.p4BranchesInGit:
1756 if not self.silent:
1757 print("\n Importing new branch %s" % fullBranch);
1758 if self.importNewBranch(branch, change - 1):
1759 parent = ""
1760 self.p4BranchesInGit.append(fullBranch)
1761 if not self.silent:
1762 print("\n Resuming with change %s" % change);
1763
1764 if self.verbose:
1765 print "parent determined through known branches: %s" % parent
e87f37ae 1766
8134f69c
SH
1767 branch = self.gitRefForBranch(branch)
1768 parent = self.gitRefForBranch(parent)
e87f37ae
SH
1769
1770 if self.verbose:
1771 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1772
1773 if len(parent) == 0 and branch in self.initialParents:
1774 parent = self.initialParents[branch]
1775 del self.initialParents[branch]
1776
1777 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1778 else:
1779 files = self.extractFilesFromCommit(description)
1780 self.commit(description, files, self.branch, self.depotPaths,
1781 self.initialParent)
1782 self.initialParent = ""
1783 except IOError:
1784 print self.gitError.read()
1785 sys.exit(1)
1786
c208a243
SH
1787 def importHeadRevision(self, revision):
1788 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1789
4e2e6ce4
PW
1790 details = {}
1791 details["user"] = "git perforce import user"
1494fcbb 1792 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
1793 % (' '.join(self.depotPaths), revision))
1794 details["change"] = revision
1795 newestRevision = 0
1796
1797 fileCnt = 0
6de040df
LD
1798 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1799
1800 for info in p4CmdList(["files"] + fileArgs):
c208a243 1801
68b28593 1802 if 'code' in info and info['code'] == 'error':
c208a243
SH
1803 sys.stderr.write("p4 returned an error: %s\n"
1804 % info['data'])
d88e707f
PW
1805 if info['data'].find("must refer to client") >= 0:
1806 sys.stderr.write("This particular p4 error is misleading.\n")
1807 sys.stderr.write("Perhaps the depot path was misspelled.\n");
1808 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 1809 sys.exit(1)
68b28593
PW
1810 if 'p4ExitCode' in info:
1811 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
1812 sys.exit(1)
1813
1814
1815 change = int(info["change"])
1816 if change > newestRevision:
1817 newestRevision = change
1818
56c09345 1819 if info["action"] in self.delete_actions:
c208a243
SH
1820 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1821 #fileCnt = fileCnt + 1
1822 continue
1823
1824 for prop in ["depotFile", "rev", "action", "type" ]:
1825 details["%s%s" % (prop, fileCnt)] = info[prop]
1826
1827 fileCnt = fileCnt + 1
1828
1829 details["change"] = newestRevision
4e2e6ce4
PW
1830
1831 # Use time from top-most change so that all git-p4 clones of
1832 # the same p4 repo have the same commit SHA1s.
1833 res = p4CmdList("describe -s %d" % newestRevision)
1834 newestTime = None
1835 for r in res:
1836 if r.has_key('time'):
1837 newestTime = int(r['time'])
1838 if newestTime is None:
1839 die("\"describe -s\" on newest change %d did not give a time")
1840 details["time"] = newestTime
1841
c208a243
SH
1842 self.updateOptionDict(details)
1843 try:
1844 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1845 except IOError:
1846 print "IO error with git fast-import. Is your git version recent enough?"
1847 print self.gitError.read()
1848
1849
3a70cdfa
TAL
1850 def getClientSpec(self):
1851 specList = p4CmdList( "client -o" )
1852 temp = {}
1853 for entry in specList:
1854 for k,v in entry.iteritems():
1855 if k.startswith("View"):
3952710b
IW
1856
1857 # p4 has these %%1 to %%9 arguments in specs to
1858 # reorder paths; which we can't handle (yet :)
1859 if re.match('%%\d', v) != None:
1860 print "Sorry, can't handle %%n arguments in client specs"
1861 sys.exit(1)
1862
3a70cdfa
TAL
1863 if v.startswith('"'):
1864 start = 1
1865 else:
1866 start = 0
1867 index = v.find("...")
3952710b
IW
1868
1869 # save the "client view"; i.e the RHS of the view
1870 # line that tells the client where to put the
1871 # files for this view.
1872 cv = v[index+3:].strip() # +3 to remove previous '...'
1873
1874 # if the client view doesn't end with a
1875 # ... wildcard, then we're going to mess up the
1876 # output directory, so fail gracefully.
1877 if not cv.endswith('...'):
1878 print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1879 sys.exit(1)
1880 cv=cv[:-3]
1881
1882 # now save the view; +index means included, -index
1883 # means it should be filtered out.
3a70cdfa
TAL
1884 v = v[start:index]
1885 if v.startswith("-"):
1886 v = v[1:]
3952710b 1887 include = -len(v)
3a70cdfa 1888 else:
3952710b
IW
1889 include = len(v)
1890
1891 temp[v] = (include, cv)
1892
3a70cdfa 1893 self.clientSpecDirs = temp.items()
3952710b 1894 self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
3a70cdfa 1895
b984733c 1896 def run(self, args):
6326aa58 1897 self.depotPaths = []
179caebf
SH
1898 self.changeRange = ""
1899 self.initialParent = ""
6326aa58 1900 self.previousDepotPaths = []
ce6f33c8 1901
29bdbac1
SH
1902 # map from branch depot path to parent branch
1903 self.knownBranches = {}
1904 self.initialParents = {}
5ca44617 1905 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
1906 if not self.syncWithOrigin:
1907 self.hasOrigin = False
29bdbac1 1908
a028a98e
SH
1909 if self.importIntoRemotes:
1910 self.refPrefix = "refs/remotes/p4/"
1911 else:
db775559 1912 self.refPrefix = "refs/heads/p4/"
a028a98e 1913
cebdf5af
HWN
1914 if self.syncWithOrigin and self.hasOrigin:
1915 if not self.silent:
1916 print "Syncing with origin first by calling git fetch origin"
1917 system("git fetch origin")
10f880f8 1918
569d1bd4 1919 if len(self.branch) == 0:
db775559 1920 self.branch = self.refPrefix + "master"
a028a98e 1921 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 1922 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 1923 system("git branch -D p4");
faf1bd20 1924 # create it /after/ importing, when master exists
0058a33a 1925 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 1926 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 1927
3cafb7d8 1928 if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
3a70cdfa
TAL
1929 self.getClientSpec()
1930
6a49f8e2
HWN
1931 # TODO: should always look at previous commits,
1932 # merge with previous imports, if possible.
1933 if args == []:
d414c74a 1934 if self.hasOrigin:
5ca44617 1935 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
1936 self.listExistingP4GitBranches()
1937
1938 if len(self.p4BranchesInGit) > 1:
1939 if not self.silent:
1940 print "Importing from/into multiple branches"
1941 self.detectBranches = True
967f72e2 1942
29bdbac1
SH
1943 if self.verbose:
1944 print "branches: %s" % self.p4BranchesInGit
1945
1946 p4Change = 0
1947 for branch in self.p4BranchesInGit:
cebdf5af 1948 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
1949
1950 settings = extractSettingsGitLog(logMsg)
29bdbac1 1951
bb6e09b2
HWN
1952 self.readOptions(settings)
1953 if (settings.has_key('depot-paths')
1954 and settings.has_key ('change')):
1955 change = int(settings['change']) + 1
29bdbac1
SH
1956 p4Change = max(p4Change, change)
1957
bb6e09b2
HWN
1958 depotPaths = sorted(settings['depot-paths'])
1959 if self.previousDepotPaths == []:
6326aa58 1960 self.previousDepotPaths = depotPaths
29bdbac1 1961 else:
6326aa58
HWN
1962 paths = []
1963 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
1964 prev_list = prev.split("/")
1965 cur_list = cur.split("/")
1966 for i in range(0, min(len(cur_list), len(prev_list))):
1967 if cur_list[i] <> prev_list[i]:
583e1707 1968 i = i - 1
6326aa58
HWN
1969 break
1970
04d277b3 1971 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
1972
1973 self.previousDepotPaths = paths
29bdbac1
SH
1974
1975 if p4Change > 0:
bb6e09b2 1976 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 1977 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
1978 if not self.detectBranches:
1979 self.initialParent = parseRevision(self.branch)
341dc1c1 1980 if not self.silent and not self.detectBranches:
967f72e2 1981 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 1982
f9162f6a
SH
1983 if not self.branch.startswith("refs/"):
1984 self.branch = "refs/heads/" + self.branch
179caebf 1985
6326aa58 1986 if len(args) == 0 and self.depotPaths:
b984733c 1987 if not self.silent:
6326aa58 1988 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 1989 else:
6326aa58 1990 if self.depotPaths and self.depotPaths != args:
cebdf5af 1991 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
1992 "This doesn't work!" % (' '.join (self.depotPaths),
1993 ' '.join (args)))
b984733c 1994 sys.exit(1)
6326aa58 1995
bb6e09b2 1996 self.depotPaths = sorted(args)
b984733c 1997
1c49fc19 1998 revision = ""
b984733c 1999 self.users = {}
b984733c 2000
6326aa58
HWN
2001 newPaths = []
2002 for p in self.depotPaths:
2003 if p.find("@") != -1:
2004 atIdx = p.index("@")
2005 self.changeRange = p[atIdx:]
2006 if self.changeRange == "@all":
2007 self.changeRange = ""
6a49f8e2 2008 elif ',' not in self.changeRange:
1c49fc19 2009 revision = self.changeRange
6326aa58 2010 self.changeRange = ""
7fcff9de 2011 p = p[:atIdx]
6326aa58
HWN
2012 elif p.find("#") != -1:
2013 hashIdx = p.index("#")
1c49fc19 2014 revision = p[hashIdx:]
7fcff9de 2015 p = p[:hashIdx]
6326aa58 2016 elif self.previousDepotPaths == []:
1c49fc19 2017 revision = "#head"
6326aa58
HWN
2018
2019 p = re.sub ("\.\.\.$", "", p)
2020 if not p.endswith("/"):
2021 p += "/"
2022
2023 newPaths.append(p)
2024
2025 self.depotPaths = newPaths
2026
b984733c 2027
b607e71e 2028 self.loadUserMapFromCache()
cb53e1f8
SH
2029 self.labels = {}
2030 if self.detectLabels:
2031 self.getLabels();
b984733c 2032
4b97ffb1 2033 if self.detectBranches:
df450923
SH
2034 ## FIXME - what's a P4 projectName ?
2035 self.projectName = self.guessProjectName()
2036
38f9f5ec
SH
2037 if self.hasOrigin:
2038 self.getBranchMappingFromGitBranches()
2039 else:
2040 self.getBranchMapping()
29bdbac1
SH
2041 if self.verbose:
2042 print "p4-git branches: %s" % self.p4BranchesInGit
2043 print "initial parents: %s" % self.initialParents
2044 for b in self.p4BranchesInGit:
2045 if b != "master":
6326aa58
HWN
2046
2047 ## FIXME
29bdbac1
SH
2048 b = b[len(self.projectName):]
2049 self.createdBranches.add(b)
4b97ffb1 2050
f291b4e3 2051 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 2052
cebdf5af 2053 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
2054 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2055 stderr=subprocess.PIPE);
08483580
SH
2056 self.gitOutput = importProcess.stdout
2057 self.gitStream = importProcess.stdin
2058 self.gitError = importProcess.stderr
b984733c 2059
1c49fc19 2060 if revision:
c208a243 2061 self.importHeadRevision(revision)
b984733c
SH
2062 else:
2063 changes = []
2064
0828ab14 2065 if len(self.changesFile) > 0:
b984733c 2066 output = open(self.changesFile).readlines()
1d7367dc 2067 changeSet = set()
b984733c
SH
2068 for line in output:
2069 changeSet.add(int(line))
2070
2071 for change in changeSet:
2072 changes.append(change)
2073
2074 changes.sort()
2075 else:
accad8e0
PW
2076 # catch "git-p4 sync" with no new branches, in a repo that
2077 # does not have any existing git-p4 branches
2078 if len(args) == 0 and not self.p4BranchesInGit:
e32e00dc 2079 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
29bdbac1 2080 if self.verbose:
86dff6b6 2081 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 2082 self.changeRange)
4f6432d8 2083 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 2084
01a9c9c5 2085 if len(self.maxChanges) > 0:
7fcff9de 2086 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 2087
b984733c 2088 if len(changes) == 0:
0828ab14 2089 if not self.silent:
341dc1c1 2090 print "No changes to import!"
1f52af6c 2091 return True
b984733c 2092
a9d1a27a
SH
2093 if not self.silent and not self.detectBranches:
2094 print "Import destination: %s" % self.branch
2095
341dc1c1
SH
2096 self.updatedBranches = set()
2097
e87f37ae 2098 self.importChanges(changes)
b984733c 2099
341dc1c1
SH
2100 if not self.silent:
2101 print ""
2102 if len(self.updatedBranches) > 0:
2103 sys.stdout.write("Updated branches: ")
2104 for b in self.updatedBranches:
2105 sys.stdout.write("%s " % b)
2106 sys.stdout.write("\n")
b984733c 2107
b984733c 2108 self.gitStream.close()
29bdbac1
SH
2109 if importProcess.wait() != 0:
2110 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
2111 self.gitOutput.close()
2112 self.gitError.close()
2113
b984733c
SH
2114 return True
2115
01ce1fe9
SH
2116class P4Rebase(Command):
2117 def __init__(self):
2118 Command.__init__(self)
01265103 2119 self.options = [ ]
cebdf5af
HWN
2120 self.description = ("Fetches the latest revision from perforce and "
2121 + "rebases the current work (branch) against it")
68c42153 2122 self.verbose = False
01ce1fe9
SH
2123
2124 def run(self, args):
2125 sync = P4Sync()
2126 sync.run([])
d7e3868c 2127
14594f4b
SH
2128 return self.rebase()
2129
2130 def rebase(self):
36ee4ee4
SH
2131 if os.system("git update-index --refresh") != 0:
2132 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2133 if len(read_pipe("git diff-index HEAD --")) > 0:
2134 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2135
d7e3868c
SH
2136 [upstream, settings] = findUpstreamBranchPoint()
2137 if len(upstream) == 0:
2138 die("Cannot find upstream branchpoint for rebase")
2139
2140 # the branchpoint may be p4/foo~3, so strip off the parent
2141 upstream = re.sub("~[0-9]+$", "", upstream)
2142
2143 print "Rebasing the current branch onto %s" % upstream
b25b2065 2144 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 2145 system("git rebase %s" % upstream)
1f52af6c 2146 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
2147 return True
2148
f9a3a4f7
SH
2149class P4Clone(P4Sync):
2150 def __init__(self):
2151 P4Sync.__init__(self)
2152 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 2153 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 2154 self.options += [
bb6e09b2
HWN
2155 optparse.make_option("--destination", dest="cloneDestination",
2156 action='store', default=None,
354081d5
TT
2157 help="where to leave result of the clone"),
2158 optparse.make_option("-/", dest="cloneExclude",
2159 action="append", type="string",
38200076
PW
2160 help="exclude depot path"),
2161 optparse.make_option("--bare", dest="cloneBare",
2162 action="store_true", default=False),
354081d5 2163 ]
bb6e09b2 2164 self.cloneDestination = None
f9a3a4f7 2165 self.needsGit = False
38200076 2166 self.cloneBare = False
f9a3a4f7 2167
354081d5
TT
2168 # This is required for the "append" cloneExclude action
2169 def ensure_value(self, attr, value):
2170 if not hasattr(self, attr) or getattr(self, attr) is None:
2171 setattr(self, attr, value)
2172 return getattr(self, attr)
2173
6a49f8e2
HWN
2174 def defaultDestination(self, args):
2175 ## TODO: use common prefix of args?
2176 depotPath = args[0]
2177 depotDir = re.sub("(@[^@]*)$", "", depotPath)
2178 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 2179 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
2180 depotDir = re.sub(r"/$", "", depotDir)
2181 return os.path.split(depotDir)[1]
2182
f9a3a4f7
SH
2183 def run(self, args):
2184 if len(args) < 1:
2185 return False
bb6e09b2
HWN
2186
2187 if self.keepRepoPath and not self.cloneDestination:
2188 sys.stderr.write("Must specify destination for --keep-path\n")
2189 sys.exit(1)
f9a3a4f7 2190
6326aa58 2191 depotPaths = args
5e100b5c
SH
2192
2193 if not self.cloneDestination and len(depotPaths) > 1:
2194 self.cloneDestination = depotPaths[-1]
2195 depotPaths = depotPaths[:-1]
2196
354081d5 2197 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
2198 for p in depotPaths:
2199 if not p.startswith("//"):
2200 return False
f9a3a4f7 2201
bb6e09b2 2202 if not self.cloneDestination:
98ad4faf 2203 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 2204
86dff6b6 2205 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 2206
c3bf3f13
KG
2207 if not os.path.exists(self.cloneDestination):
2208 os.makedirs(self.cloneDestination)
053fd0c1 2209 chdir(self.cloneDestination)
38200076
PW
2210
2211 init_cmd = [ "git", "init" ]
2212 if self.cloneBare:
2213 init_cmd.append("--bare")
2214 subprocess.check_call(init_cmd)
2215
6326aa58 2216 if not P4Sync.run(self, depotPaths):
f9a3a4f7 2217 return False
f9a3a4f7 2218 if self.branch != "master":
e9905013
TAL
2219 if self.importIntoRemotes:
2220 masterbranch = "refs/remotes/p4/master"
2221 else:
2222 masterbranch = "refs/heads/p4/master"
2223 if gitBranchExists(masterbranch):
2224 system("git branch master %s" % masterbranch)
38200076
PW
2225 if not self.cloneBare:
2226 system("git checkout -f")
8f9b2e08
SH
2227 else:
2228 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 2229
f9a3a4f7
SH
2230 return True
2231
09d89de2
SH
2232class P4Branches(Command):
2233 def __init__(self):
2234 Command.__init__(self)
2235 self.options = [ ]
2236 self.description = ("Shows the git branches that hold imports and their "
2237 + "corresponding perforce depot paths")
2238 self.verbose = False
2239
2240 def run(self, args):
5ca44617
SH
2241 if originP4BranchesExist():
2242 createOrUpdateBranchesFromOrigin()
2243
09d89de2
SH
2244 cmdline = "git rev-parse --symbolic "
2245 cmdline += " --remotes"
2246
2247 for line in read_pipe_lines(cmdline):
2248 line = line.strip()
2249
2250 if not line.startswith('p4/') or line == "p4/HEAD":
2251 continue
2252 branch = line
2253
2254 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2255 settings = extractSettingsGitLog(log)
2256
2257 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2258 return True
2259
b984733c
SH
2260class HelpFormatter(optparse.IndentedHelpFormatter):
2261 def __init__(self):
2262 optparse.IndentedHelpFormatter.__init__(self)
2263
2264 def format_description(self, description):
2265 if description:
2266 return description + "\n"
2267 else:
2268 return ""
4f5cf76a 2269
86949eef
SH
2270def printUsage(commands):
2271 print "usage: %s <command> [options]" % sys.argv[0]
2272 print ""
2273 print "valid commands: %s" % ", ".join(commands)
2274 print ""
2275 print "Try %s <command> --help for command specific help." % sys.argv[0]
2276 print ""
2277
2278commands = {
b86f7378
HWN
2279 "debug" : P4Debug,
2280 "submit" : P4Submit,
a9834f58 2281 "commit" : P4Submit,
b86f7378
HWN
2282 "sync" : P4Sync,
2283 "rebase" : P4Rebase,
2284 "clone" : P4Clone,
09d89de2
SH
2285 "rollback" : P4RollBack,
2286 "branches" : P4Branches
86949eef
SH
2287}
2288
86949eef 2289
bb6e09b2
HWN
2290def main():
2291 if len(sys.argv[1:]) == 0:
2292 printUsage(commands.keys())
2293 sys.exit(2)
4f5cf76a 2294
bb6e09b2
HWN
2295 cmd = ""
2296 cmdName = sys.argv[1]
2297 try:
b86f7378
HWN
2298 klass = commands[cmdName]
2299 cmd = klass()
bb6e09b2
HWN
2300 except KeyError:
2301 print "unknown command %s" % cmdName
2302 print ""
2303 printUsage(commands.keys())
2304 sys.exit(2)
2305
2306 options = cmd.options
b86f7378 2307 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
2308
2309 args = sys.argv[2:]
2310
2311 if len(options) > 0:
2312 options.append(optparse.make_option("--git-dir", dest="gitdir"))
2313
2314 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2315 options,
2316 description = cmd.description,
2317 formatter = HelpFormatter())
2318
2319 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2320 global verbose
2321 verbose = cmd.verbose
2322 if cmd.needsGit:
b86f7378
HWN
2323 if cmd.gitdir == None:
2324 cmd.gitdir = os.path.abspath(".git")
2325 if not isValidGitDir(cmd.gitdir):
2326 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2327 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
2328 cdup = read_pipe("git rev-parse --show-cdup").strip()
2329 if len(cdup) > 0:
053fd0c1 2330 chdir(cdup);
e20a9e53 2331
b86f7378
HWN
2332 if not isValidGitDir(cmd.gitdir):
2333 if isValidGitDir(cmd.gitdir + "/.git"):
2334 cmd.gitdir += "/.git"
bb6e09b2 2335 else:
b86f7378 2336 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 2337
b86f7378 2338 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 2339
bb6e09b2
HWN
2340 if not cmd.run(args):
2341 parser.print_help()
4f5cf76a 2342
4f5cf76a 2343
bb6e09b2
HWN
2344if __name__ == '__main__':
2345 main()