git p4: import/export of labels to/from p4
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
1d7367dc
RG
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
60df071c 13import re, shutil
8b41a97f 14
4addad22 15verbose = False
86949eef 16
06804c76
LD
17# Only labels/tags matching this will be imported/exported
18defaultLabelRegexp = r'[A-Z0-9_\-.]+$'
21a50753
AK
19
20def p4_build_cmd(cmd):
21 """Build a suitable p4 command line.
22
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
26 """
6de040df 27 real_cmd = ["p4"]
abcaf073
AK
28
29 user = gitConfig("git-p4.user")
30 if len(user) > 0:
6de040df 31 real_cmd += ["-u",user]
abcaf073
AK
32
33 password = gitConfig("git-p4.password")
34 if len(password) > 0:
6de040df 35 real_cmd += ["-P", password]
abcaf073
AK
36
37 port = gitConfig("git-p4.port")
38 if len(port) > 0:
6de040df 39 real_cmd += ["-p", port]
abcaf073
AK
40
41 host = gitConfig("git-p4.host")
42 if len(host) > 0:
41799aa2 43 real_cmd += ["-H", host]
abcaf073
AK
44
45 client = gitConfig("git-p4.client")
46 if len(client) > 0:
6de040df 47 real_cmd += ["-c", client]
abcaf073 48
6de040df
LD
49
50 if isinstance(cmd,basestring):
51 real_cmd = ' '.join(real_cmd) + ' ' + cmd
52 else:
53 real_cmd += cmd
21a50753
AK
54 return real_cmd
55
053fd0c1 56def chdir(dir):
6de040df 57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
bf1d68ff
GG
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
053fd0c1 60 os.chdir(dir)
bf1d68ff 61 os.environ['PWD'] = os.getcwd()
053fd0c1 62
86dff6b6
HWN
63def die(msg):
64 if verbose:
65 raise Exception(msg)
66 else:
67 sys.stderr.write(msg + "\n")
68 sys.exit(1)
69
6de040df 70def write_pipe(c, stdin):
4addad22 71 if verbose:
6de040df 72 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 73
6de040df
LD
74 expand = isinstance(c,basestring)
75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
76 pipe = p.stdin
77 val = pipe.write(stdin)
78 pipe.close()
79 if p.wait():
80 die('Command failed: %s' % str(c))
b016d397
HWN
81
82 return val
83
6de040df 84def p4_write_pipe(c, stdin):
d9429194 85 real_cmd = p4_build_cmd(c)
6de040df 86 return write_pipe(real_cmd, stdin)
d9429194 87
4addad22
HWN
88def read_pipe(c, ignore_error=False):
89 if verbose:
6de040df 90 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 91
6de040df
LD
92 expand = isinstance(c,basestring)
93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
94 pipe = p.stdout
b016d397 95 val = pipe.read()
6de040df
LD
96 if p.wait() and not ignore_error:
97 die('Command failed: %s' % str(c))
b016d397
HWN
98
99 return val
100
d9429194
AK
101def p4_read_pipe(c, ignore_error=False):
102 real_cmd = p4_build_cmd(c)
103 return read_pipe(real_cmd, ignore_error)
b016d397 104
bce4c5fc 105def read_pipe_lines(c):
4addad22 106 if verbose:
6de040df
LD
107 sys.stderr.write('Reading pipe: %s\n' % str(c))
108
109 expand = isinstance(c, basestring)
110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
111 pipe = p.stdout
b016d397 112 val = pipe.readlines()
6de040df
LD
113 if pipe.close() or p.wait():
114 die('Command failed: %s' % str(c))
b016d397
HWN
115
116 return val
caace111 117
2318121b
AK
118def p4_read_pipe_lines(c):
119 """Specifically invoke p4 on the command supplied. """
155af834 120 real_cmd = p4_build_cmd(c)
2318121b
AK
121 return read_pipe_lines(real_cmd)
122
6754a299 123def system(cmd):
6de040df 124 expand = isinstance(cmd,basestring)
4addad22 125 if verbose:
6de040df
LD
126 sys.stderr.write("executing %s\n" % str(cmd))
127 subprocess.check_call(cmd, shell=expand)
6754a299 128
bf9320f1
AK
129def p4_system(cmd):
130 """Specifically invoke p4 as the system command. """
155af834 131 real_cmd = p4_build_cmd(cmd)
6de040df
LD
132 expand = isinstance(real_cmd, basestring)
133 subprocess.check_call(real_cmd, shell=expand)
134
135def p4_integrate(src, dest):
136 p4_system(["integrate", "-Dt", src, dest])
137
138def p4_sync(path):
139 p4_system(["sync", path])
140
141def p4_add(f):
142 p4_system(["add", f])
143
144def p4_delete(f):
145 p4_system(["delete", f])
146
147def p4_edit(f):
148 p4_system(["edit", f])
149
150def p4_revert(f):
151 p4_system(["revert", f])
152
153def p4_reopen(type, file):
154 p4_system(["reopen", "-t", type, file])
bf9320f1 155
9cffb8c8
PW
156#
157# Canonicalize the p4 type and return a tuple of the
158# base type, plus any modifiers. See "p4 help filetypes"
159# for a list and explanation.
160#
161def split_p4_type(p4type):
162
163 p4_filetypes_historical = {
164 "ctempobj": "binary+Sw",
165 "ctext": "text+C",
166 "cxtext": "text+Cx",
167 "ktext": "text+k",
168 "kxtext": "text+kx",
169 "ltext": "text+F",
170 "tempobj": "binary+FSw",
171 "ubinary": "binary+F",
172 "uresource": "resource+F",
173 "uxbinary": "binary+Fx",
174 "xbinary": "binary+x",
175 "xltext": "text+Fx",
176 "xtempobj": "binary+Swx",
177 "xtext": "text+x",
178 "xunicode": "unicode+x",
179 "xutf16": "utf16+x",
180 }
181 if p4type in p4_filetypes_historical:
182 p4type = p4_filetypes_historical[p4type]
183 mods = ""
184 s = p4type.split("+")
185 base = s[0]
186 mods = ""
187 if len(s) > 1:
188 mods = s[1]
189 return (base, mods)
b9fc6ea9 190
60df071c
LD
191#
192# return the raw p4 type of a file (text, text+ko, etc)
193#
194def p4_type(file):
195 results = p4CmdList(["fstat", "-T", "headType", file])
196 return results[0]['headType']
197
198#
199# Given a type base and modifier, return a regexp matching
200# the keywords that can be expanded in the file
201#
202def p4_keywords_regexp_for_type(base, type_mods):
203 if base in ("text", "unicode", "binary"):
204 kwords = None
205 if "ko" in type_mods:
206 kwords = 'Id|Header'
207 elif "k" in type_mods:
208 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
209 else:
210 return None
211 pattern = r"""
212 \$ # Starts with a dollar, followed by...
213 (%s) # one of the keywords, followed by...
214 (:[^$]+)? # possibly an old expansion, followed by...
215 \$ # another dollar
216 """ % kwords
217 return pattern
218 else:
219 return None
220
221#
222# Given a file, return a regexp matching the possible
223# RCS keywords that will be expanded, or None for files
224# with kw expansion turned off.
225#
226def p4_keywords_regexp_for_file(file):
227 if not os.path.exists(file):
228 return None
229 else:
230 (type_base, type_mods) = split_p4_type(p4_type(file))
231 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 232
c65b670e
CP
233def setP4ExecBit(file, mode):
234 # Reopens an already open file and changes the execute bit to match
235 # the execute bit setting in the passed in mode.
236
237 p4Type = "+x"
238
239 if not isModeExec(mode):
240 p4Type = getP4OpenedType(file)
241 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
242 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
243 if p4Type[-1] == "+":
244 p4Type = p4Type[0:-1]
245
6de040df 246 p4_reopen(p4Type, file)
c65b670e
CP
247
248def getP4OpenedType(file):
249 # Returns the perforce file type for the given file.
250
6de040df 251 result = p4_read_pipe(["opened", file])
f3e5ae4f 252 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
253 if match:
254 return match.group(1)
255 else:
f3e5ae4f 256 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 257
06804c76
LD
258# Return the set of all p4 labels
259def getP4Labels(depotPaths):
260 labels = set()
261 if isinstance(depotPaths,basestring):
262 depotPaths = [depotPaths]
263
264 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
265 label = l['label']
266 labels.add(label)
267
268 return labels
269
270# Return the set of all git tags
271def getGitTags():
272 gitTags = set()
273 for line in read_pipe_lines(["git", "tag"]):
274 tag = line.strip()
275 gitTags.add(tag)
276 return gitTags
277
b43b0a3c
CP
278def diffTreePattern():
279 # This is a simple generator for the diff tree regex pattern. This could be
280 # a class variable if this and parseDiffTreeEntry were a part of a class.
281 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
282 while True:
283 yield pattern
284
285def parseDiffTreeEntry(entry):
286 """Parses a single diff tree entry into its component elements.
287
288 See git-diff-tree(1) manpage for details about the format of the diff
289 output. This method returns a dictionary with the following elements:
290
291 src_mode - The mode of the source file
292 dst_mode - The mode of the destination file
293 src_sha1 - The sha1 for the source file
294 dst_sha1 - The sha1 fr the destination file
295 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
296 status_score - The score for the status (applicable for 'C' and 'R'
297 statuses). This is None if there is no score.
298 src - The path for the source file.
299 dst - The path for the destination file. This is only present for
300 copy or renames. If it is not present, this is None.
301
302 If the pattern is not matched, None is returned."""
303
304 match = diffTreePattern().next().match(entry)
305 if match:
306 return {
307 'src_mode': match.group(1),
308 'dst_mode': match.group(2),
309 'src_sha1': match.group(3),
310 'dst_sha1': match.group(4),
311 'status': match.group(5),
312 'status_score': match.group(6),
313 'src': match.group(7),
314 'dst': match.group(10)
315 }
316 return None
317
c65b670e
CP
318def isModeExec(mode):
319 # Returns True if the given git mode represents an executable file,
320 # otherwise False.
321 return mode[-3:] == "755"
322
323def isModeExecChanged(src_mode, dst_mode):
324 return isModeExec(src_mode) != isModeExec(dst_mode)
325
b932705b 326def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
327
328 if isinstance(cmd,basestring):
329 cmd = "-G " + cmd
330 expand = True
331 else:
332 cmd = ["-G"] + cmd
333 expand = False
334
335 cmd = p4_build_cmd(cmd)
6a49f8e2 336 if verbose:
6de040df 337 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
338
339 # Use a temporary file to avoid deadlocks without
340 # subprocess.communicate(), which would put another copy
341 # of stdout into memory.
342 stdin_file = None
343 if stdin is not None:
344 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
345 if isinstance(stdin,basestring):
346 stdin_file.write(stdin)
347 else:
348 for i in stdin:
349 stdin_file.write(i + '\n')
9f90c733
SL
350 stdin_file.flush()
351 stdin_file.seek(0)
352
6de040df
LD
353 p4 = subprocess.Popen(cmd,
354 shell=expand,
9f90c733
SL
355 stdin=stdin_file,
356 stdout=subprocess.PIPE)
86949eef
SH
357
358 result = []
359 try:
360 while True:
9f90c733 361 entry = marshal.load(p4.stdout)
c3f6163b
AG
362 if cb is not None:
363 cb(entry)
364 else:
365 result.append(entry)
86949eef
SH
366 except EOFError:
367 pass
9f90c733
SL
368 exitCode = p4.wait()
369 if exitCode != 0:
ac3e0d79
SH
370 entry = {}
371 entry["p4ExitCode"] = exitCode
372 result.append(entry)
86949eef
SH
373
374 return result
375
376def p4Cmd(cmd):
377 list = p4CmdList(cmd)
378 result = {}
379 for entry in list:
380 result.update(entry)
381 return result;
382
cb2c9db5
SH
383def p4Where(depotPath):
384 if not depotPath.endswith("/"):
385 depotPath += "/"
7f705dc3 386 depotPath = depotPath + "..."
6de040df 387 outputList = p4CmdList(["where", depotPath])
7f705dc3
TAL
388 output = None
389 for entry in outputList:
75bc9573
TAL
390 if "depotFile" in entry:
391 if entry["depotFile"] == depotPath:
392 output = entry
393 break
394 elif "data" in entry:
395 data = entry.get("data")
396 space = data.find(" ")
397 if data[:space] == depotPath:
398 output = entry
399 break
7f705dc3
TAL
400 if output == None:
401 return ""
dc524036
SH
402 if output["code"] == "error":
403 return ""
cb2c9db5
SH
404 clientPath = ""
405 if "path" in output:
406 clientPath = output.get("path")
407 elif "data" in output:
408 data = output.get("data")
409 lastSpace = data.rfind(" ")
410 clientPath = data[lastSpace + 1:]
411
412 if clientPath.endswith("..."):
413 clientPath = clientPath[:-3]
414 return clientPath
415
86949eef 416def currentGitBranch():
b25b2065 417 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 418
4f5cf76a 419def isValidGitDir(path):
bb6e09b2
HWN
420 if (os.path.exists(path + "/HEAD")
421 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
422 return True;
423 return False
424
463e8af6 425def parseRevision(ref):
b25b2065 426 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 427
28755dba
PW
428def branchExists(ref):
429 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
430 ignore_error=True)
431 return len(rev) > 0
432
6ae8de88
SH
433def extractLogMessageFromGitCommit(commit):
434 logMessage = ""
b016d397
HWN
435
436 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 437 foundTitle = False
b016d397 438 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
439 if not foundTitle:
440 if len(log) == 1:
1c094184 441 foundTitle = True
6ae8de88
SH
442 continue
443
444 logMessage += log
445 return logMessage
446
bb6e09b2 447def extractSettingsGitLog(log):
6ae8de88
SH
448 values = {}
449 for line in log.split("\n"):
450 line = line.strip()
6326aa58
HWN
451 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
452 if not m:
453 continue
454
455 assignments = m.group(1).split (':')
456 for a in assignments:
457 vals = a.split ('=')
458 key = vals[0].strip()
459 val = ('='.join (vals[1:])).strip()
460 if val.endswith ('\"') and val.startswith('"'):
461 val = val[1:-1]
462
463 values[key] = val
464
845b42cb
SH
465 paths = values.get("depot-paths")
466 if not paths:
467 paths = values.get("depot-path")
a3fdd579
SH
468 if paths:
469 values['depot-paths'] = paths.split(',')
bb6e09b2 470 return values
6ae8de88 471
8136a639 472def gitBranchExists(branch):
bb6e09b2
HWN
473 proc = subprocess.Popen(["git", "rev-parse", branch],
474 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 475 return proc.wait() == 0;
8136a639 476
36bd8446 477_gitConfig = {}
99f790f2 478def gitConfig(key, args = None): # set args to "--bool", for instance
36bd8446 479 if not _gitConfig.has_key(key):
99f790f2
TAL
480 argsFilter = ""
481 if args != None:
482 argsFilter = "%s " % args
483 cmd = "git config %s%s" % (argsFilter, key)
484 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
36bd8446 485 return _gitConfig[key]
01265103 486
7199cf13
VA
487def gitConfigList(key):
488 if not _gitConfig.has_key(key):
489 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
490 return _gitConfig[key]
491
062410bb
SH
492def p4BranchesInGit(branchesAreInRemotes = True):
493 branches = {}
494
495 cmdline = "git rev-parse --symbolic "
496 if branchesAreInRemotes:
497 cmdline += " --remotes"
498 else:
499 cmdline += " --branches"
500
501 for line in read_pipe_lines(cmdline):
502 line = line.strip()
503
504 ## only import to p4/
505 if not line.startswith('p4/') or line == "p4/HEAD":
506 continue
507 branch = line
508
509 # strip off p4
510 branch = re.sub ("^p4/", "", line)
511
512 branches[branch] = parseRevision(line)
513 return branches
514
9ceab363 515def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
516 branches = p4BranchesInGit()
517 # map from depot-path to branch name
518 branchByDepotPath = {}
519 for branch in branches.keys():
520 tip = branches[branch]
521 log = extractLogMessageFromGitCommit(tip)
522 settings = extractSettingsGitLog(log)
523 if settings.has_key("depot-paths"):
524 paths = ",".join(settings["depot-paths"])
525 branchByDepotPath[paths] = "remotes/p4/" + branch
526
27d2d811 527 settings = None
27d2d811
SH
528 parent = 0
529 while parent < 65535:
9ceab363 530 commit = head + "~%s" % parent
27d2d811
SH
531 log = extractLogMessageFromGitCommit(commit)
532 settings = extractSettingsGitLog(log)
86506fe5
SH
533 if settings.has_key("depot-paths"):
534 paths = ",".join(settings["depot-paths"])
535 if branchByDepotPath.has_key(paths):
536 return [branchByDepotPath[paths], settings]
27d2d811 537
86506fe5 538 parent = parent + 1
27d2d811 539
86506fe5 540 return ["", settings]
27d2d811 541
5ca44617
SH
542def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
543 if not silent:
544 print ("Creating/updating branch(es) in %s based on origin branch(es)"
545 % localRefPrefix)
546
547 originPrefix = "origin/p4/"
548
549 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
550 line = line.strip()
551 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
552 continue
553
554 headName = line[len(originPrefix):]
555 remoteHead = localRefPrefix + headName
556 originHead = line
557
558 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
559 if (not original.has_key('depot-paths')
560 or not original.has_key('change')):
561 continue
562
563 update = False
564 if not gitBranchExists(remoteHead):
565 if verbose:
566 print "creating %s" % remoteHead
567 update = True
568 else:
569 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
570 if settings.has_key('change') > 0:
571 if settings['depot-paths'] == original['depot-paths']:
572 originP4Change = int(original['change'])
573 p4Change = int(settings['change'])
574 if originP4Change > p4Change:
575 print ("%s (%s) is newer than %s (%s). "
576 "Updating p4 branch from origin."
577 % (originHead, originP4Change,
578 remoteHead, p4Change))
579 update = True
580 else:
581 print ("Ignoring: %s was imported from %s while "
582 "%s was imported from %s"
583 % (originHead, ','.join(original['depot-paths']),
584 remoteHead, ','.join(settings['depot-paths'])))
585
586 if update:
587 system("git update-ref %s %s" % (remoteHead, originHead))
588
589def originP4BranchesExist():
590 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
591
4f6432d8
SH
592def p4ChangesForPaths(depotPaths, changeRange):
593 assert depotPaths
6de040df
LD
594 cmd = ['changes']
595 for p in depotPaths:
596 cmd += ["%s...%s" % (p, changeRange)]
597 output = p4_read_pipe_lines(cmd)
4f6432d8 598
b4b0ba06 599 changes = {}
4f6432d8 600 for line in output:
c3f6163b
AG
601 changeNum = int(line.split(" ")[1])
602 changes[changeNum] = True
4f6432d8 603
b4b0ba06
PW
604 changelist = changes.keys()
605 changelist.sort()
606 return changelist
4f6432d8 607
d53de8b9
TAL
608def p4PathStartsWith(path, prefix):
609 # This method tries to remedy a potential mixed-case issue:
610 #
611 # If UserA adds //depot/DirA/file1
612 # and UserB adds //depot/dira/file2
613 #
614 # we may or may not have a problem. If you have core.ignorecase=true,
615 # we treat DirA and dira as the same directory
616 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
617 if ignorecase:
618 return path.lower().startswith(prefix.lower())
619 return path.startswith(prefix)
620
543987bd
PW
621def getClientSpec():
622 """Look at the p4 client spec, create a View() object that contains
623 all the mappings, and return it."""
624
625 specList = p4CmdList("client -o")
626 if len(specList) != 1:
627 die('Output from "client -o" is %d lines, expecting 1' %
628 len(specList))
629
630 # dictionary of all client parameters
631 entry = specList[0]
632
633 # just the keys that start with "View"
634 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
635
636 # hold this new View
637 view = View()
638
639 # append the lines, in order, to the view
640 for view_num in range(len(view_keys)):
641 k = "View%d" % view_num
642 if k not in view_keys:
643 die("Expected view key %s missing" % k)
644 view.append(entry[k])
645
646 return view
647
648def getClientRoot():
649 """Grab the client directory."""
650
651 output = p4CmdList("client -o")
652 if len(output) != 1:
653 die('Output from "client -o" is %d lines, expecting 1' % len(output))
654
655 entry = output[0]
656 if "Root" not in entry:
657 die('Client has no "Root"')
658
659 return entry["Root"]
660
b984733c
SH
661class Command:
662 def __init__(self):
663 self.usage = "usage: %prog [options]"
8910ac0e 664 self.needsGit = True
b984733c 665
3ea2cfd4
LD
666class P4UserMap:
667 def __init__(self):
668 self.userMapFromPerforceServer = False
affb474f
LD
669 self.myP4UserId = None
670
671 def p4UserId(self):
672 if self.myP4UserId:
673 return self.myP4UserId
674
675 results = p4CmdList("user -o")
676 for r in results:
677 if r.has_key('User'):
678 self.myP4UserId = r['User']
679 return r['User']
680 die("Could not find your p4 user id")
681
682 def p4UserIsMe(self, p4User):
683 # return True if the given p4 user is actually me
684 me = self.p4UserId()
685 if not p4User or p4User != me:
686 return False
687 else:
688 return True
3ea2cfd4
LD
689
690 def getUserCacheFilename(self):
691 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
692 return home + "/.gitp4-usercache.txt"
693
694 def getUserMapFromPerforceServer(self):
695 if self.userMapFromPerforceServer:
696 return
697 self.users = {}
698 self.emails = {}
699
700 for output in p4CmdList("users"):
701 if not output.has_key("User"):
702 continue
703 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
704 self.emails[output["Email"]] = output["User"]
705
706
707 s = ''
708 for (key, val) in self.users.items():
709 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
710
711 open(self.getUserCacheFilename(), "wb").write(s)
712 self.userMapFromPerforceServer = True
713
714 def loadUserMapFromCache(self):
715 self.users = {}
716 self.userMapFromPerforceServer = False
717 try:
718 cache = open(self.getUserCacheFilename(), "rb")
719 lines = cache.readlines()
720 cache.close()
721 for line in lines:
722 entry = line.strip().split("\t")
723 self.users[entry[0]] = entry[1]
724 except IOError:
725 self.getUserMapFromPerforceServer()
726
b984733c 727class P4Debug(Command):
86949eef 728 def __init__(self):
6ae8de88 729 Command.__init__(self)
86949eef 730 self.options = [
b1ce9447
HWN
731 optparse.make_option("--verbose", dest="verbose", action="store_true",
732 default=False),
4addad22 733 ]
c8c39116 734 self.description = "A tool to debug the output of p4 -G."
8910ac0e 735 self.needsGit = False
b1ce9447 736 self.verbose = False
86949eef
SH
737
738 def run(self, args):
b1ce9447 739 j = 0
6de040df 740 for output in p4CmdList(args):
b1ce9447
HWN
741 print 'Element: %d' % j
742 j += 1
86949eef 743 print output
b984733c 744 return True
86949eef 745
5834684d
SH
746class P4RollBack(Command):
747 def __init__(self):
748 Command.__init__(self)
749 self.options = [
0c66a783
SH
750 optparse.make_option("--verbose", dest="verbose", action="store_true"),
751 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
752 ]
753 self.description = "A tool to debug the multi-branch import. Don't use :)"
52102d47 754 self.verbose = False
0c66a783 755 self.rollbackLocalBranches = False
5834684d
SH
756
757 def run(self, args):
758 if len(args) != 1:
759 return False
760 maxChange = int(args[0])
0c66a783 761
ad192f28 762 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
763 die("Problems executing p4");
764
0c66a783
SH
765 if self.rollbackLocalBranches:
766 refPrefix = "refs/heads/"
b016d397 767 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
768 else:
769 refPrefix = "refs/remotes/"
b016d397 770 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
771
772 for line in lines:
773 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
774 line = line.strip()
775 ref = refPrefix + line
5834684d 776 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
777 settings = extractSettingsGitLog(log)
778
779 depotPaths = settings['depot-paths']
780 change = settings['change']
781
5834684d 782 changed = False
52102d47 783
6326aa58
HWN
784 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
785 for p in depotPaths]))) == 0:
52102d47
SH
786 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
787 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
788 continue
789
bb6e09b2 790 while change and int(change) > maxChange:
5834684d 791 changed = True
52102d47
SH
792 if self.verbose:
793 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
794 system("git update-ref %s \"%s^\"" % (ref, ref))
795 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
796 settings = extractSettingsGitLog(log)
797
798
799 depotPaths = settings['depot-paths']
800 change = settings['change']
5834684d
SH
801
802 if changed:
52102d47 803 print "%s rewound to %s" % (ref, change)
5834684d
SH
804
805 return True
806
3ea2cfd4 807class P4Submit(Command, P4UserMap):
4f5cf76a 808 def __init__(self):
b984733c 809 Command.__init__(self)
3ea2cfd4 810 P4UserMap.__init__(self)
4f5cf76a 811 self.options = [
4addad22 812 optparse.make_option("--verbose", dest="verbose", action="store_true"),
4f5cf76a 813 optparse.make_option("--origin", dest="origin"),
ae901090 814 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
815 # preserve the user, requires relevant p4 permissions
816 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 817 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
4f5cf76a
SH
818 ]
819 self.description = "Submit changes from git to the perforce depot."
c9b50e63 820 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 821 self.interactive = True
9512497b 822 self.origin = ""
ae901090 823 self.detectRenames = False
b0d10df7 824 self.verbose = False
3ea2cfd4 825 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
f7baba8b 826 self.isWindows = (platform.system() == "Windows")
06804c76 827 self.exportLabels = False
4f5cf76a 828
4f5cf76a
SH
829 def check(self):
830 if len(p4CmdList("opened ...")) > 0:
831 die("You have files opened with perforce! Close them before starting the sync.")
832
edae1e2f
SH
833 # replaces everything between 'Description:' and the next P4 submit template field with the
834 # commit message
4f5cf76a
SH
835 def prepareLogMessage(self, template, message):
836 result = ""
837
edae1e2f
SH
838 inDescriptionSection = False
839
4f5cf76a
SH
840 for line in template.split("\n"):
841 if line.startswith("#"):
842 result += line + "\n"
843 continue
844
edae1e2f 845 if inDescriptionSection:
c9dbab04 846 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f
SH
847 inDescriptionSection = False
848 else:
849 continue
850 else:
851 if line.startswith("Description:"):
852 inDescriptionSection = True
853 line += "\n"
854 for messageLine in message.split("\n"):
855 line += "\t" + messageLine + "\n"
856
857 result += line + "\n"
4f5cf76a
SH
858
859 return result
860
60df071c
LD
861 def patchRCSKeywords(self, file, pattern):
862 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
863 (handle, outFileName) = tempfile.mkstemp(dir='.')
864 try:
865 outFile = os.fdopen(handle, "w+")
866 inFile = open(file, "r")
867 regexp = re.compile(pattern, re.VERBOSE)
868 for line in inFile.readlines():
869 line = regexp.sub(r'$\1$', line)
870 outFile.write(line)
871 inFile.close()
872 outFile.close()
873 # Forcibly overwrite the original file
874 os.unlink(file)
875 shutil.move(outFileName, file)
876 except:
877 # cleanup our temporary file
878 os.unlink(outFileName)
879 print "Failed to strip RCS keywords in %s" % file
880 raise
881
882 print "Patched up RCS keywords in %s" % file
883
3ea2cfd4
LD
884 def p4UserForCommit(self,id):
885 # Return the tuple (perforce user,git email) for a given git commit id
886 self.getUserMapFromPerforceServer()
887 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
888 gitEmail = gitEmail.strip()
889 if not self.emails.has_key(gitEmail):
890 return (None,gitEmail)
891 else:
892 return (self.emails[gitEmail],gitEmail)
893
894 def checkValidP4Users(self,commits):
895 # check if any git authors cannot be mapped to p4 users
896 for id in commits:
897 (user,email) = self.p4UserForCommit(id)
898 if not user:
899 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
900 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
901 print "%s" % msg
902 else:
903 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
904
905 def lastP4Changelist(self):
906 # Get back the last changelist number submitted in this client spec. This
907 # then gets used to patch up the username in the change. If the same
908 # client spec is being used by multiple processes then this might go
909 # wrong.
910 results = p4CmdList("client -o") # find the current client
911 client = None
912 for r in results:
913 if r.has_key('Client'):
914 client = r['Client']
915 break
916 if not client:
917 die("could not get client spec")
6de040df 918 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
919 for r in results:
920 if r.has_key('change'):
921 return r['change']
922 die("Could not get changelist number for last submit - cannot patch up user details")
923
924 def modifyChangelistUser(self, changelist, newUser):
925 # fixup the user field of a changelist after it has been submitted.
926 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
927 if len(changes) != 1:
928 die("Bad output from p4 change modifying %s to user %s" %
929 (changelist, newUser))
930
931 c = changes[0]
932 if c['User'] == newUser: return # nothing to do
933 c['User'] = newUser
934 input = marshal.dumps(c)
935
3ea2cfd4
LD
936 result = p4CmdList("change -f -i", stdin=input)
937 for r in result:
938 if r.has_key('code'):
939 if r['code'] == 'error':
940 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
941 if r.has_key('data'):
942 print("Updated user field for changelist %s to %s" % (changelist, newUser))
943 return
944 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
945
946 def canChangeChangelists(self):
947 # check to see if we have p4 admin or super-user permissions, either of
948 # which are required to modify changelists.
52a4880b 949 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
950 for r in results:
951 if r.has_key('perm'):
952 if r['perm'] == 'admin':
953 return 1
954 if r['perm'] == 'super':
955 return 1
956 return 0
957
ea99c3ae
SH
958 def prepareSubmitTemplate(self):
959 # remove lines in the Files section that show changes to files outside the depot path we're committing into
960 template = ""
961 inFilesSection = False
6de040df 962 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
963 if line.endswith("\r\n"):
964 line = line[:-2] + "\n"
ea99c3ae
SH
965 if inFilesSection:
966 if line.startswith("\t"):
967 # path starts and ends with a tab
968 path = line[1:]
969 lastTab = path.rfind("\t")
970 if lastTab != -1:
971 path = path[:lastTab]
d53de8b9 972 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
973 continue
974 else:
975 inFilesSection = False
976 else:
977 if line.startswith("Files:"):
978 inFilesSection = True
979
980 template += line
981
982 return template
983
7c766e57
PW
984 def edit_template(self, template_file):
985 """Invoke the editor to let the user change the submission
986 message. Return true if okay to continue with the submit."""
987
988 # if configured to skip the editing part, just submit
989 if gitConfig("git-p4.skipSubmitEdit") == "true":
990 return True
991
992 # look at the modification time, to check later if the user saved
993 # the file
994 mtime = os.stat(template_file).st_mtime
995
996 # invoke the editor
997 if os.environ.has_key("P4EDITOR"):
998 editor = os.environ.get("P4EDITOR")
999 else:
1000 editor = read_pipe("git var GIT_EDITOR").strip()
1001 system(editor + " " + template_file)
1002
1003 # If the file was not saved, prompt to see if this patch should
1004 # be skipped. But skip this verification step if configured so.
1005 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1006 return True
1007
d1652049
PW
1008 # modification time updated means user saved the file
1009 if os.stat(template_file).st_mtime > mtime:
1010 return True
1011
1012 while True:
1013 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1014 if response == 'y':
1015 return True
1016 if response == 'n':
1017 return False
7c766e57 1018
7cb5cbef 1019 def applyCommit(self, id):
0e36f2d7 1020 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
ae901090 1021
848de9c3 1022 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1023
ae901090
VA
1024 if not self.detectRenames:
1025 # If not explicitly set check the config variable
0a9feffc 1026 self.detectRenames = gitConfig("git-p4.detectRenames")
ae901090 1027
0a9feffc
VA
1028 if self.detectRenames.lower() == "false" or self.detectRenames == "":
1029 diffOpts = ""
1030 elif self.detectRenames.lower() == "true":
ae901090
VA
1031 diffOpts = "-M"
1032 else:
0a9feffc 1033 diffOpts = "-M%s" % self.detectRenames
ae901090 1034
0a9feffc
VA
1035 detectCopies = gitConfig("git-p4.detectCopies")
1036 if detectCopies.lower() == "true":
4fddb41b 1037 diffOpts += " -C"
0a9feffc
VA
1038 elif detectCopies != "" and detectCopies.lower() != "false":
1039 diffOpts += " -C%s" % detectCopies
4fddb41b 1040
68cbcf1b 1041 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
4fddb41b
VA
1042 diffOpts += " --find-copies-harder"
1043
0e36f2d7 1044 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
1045 filesToAdd = set()
1046 filesToDelete = set()
d336c158 1047 editedFiles = set()
c65b670e 1048 filesToChangeExecBit = {}
60df071c 1049
4f5cf76a 1050 for line in diff:
b43b0a3c
CP
1051 diff = parseDiffTreeEntry(line)
1052 modifier = diff['status']
1053 path = diff['src']
4f5cf76a 1054 if modifier == "M":
6de040df 1055 p4_edit(path)
c65b670e
CP
1056 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1057 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1058 editedFiles.add(path)
4f5cf76a
SH
1059 elif modifier == "A":
1060 filesToAdd.add(path)
c65b670e 1061 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1062 if path in filesToDelete:
1063 filesToDelete.remove(path)
1064 elif modifier == "D":
1065 filesToDelete.add(path)
1066 if path in filesToAdd:
1067 filesToAdd.remove(path)
4fddb41b
VA
1068 elif modifier == "C":
1069 src, dest = diff['src'], diff['dst']
6de040df 1070 p4_integrate(src, dest)
4fddb41b 1071 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1072 p4_edit(dest)
4fddb41b 1073 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1074 p4_edit(dest)
4fddb41b
VA
1075 filesToChangeExecBit[dest] = diff['dst_mode']
1076 os.unlink(dest)
1077 editedFiles.add(dest)
d9a5f25b 1078 elif modifier == "R":
b43b0a3c 1079 src, dest = diff['src'], diff['dst']
6de040df 1080 p4_integrate(src, dest)
ae901090 1081 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1082 p4_edit(dest)
c65b670e 1083 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1084 p4_edit(dest)
c65b670e 1085 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
1086 os.unlink(dest)
1087 editedFiles.add(dest)
1088 filesToDelete.add(src)
4f5cf76a
SH
1089 else:
1090 die("unknown modifier %s for %s" % (modifier, path))
1091
0e36f2d7 1092 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 1093 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1094 tryPatchCmd = patchcmd + "--check -"
1095 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1096 patch_succeeded = True
51a2640a 1097
47a130b7 1098 if os.system(tryPatchCmd) != 0:
60df071c
LD
1099 fixed_rcs_keywords = False
1100 patch_succeeded = False
51a2640a 1101 print "Unfortunately applying the change failed!"
60df071c
LD
1102
1103 # Patch failed, maybe it's just RCS keyword woes. Look through
1104 # the patch to see if that's possible.
1105 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1106 file = None
1107 pattern = None
1108 kwfiles = {}
1109 for file in editedFiles | filesToDelete:
1110 # did this file's delta contain RCS keywords?
1111 pattern = p4_keywords_regexp_for_file(file)
1112
1113 if pattern:
1114 # this file is a possibility...look for RCS keywords.
1115 regexp = re.compile(pattern, re.VERBOSE)
1116 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1117 if regexp.search(line):
1118 if verbose:
1119 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1120 kwfiles[file] = pattern
1121 break
1122
1123 for file in kwfiles:
1124 if verbose:
1125 print "zapping %s with %s" % (line,pattern)
1126 self.patchRCSKeywords(file, kwfiles[file])
1127 fixed_rcs_keywords = True
1128
1129 if fixed_rcs_keywords:
1130 print "Retrying the patch with RCS keywords cleaned up"
1131 if os.system(tryPatchCmd) == 0:
1132 patch_succeeded = True
1133
1134 if not patch_succeeded:
51a2640a
SH
1135 print "What do you want to do?"
1136 response = "x"
1137 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
1138 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1139 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
1140 if response == "s":
1141 print "Skipping! Good luck with the next patches..."
20947149 1142 for f in editedFiles:
6de040df 1143 p4_revert(f)
20947149 1144 for f in filesToAdd:
6de040df 1145 os.remove(f)
51a2640a
SH
1146 return
1147 elif response == "a":
47a130b7 1148 os.system(applyPatchCmd)
51a2640a
SH
1149 if len(filesToAdd) > 0:
1150 print "You may also want to call p4 add on the following files:"
1151 print " ".join(filesToAdd)
1152 if len(filesToDelete):
1153 print "The following files should be scheduled for deletion with p4 delete:"
1154 print " ".join(filesToDelete)
cebdf5af 1155 die("Please resolve and submit the conflict manually and "
9dcb9f24 1156 + "continue afterwards with git p4 submit --continue")
51a2640a
SH
1157 elif response == "w":
1158 system(diffcmd + " > patch.txt")
1159 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af 1160 die("Please resolve and submit the conflict manually and "
9dcb9f24 1161 "continue afterwards with git p4 submit --continue")
51a2640a 1162
47a130b7 1163 system(applyPatchCmd)
4f5cf76a
SH
1164
1165 for f in filesToAdd:
6de040df 1166 p4_add(f)
4f5cf76a 1167 for f in filesToDelete:
6de040df
LD
1168 p4_revert(f)
1169 p4_delete(f)
4f5cf76a 1170
c65b670e
CP
1171 # Set/clear executable bits
1172 for f in filesToChangeExecBit.keys():
1173 mode = filesToChangeExecBit[f]
1174 setP4ExecBit(f, mode)
1175
0e36f2d7 1176 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1177 logMessage = logMessage.strip()
4f5cf76a 1178
ea99c3ae 1179 template = self.prepareSubmitTemplate()
4f5cf76a
SH
1180
1181 if self.interactive:
1182 submitTemplate = self.prepareLogMessage(template, logMessage)
ecdba36d
LD
1183
1184 if self.preserveUser:
1185 submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1186
67abd417
SB
1187 if os.environ.has_key("P4DIFF"):
1188 del(os.environ["P4DIFF"])
8b130262
AW
1189 diff = ""
1190 for editedFile in editedFiles:
6de040df 1191 diff += p4_read_pipe(['diff', '-du', editedFile])
4f5cf76a 1192
f3e5ae4f 1193 newdiff = ""
4f5cf76a 1194 for newFile in filesToAdd:
f3e5ae4f
MSO
1195 newdiff += "==== new file ====\n"
1196 newdiff += "--- /dev/null\n"
1197 newdiff += "+++ %s\n" % newFile
4f5cf76a
SH
1198 f = open(newFile, "r")
1199 for line in f.readlines():
f3e5ae4f 1200 newdiff += "+" + line
4f5cf76a
SH
1201 f.close()
1202
848de9c3
LD
1203 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1204 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
9dcb9f24
PW
1205 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1206 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
848de9c3 1207
f3e5ae4f 1208 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a 1209
7c766e57 1210 (handle, fileName) = tempfile.mkstemp()
e96e400f 1211 tmpFile = os.fdopen(handle, "w+")
f3e5ae4f
MSO
1212 if self.isWindows:
1213 submitTemplate = submitTemplate.replace("\n", "\r\n")
1214 separatorLine = separatorLine.replace("\n", "\r\n")
1215 newdiff = newdiff.replace("\n", "\r\n")
1216 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
e96e400f 1217 tmpFile.close()
e96e400f 1218
7c766e57
PW
1219 if self.edit_template(fileName):
1220 # read the edited message and submit
cdc7e388
SH
1221 tmpFile = open(fileName, "rb")
1222 message = tmpFile.read()
1223 tmpFile.close()
1224 submitTemplate = message[:message.index(separatorLine)]
1225 if self.isWindows:
1226 submitTemplate = submitTemplate.replace("\r\n", "\n")
6de040df 1227 p4_write_pipe(['submit', '-i'], submitTemplate)
3ea2cfd4
LD
1228
1229 if self.preserveUser:
1230 if p4User:
1231 # Get last changelist number. Cannot easily get it from
7c766e57
PW
1232 # the submit command output as the output is
1233 # unmarshalled.
3ea2cfd4
LD
1234 changelist = self.lastP4Changelist()
1235 self.modifyChangelistUser(changelist, p4User)
cdc7e388 1236 else:
7c766e57 1237 # skip this patch
d1652049 1238 print "Submission cancelled, undoing p4 changes."
cdc7e388 1239 for f in editedFiles:
6de040df 1240 p4_revert(f)
cdc7e388 1241 for f in filesToAdd:
6de040df
LD
1242 p4_revert(f)
1243 os.remove(f)
cdc7e388
SH
1244
1245 os.remove(fileName)
4f5cf76a
SH
1246 else:
1247 fileName = "submit.txt"
1248 file = open(fileName, "w+")
1249 file.write(self.prepareLogMessage(template, logMessage))
1250 file.close()
cebdf5af
HWN
1251 print ("Perforce submit template written as %s. "
1252 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1253 % (fileName, fileName))
4f5cf76a 1254
06804c76
LD
1255 # Export git tags as p4 labels. Create a p4 label and then tag
1256 # with that.
1257 def exportGitTags(self, gitTags):
1258 validTagRegexp = gitConfig("git-p4.validTagRegexp")
1259 if len(validTagRegexp) == 0:
1260 validTagRegexp = defaultLabelRegexp
1261 m = re.compile(validTagRegexp)
1262 commit_re = re.compile(r'\s*\[git-p4:.*change = (\d+)\s*\]')
1263
1264 for name in gitTags:
1265
1266 if not m.match(name):
1267 if verbose:
1268 print "tag %s does not match regexp %s" % (name, validTagRegexp)
1269 continue
1270
1271 # Get the p4 commit this corresponds to
1272 changelist = None
1273 for l in read_pipe_lines(["git", "log", "--max-count=1", name]):
1274 match = commit_re.match(l)
1275 if match:
1276 changelist = match.group(1)
1277
1278 if not changelist:
1279 # a tag pointing to something not sent to p4; ignore
1280 if verbose:
1281 print "git tag %s does not give a p4 commit" % name
1282 continue
1283
1284 # Get the tag details.
1285 inHeader = True
1286 isAnnotated = False
1287 body = []
1288 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1289 l = l.strip()
1290 if inHeader:
1291 if re.match(r'tag\s+', l):
1292 isAnnotated = True
1293 elif re.match(r'\s*$', l):
1294 inHeader = False
1295 continue
1296 else:
1297 body.append(l)
1298
1299 if not isAnnotated:
1300 body = ["lightweight tag imported by git p4\n"]
1301
1302 # Create the label - use the same view as the client spec we are using
1303 clientSpec = getClientSpec()
1304
1305 labelTemplate = "Label: %s\n" % name
1306 labelTemplate += "Description:\n"
1307 for b in body:
1308 labelTemplate += "\t" + b + "\n"
1309 labelTemplate += "View:\n"
1310 for mapping in clientSpec.mappings:
1311 labelTemplate += "\t%s\n" % mapping.depot_side.path
1312
1313 p4_write_pipe(["label", "-i"], labelTemplate)
1314
1315 # Use the label
1316 p4_system(["tag", "-l", name] +
1317 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1318
1319 if verbose:
1320 print "created p4 label for tag %s" % name
1321
4f5cf76a 1322 def run(self, args):
c9b50e63
SH
1323 if len(args) == 0:
1324 self.master = currentGitBranch()
4280e533 1325 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1326 die("Detecting current git branch failed!")
1327 elif len(args) == 1:
1328 self.master = args[0]
28755dba
PW
1329 if not branchExists(self.master):
1330 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1331 else:
1332 return False
1333
4c2d5d72
JX
1334 allowSubmit = gitConfig("git-p4.allowSubmit")
1335 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1336 die("%s is not in git-p4.allowSubmit" % self.master)
1337
27d2d811 1338 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1339 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1340 if len(self.origin) == 0:
1341 self.origin = upstream
a3fdd579 1342
3ea2cfd4
LD
1343 if self.preserveUser:
1344 if not self.canChangeChangelists():
1345 die("Cannot preserve user names without p4 super-user or admin permissions")
1346
a3fdd579
SH
1347 if self.verbose:
1348 print "Origin branch is " + self.origin
9512497b 1349
ea99c3ae 1350 if len(self.depotPath) == 0:
9512497b
SH
1351 print "Internal error: cannot locate perforce depot path from existing branches"
1352 sys.exit(128)
1353
543987bd
PW
1354 self.useClientSpec = False
1355 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1356 self.useClientSpec = True
1357 if self.useClientSpec:
1358 self.clientSpecDirs = getClientSpec()
9512497b 1359
543987bd
PW
1360 if self.useClientSpec:
1361 # all files are relative to the client spec
1362 self.clientPath = getClientRoot()
1363 else:
1364 self.clientPath = p4Where(self.depotPath)
9512497b 1365
543987bd
PW
1366 if self.clientPath == "":
1367 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1368
ea99c3ae 1369 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1370 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1371
0591cfa8
GG
1372 # ensure the clientPath exists
1373 if not os.path.exists(self.clientPath):
1374 os.makedirs(self.clientPath)
1375
053fd0c1 1376 chdir(self.clientPath)
6a01298a 1377 print "Synchronizing p4 checkout..."
6de040df 1378 p4_sync("...")
4f5cf76a 1379 self.check()
4f5cf76a 1380
4c750c0d
SH
1381 commits = []
1382 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1383 commits.append(line.strip())
1384 commits.reverse()
4f5cf76a 1385
848de9c3
LD
1386 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1387 self.checkAuthorship = False
1388 else:
1389 self.checkAuthorship = True
1390
3ea2cfd4
LD
1391 if self.preserveUser:
1392 self.checkValidP4Users(commits)
1393
4f5cf76a 1394 while len(commits) > 0:
4f5cf76a
SH
1395 commit = commits[0]
1396 commits = commits[1:]
7cb5cbef 1397 self.applyCommit(commit)
4f5cf76a
SH
1398 if not self.interactive:
1399 break
1400
4f5cf76a 1401 if len(commits) == 0:
4c750c0d 1402 print "All changes applied!"
053fd0c1 1403 chdir(self.oldWorkingDirectory)
14594f4b 1404
4c750c0d
SH
1405 sync = P4Sync()
1406 sync.run([])
14594f4b 1407
4c750c0d
SH
1408 rebase = P4Rebase()
1409 rebase.rebase()
4f5cf76a 1410
06804c76
LD
1411 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1412 self.exportLabels = true
1413
1414 if self.exportLabels:
1415 p4Labels = getP4Labels(self.depotPath)
1416 gitTags = getGitTags()
1417
1418 missingGitTags = gitTags - p4Labels
1419 self.exportGitTags(missingGitTags)
1420
b984733c
SH
1421 return True
1422
ecb7cf98
PW
1423class View(object):
1424 """Represent a p4 view ("p4 help views"), and map files in a
1425 repo according to the view."""
1426
1427 class Path(object):
1428 """A depot or client path, possibly containing wildcards.
1429 The only one supported is ... at the end, currently.
1430 Initialize with the full path, with //depot or //client."""
1431
1432 def __init__(self, path, is_depot):
1433 self.path = path
1434 self.is_depot = is_depot
1435 self.find_wildcards()
1436 # remember the prefix bit, useful for relative mappings
1437 m = re.match("(//[^/]+/)", self.path)
1438 if not m:
1439 die("Path %s does not start with //prefix/" % self.path)
1440 prefix = m.group(1)
1441 if not self.is_depot:
1442 # strip //client/ on client paths
1443 self.path = self.path[len(prefix):]
1444
1445 def find_wildcards(self):
1446 """Make sure wildcards are valid, and set up internal
1447 variables."""
1448
1449 self.ends_triple_dot = False
1450 # There are three wildcards allowed in p4 views
1451 # (see "p4 help views"). This code knows how to
1452 # handle "..." (only at the end), but cannot deal with
1453 # "%%n" or "*". Only check the depot_side, as p4 should
1454 # validate that the client_side matches too.
1455 if re.search(r'%%[1-9]', self.path):
1456 die("Can't handle %%n wildcards in view: %s" % self.path)
1457 if self.path.find("*") >= 0:
1458 die("Can't handle * wildcards in view: %s" % self.path)
1459 triple_dot_index = self.path.find("...")
1460 if triple_dot_index >= 0:
43b82bd9
PW
1461 if triple_dot_index != len(self.path) - 3:
1462 die("Can handle only single ... wildcard, at end: %s" %
ecb7cf98
PW
1463 self.path)
1464 self.ends_triple_dot = True
1465
1466 def ensure_compatible(self, other_path):
1467 """Make sure the wildcards agree."""
1468 if self.ends_triple_dot != other_path.ends_triple_dot:
1469 die("Both paths must end with ... if either does;\n" +
1470 "paths: %s %s" % (self.path, other_path.path))
1471
1472 def match_wildcards(self, test_path):
1473 """See if this test_path matches us, and fill in the value
1474 of the wildcards if so. Returns a tuple of
1475 (True|False, wildcards[]). For now, only the ... at end
1476 is supported, so at most one wildcard."""
1477 if self.ends_triple_dot:
1478 dotless = self.path[:-3]
1479 if test_path.startswith(dotless):
1480 wildcard = test_path[len(dotless):]
1481 return (True, [ wildcard ])
1482 else:
1483 if test_path == self.path:
1484 return (True, [])
1485 return (False, [])
1486
1487 def match(self, test_path):
1488 """Just return if it matches; don't bother with the wildcards."""
1489 b, _ = self.match_wildcards(test_path)
1490 return b
1491
1492 def fill_in_wildcards(self, wildcards):
1493 """Return the relative path, with the wildcards filled in
1494 if there are any."""
1495 if self.ends_triple_dot:
1496 return self.path[:-3] + wildcards[0]
1497 else:
1498 return self.path
1499
1500 class Mapping(object):
1501 def __init__(self, depot_side, client_side, overlay, exclude):
1502 # depot_side is without the trailing /... if it had one
1503 self.depot_side = View.Path(depot_side, is_depot=True)
1504 self.client_side = View.Path(client_side, is_depot=False)
1505 self.overlay = overlay # started with "+"
1506 self.exclude = exclude # started with "-"
1507 assert not (self.overlay and self.exclude)
1508 self.depot_side.ensure_compatible(self.client_side)
1509
1510 def __str__(self):
1511 c = " "
1512 if self.overlay:
1513 c = "+"
1514 if self.exclude:
1515 c = "-"
1516 return "View.Mapping: %s%s -> %s" % \
329afb8e 1517 (c, self.depot_side.path, self.client_side.path)
ecb7cf98
PW
1518
1519 def map_depot_to_client(self, depot_path):
1520 """Calculate the client path if using this mapping on the
1521 given depot path; does not consider the effect of other
1522 mappings in a view. Even excluded mappings are returned."""
1523 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1524 if not matches:
1525 return ""
1526 client_path = self.client_side.fill_in_wildcards(wildcards)
1527 return client_path
1528
1529 #
1530 # View methods
1531 #
1532 def __init__(self):
1533 self.mappings = []
1534
1535 def append(self, view_line):
1536 """Parse a view line, splitting it into depot and client
1537 sides. Append to self.mappings, preserving order."""
1538
1539 # Split the view line into exactly two words. P4 enforces
1540 # structure on these lines that simplifies this quite a bit.
1541 #
1542 # Either or both words may be double-quoted.
1543 # Single quotes do not matter.
1544 # Double-quote marks cannot occur inside the words.
1545 # A + or - prefix is also inside the quotes.
1546 # There are no quotes unless they contain a space.
1547 # The line is already white-space stripped.
1548 # The two words are separated by a single space.
1549 #
1550 if view_line[0] == '"':
1551 # First word is double quoted. Find its end.
1552 close_quote_index = view_line.find('"', 1)
1553 if close_quote_index <= 0:
1554 die("No first-word closing quote found: %s" % view_line)
1555 depot_side = view_line[1:close_quote_index]
1556 # skip closing quote and space
1557 rhs_index = close_quote_index + 1 + 1
1558 else:
1559 space_index = view_line.find(" ")
1560 if space_index <= 0:
1561 die("No word-splitting space found: %s" % view_line)
1562 depot_side = view_line[0:space_index]
1563 rhs_index = space_index + 1
1564
1565 if view_line[rhs_index] == '"':
1566 # Second word is double quoted. Make sure there is a
1567 # double quote at the end too.
1568 if not view_line.endswith('"'):
1569 die("View line with rhs quote should end with one: %s" %
1570 view_line)
1571 # skip the quotes
1572 client_side = view_line[rhs_index+1:-1]
1573 else:
1574 client_side = view_line[rhs_index:]
1575
1576 # prefix + means overlay on previous mapping
1577 overlay = False
1578 if depot_side.startswith("+"):
1579 overlay = True
1580 depot_side = depot_side[1:]
1581
1582 # prefix - means exclude this path
1583 exclude = False
1584 if depot_side.startswith("-"):
1585 exclude = True
1586 depot_side = depot_side[1:]
1587
1588 m = View.Mapping(depot_side, client_side, overlay, exclude)
1589 self.mappings.append(m)
1590
1591 def map_in_client(self, depot_path):
1592 """Return the relative location in the client where this
1593 depot file should live. Returns "" if the file should
1594 not be mapped in the client."""
1595
1596 paths_filled = []
1597 client_path = ""
1598
1599 # look at later entries first
1600 for m in self.mappings[::-1]:
1601
1602 # see where will this path end up in the client
1603 p = m.map_depot_to_client(depot_path)
1604
1605 if p == "":
1606 # Depot path does not belong in client. Must remember
1607 # this, as previous items should not cause files to
1608 # exist in this path either. Remember that the list is
1609 # being walked from the end, which has higher precedence.
1610 # Overlap mappings do not exclude previous mappings.
1611 if not m.overlay:
1612 paths_filled.append(m.client_side)
1613
1614 else:
1615 # This mapping matched; no need to search any further.
1616 # But, the mapping could be rejected if the client path
6ee9a999
PW
1617 # has already been claimed by an earlier mapping (i.e.
1618 # one later in the list, which we are walking backwards).
ecb7cf98
PW
1619 already_mapped_in_client = False
1620 for f in paths_filled:
1621 # this is View.Path.match
1622 if f.match(p):
1623 already_mapped_in_client = True
1624 break
1625 if not already_mapped_in_client:
1626 # Include this file, unless it is from a line that
1627 # explicitly said to exclude it.
1628 if not m.exclude:
1629 client_path = p
1630
1631 # a match, even if rejected, always stops the search
1632 break
1633
1634 return client_path
1635
3ea2cfd4 1636class P4Sync(Command, P4UserMap):
56c09345
PW
1637 delete_actions = ( "delete", "move/delete", "purge" )
1638
b984733c
SH
1639 def __init__(self):
1640 Command.__init__(self)
3ea2cfd4 1641 P4UserMap.__init__(self)
b984733c
SH
1642 self.options = [
1643 optparse.make_option("--branch", dest="branch"),
1644 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1645 optparse.make_option("--changesfile", dest="changesFile"),
1646 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1647 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 1648 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
a028a98e 1649 optparse.make_option("--verbose", dest="verbose", action="store_true"),
d2c6dd30
HWN
1650 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1651 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 1652 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 1653 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1654 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1655 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1656 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
1657 ]
1658 self.description = """Imports from Perforce into a git repository.\n
1659 example:
1660 //depot/my/project/ -- to import the current head
1661 //depot/my/project/@all -- to import everything
1662 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1663
1664 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1665
1666 self.usage += " //depot/path[@revRange]"
b984733c 1667 self.silent = False
1d7367dc
RG
1668 self.createdBranches = set()
1669 self.committedChanges = set()
569d1bd4 1670 self.branch = ""
b984733c 1671 self.detectBranches = False
cb53e1f8 1672 self.detectLabels = False
06804c76 1673 self.importLabels = False
b984733c 1674 self.changesFile = ""
01265103 1675 self.syncWithOrigin = True
4b97ffb1 1676 self.verbose = False
a028a98e 1677 self.importIntoRemotes = True
01a9c9c5 1678 self.maxChanges = ""
c1f9197f 1679 self.isWindows = (platform.system() == "Windows")
8b41a97f 1680 self.keepRepoPath = False
6326aa58 1681 self.depotPaths = None
3c699645 1682 self.p4BranchesInGit = []
354081d5 1683 self.cloneExclude = []
3a70cdfa 1684 self.useClientSpec = False
a93d33ee 1685 self.useClientSpec_from_options = False
ecb7cf98 1686 self.clientSpecDirs = None
fed23693
VA
1687 self.tempBranches = []
1688 self.tempBranchLocation = "git-p4-tmp"
b984733c 1689
01265103
SH
1690 if gitConfig("git-p4.syncFromOrigin") == "false":
1691 self.syncWithOrigin = False
1692
084f6306
PW
1693 #
1694 # P4 wildcards are not allowed in filenames. P4 complains
1695 # if you simply add them, but you can force it with "-f", in
1696 # which case it translates them into %xx encoding internally.
1697 # Search for and fix just these four characters. Do % last so
1698 # that fixing it does not inadvertently create new %-escapes.
1699 #
1700 def wildcard_decode(self, path):
1701 # Cannot have * in a filename in windows; untested as to
1702 # what p4 would do in such a case.
1703 if not self.isWindows:
1704 path = path.replace("%2A", "*")
1705 path = path.replace("%23", "#") \
1706 .replace("%40", "@") \
1707 .replace("%25", "%")
1708 return path
1709
fed23693
VA
1710 # Force a checkpoint in fast-import and wait for it to finish
1711 def checkpoint(self):
1712 self.gitStream.write("checkpoint\n\n")
1713 self.gitStream.write("progress checkpoint\n\n")
1714 out = self.gitOutput.readline()
1715 if self.verbose:
1716 print "checkpoint finished: " + out
1717
b984733c 1718 def extractFilesFromCommit(self, commit):
354081d5
TT
1719 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1720 for path in self.cloneExclude]
b984733c
SH
1721 files = []
1722 fnum = 0
1723 while commit.has_key("depotFile%s" % fnum):
1724 path = commit["depotFile%s" % fnum]
6326aa58 1725
354081d5 1726 if [p for p in self.cloneExclude
d53de8b9 1727 if p4PathStartsWith(path, p)]:
354081d5
TT
1728 found = False
1729 else:
1730 found = [p for p in self.depotPaths
d53de8b9 1731 if p4PathStartsWith(path, p)]
6326aa58 1732 if not found:
b984733c
SH
1733 fnum = fnum + 1
1734 continue
1735
1736 file = {}
1737 file["path"] = path
1738 file["rev"] = commit["rev%s" % fnum]
1739 file["action"] = commit["action%s" % fnum]
1740 file["type"] = commit["type%s" % fnum]
1741 files.append(file)
1742 fnum = fnum + 1
1743 return files
1744
6326aa58 1745 def stripRepoPath(self, path, prefixes):
3952710b 1746 if self.useClientSpec:
ecb7cf98 1747 return self.clientSpecDirs.map_in_client(path)
3952710b 1748
8b41a97f 1749 if self.keepRepoPath:
6326aa58
HWN
1750 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1751
1752 for p in prefixes:
d53de8b9 1753 if p4PathStartsWith(path, p):
6326aa58 1754 path = path[len(p):]
8b41a97f 1755
6326aa58 1756 return path
6754a299 1757
71b112d4 1758 def splitFilesIntoBranches(self, commit):
d5904674 1759 branches = {}
71b112d4
SH
1760 fnum = 0
1761 while commit.has_key("depotFile%s" % fnum):
1762 path = commit["depotFile%s" % fnum]
6326aa58 1763 found = [p for p in self.depotPaths
d53de8b9 1764 if p4PathStartsWith(path, p)]
6326aa58 1765 if not found:
71b112d4
SH
1766 fnum = fnum + 1
1767 continue
1768
1769 file = {}
1770 file["path"] = path
1771 file["rev"] = commit["rev%s" % fnum]
1772 file["action"] = commit["action%s" % fnum]
1773 file["type"] = commit["type%s" % fnum]
1774 fnum = fnum + 1
1775
6326aa58 1776 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 1777
4b97ffb1 1778 for branch in self.knownBranches.keys():
6754a299
HWN
1779
1780 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1781 if relPath.startswith(branch + "/"):
d5904674
SH
1782 if branch not in branches:
1783 branches[branch] = []
71b112d4 1784 branches[branch].append(file)
6555b2cc 1785 break
b984733c
SH
1786
1787 return branches
1788
b932705b
LD
1789 # output one file from the P4 stream
1790 # - helper for streamP4Files
1791
1792 def streamOneP4File(self, file, contents):
b932705b 1793 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
084f6306 1794 relPath = self.wildcard_decode(relPath)
b932705b
LD
1795 if verbose:
1796 sys.stderr.write("%s\n" % relPath)
1797
9cffb8c8
PW
1798 (type_base, type_mods) = split_p4_type(file["type"])
1799
1800 git_mode = "100644"
1801 if "x" in type_mods:
1802 git_mode = "100755"
1803 if type_base == "symlink":
1804 git_mode = "120000"
1805 # p4 print on a symlink contains "target\n"; remove the newline
b39c3612
EP
1806 data = ''.join(contents)
1807 contents = [data[:-1]]
b932705b 1808
9cffb8c8 1809 if type_base == "utf16":
55aa5714
PW
1810 # p4 delivers different text in the python output to -G
1811 # than it does when using "print -o", or normal p4 client
1812 # operations. utf16 is converted to ascii or utf8, perhaps.
1813 # But ascii text saved as -t utf16 is completely mangled.
1814 # Invoke print -o to get the real contents.
6de040df 1815 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
55aa5714
PW
1816 contents = [ text ]
1817
9f7ef0ea
PW
1818 if type_base == "apple":
1819 # Apple filetype files will be streamed as a concatenation of
1820 # its appledouble header and the contents. This is useless
1821 # on both macs and non-macs. If using "print -q -o xx", it
1822 # will create "xx" with the data, and "%xx" with the header.
1823 # This is also not very useful.
1824 #
1825 # Ideally, someday, this script can learn how to generate
1826 # appledouble files directly and import those to git, but
1827 # non-mac machines can never find a use for apple filetype.
1828 print "\nIgnoring apple filetype file %s" % file['depotFile']
1829 return
1830
9cffb8c8
PW
1831 # Perhaps windows wants unicode, utf16 newlines translated too;
1832 # but this is not doing it.
1833 if self.isWindows and type_base == "text":
b932705b
LD
1834 mangled = []
1835 for data in contents:
1836 data = data.replace("\r\n", "\n")
1837 mangled.append(data)
1838 contents = mangled
1839
55aa5714
PW
1840 # Note that we do not try to de-mangle keywords on utf16 files,
1841 # even though in theory somebody may want that.
60df071c
LD
1842 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1843 if pattern:
1844 regexp = re.compile(pattern, re.VERBOSE)
1845 text = ''.join(contents)
1846 text = regexp.sub(r'$\1$', text)
1847 contents = [ text ]
b932705b 1848
9cffb8c8 1849 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
1850
1851 # total length...
1852 length = 0
1853 for d in contents:
1854 length = length + len(d)
1855
1856 self.gitStream.write("data %d\n" % length)
1857 for d in contents:
1858 self.gitStream.write(d)
1859 self.gitStream.write("\n")
1860
1861 def streamOneP4Deletion(self, file):
1862 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1863 if verbose:
1864 sys.stderr.write("delete %s\n" % relPath)
1865 self.gitStream.write("D %s\n" % relPath)
1866
1867 # handle another chunk of streaming data
1868 def streamP4FilesCb(self, marshalled):
1869
c3f6163b
AG
1870 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1871 # start of a new file - output the old one first
1872 self.streamOneP4File(self.stream_file, self.stream_contents)
1873 self.stream_file = {}
1874 self.stream_contents = []
1875 self.stream_have_file_info = False
b932705b 1876
c3f6163b
AG
1877 # pick up the new file information... for the
1878 # 'data' field we need to append to our array
1879 for k in marshalled.keys():
1880 if k == 'data':
1881 self.stream_contents.append(marshalled['data'])
1882 else:
1883 self.stream_file[k] = marshalled[k]
b932705b 1884
c3f6163b 1885 self.stream_have_file_info = True
b932705b
LD
1886
1887 # Stream directly from "p4 files" into "git fast-import"
1888 def streamP4Files(self, files):
30b5940b
SH
1889 filesForCommit = []
1890 filesToRead = []
b932705b 1891 filesToDelete = []
30b5940b 1892
3a70cdfa 1893 for f in files:
ecb7cf98
PW
1894 # if using a client spec, only add the files that have
1895 # a path in the client
1896 if self.clientSpecDirs:
1897 if self.clientSpecDirs.map_in_client(f['path']) == "":
1898 continue
3a70cdfa 1899
ecb7cf98
PW
1900 filesForCommit.append(f)
1901 if f['action'] in self.delete_actions:
1902 filesToDelete.append(f)
1903 else:
1904 filesToRead.append(f)
6a49f8e2 1905
b932705b
LD
1906 # deleted files...
1907 for f in filesToDelete:
1908 self.streamOneP4Deletion(f)
1b9a4684 1909
b932705b
LD
1910 if len(filesToRead) > 0:
1911 self.stream_file = {}
1912 self.stream_contents = []
1913 self.stream_have_file_info = False
8ff45f2a 1914
c3f6163b
AG
1915 # curry self argument
1916 def streamP4FilesCbSelf(entry):
1917 self.streamP4FilesCb(entry)
6a49f8e2 1918
6de040df
LD
1919 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1920
1921 p4CmdList(["-x", "-", "print"],
1922 stdin=fileArgs,
1923 cb=streamP4FilesCbSelf)
30b5940b 1924
b932705b
LD
1925 # do the last chunk
1926 if self.stream_file.has_key('depotFile'):
1927 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 1928
affb474f
LD
1929 def make_email(self, userid):
1930 if userid in self.users:
1931 return self.users[userid]
1932 else:
1933 return "%s <a@b>" % userid
1934
06804c76
LD
1935 # Stream a p4 tag
1936 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
1937 if verbose:
1938 print "writing tag %s for commit %s" % (labelName, commit)
1939 gitStream.write("tag %s\n" % labelName)
1940 gitStream.write("from %s\n" % commit)
1941
1942 if labelDetails.has_key('Owner'):
1943 owner = labelDetails["Owner"]
1944 else:
1945 owner = None
1946
1947 # Try to use the owner of the p4 label, or failing that,
1948 # the current p4 user id.
1949 if owner:
1950 email = self.make_email(owner)
1951 else:
1952 email = self.make_email(self.p4UserId())
1953 tagger = "%s %s %s" % (email, epoch, self.tz)
1954
1955 gitStream.write("tagger %s\n" % tagger)
1956
1957 print "labelDetails=",labelDetails
1958 if labelDetails.has_key('Description'):
1959 description = labelDetails['Description']
1960 else:
1961 description = 'Label from git p4'
1962
1963 gitStream.write("data %d\n" % len(description))
1964 gitStream.write(description)
1965 gitStream.write("\n")
1966
6326aa58 1967 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
1968 epoch = details["time"]
1969 author = details["user"]
c3f6163b 1970 self.branchPrefixes = branchPrefixes
b984733c 1971
4b97ffb1
SH
1972 if self.verbose:
1973 print "commit into %s" % branch
1974
96e07dd2
HWN
1975 # start with reading files; if that fails, we should not
1976 # create a commit.
1977 new_files = []
1978 for f in files:
d53de8b9 1979 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
1980 new_files.append (f)
1981 else:
afa1dd9a 1982 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 1983
b984733c 1984 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 1985# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
1986 self.committedChanges.add(int(details["change"]))
1987 committer = ""
b607e71e
SH
1988 if author not in self.users:
1989 self.getUserMapFromPerforceServer()
affb474f 1990 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
1991
1992 self.gitStream.write("committer %s\n" % committer)
1993
1994 self.gitStream.write("data <<EOT\n")
1995 self.gitStream.write(details["desc"])
6581de09
SH
1996 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1997 % (','.join (branchPrefixes), details["change"]))
1998 if len(details['options']) > 0:
1999 self.gitStream.write(": options = %s" % details['options'])
2000 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2001
2002 if len(parent) > 0:
4b97ffb1
SH
2003 if self.verbose:
2004 print "parent %s" % parent
b984733c
SH
2005 self.gitStream.write("from %s\n" % parent)
2006
b932705b 2007 self.streamP4Files(new_files)
b984733c
SH
2008 self.gitStream.write("\n")
2009
1f4ba1cb
SH
2010 change = int(details["change"])
2011
9bda3a85 2012 if self.labels.has_key(change):
1f4ba1cb
SH
2013 label = self.labels[change]
2014 labelDetails = label[0]
2015 labelRevisions = label[1]
71b112d4
SH
2016 if self.verbose:
2017 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2018
6de040df
LD
2019 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2020 for p in branchPrefixes])
1f4ba1cb
SH
2021
2022 if len(files) == len(labelRevisions):
2023
2024 cleanedFiles = {}
2025 for info in files:
56c09345 2026 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2027 continue
2028 cleanedFiles[info["depotFile"]] = info["rev"]
2029
2030 if cleanedFiles == labelRevisions:
06804c76 2031 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2032
2033 else:
a46668fa 2034 if not self.silent:
cebdf5af
HWN
2035 print ("Tag %s does not match with change %s: files do not match."
2036 % (labelDetails["label"], change))
1f4ba1cb
SH
2037
2038 else:
a46668fa 2039 if not self.silent:
cebdf5af
HWN
2040 print ("Tag %s does not match with change %s: file count is different."
2041 % (labelDetails["label"], change))
b984733c 2042
06804c76 2043 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2044 def getLabels(self):
2045 self.labels = {}
2046
52a4880b 2047 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2048 if len(l) > 0 and not self.silent:
183f8436 2049 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2050
2051 for output in l:
1f4ba1cb
SH
2052 label = output["label"]
2053 revisions = {}
2054 newestChange = 0
71b112d4
SH
2055 if self.verbose:
2056 print "Querying files for label %s" % label
6de040df
LD
2057 for file in p4CmdList(["files"] +
2058 ["%s...@%s" % (p, label)
2059 for p in self.depotPaths]):
1f4ba1cb
SH
2060 revisions[file["depotFile"]] = file["rev"]
2061 change = int(file["change"])
2062 if change > newestChange:
2063 newestChange = change
2064
9bda3a85
SH
2065 self.labels[newestChange] = [output, revisions]
2066
2067 if self.verbose:
2068 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2069
06804c76
LD
2070 # Import p4 labels as git tags. A direct mapping does not
2071 # exist, so assume that if all the files are at the same revision
2072 # then we can use that, or it's something more complicated we should
2073 # just ignore.
2074 def importP4Labels(self, stream, p4Labels):
2075 if verbose:
2076 print "import p4 labels: " + ' '.join(p4Labels)
2077
2078 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2079 validLabelRegexp = gitConfig("git-p4.validLabelRegexp")
2080 if len(validLabelRegexp) == 0:
2081 validLabelRegexp = defaultLabelRegexp
2082 m = re.compile(validLabelRegexp)
2083
2084 for name in p4Labels:
2085 commitFound = False
2086
2087 if not m.match(name):
2088 if verbose:
2089 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2090 continue
2091
2092 if name in ignoredP4Labels:
2093 continue
2094
2095 labelDetails = p4CmdList(['label', "-o", name])[0]
2096
2097 # get the most recent changelist for each file in this label
2098 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2099 for p in self.depotPaths])
2100
2101 if change.has_key('change'):
2102 # find the corresponding git commit; take the oldest commit
2103 changelist = int(change['change'])
2104 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2105 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2106 if len(gitCommit) == 0:
2107 print "could not find git commit for changelist %d" % changelist
2108 else:
2109 gitCommit = gitCommit.strip()
2110 commitFound = True
2111 # Convert from p4 time format
2112 try:
2113 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2114 except ValueError:
2115 print "Could not convert label time %s" % labelDetail['Update']
2116 tmwhen = 1
2117
2118 when = int(time.mktime(tmwhen))
2119 self.streamTag(stream, name, labelDetails, gitCommit, when)
2120 if verbose:
2121 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2122 else:
2123 if verbose:
2124 print "Label %s has no changelists - possibly deleted?" % name
2125
2126 if not commitFound:
2127 # We can't import this label; don't try again as it will get very
2128 # expensive repeatedly fetching all the files for labels that will
2129 # never be imported. If the label is moved in the future, the
2130 # ignore will need to be removed manually.
2131 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2132
86dff6b6
HWN
2133 def guessProjectName(self):
2134 for p in self.depotPaths:
6e5295c4
SH
2135 if p.endswith("/"):
2136 p = p[:-1]
2137 p = p[p.strip().rfind("/") + 1:]
2138 if not p.endswith("/"):
2139 p += "/"
2140 return p
86dff6b6 2141
4b97ffb1 2142 def getBranchMapping(self):
6555b2cc
SH
2143 lostAndFoundBranches = set()
2144
8ace74c0
VA
2145 user = gitConfig("git-p4.branchUser")
2146 if len(user) > 0:
2147 command = "branches -u %s" % user
2148 else:
2149 command = "branches"
2150
2151 for info in p4CmdList(command):
52a4880b 2152 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2153 viewIdx = 0
2154 while details.has_key("View%s" % viewIdx):
2155 paths = details["View%s" % viewIdx].split(" ")
2156 viewIdx = viewIdx + 1
2157 # require standard //depot/foo/... //depot/bar/... mapping
2158 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2159 continue
2160 source = paths[0]
2161 destination = paths[1]
6509e19c 2162 ## HACK
d53de8b9 2163 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2164 source = source[len(self.depotPaths[0]):-4]
2165 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2166
1a2edf4e
SH
2167 if destination in self.knownBranches:
2168 if not self.silent:
2169 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2170 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2171 continue
2172
6555b2cc
SH
2173 self.knownBranches[destination] = source
2174
2175 lostAndFoundBranches.discard(destination)
2176
29bdbac1 2177 if source not in self.knownBranches:
6555b2cc
SH
2178 lostAndFoundBranches.add(source)
2179
7199cf13
VA
2180 # Perforce does not strictly require branches to be defined, so we also
2181 # check git config for a branch list.
2182 #
2183 # Example of branch definition in git config file:
2184 # [git-p4]
2185 # branchList=main:branchA
2186 # branchList=main:branchB
2187 # branchList=branchA:branchC
2188 configBranches = gitConfigList("git-p4.branchList")
2189 for branch in configBranches:
2190 if branch:
2191 (source, destination) = branch.split(":")
2192 self.knownBranches[destination] = source
2193
2194 lostAndFoundBranches.discard(destination)
2195
2196 if source not in self.knownBranches:
2197 lostAndFoundBranches.add(source)
2198
6555b2cc
SH
2199
2200 for branch in lostAndFoundBranches:
2201 self.knownBranches[branch] = branch
29bdbac1 2202
38f9f5ec
SH
2203 def getBranchMappingFromGitBranches(self):
2204 branches = p4BranchesInGit(self.importIntoRemotes)
2205 for branch in branches.keys():
2206 if branch == "master":
2207 branch = "main"
2208 else:
2209 branch = branch[len(self.projectName):]
2210 self.knownBranches[branch] = branch
2211
29bdbac1 2212 def listExistingP4GitBranches(self):
144ff46b
SH
2213 # branches holds mapping from name to commit
2214 branches = p4BranchesInGit(self.importIntoRemotes)
2215 self.p4BranchesInGit = branches.keys()
2216 for branch in branches.keys():
2217 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 2218
bb6e09b2
HWN
2219 def updateOptionDict(self, d):
2220 option_keys = {}
2221 if self.keepRepoPath:
2222 option_keys['keepRepoPath'] = 1
2223
2224 d["options"] = ' '.join(sorted(option_keys.keys()))
2225
2226 def readOptions(self, d):
2227 self.keepRepoPath = (d.has_key('options')
2228 and ('keepRepoPath' in d['options']))
6326aa58 2229
8134f69c
SH
2230 def gitRefForBranch(self, branch):
2231 if branch == "main":
2232 return self.refPrefix + "master"
2233
2234 if len(branch) <= 0:
2235 return branch
2236
2237 return self.refPrefix + self.projectName + branch
2238
1ca3d710
SH
2239 def gitCommitByP4Change(self, ref, change):
2240 if self.verbose:
2241 print "looking in ref " + ref + " for change %s using bisect..." % change
2242
2243 earliestCommit = ""
2244 latestCommit = parseRevision(ref)
2245
2246 while True:
2247 if self.verbose:
2248 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2249 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2250 if len(next) == 0:
2251 if self.verbose:
2252 print "argh"
2253 return ""
2254 log = extractLogMessageFromGitCommit(next)
2255 settings = extractSettingsGitLog(log)
2256 currentChange = int(settings['change'])
2257 if self.verbose:
2258 print "current change %s" % currentChange
2259
2260 if currentChange == change:
2261 if self.verbose:
2262 print "found %s" % next
2263 return next
2264
2265 if currentChange < change:
2266 earliestCommit = "^%s" % next
2267 else:
2268 latestCommit = "%s" % next
2269
2270 return ""
2271
2272 def importNewBranch(self, branch, maxChange):
2273 # make fast-import flush all changes to disk and update the refs using the checkpoint
2274 # command so that we can try to find the branch parent in the git history
2275 self.gitStream.write("checkpoint\n\n");
2276 self.gitStream.flush();
2277 branchPrefix = self.depotPaths[0] + branch + "/"
2278 range = "@1,%s" % maxChange
2279 #print "prefix" + branchPrefix
2280 changes = p4ChangesForPaths([branchPrefix], range)
2281 if len(changes) <= 0:
2282 return False
2283 firstChange = changes[0]
2284 #print "first change in branch: %s" % firstChange
2285 sourceBranch = self.knownBranches[branch]
2286 sourceDepotPath = self.depotPaths[0] + sourceBranch
2287 sourceRef = self.gitRefForBranch(sourceBranch)
2288 #print "source " + sourceBranch
2289
52a4880b 2290 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2291 #print "branch parent: %s" % branchParentChange
2292 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2293 if len(gitParent) > 0:
2294 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2295 #print "parent git commit: %s" % gitParent
2296
2297 self.importChanges(changes)
2298 return True
2299
fed23693
VA
2300 def searchParent(self, parent, branch, target):
2301 parentFound = False
2302 for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2303 blob = blob.strip()
2304 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2305 parentFound = True
2306 if self.verbose:
2307 print "Found parent of %s in commit %s" % (branch, blob)
2308 break
2309 if parentFound:
2310 return blob
2311 else:
2312 return None
2313
e87f37ae
SH
2314 def importChanges(self, changes):
2315 cnt = 1
2316 for change in changes:
3558f32f 2317 description = p4Cmd(["describe", str(change)])
e87f37ae
SH
2318 self.updateOptionDict(description)
2319
2320 if not self.silent:
2321 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2322 sys.stdout.flush()
2323 cnt = cnt + 1
2324
2325 try:
2326 if self.detectBranches:
2327 branches = self.splitFilesIntoBranches(description)
2328 for branch in branches.keys():
2329 ## HACK --hwn
2330 branchPrefix = self.depotPaths[0] + branch + "/"
2331
2332 parent = ""
2333
2334 filesForCommit = branches[branch]
2335
2336 if self.verbose:
2337 print "branch is %s" % branch
2338
2339 self.updatedBranches.add(branch)
2340
2341 if branch not in self.createdBranches:
2342 self.createdBranches.add(branch)
2343 parent = self.knownBranches[branch]
2344 if parent == branch:
2345 parent = ""
1ca3d710
SH
2346 else:
2347 fullBranch = self.projectName + branch
2348 if fullBranch not in self.p4BranchesInGit:
2349 if not self.silent:
2350 print("\n Importing new branch %s" % fullBranch);
2351 if self.importNewBranch(branch, change - 1):
2352 parent = ""
2353 self.p4BranchesInGit.append(fullBranch)
2354 if not self.silent:
2355 print("\n Resuming with change %s" % change);
2356
2357 if self.verbose:
2358 print "parent determined through known branches: %s" % parent
e87f37ae 2359
8134f69c
SH
2360 branch = self.gitRefForBranch(branch)
2361 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2362
2363 if self.verbose:
2364 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2365
2366 if len(parent) == 0 and branch in self.initialParents:
2367 parent = self.initialParents[branch]
2368 del self.initialParents[branch]
2369
fed23693
VA
2370 blob = None
2371 if len(parent) > 0:
2372 tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2373 if self.verbose:
2374 print "Creating temporary branch: " + tempBranch
2375 self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2376 self.tempBranches.append(tempBranch)
2377 self.checkpoint()
2378 blob = self.searchParent(parent, branch, tempBranch)
2379 if blob:
2380 self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2381 else:
2382 if self.verbose:
2383 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2384 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
e87f37ae
SH
2385 else:
2386 files = self.extractFilesFromCommit(description)
2387 self.commit(description, files, self.branch, self.depotPaths,
2388 self.initialParent)
2389 self.initialParent = ""
2390 except IOError:
2391 print self.gitError.read()
2392 sys.exit(1)
2393
c208a243
SH
2394 def importHeadRevision(self, revision):
2395 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2396
4e2e6ce4
PW
2397 details = {}
2398 details["user"] = "git perforce import user"
1494fcbb 2399 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2400 % (' '.join(self.depotPaths), revision))
2401 details["change"] = revision
2402 newestRevision = 0
2403
2404 fileCnt = 0
6de040df
LD
2405 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2406
2407 for info in p4CmdList(["files"] + fileArgs):
c208a243 2408
68b28593 2409 if 'code' in info and info['code'] == 'error':
c208a243
SH
2410 sys.stderr.write("p4 returned an error: %s\n"
2411 % info['data'])
d88e707f
PW
2412 if info['data'].find("must refer to client") >= 0:
2413 sys.stderr.write("This particular p4 error is misleading.\n")
2414 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2415 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2416 sys.exit(1)
68b28593
PW
2417 if 'p4ExitCode' in info:
2418 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2419 sys.exit(1)
2420
2421
2422 change = int(info["change"])
2423 if change > newestRevision:
2424 newestRevision = change
2425
56c09345 2426 if info["action"] in self.delete_actions:
c208a243
SH
2427 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2428 #fileCnt = fileCnt + 1
2429 continue
2430
2431 for prop in ["depotFile", "rev", "action", "type" ]:
2432 details["%s%s" % (prop, fileCnt)] = info[prop]
2433
2434 fileCnt = fileCnt + 1
2435
2436 details["change"] = newestRevision
4e2e6ce4 2437
9dcb9f24 2438 # Use time from top-most change so that all git p4 clones of
4e2e6ce4
PW
2439 # the same p4 repo have the same commit SHA1s.
2440 res = p4CmdList("describe -s %d" % newestRevision)
2441 newestTime = None
2442 for r in res:
2443 if r.has_key('time'):
2444 newestTime = int(r['time'])
2445 if newestTime is None:
2446 die("\"describe -s\" on newest change %d did not give a time")
2447 details["time"] = newestTime
2448
c208a243
SH
2449 self.updateOptionDict(details)
2450 try:
2451 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2452 except IOError:
2453 print "IO error with git fast-import. Is your git version recent enough?"
2454 print self.gitError.read()
2455
2456
b984733c 2457 def run(self, args):
6326aa58 2458 self.depotPaths = []
179caebf
SH
2459 self.changeRange = ""
2460 self.initialParent = ""
6326aa58 2461 self.previousDepotPaths = []
ce6f33c8 2462
29bdbac1
SH
2463 # map from branch depot path to parent branch
2464 self.knownBranches = {}
2465 self.initialParents = {}
5ca44617 2466 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
2467 if not self.syncWithOrigin:
2468 self.hasOrigin = False
29bdbac1 2469
a028a98e
SH
2470 if self.importIntoRemotes:
2471 self.refPrefix = "refs/remotes/p4/"
2472 else:
db775559 2473 self.refPrefix = "refs/heads/p4/"
a028a98e 2474
cebdf5af
HWN
2475 if self.syncWithOrigin and self.hasOrigin:
2476 if not self.silent:
2477 print "Syncing with origin first by calling git fetch origin"
2478 system("git fetch origin")
10f880f8 2479
569d1bd4 2480 if len(self.branch) == 0:
db775559 2481 self.branch = self.refPrefix + "master"
a028a98e 2482 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2483 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 2484 system("git branch -D p4");
faf1bd20 2485 # create it /after/ importing, when master exists
0058a33a 2486 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 2487 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 2488
a93d33ee
PW
2489 # accept either the command-line option, or the configuration variable
2490 if self.useClientSpec:
2491 # will use this after clone to set the variable
2492 self.useClientSpec_from_options = True
2493 else:
09fca77b
PW
2494 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2495 self.useClientSpec = True
2496 if self.useClientSpec:
543987bd 2497 self.clientSpecDirs = getClientSpec()
3a70cdfa 2498
6a49f8e2
HWN
2499 # TODO: should always look at previous commits,
2500 # merge with previous imports, if possible.
2501 if args == []:
d414c74a 2502 if self.hasOrigin:
5ca44617 2503 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
2504 self.listExistingP4GitBranches()
2505
2506 if len(self.p4BranchesInGit) > 1:
2507 if not self.silent:
2508 print "Importing from/into multiple branches"
2509 self.detectBranches = True
967f72e2 2510
29bdbac1
SH
2511 if self.verbose:
2512 print "branches: %s" % self.p4BranchesInGit
2513
2514 p4Change = 0
2515 for branch in self.p4BranchesInGit:
cebdf5af 2516 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2517
2518 settings = extractSettingsGitLog(logMsg)
29bdbac1 2519
bb6e09b2
HWN
2520 self.readOptions(settings)
2521 if (settings.has_key('depot-paths')
2522 and settings.has_key ('change')):
2523 change = int(settings['change']) + 1
29bdbac1
SH
2524 p4Change = max(p4Change, change)
2525
bb6e09b2
HWN
2526 depotPaths = sorted(settings['depot-paths'])
2527 if self.previousDepotPaths == []:
6326aa58 2528 self.previousDepotPaths = depotPaths
29bdbac1 2529 else:
6326aa58
HWN
2530 paths = []
2531 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2532 prev_list = prev.split("/")
2533 cur_list = cur.split("/")
2534 for i in range(0, min(len(cur_list), len(prev_list))):
2535 if cur_list[i] <> prev_list[i]:
583e1707 2536 i = i - 1
6326aa58
HWN
2537 break
2538
04d277b3 2539 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2540
2541 self.previousDepotPaths = paths
29bdbac1
SH
2542
2543 if p4Change > 0:
bb6e09b2 2544 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2545 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
2546 if not self.detectBranches:
2547 self.initialParent = parseRevision(self.branch)
341dc1c1 2548 if not self.silent and not self.detectBranches:
967f72e2 2549 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2550
f9162f6a
SH
2551 if not self.branch.startswith("refs/"):
2552 self.branch = "refs/heads/" + self.branch
179caebf 2553
6326aa58 2554 if len(args) == 0 and self.depotPaths:
b984733c 2555 if not self.silent:
6326aa58 2556 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2557 else:
6326aa58 2558 if self.depotPaths and self.depotPaths != args:
cebdf5af 2559 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2560 "This doesn't work!" % (' '.join (self.depotPaths),
2561 ' '.join (args)))
b984733c 2562 sys.exit(1)
6326aa58 2563
bb6e09b2 2564 self.depotPaths = sorted(args)
b984733c 2565
1c49fc19 2566 revision = ""
b984733c 2567 self.users = {}
b984733c 2568
58c8bc7c
PW
2569 # Make sure no revision specifiers are used when --changesfile
2570 # is specified.
2571 bad_changesfile = False
2572 if len(self.changesFile) > 0:
2573 for p in self.depotPaths:
2574 if p.find("@") >= 0 or p.find("#") >= 0:
2575 bad_changesfile = True
2576 break
2577 if bad_changesfile:
2578 die("Option --changesfile is incompatible with revision specifiers")
2579
6326aa58
HWN
2580 newPaths = []
2581 for p in self.depotPaths:
2582 if p.find("@") != -1:
2583 atIdx = p.index("@")
2584 self.changeRange = p[atIdx:]
2585 if self.changeRange == "@all":
2586 self.changeRange = ""
6a49f8e2 2587 elif ',' not in self.changeRange:
1c49fc19 2588 revision = self.changeRange
6326aa58 2589 self.changeRange = ""
7fcff9de 2590 p = p[:atIdx]
6326aa58
HWN
2591 elif p.find("#") != -1:
2592 hashIdx = p.index("#")
1c49fc19 2593 revision = p[hashIdx:]
7fcff9de 2594 p = p[:hashIdx]
6326aa58 2595 elif self.previousDepotPaths == []:
58c8bc7c
PW
2596 # pay attention to changesfile, if given, else import
2597 # the entire p4 tree at the head revision
2598 if len(self.changesFile) == 0:
2599 revision = "#head"
6326aa58
HWN
2600
2601 p = re.sub ("\.\.\.$", "", p)
2602 if not p.endswith("/"):
2603 p += "/"
2604
2605 newPaths.append(p)
2606
2607 self.depotPaths = newPaths
2608
b607e71e 2609 self.loadUserMapFromCache()
cb53e1f8
SH
2610 self.labels = {}
2611 if self.detectLabels:
2612 self.getLabels();
b984733c 2613
4b97ffb1 2614 if self.detectBranches:
df450923
SH
2615 ## FIXME - what's a P4 projectName ?
2616 self.projectName = self.guessProjectName()
2617
38f9f5ec
SH
2618 if self.hasOrigin:
2619 self.getBranchMappingFromGitBranches()
2620 else:
2621 self.getBranchMapping()
29bdbac1
SH
2622 if self.verbose:
2623 print "p4-git branches: %s" % self.p4BranchesInGit
2624 print "initial parents: %s" % self.initialParents
2625 for b in self.p4BranchesInGit:
2626 if b != "master":
6326aa58
HWN
2627
2628 ## FIXME
29bdbac1
SH
2629 b = b[len(self.projectName):]
2630 self.createdBranches.add(b)
4b97ffb1 2631
f291b4e3 2632 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 2633
cebdf5af 2634 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
2635 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2636 stderr=subprocess.PIPE);
08483580
SH
2637 self.gitOutput = importProcess.stdout
2638 self.gitStream = importProcess.stdin
2639 self.gitError = importProcess.stderr
b984733c 2640
1c49fc19 2641 if revision:
c208a243 2642 self.importHeadRevision(revision)
b984733c
SH
2643 else:
2644 changes = []
2645
0828ab14 2646 if len(self.changesFile) > 0:
b984733c 2647 output = open(self.changesFile).readlines()
1d7367dc 2648 changeSet = set()
b984733c
SH
2649 for line in output:
2650 changeSet.add(int(line))
2651
2652 for change in changeSet:
2653 changes.append(change)
2654
2655 changes.sort()
2656 else:
9dcb9f24
PW
2657 # catch "git p4 sync" with no new branches, in a repo that
2658 # does not have any existing p4 branches
accad8e0 2659 if len(args) == 0 and not self.p4BranchesInGit:
e32e00dc 2660 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
29bdbac1 2661 if self.verbose:
86dff6b6 2662 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 2663 self.changeRange)
4f6432d8 2664 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 2665
01a9c9c5 2666 if len(self.maxChanges) > 0:
7fcff9de 2667 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 2668
b984733c 2669 if len(changes) == 0:
0828ab14 2670 if not self.silent:
341dc1c1 2671 print "No changes to import!"
06804c76
LD
2672 else:
2673 if not self.silent and not self.detectBranches:
2674 print "Import destination: %s" % self.branch
2675
2676 self.updatedBranches = set()
b984733c 2677
06804c76 2678 self.importChanges(changes)
a9d1a27a 2679
06804c76
LD
2680 if not self.silent:
2681 print ""
2682 if len(self.updatedBranches) > 0:
2683 sys.stdout.write("Updated branches: ")
2684 for b in self.updatedBranches:
2685 sys.stdout.write("%s " % b)
2686 sys.stdout.write("\n")
341dc1c1 2687
06804c76
LD
2688 if gitConfig("git-p4.importLabels", "--bool") == "true":
2689 self.importLabels = true
b984733c 2690
06804c76
LD
2691 if self.importLabels:
2692 p4Labels = getP4Labels(self.depotPaths)
2693 gitTags = getGitTags()
2694
2695 missingP4Labels = p4Labels - gitTags
2696 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 2697
b984733c 2698 self.gitStream.close()
29bdbac1
SH
2699 if importProcess.wait() != 0:
2700 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
2701 self.gitOutput.close()
2702 self.gitError.close()
2703
fed23693
VA
2704 # Cleanup temporary branches created during import
2705 if self.tempBranches != []:
2706 for branch in self.tempBranches:
2707 read_pipe("git update-ref -d %s" % branch)
2708 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2709
b984733c
SH
2710 return True
2711
01ce1fe9
SH
2712class P4Rebase(Command):
2713 def __init__(self):
2714 Command.__init__(self)
06804c76
LD
2715 self.options = [
2716 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2717 optparse.make_option("--verbose", dest="verbose", action="store_true"),
2718 ]
2719 self.verbose = False
2720 self.importLabels = False
cebdf5af
HWN
2721 self.description = ("Fetches the latest revision from perforce and "
2722 + "rebases the current work (branch) against it")
01ce1fe9
SH
2723
2724 def run(self, args):
2725 sync = P4Sync()
06804c76 2726 sync.importLabels = self.importLabels
01ce1fe9 2727 sync.run([])
d7e3868c 2728
14594f4b
SH
2729 return self.rebase()
2730
2731 def rebase(self):
36ee4ee4
SH
2732 if os.system("git update-index --refresh") != 0:
2733 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2734 if len(read_pipe("git diff-index HEAD --")) > 0:
2735 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2736
d7e3868c
SH
2737 [upstream, settings] = findUpstreamBranchPoint()
2738 if len(upstream) == 0:
2739 die("Cannot find upstream branchpoint for rebase")
2740
2741 # the branchpoint may be p4/foo~3, so strip off the parent
2742 upstream = re.sub("~[0-9]+$", "", upstream)
2743
2744 print "Rebasing the current branch onto %s" % upstream
b25b2065 2745 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 2746 system("git rebase %s" % upstream)
1f52af6c 2747 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
2748 return True
2749
f9a3a4f7
SH
2750class P4Clone(P4Sync):
2751 def __init__(self):
2752 P4Sync.__init__(self)
2753 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 2754 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 2755 self.options += [
bb6e09b2
HWN
2756 optparse.make_option("--destination", dest="cloneDestination",
2757 action='store', default=None,
354081d5
TT
2758 help="where to leave result of the clone"),
2759 optparse.make_option("-/", dest="cloneExclude",
2760 action="append", type="string",
38200076
PW
2761 help="exclude depot path"),
2762 optparse.make_option("--bare", dest="cloneBare",
2763 action="store_true", default=False),
354081d5 2764 ]
bb6e09b2 2765 self.cloneDestination = None
f9a3a4f7 2766 self.needsGit = False
38200076 2767 self.cloneBare = False
f9a3a4f7 2768
354081d5
TT
2769 # This is required for the "append" cloneExclude action
2770 def ensure_value(self, attr, value):
2771 if not hasattr(self, attr) or getattr(self, attr) is None:
2772 setattr(self, attr, value)
2773 return getattr(self, attr)
2774
6a49f8e2
HWN
2775 def defaultDestination(self, args):
2776 ## TODO: use common prefix of args?
2777 depotPath = args[0]
2778 depotDir = re.sub("(@[^@]*)$", "", depotPath)
2779 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 2780 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
2781 depotDir = re.sub(r"/$", "", depotDir)
2782 return os.path.split(depotDir)[1]
2783
f9a3a4f7
SH
2784 def run(self, args):
2785 if len(args) < 1:
2786 return False
bb6e09b2
HWN
2787
2788 if self.keepRepoPath and not self.cloneDestination:
2789 sys.stderr.write("Must specify destination for --keep-path\n")
2790 sys.exit(1)
f9a3a4f7 2791
6326aa58 2792 depotPaths = args
5e100b5c
SH
2793
2794 if not self.cloneDestination and len(depotPaths) > 1:
2795 self.cloneDestination = depotPaths[-1]
2796 depotPaths = depotPaths[:-1]
2797
354081d5 2798 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
2799 for p in depotPaths:
2800 if not p.startswith("//"):
2801 return False
f9a3a4f7 2802
bb6e09b2 2803 if not self.cloneDestination:
98ad4faf 2804 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 2805
86dff6b6 2806 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 2807
c3bf3f13
KG
2808 if not os.path.exists(self.cloneDestination):
2809 os.makedirs(self.cloneDestination)
053fd0c1 2810 chdir(self.cloneDestination)
38200076
PW
2811
2812 init_cmd = [ "git", "init" ]
2813 if self.cloneBare:
2814 init_cmd.append("--bare")
2815 subprocess.check_call(init_cmd)
2816
6326aa58 2817 if not P4Sync.run(self, depotPaths):
f9a3a4f7 2818 return False
f9a3a4f7 2819 if self.branch != "master":
e9905013
TAL
2820 if self.importIntoRemotes:
2821 masterbranch = "refs/remotes/p4/master"
2822 else:
2823 masterbranch = "refs/heads/p4/master"
2824 if gitBranchExists(masterbranch):
2825 system("git branch master %s" % masterbranch)
38200076
PW
2826 if not self.cloneBare:
2827 system("git checkout -f")
8f9b2e08
SH
2828 else:
2829 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 2830
a93d33ee
PW
2831 # auto-set this variable if invoked with --use-client-spec
2832 if self.useClientSpec_from_options:
2833 system("git config --bool git-p4.useclientspec true")
2834
f9a3a4f7
SH
2835 return True
2836
09d89de2
SH
2837class P4Branches(Command):
2838 def __init__(self):
2839 Command.__init__(self)
2840 self.options = [ ]
2841 self.description = ("Shows the git branches that hold imports and their "
2842 + "corresponding perforce depot paths")
2843 self.verbose = False
2844
2845 def run(self, args):
5ca44617
SH
2846 if originP4BranchesExist():
2847 createOrUpdateBranchesFromOrigin()
2848
09d89de2
SH
2849 cmdline = "git rev-parse --symbolic "
2850 cmdline += " --remotes"
2851
2852 for line in read_pipe_lines(cmdline):
2853 line = line.strip()
2854
2855 if not line.startswith('p4/') or line == "p4/HEAD":
2856 continue
2857 branch = line
2858
2859 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2860 settings = extractSettingsGitLog(log)
2861
2862 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2863 return True
2864
b984733c
SH
2865class HelpFormatter(optparse.IndentedHelpFormatter):
2866 def __init__(self):
2867 optparse.IndentedHelpFormatter.__init__(self)
2868
2869 def format_description(self, description):
2870 if description:
2871 return description + "\n"
2872 else:
2873 return ""
4f5cf76a 2874
86949eef
SH
2875def printUsage(commands):
2876 print "usage: %s <command> [options]" % sys.argv[0]
2877 print ""
2878 print "valid commands: %s" % ", ".join(commands)
2879 print ""
2880 print "Try %s <command> --help for command specific help." % sys.argv[0]
2881 print ""
2882
2883commands = {
b86f7378
HWN
2884 "debug" : P4Debug,
2885 "submit" : P4Submit,
a9834f58 2886 "commit" : P4Submit,
b86f7378
HWN
2887 "sync" : P4Sync,
2888 "rebase" : P4Rebase,
2889 "clone" : P4Clone,
09d89de2
SH
2890 "rollback" : P4RollBack,
2891 "branches" : P4Branches
86949eef
SH
2892}
2893
86949eef 2894
bb6e09b2
HWN
2895def main():
2896 if len(sys.argv[1:]) == 0:
2897 printUsage(commands.keys())
2898 sys.exit(2)
4f5cf76a 2899
bb6e09b2
HWN
2900 cmd = ""
2901 cmdName = sys.argv[1]
2902 try:
b86f7378
HWN
2903 klass = commands[cmdName]
2904 cmd = klass()
bb6e09b2
HWN
2905 except KeyError:
2906 print "unknown command %s" % cmdName
2907 print ""
2908 printUsage(commands.keys())
2909 sys.exit(2)
2910
2911 options = cmd.options
b86f7378 2912 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
2913
2914 args = sys.argv[2:]
2915
2916 if len(options) > 0:
ef86890c
PW
2917 if cmd.needsGit:
2918 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2
HWN
2919
2920 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2921 options,
2922 description = cmd.description,
2923 formatter = HelpFormatter())
2924
2925 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2926 global verbose
2927 verbose = cmd.verbose
2928 if cmd.needsGit:
b86f7378
HWN
2929 if cmd.gitdir == None:
2930 cmd.gitdir = os.path.abspath(".git")
2931 if not isValidGitDir(cmd.gitdir):
2932 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2933 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
2934 cdup = read_pipe("git rev-parse --show-cdup").strip()
2935 if len(cdup) > 0:
053fd0c1 2936 chdir(cdup);
e20a9e53 2937
b86f7378
HWN
2938 if not isValidGitDir(cmd.gitdir):
2939 if isValidGitDir(cmd.gitdir + "/.git"):
2940 cmd.gitdir += "/.git"
bb6e09b2 2941 else:
b86f7378 2942 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 2943
b86f7378 2944 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 2945
bb6e09b2
HWN
2946 if not cmd.run(args):
2947 parser.print_help()
09fca77b 2948 sys.exit(2)
4f5cf76a 2949
4f5cf76a 2950
bb6e09b2
HWN
2951if __name__ == '__main__':
2952 main()