git p4: fix writable file after rename or copy
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
1d7367dc
RG
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
60df071c 13import re, shutil
8b41a97f 14
4addad22 15verbose = False
86949eef 16
06804c76 17# Only labels/tags matching this will be imported/exported
c8942a22 18defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753
AK
19
20def p4_build_cmd(cmd):
21 """Build a suitable p4 command line.
22
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
26 """
6de040df 27 real_cmd = ["p4"]
abcaf073
AK
28
29 user = gitConfig("git-p4.user")
30 if len(user) > 0:
6de040df 31 real_cmd += ["-u",user]
abcaf073
AK
32
33 password = gitConfig("git-p4.password")
34 if len(password) > 0:
6de040df 35 real_cmd += ["-P", password]
abcaf073
AK
36
37 port = gitConfig("git-p4.port")
38 if len(port) > 0:
6de040df 39 real_cmd += ["-p", port]
abcaf073
AK
40
41 host = gitConfig("git-p4.host")
42 if len(host) > 0:
41799aa2 43 real_cmd += ["-H", host]
abcaf073
AK
44
45 client = gitConfig("git-p4.client")
46 if len(client) > 0:
6de040df 47 real_cmd += ["-c", client]
abcaf073 48
6de040df
LD
49
50 if isinstance(cmd,basestring):
51 real_cmd = ' '.join(real_cmd) + ' ' + cmd
52 else:
53 real_cmd += cmd
21a50753
AK
54 return real_cmd
55
053fd0c1 56def chdir(dir):
6de040df 57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
bf1d68ff
GG
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
053fd0c1 60 os.chdir(dir)
bf1d68ff 61 os.environ['PWD'] = os.getcwd()
053fd0c1 62
86dff6b6
HWN
63def die(msg):
64 if verbose:
65 raise Exception(msg)
66 else:
67 sys.stderr.write(msg + "\n")
68 sys.exit(1)
69
6de040df 70def write_pipe(c, stdin):
4addad22 71 if verbose:
6de040df 72 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 73
6de040df
LD
74 expand = isinstance(c,basestring)
75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
76 pipe = p.stdin
77 val = pipe.write(stdin)
78 pipe.close()
79 if p.wait():
80 die('Command failed: %s' % str(c))
b016d397
HWN
81
82 return val
83
6de040df 84def p4_write_pipe(c, stdin):
d9429194 85 real_cmd = p4_build_cmd(c)
6de040df 86 return write_pipe(real_cmd, stdin)
d9429194 87
4addad22
HWN
88def read_pipe(c, ignore_error=False):
89 if verbose:
6de040df 90 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 91
6de040df
LD
92 expand = isinstance(c,basestring)
93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
94 pipe = p.stdout
b016d397 95 val = pipe.read()
6de040df
LD
96 if p.wait() and not ignore_error:
97 die('Command failed: %s' % str(c))
b016d397
HWN
98
99 return val
100
d9429194
AK
101def p4_read_pipe(c, ignore_error=False):
102 real_cmd = p4_build_cmd(c)
103 return read_pipe(real_cmd, ignore_error)
b016d397 104
bce4c5fc 105def read_pipe_lines(c):
4addad22 106 if verbose:
6de040df
LD
107 sys.stderr.write('Reading pipe: %s\n' % str(c))
108
109 expand = isinstance(c, basestring)
110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
111 pipe = p.stdout
b016d397 112 val = pipe.readlines()
6de040df
LD
113 if pipe.close() or p.wait():
114 die('Command failed: %s' % str(c))
b016d397
HWN
115
116 return val
caace111 117
2318121b
AK
118def p4_read_pipe_lines(c):
119 """Specifically invoke p4 on the command supplied. """
155af834 120 real_cmd = p4_build_cmd(c)
2318121b
AK
121 return read_pipe_lines(real_cmd)
122
6754a299 123def system(cmd):
6de040df 124 expand = isinstance(cmd,basestring)
4addad22 125 if verbose:
6de040df
LD
126 sys.stderr.write("executing %s\n" % str(cmd))
127 subprocess.check_call(cmd, shell=expand)
6754a299 128
bf9320f1
AK
129def p4_system(cmd):
130 """Specifically invoke p4 as the system command. """
155af834 131 real_cmd = p4_build_cmd(cmd)
6de040df
LD
132 expand = isinstance(real_cmd, basestring)
133 subprocess.check_call(real_cmd, shell=expand)
134
135def p4_integrate(src, dest):
136 p4_system(["integrate", "-Dt", src, dest])
137
8d7ec362
PW
138def p4_sync(f, *options):
139 p4_system(["sync"] + list(options) + [f])
6de040df
LD
140
141def p4_add(f):
142 p4_system(["add", f])
143
144def p4_delete(f):
145 p4_system(["delete", f])
146
147def p4_edit(f):
148 p4_system(["edit", f])
149
150def p4_revert(f):
151 p4_system(["revert", f])
152
153def p4_reopen(type, file):
154 p4_system(["reopen", "-t", type, file])
bf9320f1 155
9cffb8c8
PW
156#
157# Canonicalize the p4 type and return a tuple of the
158# base type, plus any modifiers. See "p4 help filetypes"
159# for a list and explanation.
160#
161def split_p4_type(p4type):
162
163 p4_filetypes_historical = {
164 "ctempobj": "binary+Sw",
165 "ctext": "text+C",
166 "cxtext": "text+Cx",
167 "ktext": "text+k",
168 "kxtext": "text+kx",
169 "ltext": "text+F",
170 "tempobj": "binary+FSw",
171 "ubinary": "binary+F",
172 "uresource": "resource+F",
173 "uxbinary": "binary+Fx",
174 "xbinary": "binary+x",
175 "xltext": "text+Fx",
176 "xtempobj": "binary+Swx",
177 "xtext": "text+x",
178 "xunicode": "unicode+x",
179 "xutf16": "utf16+x",
180 }
181 if p4type in p4_filetypes_historical:
182 p4type = p4_filetypes_historical[p4type]
183 mods = ""
184 s = p4type.split("+")
185 base = s[0]
186 mods = ""
187 if len(s) > 1:
188 mods = s[1]
189 return (base, mods)
b9fc6ea9 190
60df071c
LD
191#
192# return the raw p4 type of a file (text, text+ko, etc)
193#
194def p4_type(file):
195 results = p4CmdList(["fstat", "-T", "headType", file])
196 return results[0]['headType']
197
198#
199# Given a type base and modifier, return a regexp matching
200# the keywords that can be expanded in the file
201#
202def p4_keywords_regexp_for_type(base, type_mods):
203 if base in ("text", "unicode", "binary"):
204 kwords = None
205 if "ko" in type_mods:
206 kwords = 'Id|Header'
207 elif "k" in type_mods:
208 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
209 else:
210 return None
211 pattern = r"""
212 \$ # Starts with a dollar, followed by...
213 (%s) # one of the keywords, followed by...
214 (:[^$]+)? # possibly an old expansion, followed by...
215 \$ # another dollar
216 """ % kwords
217 return pattern
218 else:
219 return None
220
221#
222# Given a file, return a regexp matching the possible
223# RCS keywords that will be expanded, or None for files
224# with kw expansion turned off.
225#
226def p4_keywords_regexp_for_file(file):
227 if not os.path.exists(file):
228 return None
229 else:
230 (type_base, type_mods) = split_p4_type(p4_type(file))
231 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 232
c65b670e
CP
233def setP4ExecBit(file, mode):
234 # Reopens an already open file and changes the execute bit to match
235 # the execute bit setting in the passed in mode.
236
237 p4Type = "+x"
238
239 if not isModeExec(mode):
240 p4Type = getP4OpenedType(file)
241 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
242 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
243 if p4Type[-1] == "+":
244 p4Type = p4Type[0:-1]
245
6de040df 246 p4_reopen(p4Type, file)
c65b670e
CP
247
248def getP4OpenedType(file):
249 # Returns the perforce file type for the given file.
250
6de040df 251 result = p4_read_pipe(["opened", file])
f3e5ae4f 252 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
253 if match:
254 return match.group(1)
255 else:
f3e5ae4f 256 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 257
06804c76
LD
258# Return the set of all p4 labels
259def getP4Labels(depotPaths):
260 labels = set()
261 if isinstance(depotPaths,basestring):
262 depotPaths = [depotPaths]
263
264 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
265 label = l['label']
266 labels.add(label)
267
268 return labels
269
270# Return the set of all git tags
271def getGitTags():
272 gitTags = set()
273 for line in read_pipe_lines(["git", "tag"]):
274 tag = line.strip()
275 gitTags.add(tag)
276 return gitTags
277
b43b0a3c
CP
278def diffTreePattern():
279 # This is a simple generator for the diff tree regex pattern. This could be
280 # a class variable if this and parseDiffTreeEntry were a part of a class.
281 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
282 while True:
283 yield pattern
284
285def parseDiffTreeEntry(entry):
286 """Parses a single diff tree entry into its component elements.
287
288 See git-diff-tree(1) manpage for details about the format of the diff
289 output. This method returns a dictionary with the following elements:
290
291 src_mode - The mode of the source file
292 dst_mode - The mode of the destination file
293 src_sha1 - The sha1 for the source file
294 dst_sha1 - The sha1 fr the destination file
295 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
296 status_score - The score for the status (applicable for 'C' and 'R'
297 statuses). This is None if there is no score.
298 src - The path for the source file.
299 dst - The path for the destination file. This is only present for
300 copy or renames. If it is not present, this is None.
301
302 If the pattern is not matched, None is returned."""
303
304 match = diffTreePattern().next().match(entry)
305 if match:
306 return {
307 'src_mode': match.group(1),
308 'dst_mode': match.group(2),
309 'src_sha1': match.group(3),
310 'dst_sha1': match.group(4),
311 'status': match.group(5),
312 'status_score': match.group(6),
313 'src': match.group(7),
314 'dst': match.group(10)
315 }
316 return None
317
c65b670e
CP
318def isModeExec(mode):
319 # Returns True if the given git mode represents an executable file,
320 # otherwise False.
321 return mode[-3:] == "755"
322
323def isModeExecChanged(src_mode, dst_mode):
324 return isModeExec(src_mode) != isModeExec(dst_mode)
325
b932705b 326def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
327
328 if isinstance(cmd,basestring):
329 cmd = "-G " + cmd
330 expand = True
331 else:
332 cmd = ["-G"] + cmd
333 expand = False
334
335 cmd = p4_build_cmd(cmd)
6a49f8e2 336 if verbose:
6de040df 337 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
338
339 # Use a temporary file to avoid deadlocks without
340 # subprocess.communicate(), which would put another copy
341 # of stdout into memory.
342 stdin_file = None
343 if stdin is not None:
344 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
345 if isinstance(stdin,basestring):
346 stdin_file.write(stdin)
347 else:
348 for i in stdin:
349 stdin_file.write(i + '\n')
9f90c733
SL
350 stdin_file.flush()
351 stdin_file.seek(0)
352
6de040df
LD
353 p4 = subprocess.Popen(cmd,
354 shell=expand,
9f90c733
SL
355 stdin=stdin_file,
356 stdout=subprocess.PIPE)
86949eef
SH
357
358 result = []
359 try:
360 while True:
9f90c733 361 entry = marshal.load(p4.stdout)
c3f6163b
AG
362 if cb is not None:
363 cb(entry)
364 else:
365 result.append(entry)
86949eef
SH
366 except EOFError:
367 pass
9f90c733
SL
368 exitCode = p4.wait()
369 if exitCode != 0:
ac3e0d79
SH
370 entry = {}
371 entry["p4ExitCode"] = exitCode
372 result.append(entry)
86949eef
SH
373
374 return result
375
376def p4Cmd(cmd):
377 list = p4CmdList(cmd)
378 result = {}
379 for entry in list:
380 result.update(entry)
381 return result;
382
cb2c9db5
SH
383def p4Where(depotPath):
384 if not depotPath.endswith("/"):
385 depotPath += "/"
7f705dc3 386 depotPath = depotPath + "..."
6de040df 387 outputList = p4CmdList(["where", depotPath])
7f705dc3
TAL
388 output = None
389 for entry in outputList:
75bc9573
TAL
390 if "depotFile" in entry:
391 if entry["depotFile"] == depotPath:
392 output = entry
393 break
394 elif "data" in entry:
395 data = entry.get("data")
396 space = data.find(" ")
397 if data[:space] == depotPath:
398 output = entry
399 break
7f705dc3
TAL
400 if output == None:
401 return ""
dc524036
SH
402 if output["code"] == "error":
403 return ""
cb2c9db5
SH
404 clientPath = ""
405 if "path" in output:
406 clientPath = output.get("path")
407 elif "data" in output:
408 data = output.get("data")
409 lastSpace = data.rfind(" ")
410 clientPath = data[lastSpace + 1:]
411
412 if clientPath.endswith("..."):
413 clientPath = clientPath[:-3]
414 return clientPath
415
86949eef 416def currentGitBranch():
b25b2065 417 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 418
4f5cf76a 419def isValidGitDir(path):
bb6e09b2
HWN
420 if (os.path.exists(path + "/HEAD")
421 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
422 return True;
423 return False
424
463e8af6 425def parseRevision(ref):
b25b2065 426 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 427
28755dba
PW
428def branchExists(ref):
429 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
430 ignore_error=True)
431 return len(rev) > 0
432
6ae8de88
SH
433def extractLogMessageFromGitCommit(commit):
434 logMessage = ""
b016d397
HWN
435
436 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 437 foundTitle = False
b016d397 438 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
439 if not foundTitle:
440 if len(log) == 1:
1c094184 441 foundTitle = True
6ae8de88
SH
442 continue
443
444 logMessage += log
445 return logMessage
446
bb6e09b2 447def extractSettingsGitLog(log):
6ae8de88
SH
448 values = {}
449 for line in log.split("\n"):
450 line = line.strip()
6326aa58
HWN
451 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
452 if not m:
453 continue
454
455 assignments = m.group(1).split (':')
456 for a in assignments:
457 vals = a.split ('=')
458 key = vals[0].strip()
459 val = ('='.join (vals[1:])).strip()
460 if val.endswith ('\"') and val.startswith('"'):
461 val = val[1:-1]
462
463 values[key] = val
464
845b42cb
SH
465 paths = values.get("depot-paths")
466 if not paths:
467 paths = values.get("depot-path")
a3fdd579
SH
468 if paths:
469 values['depot-paths'] = paths.split(',')
bb6e09b2 470 return values
6ae8de88 471
8136a639 472def gitBranchExists(branch):
bb6e09b2
HWN
473 proc = subprocess.Popen(["git", "rev-parse", branch],
474 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 475 return proc.wait() == 0;
8136a639 476
36bd8446 477_gitConfig = {}
99f790f2 478def gitConfig(key, args = None): # set args to "--bool", for instance
36bd8446 479 if not _gitConfig.has_key(key):
99f790f2
TAL
480 argsFilter = ""
481 if args != None:
482 argsFilter = "%s " % args
483 cmd = "git config %s%s" % (argsFilter, key)
484 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
36bd8446 485 return _gitConfig[key]
01265103 486
7199cf13
VA
487def gitConfigList(key):
488 if not _gitConfig.has_key(key):
489 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
490 return _gitConfig[key]
491
062410bb
SH
492def p4BranchesInGit(branchesAreInRemotes = True):
493 branches = {}
494
495 cmdline = "git rev-parse --symbolic "
496 if branchesAreInRemotes:
497 cmdline += " --remotes"
498 else:
499 cmdline += " --branches"
500
501 for line in read_pipe_lines(cmdline):
502 line = line.strip()
503
504 ## only import to p4/
505 if not line.startswith('p4/') or line == "p4/HEAD":
506 continue
507 branch = line
508
509 # strip off p4
510 branch = re.sub ("^p4/", "", line)
511
512 branches[branch] = parseRevision(line)
513 return branches
514
9ceab363 515def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
516 branches = p4BranchesInGit()
517 # map from depot-path to branch name
518 branchByDepotPath = {}
519 for branch in branches.keys():
520 tip = branches[branch]
521 log = extractLogMessageFromGitCommit(tip)
522 settings = extractSettingsGitLog(log)
523 if settings.has_key("depot-paths"):
524 paths = ",".join(settings["depot-paths"])
525 branchByDepotPath[paths] = "remotes/p4/" + branch
526
27d2d811 527 settings = None
27d2d811
SH
528 parent = 0
529 while parent < 65535:
9ceab363 530 commit = head + "~%s" % parent
27d2d811
SH
531 log = extractLogMessageFromGitCommit(commit)
532 settings = extractSettingsGitLog(log)
86506fe5
SH
533 if settings.has_key("depot-paths"):
534 paths = ",".join(settings["depot-paths"])
535 if branchByDepotPath.has_key(paths):
536 return [branchByDepotPath[paths], settings]
27d2d811 537
86506fe5 538 parent = parent + 1
27d2d811 539
86506fe5 540 return ["", settings]
27d2d811 541
5ca44617
SH
542def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
543 if not silent:
544 print ("Creating/updating branch(es) in %s based on origin branch(es)"
545 % localRefPrefix)
546
547 originPrefix = "origin/p4/"
548
549 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
550 line = line.strip()
551 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
552 continue
553
554 headName = line[len(originPrefix):]
555 remoteHead = localRefPrefix + headName
556 originHead = line
557
558 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
559 if (not original.has_key('depot-paths')
560 or not original.has_key('change')):
561 continue
562
563 update = False
564 if not gitBranchExists(remoteHead):
565 if verbose:
566 print "creating %s" % remoteHead
567 update = True
568 else:
569 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
570 if settings.has_key('change') > 0:
571 if settings['depot-paths'] == original['depot-paths']:
572 originP4Change = int(original['change'])
573 p4Change = int(settings['change'])
574 if originP4Change > p4Change:
575 print ("%s (%s) is newer than %s (%s). "
576 "Updating p4 branch from origin."
577 % (originHead, originP4Change,
578 remoteHead, p4Change))
579 update = True
580 else:
581 print ("Ignoring: %s was imported from %s while "
582 "%s was imported from %s"
583 % (originHead, ','.join(original['depot-paths']),
584 remoteHead, ','.join(settings['depot-paths'])))
585
586 if update:
587 system("git update-ref %s %s" % (remoteHead, originHead))
588
589def originP4BranchesExist():
590 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
591
4f6432d8
SH
592def p4ChangesForPaths(depotPaths, changeRange):
593 assert depotPaths
6de040df
LD
594 cmd = ['changes']
595 for p in depotPaths:
596 cmd += ["%s...%s" % (p, changeRange)]
597 output = p4_read_pipe_lines(cmd)
4f6432d8 598
b4b0ba06 599 changes = {}
4f6432d8 600 for line in output:
c3f6163b
AG
601 changeNum = int(line.split(" ")[1])
602 changes[changeNum] = True
4f6432d8 603
b4b0ba06
PW
604 changelist = changes.keys()
605 changelist.sort()
606 return changelist
4f6432d8 607
d53de8b9
TAL
608def p4PathStartsWith(path, prefix):
609 # This method tries to remedy a potential mixed-case issue:
610 #
611 # If UserA adds //depot/DirA/file1
612 # and UserB adds //depot/dira/file2
613 #
614 # we may or may not have a problem. If you have core.ignorecase=true,
615 # we treat DirA and dira as the same directory
616 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
617 if ignorecase:
618 return path.lower().startswith(prefix.lower())
619 return path.startswith(prefix)
620
543987bd
PW
621def getClientSpec():
622 """Look at the p4 client spec, create a View() object that contains
623 all the mappings, and return it."""
624
625 specList = p4CmdList("client -o")
626 if len(specList) != 1:
627 die('Output from "client -o" is %d lines, expecting 1' %
628 len(specList))
629
630 # dictionary of all client parameters
631 entry = specList[0]
632
633 # just the keys that start with "View"
634 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
635
636 # hold this new View
637 view = View()
638
639 # append the lines, in order, to the view
640 for view_num in range(len(view_keys)):
641 k = "View%d" % view_num
642 if k not in view_keys:
643 die("Expected view key %s missing" % k)
644 view.append(entry[k])
645
646 return view
647
648def getClientRoot():
649 """Grab the client directory."""
650
651 output = p4CmdList("client -o")
652 if len(output) != 1:
653 die('Output from "client -o" is %d lines, expecting 1' % len(output))
654
655 entry = output[0]
656 if "Root" not in entry:
657 die('Client has no "Root"')
658
659 return entry["Root"]
660
b984733c
SH
661class Command:
662 def __init__(self):
663 self.usage = "usage: %prog [options]"
8910ac0e 664 self.needsGit = True
6a10b6aa 665 self.verbose = False
b984733c 666
3ea2cfd4
LD
667class P4UserMap:
668 def __init__(self):
669 self.userMapFromPerforceServer = False
affb474f
LD
670 self.myP4UserId = None
671
672 def p4UserId(self):
673 if self.myP4UserId:
674 return self.myP4UserId
675
676 results = p4CmdList("user -o")
677 for r in results:
678 if r.has_key('User'):
679 self.myP4UserId = r['User']
680 return r['User']
681 die("Could not find your p4 user id")
682
683 def p4UserIsMe(self, p4User):
684 # return True if the given p4 user is actually me
685 me = self.p4UserId()
686 if not p4User or p4User != me:
687 return False
688 else:
689 return True
3ea2cfd4
LD
690
691 def getUserCacheFilename(self):
692 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
693 return home + "/.gitp4-usercache.txt"
694
695 def getUserMapFromPerforceServer(self):
696 if self.userMapFromPerforceServer:
697 return
698 self.users = {}
699 self.emails = {}
700
701 for output in p4CmdList("users"):
702 if not output.has_key("User"):
703 continue
704 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
705 self.emails[output["Email"]] = output["User"]
706
707
708 s = ''
709 for (key, val) in self.users.items():
710 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
711
712 open(self.getUserCacheFilename(), "wb").write(s)
713 self.userMapFromPerforceServer = True
714
715 def loadUserMapFromCache(self):
716 self.users = {}
717 self.userMapFromPerforceServer = False
718 try:
719 cache = open(self.getUserCacheFilename(), "rb")
720 lines = cache.readlines()
721 cache.close()
722 for line in lines:
723 entry = line.strip().split("\t")
724 self.users[entry[0]] = entry[1]
725 except IOError:
726 self.getUserMapFromPerforceServer()
727
b984733c 728class P4Debug(Command):
86949eef 729 def __init__(self):
6ae8de88 730 Command.__init__(self)
6a10b6aa 731 self.options = []
c8c39116 732 self.description = "A tool to debug the output of p4 -G."
8910ac0e 733 self.needsGit = False
86949eef
SH
734
735 def run(self, args):
b1ce9447 736 j = 0
6de040df 737 for output in p4CmdList(args):
b1ce9447
HWN
738 print 'Element: %d' % j
739 j += 1
86949eef 740 print output
b984733c 741 return True
86949eef 742
5834684d
SH
743class P4RollBack(Command):
744 def __init__(self):
745 Command.__init__(self)
746 self.options = [
0c66a783 747 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
748 ]
749 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 750 self.rollbackLocalBranches = False
5834684d
SH
751
752 def run(self, args):
753 if len(args) != 1:
754 return False
755 maxChange = int(args[0])
0c66a783 756
ad192f28 757 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
758 die("Problems executing p4");
759
0c66a783
SH
760 if self.rollbackLocalBranches:
761 refPrefix = "refs/heads/"
b016d397 762 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
763 else:
764 refPrefix = "refs/remotes/"
b016d397 765 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
766
767 for line in lines:
768 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
769 line = line.strip()
770 ref = refPrefix + line
5834684d 771 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
772 settings = extractSettingsGitLog(log)
773
774 depotPaths = settings['depot-paths']
775 change = settings['change']
776
5834684d 777 changed = False
52102d47 778
6326aa58
HWN
779 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
780 for p in depotPaths]))) == 0:
52102d47
SH
781 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
782 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
783 continue
784
bb6e09b2 785 while change and int(change) > maxChange:
5834684d 786 changed = True
52102d47
SH
787 if self.verbose:
788 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
789 system("git update-ref %s \"%s^\"" % (ref, ref))
790 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
791 settings = extractSettingsGitLog(log)
792
793
794 depotPaths = settings['depot-paths']
795 change = settings['change']
5834684d
SH
796
797 if changed:
52102d47 798 print "%s rewound to %s" % (ref, change)
5834684d
SH
799
800 return True
801
3ea2cfd4 802class P4Submit(Command, P4UserMap):
4f5cf76a 803 def __init__(self):
b984733c 804 Command.__init__(self)
3ea2cfd4 805 P4UserMap.__init__(self)
4f5cf76a 806 self.options = [
4f5cf76a 807 optparse.make_option("--origin", dest="origin"),
ae901090 808 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
809 # preserve the user, requires relevant p4 permissions
810 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 811 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
4f5cf76a
SH
812 ]
813 self.description = "Submit changes from git to the perforce depot."
c9b50e63 814 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 815 self.interactive = True
9512497b 816 self.origin = ""
ae901090 817 self.detectRenames = False
3ea2cfd4 818 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
f7baba8b 819 self.isWindows = (platform.system() == "Windows")
06804c76 820 self.exportLabels = False
4f5cf76a 821
4f5cf76a
SH
822 def check(self):
823 if len(p4CmdList("opened ...")) > 0:
824 die("You have files opened with perforce! Close them before starting the sync.")
825
edae1e2f
SH
826 # replaces everything between 'Description:' and the next P4 submit template field with the
827 # commit message
4f5cf76a
SH
828 def prepareLogMessage(self, template, message):
829 result = ""
830
edae1e2f
SH
831 inDescriptionSection = False
832
4f5cf76a
SH
833 for line in template.split("\n"):
834 if line.startswith("#"):
835 result += line + "\n"
836 continue
837
edae1e2f 838 if inDescriptionSection:
c9dbab04 839 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f
SH
840 inDescriptionSection = False
841 else:
842 continue
843 else:
844 if line.startswith("Description:"):
845 inDescriptionSection = True
846 line += "\n"
847 for messageLine in message.split("\n"):
848 line += "\t" + messageLine + "\n"
849
850 result += line + "\n"
4f5cf76a
SH
851
852 return result
853
60df071c
LD
854 def patchRCSKeywords(self, file, pattern):
855 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
856 (handle, outFileName) = tempfile.mkstemp(dir='.')
857 try:
858 outFile = os.fdopen(handle, "w+")
859 inFile = open(file, "r")
860 regexp = re.compile(pattern, re.VERBOSE)
861 for line in inFile.readlines():
862 line = regexp.sub(r'$\1$', line)
863 outFile.write(line)
864 inFile.close()
865 outFile.close()
866 # Forcibly overwrite the original file
867 os.unlink(file)
868 shutil.move(outFileName, file)
869 except:
870 # cleanup our temporary file
871 os.unlink(outFileName)
872 print "Failed to strip RCS keywords in %s" % file
873 raise
874
875 print "Patched up RCS keywords in %s" % file
876
3ea2cfd4
LD
877 def p4UserForCommit(self,id):
878 # Return the tuple (perforce user,git email) for a given git commit id
879 self.getUserMapFromPerforceServer()
880 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
881 gitEmail = gitEmail.strip()
882 if not self.emails.has_key(gitEmail):
883 return (None,gitEmail)
884 else:
885 return (self.emails[gitEmail],gitEmail)
886
887 def checkValidP4Users(self,commits):
888 # check if any git authors cannot be mapped to p4 users
889 for id in commits:
890 (user,email) = self.p4UserForCommit(id)
891 if not user:
892 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
893 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
894 print "%s" % msg
895 else:
896 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
897
898 def lastP4Changelist(self):
899 # Get back the last changelist number submitted in this client spec. This
900 # then gets used to patch up the username in the change. If the same
901 # client spec is being used by multiple processes then this might go
902 # wrong.
903 results = p4CmdList("client -o") # find the current client
904 client = None
905 for r in results:
906 if r.has_key('Client'):
907 client = r['Client']
908 break
909 if not client:
910 die("could not get client spec")
6de040df 911 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
912 for r in results:
913 if r.has_key('change'):
914 return r['change']
915 die("Could not get changelist number for last submit - cannot patch up user details")
916
917 def modifyChangelistUser(self, changelist, newUser):
918 # fixup the user field of a changelist after it has been submitted.
919 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
920 if len(changes) != 1:
921 die("Bad output from p4 change modifying %s to user %s" %
922 (changelist, newUser))
923
924 c = changes[0]
925 if c['User'] == newUser: return # nothing to do
926 c['User'] = newUser
927 input = marshal.dumps(c)
928
3ea2cfd4
LD
929 result = p4CmdList("change -f -i", stdin=input)
930 for r in result:
931 if r.has_key('code'):
932 if r['code'] == 'error':
933 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
934 if r.has_key('data'):
935 print("Updated user field for changelist %s to %s" % (changelist, newUser))
936 return
937 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
938
939 def canChangeChangelists(self):
940 # check to see if we have p4 admin or super-user permissions, either of
941 # which are required to modify changelists.
52a4880b 942 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
943 for r in results:
944 if r.has_key('perm'):
945 if r['perm'] == 'admin':
946 return 1
947 if r['perm'] == 'super':
948 return 1
949 return 0
950
ea99c3ae
SH
951 def prepareSubmitTemplate(self):
952 # remove lines in the Files section that show changes to files outside the depot path we're committing into
953 template = ""
954 inFilesSection = False
6de040df 955 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
956 if line.endswith("\r\n"):
957 line = line[:-2] + "\n"
ea99c3ae
SH
958 if inFilesSection:
959 if line.startswith("\t"):
960 # path starts and ends with a tab
961 path = line[1:]
962 lastTab = path.rfind("\t")
963 if lastTab != -1:
964 path = path[:lastTab]
d53de8b9 965 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
966 continue
967 else:
968 inFilesSection = False
969 else:
970 if line.startswith("Files:"):
971 inFilesSection = True
972
973 template += line
974
975 return template
976
7c766e57
PW
977 def edit_template(self, template_file):
978 """Invoke the editor to let the user change the submission
979 message. Return true if okay to continue with the submit."""
980
981 # if configured to skip the editing part, just submit
982 if gitConfig("git-p4.skipSubmitEdit") == "true":
983 return True
984
985 # look at the modification time, to check later if the user saved
986 # the file
987 mtime = os.stat(template_file).st_mtime
988
989 # invoke the editor
f95ceaf0 990 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
991 editor = os.environ.get("P4EDITOR")
992 else:
993 editor = read_pipe("git var GIT_EDITOR").strip()
994 system(editor + " " + template_file)
995
996 # If the file was not saved, prompt to see if this patch should
997 # be skipped. But skip this verification step if configured so.
998 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
999 return True
1000
d1652049
PW
1001 # modification time updated means user saved the file
1002 if os.stat(template_file).st_mtime > mtime:
1003 return True
1004
1005 while True:
1006 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1007 if response == 'y':
1008 return True
1009 if response == 'n':
1010 return False
7c766e57 1011
7cb5cbef 1012 def applyCommit(self, id):
0e36f2d7 1013 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
ae901090 1014
848de9c3 1015 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1016
ae901090
VA
1017 if not self.detectRenames:
1018 # If not explicitly set check the config variable
0a9feffc 1019 self.detectRenames = gitConfig("git-p4.detectRenames")
ae901090 1020
0a9feffc
VA
1021 if self.detectRenames.lower() == "false" or self.detectRenames == "":
1022 diffOpts = ""
1023 elif self.detectRenames.lower() == "true":
ae901090
VA
1024 diffOpts = "-M"
1025 else:
0a9feffc 1026 diffOpts = "-M%s" % self.detectRenames
ae901090 1027
0a9feffc
VA
1028 detectCopies = gitConfig("git-p4.detectCopies")
1029 if detectCopies.lower() == "true":
4fddb41b 1030 diffOpts += " -C"
0a9feffc
VA
1031 elif detectCopies != "" and detectCopies.lower() != "false":
1032 diffOpts += " -C%s" % detectCopies
4fddb41b 1033
68cbcf1b 1034 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
4fddb41b
VA
1035 diffOpts += " --find-copies-harder"
1036
0e36f2d7 1037 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
1038 filesToAdd = set()
1039 filesToDelete = set()
d336c158 1040 editedFiles = set()
b6ad6dcc 1041 pureRenameCopy = set()
c65b670e 1042 filesToChangeExecBit = {}
60df071c 1043
4f5cf76a 1044 for line in diff:
b43b0a3c
CP
1045 diff = parseDiffTreeEntry(line)
1046 modifier = diff['status']
1047 path = diff['src']
4f5cf76a 1048 if modifier == "M":
6de040df 1049 p4_edit(path)
c65b670e
CP
1050 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1051 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1052 editedFiles.add(path)
4f5cf76a
SH
1053 elif modifier == "A":
1054 filesToAdd.add(path)
c65b670e 1055 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1056 if path in filesToDelete:
1057 filesToDelete.remove(path)
1058 elif modifier == "D":
1059 filesToDelete.add(path)
1060 if path in filesToAdd:
1061 filesToAdd.remove(path)
4fddb41b
VA
1062 elif modifier == "C":
1063 src, dest = diff['src'], diff['dst']
6de040df 1064 p4_integrate(src, dest)
b6ad6dcc 1065 pureRenameCopy.add(dest)
4fddb41b 1066 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1067 p4_edit(dest)
b6ad6dcc 1068 pureRenameCopy.discard(dest)
4fddb41b 1069 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1070 p4_edit(dest)
b6ad6dcc 1071 pureRenameCopy.discard(dest)
4fddb41b
VA
1072 filesToChangeExecBit[dest] = diff['dst_mode']
1073 os.unlink(dest)
1074 editedFiles.add(dest)
d9a5f25b 1075 elif modifier == "R":
b43b0a3c 1076 src, dest = diff['src'], diff['dst']
6de040df 1077 p4_integrate(src, dest)
ae901090 1078 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1079 p4_edit(dest)
b6ad6dcc
PW
1080 else:
1081 pureRenameCopy.add(dest)
c65b670e 1082 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1083 p4_edit(dest)
c65b670e 1084 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
1085 os.unlink(dest)
1086 editedFiles.add(dest)
1087 filesToDelete.add(src)
4f5cf76a
SH
1088 else:
1089 die("unknown modifier %s for %s" % (modifier, path))
1090
0e36f2d7 1091 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 1092 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1093 tryPatchCmd = patchcmd + "--check -"
1094 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1095 patch_succeeded = True
51a2640a 1096
47a130b7 1097 if os.system(tryPatchCmd) != 0:
60df071c
LD
1098 fixed_rcs_keywords = False
1099 patch_succeeded = False
51a2640a 1100 print "Unfortunately applying the change failed!"
60df071c
LD
1101
1102 # Patch failed, maybe it's just RCS keyword woes. Look through
1103 # the patch to see if that's possible.
1104 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1105 file = None
1106 pattern = None
1107 kwfiles = {}
1108 for file in editedFiles | filesToDelete:
1109 # did this file's delta contain RCS keywords?
1110 pattern = p4_keywords_regexp_for_file(file)
1111
1112 if pattern:
1113 # this file is a possibility...look for RCS keywords.
1114 regexp = re.compile(pattern, re.VERBOSE)
1115 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1116 if regexp.search(line):
1117 if verbose:
1118 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1119 kwfiles[file] = pattern
1120 break
1121
1122 for file in kwfiles:
1123 if verbose:
1124 print "zapping %s with %s" % (line,pattern)
1125 self.patchRCSKeywords(file, kwfiles[file])
1126 fixed_rcs_keywords = True
1127
1128 if fixed_rcs_keywords:
1129 print "Retrying the patch with RCS keywords cleaned up"
1130 if os.system(tryPatchCmd) == 0:
1131 patch_succeeded = True
1132
1133 if not patch_succeeded:
51a2640a
SH
1134 print "What do you want to do?"
1135 response = "x"
1136 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
1137 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1138 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
1139 if response == "s":
1140 print "Skipping! Good luck with the next patches..."
20947149 1141 for f in editedFiles:
6de040df 1142 p4_revert(f)
20947149 1143 for f in filesToAdd:
6de040df 1144 os.remove(f)
51a2640a
SH
1145 return
1146 elif response == "a":
47a130b7 1147 os.system(applyPatchCmd)
51a2640a
SH
1148 if len(filesToAdd) > 0:
1149 print "You may also want to call p4 add on the following files:"
1150 print " ".join(filesToAdd)
1151 if len(filesToDelete):
1152 print "The following files should be scheduled for deletion with p4 delete:"
1153 print " ".join(filesToDelete)
cebdf5af 1154 die("Please resolve and submit the conflict manually and "
9dcb9f24 1155 + "continue afterwards with git p4 submit --continue")
51a2640a
SH
1156 elif response == "w":
1157 system(diffcmd + " > patch.txt")
1158 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af 1159 die("Please resolve and submit the conflict manually and "
9dcb9f24 1160 "continue afterwards with git p4 submit --continue")
51a2640a 1161
47a130b7 1162 system(applyPatchCmd)
4f5cf76a
SH
1163
1164 for f in filesToAdd:
6de040df 1165 p4_add(f)
4f5cf76a 1166 for f in filesToDelete:
6de040df
LD
1167 p4_revert(f)
1168 p4_delete(f)
4f5cf76a 1169
c65b670e
CP
1170 # Set/clear executable bits
1171 for f in filesToChangeExecBit.keys():
1172 mode = filesToChangeExecBit[f]
1173 setP4ExecBit(f, mode)
1174
0e36f2d7 1175 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1176 logMessage = logMessage.strip()
4f5cf76a 1177
ea99c3ae 1178 template = self.prepareSubmitTemplate()
4f5cf76a
SH
1179
1180 if self.interactive:
1181 submitTemplate = self.prepareLogMessage(template, logMessage)
ecdba36d
LD
1182
1183 if self.preserveUser:
1184 submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1185
67abd417
SB
1186 if os.environ.has_key("P4DIFF"):
1187 del(os.environ["P4DIFF"])
8b130262
AW
1188 diff = ""
1189 for editedFile in editedFiles:
6de040df 1190 diff += p4_read_pipe(['diff', '-du', editedFile])
4f5cf76a 1191
f3e5ae4f 1192 newdiff = ""
4f5cf76a 1193 for newFile in filesToAdd:
f3e5ae4f
MSO
1194 newdiff += "==== new file ====\n"
1195 newdiff += "--- /dev/null\n"
1196 newdiff += "+++ %s\n" % newFile
4f5cf76a
SH
1197 f = open(newFile, "r")
1198 for line in f.readlines():
f3e5ae4f 1199 newdiff += "+" + line
4f5cf76a
SH
1200 f.close()
1201
848de9c3
LD
1202 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1203 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
9dcb9f24
PW
1204 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1205 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
848de9c3 1206
f3e5ae4f 1207 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a 1208
7c766e57 1209 (handle, fileName) = tempfile.mkstemp()
e96e400f 1210 tmpFile = os.fdopen(handle, "w+")
f3e5ae4f
MSO
1211 if self.isWindows:
1212 submitTemplate = submitTemplate.replace("\n", "\r\n")
1213 separatorLine = separatorLine.replace("\n", "\r\n")
1214 newdiff = newdiff.replace("\n", "\r\n")
1215 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
e96e400f 1216 tmpFile.close()
e96e400f 1217
7c766e57
PW
1218 if self.edit_template(fileName):
1219 # read the edited message and submit
cdc7e388
SH
1220 tmpFile = open(fileName, "rb")
1221 message = tmpFile.read()
1222 tmpFile.close()
1223 submitTemplate = message[:message.index(separatorLine)]
1224 if self.isWindows:
1225 submitTemplate = submitTemplate.replace("\r\n", "\n")
6de040df 1226 p4_write_pipe(['submit', '-i'], submitTemplate)
3ea2cfd4
LD
1227
1228 if self.preserveUser:
1229 if p4User:
1230 # Get last changelist number. Cannot easily get it from
7c766e57
PW
1231 # the submit command output as the output is
1232 # unmarshalled.
3ea2cfd4
LD
1233 changelist = self.lastP4Changelist()
1234 self.modifyChangelistUser(changelist, p4User)
b6ad6dcc
PW
1235
1236 # The rename/copy happened by applying a patch that created a
1237 # new file. This leaves it writable, which confuses p4.
1238 for f in pureRenameCopy:
1239 p4_sync(f, "-f")
1240
cdc7e388 1241 else:
7c766e57 1242 # skip this patch
d1652049 1243 print "Submission cancelled, undoing p4 changes."
cdc7e388 1244 for f in editedFiles:
6de040df 1245 p4_revert(f)
cdc7e388 1246 for f in filesToAdd:
6de040df
LD
1247 p4_revert(f)
1248 os.remove(f)
cdc7e388
SH
1249
1250 os.remove(fileName)
4f5cf76a
SH
1251 else:
1252 fileName = "submit.txt"
1253 file = open(fileName, "w+")
1254 file.write(self.prepareLogMessage(template, logMessage))
1255 file.close()
cebdf5af
HWN
1256 print ("Perforce submit template written as %s. "
1257 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1258 % (fileName, fileName))
4f5cf76a 1259
06804c76
LD
1260 # Export git tags as p4 labels. Create a p4 label and then tag
1261 # with that.
1262 def exportGitTags(self, gitTags):
c8942a22
LD
1263 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1264 if len(validLabelRegexp) == 0:
1265 validLabelRegexp = defaultLabelRegexp
1266 m = re.compile(validLabelRegexp)
06804c76
LD
1267
1268 for name in gitTags:
1269
1270 if not m.match(name):
1271 if verbose:
1272 print "tag %s does not match regexp %s" % (name, validTagRegexp)
1273 continue
1274
1275 # Get the p4 commit this corresponds to
c8942a22
LD
1276 logMessage = extractLogMessageFromGitCommit(name)
1277 values = extractSettingsGitLog(logMessage)
06804c76 1278
c8942a22 1279 if not values.has_key('change'):
06804c76
LD
1280 # a tag pointing to something not sent to p4; ignore
1281 if verbose:
1282 print "git tag %s does not give a p4 commit" % name
1283 continue
c8942a22
LD
1284 else:
1285 changelist = values['change']
06804c76
LD
1286
1287 # Get the tag details.
1288 inHeader = True
1289 isAnnotated = False
1290 body = []
1291 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1292 l = l.strip()
1293 if inHeader:
1294 if re.match(r'tag\s+', l):
1295 isAnnotated = True
1296 elif re.match(r'\s*$', l):
1297 inHeader = False
1298 continue
1299 else:
1300 body.append(l)
1301
1302 if not isAnnotated:
1303 body = ["lightweight tag imported by git p4\n"]
1304
1305 # Create the label - use the same view as the client spec we are using
1306 clientSpec = getClientSpec()
1307
1308 labelTemplate = "Label: %s\n" % name
1309 labelTemplate += "Description:\n"
1310 for b in body:
1311 labelTemplate += "\t" + b + "\n"
1312 labelTemplate += "View:\n"
1313 for mapping in clientSpec.mappings:
1314 labelTemplate += "\t%s\n" % mapping.depot_side.path
1315
1316 p4_write_pipe(["label", "-i"], labelTemplate)
1317
1318 # Use the label
1319 p4_system(["tag", "-l", name] +
1320 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1321
1322 if verbose:
1323 print "created p4 label for tag %s" % name
1324
4f5cf76a 1325 def run(self, args):
c9b50e63
SH
1326 if len(args) == 0:
1327 self.master = currentGitBranch()
4280e533 1328 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1329 die("Detecting current git branch failed!")
1330 elif len(args) == 1:
1331 self.master = args[0]
28755dba
PW
1332 if not branchExists(self.master):
1333 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1334 else:
1335 return False
1336
4c2d5d72
JX
1337 allowSubmit = gitConfig("git-p4.allowSubmit")
1338 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1339 die("%s is not in git-p4.allowSubmit" % self.master)
1340
27d2d811 1341 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1342 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1343 if len(self.origin) == 0:
1344 self.origin = upstream
a3fdd579 1345
3ea2cfd4
LD
1346 if self.preserveUser:
1347 if not self.canChangeChangelists():
1348 die("Cannot preserve user names without p4 super-user or admin permissions")
1349
a3fdd579
SH
1350 if self.verbose:
1351 print "Origin branch is " + self.origin
9512497b 1352
ea99c3ae 1353 if len(self.depotPath) == 0:
9512497b
SH
1354 print "Internal error: cannot locate perforce depot path from existing branches"
1355 sys.exit(128)
1356
543987bd
PW
1357 self.useClientSpec = False
1358 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1359 self.useClientSpec = True
1360 if self.useClientSpec:
1361 self.clientSpecDirs = getClientSpec()
9512497b 1362
543987bd
PW
1363 if self.useClientSpec:
1364 # all files are relative to the client spec
1365 self.clientPath = getClientRoot()
1366 else:
1367 self.clientPath = p4Where(self.depotPath)
9512497b 1368
543987bd
PW
1369 if self.clientPath == "":
1370 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1371
ea99c3ae 1372 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1373 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1374
0591cfa8 1375 # ensure the clientPath exists
8d7ec362 1376 new_client_dir = False
0591cfa8 1377 if not os.path.exists(self.clientPath):
8d7ec362 1378 new_client_dir = True
0591cfa8
GG
1379 os.makedirs(self.clientPath)
1380
053fd0c1 1381 chdir(self.clientPath)
6a01298a 1382 print "Synchronizing p4 checkout..."
8d7ec362
PW
1383 if new_client_dir:
1384 # old one was destroyed, and maybe nobody told p4
1385 p4_sync("...", "-f")
1386 else:
1387 p4_sync("...")
4f5cf76a 1388 self.check()
4f5cf76a 1389
4c750c0d
SH
1390 commits = []
1391 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1392 commits.append(line.strip())
1393 commits.reverse()
4f5cf76a 1394
848de9c3
LD
1395 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1396 self.checkAuthorship = False
1397 else:
1398 self.checkAuthorship = True
1399
3ea2cfd4
LD
1400 if self.preserveUser:
1401 self.checkValidP4Users(commits)
1402
4f5cf76a 1403 while len(commits) > 0:
4f5cf76a
SH
1404 commit = commits[0]
1405 commits = commits[1:]
7cb5cbef 1406 self.applyCommit(commit)
4f5cf76a
SH
1407 if not self.interactive:
1408 break
1409
4f5cf76a 1410 if len(commits) == 0:
4c750c0d 1411 print "All changes applied!"
053fd0c1 1412 chdir(self.oldWorkingDirectory)
14594f4b 1413
4c750c0d
SH
1414 sync = P4Sync()
1415 sync.run([])
14594f4b 1416
4c750c0d
SH
1417 rebase = P4Rebase()
1418 rebase.rebase()
4f5cf76a 1419
06804c76
LD
1420 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1421 self.exportLabels = true
1422
1423 if self.exportLabels:
1424 p4Labels = getP4Labels(self.depotPath)
1425 gitTags = getGitTags()
1426
1427 missingGitTags = gitTags - p4Labels
1428 self.exportGitTags(missingGitTags)
1429
b984733c
SH
1430 return True
1431
ecb7cf98
PW
1432class View(object):
1433 """Represent a p4 view ("p4 help views"), and map files in a
1434 repo according to the view."""
1435
1436 class Path(object):
1437 """A depot or client path, possibly containing wildcards.
1438 The only one supported is ... at the end, currently.
1439 Initialize with the full path, with //depot or //client."""
1440
1441 def __init__(self, path, is_depot):
1442 self.path = path
1443 self.is_depot = is_depot
1444 self.find_wildcards()
1445 # remember the prefix bit, useful for relative mappings
1446 m = re.match("(//[^/]+/)", self.path)
1447 if not m:
1448 die("Path %s does not start with //prefix/" % self.path)
1449 prefix = m.group(1)
1450 if not self.is_depot:
1451 # strip //client/ on client paths
1452 self.path = self.path[len(prefix):]
1453
1454 def find_wildcards(self):
1455 """Make sure wildcards are valid, and set up internal
1456 variables."""
1457
1458 self.ends_triple_dot = False
1459 # There are three wildcards allowed in p4 views
1460 # (see "p4 help views"). This code knows how to
1461 # handle "..." (only at the end), but cannot deal with
1462 # "%%n" or "*". Only check the depot_side, as p4 should
1463 # validate that the client_side matches too.
1464 if re.search(r'%%[1-9]', self.path):
1465 die("Can't handle %%n wildcards in view: %s" % self.path)
1466 if self.path.find("*") >= 0:
1467 die("Can't handle * wildcards in view: %s" % self.path)
1468 triple_dot_index = self.path.find("...")
1469 if triple_dot_index >= 0:
43b82bd9
PW
1470 if triple_dot_index != len(self.path) - 3:
1471 die("Can handle only single ... wildcard, at end: %s" %
ecb7cf98
PW
1472 self.path)
1473 self.ends_triple_dot = True
1474
1475 def ensure_compatible(self, other_path):
1476 """Make sure the wildcards agree."""
1477 if self.ends_triple_dot != other_path.ends_triple_dot:
1478 die("Both paths must end with ... if either does;\n" +
1479 "paths: %s %s" % (self.path, other_path.path))
1480
1481 def match_wildcards(self, test_path):
1482 """See if this test_path matches us, and fill in the value
1483 of the wildcards if so. Returns a tuple of
1484 (True|False, wildcards[]). For now, only the ... at end
1485 is supported, so at most one wildcard."""
1486 if self.ends_triple_dot:
1487 dotless = self.path[:-3]
1488 if test_path.startswith(dotless):
1489 wildcard = test_path[len(dotless):]
1490 return (True, [ wildcard ])
1491 else:
1492 if test_path == self.path:
1493 return (True, [])
1494 return (False, [])
1495
1496 def match(self, test_path):
1497 """Just return if it matches; don't bother with the wildcards."""
1498 b, _ = self.match_wildcards(test_path)
1499 return b
1500
1501 def fill_in_wildcards(self, wildcards):
1502 """Return the relative path, with the wildcards filled in
1503 if there are any."""
1504 if self.ends_triple_dot:
1505 return self.path[:-3] + wildcards[0]
1506 else:
1507 return self.path
1508
1509 class Mapping(object):
1510 def __init__(self, depot_side, client_side, overlay, exclude):
1511 # depot_side is without the trailing /... if it had one
1512 self.depot_side = View.Path(depot_side, is_depot=True)
1513 self.client_side = View.Path(client_side, is_depot=False)
1514 self.overlay = overlay # started with "+"
1515 self.exclude = exclude # started with "-"
1516 assert not (self.overlay and self.exclude)
1517 self.depot_side.ensure_compatible(self.client_side)
1518
1519 def __str__(self):
1520 c = " "
1521 if self.overlay:
1522 c = "+"
1523 if self.exclude:
1524 c = "-"
1525 return "View.Mapping: %s%s -> %s" % \
329afb8e 1526 (c, self.depot_side.path, self.client_side.path)
ecb7cf98
PW
1527
1528 def map_depot_to_client(self, depot_path):
1529 """Calculate the client path if using this mapping on the
1530 given depot path; does not consider the effect of other
1531 mappings in a view. Even excluded mappings are returned."""
1532 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1533 if not matches:
1534 return ""
1535 client_path = self.client_side.fill_in_wildcards(wildcards)
1536 return client_path
1537
1538 #
1539 # View methods
1540 #
1541 def __init__(self):
1542 self.mappings = []
1543
1544 def append(self, view_line):
1545 """Parse a view line, splitting it into depot and client
1546 sides. Append to self.mappings, preserving order."""
1547
1548 # Split the view line into exactly two words. P4 enforces
1549 # structure on these lines that simplifies this quite a bit.
1550 #
1551 # Either or both words may be double-quoted.
1552 # Single quotes do not matter.
1553 # Double-quote marks cannot occur inside the words.
1554 # A + or - prefix is also inside the quotes.
1555 # There are no quotes unless they contain a space.
1556 # The line is already white-space stripped.
1557 # The two words are separated by a single space.
1558 #
1559 if view_line[0] == '"':
1560 # First word is double quoted. Find its end.
1561 close_quote_index = view_line.find('"', 1)
1562 if close_quote_index <= 0:
1563 die("No first-word closing quote found: %s" % view_line)
1564 depot_side = view_line[1:close_quote_index]
1565 # skip closing quote and space
1566 rhs_index = close_quote_index + 1 + 1
1567 else:
1568 space_index = view_line.find(" ")
1569 if space_index <= 0:
1570 die("No word-splitting space found: %s" % view_line)
1571 depot_side = view_line[0:space_index]
1572 rhs_index = space_index + 1
1573
1574 if view_line[rhs_index] == '"':
1575 # Second word is double quoted. Make sure there is a
1576 # double quote at the end too.
1577 if not view_line.endswith('"'):
1578 die("View line with rhs quote should end with one: %s" %
1579 view_line)
1580 # skip the quotes
1581 client_side = view_line[rhs_index+1:-1]
1582 else:
1583 client_side = view_line[rhs_index:]
1584
1585 # prefix + means overlay on previous mapping
1586 overlay = False
1587 if depot_side.startswith("+"):
1588 overlay = True
1589 depot_side = depot_side[1:]
1590
1591 # prefix - means exclude this path
1592 exclude = False
1593 if depot_side.startswith("-"):
1594 exclude = True
1595 depot_side = depot_side[1:]
1596
1597 m = View.Mapping(depot_side, client_side, overlay, exclude)
1598 self.mappings.append(m)
1599
1600 def map_in_client(self, depot_path):
1601 """Return the relative location in the client where this
1602 depot file should live. Returns "" if the file should
1603 not be mapped in the client."""
1604
1605 paths_filled = []
1606 client_path = ""
1607
1608 # look at later entries first
1609 for m in self.mappings[::-1]:
1610
1611 # see where will this path end up in the client
1612 p = m.map_depot_to_client(depot_path)
1613
1614 if p == "":
1615 # Depot path does not belong in client. Must remember
1616 # this, as previous items should not cause files to
1617 # exist in this path either. Remember that the list is
1618 # being walked from the end, which has higher precedence.
1619 # Overlap mappings do not exclude previous mappings.
1620 if not m.overlay:
1621 paths_filled.append(m.client_side)
1622
1623 else:
1624 # This mapping matched; no need to search any further.
1625 # But, the mapping could be rejected if the client path
6ee9a999
PW
1626 # has already been claimed by an earlier mapping (i.e.
1627 # one later in the list, which we are walking backwards).
ecb7cf98
PW
1628 already_mapped_in_client = False
1629 for f in paths_filled:
1630 # this is View.Path.match
1631 if f.match(p):
1632 already_mapped_in_client = True
1633 break
1634 if not already_mapped_in_client:
1635 # Include this file, unless it is from a line that
1636 # explicitly said to exclude it.
1637 if not m.exclude:
1638 client_path = p
1639
1640 # a match, even if rejected, always stops the search
1641 break
1642
1643 return client_path
1644
3ea2cfd4 1645class P4Sync(Command, P4UserMap):
56c09345
PW
1646 delete_actions = ( "delete", "move/delete", "purge" )
1647
b984733c
SH
1648 def __init__(self):
1649 Command.__init__(self)
3ea2cfd4 1650 P4UserMap.__init__(self)
b984733c
SH
1651 self.options = [
1652 optparse.make_option("--branch", dest="branch"),
1653 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1654 optparse.make_option("--changesfile", dest="changesFile"),
1655 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1656 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 1657 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
1658 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1659 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 1660 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 1661 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1662 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1663 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1664 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
1665 ]
1666 self.description = """Imports from Perforce into a git repository.\n
1667 example:
1668 //depot/my/project/ -- to import the current head
1669 //depot/my/project/@all -- to import everything
1670 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1671
1672 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1673
1674 self.usage += " //depot/path[@revRange]"
b984733c 1675 self.silent = False
1d7367dc
RG
1676 self.createdBranches = set()
1677 self.committedChanges = set()
569d1bd4 1678 self.branch = ""
b984733c 1679 self.detectBranches = False
cb53e1f8 1680 self.detectLabels = False
06804c76 1681 self.importLabels = False
b984733c 1682 self.changesFile = ""
01265103 1683 self.syncWithOrigin = True
a028a98e 1684 self.importIntoRemotes = True
01a9c9c5 1685 self.maxChanges = ""
c1f9197f 1686 self.isWindows = (platform.system() == "Windows")
8b41a97f 1687 self.keepRepoPath = False
6326aa58 1688 self.depotPaths = None
3c699645 1689 self.p4BranchesInGit = []
354081d5 1690 self.cloneExclude = []
3a70cdfa 1691 self.useClientSpec = False
a93d33ee 1692 self.useClientSpec_from_options = False
ecb7cf98 1693 self.clientSpecDirs = None
fed23693
VA
1694 self.tempBranches = []
1695 self.tempBranchLocation = "git-p4-tmp"
b984733c 1696
01265103
SH
1697 if gitConfig("git-p4.syncFromOrigin") == "false":
1698 self.syncWithOrigin = False
1699
084f6306
PW
1700 #
1701 # P4 wildcards are not allowed in filenames. P4 complains
1702 # if you simply add them, but you can force it with "-f", in
1703 # which case it translates them into %xx encoding internally.
1704 # Search for and fix just these four characters. Do % last so
1705 # that fixing it does not inadvertently create new %-escapes.
1706 #
1707 def wildcard_decode(self, path):
1708 # Cannot have * in a filename in windows; untested as to
1709 # what p4 would do in such a case.
1710 if not self.isWindows:
1711 path = path.replace("%2A", "*")
1712 path = path.replace("%23", "#") \
1713 .replace("%40", "@") \
1714 .replace("%25", "%")
1715 return path
1716
fed23693
VA
1717 # Force a checkpoint in fast-import and wait for it to finish
1718 def checkpoint(self):
1719 self.gitStream.write("checkpoint\n\n")
1720 self.gitStream.write("progress checkpoint\n\n")
1721 out = self.gitOutput.readline()
1722 if self.verbose:
1723 print "checkpoint finished: " + out
1724
b984733c 1725 def extractFilesFromCommit(self, commit):
354081d5
TT
1726 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1727 for path in self.cloneExclude]
b984733c
SH
1728 files = []
1729 fnum = 0
1730 while commit.has_key("depotFile%s" % fnum):
1731 path = commit["depotFile%s" % fnum]
6326aa58 1732
354081d5 1733 if [p for p in self.cloneExclude
d53de8b9 1734 if p4PathStartsWith(path, p)]:
354081d5
TT
1735 found = False
1736 else:
1737 found = [p for p in self.depotPaths
d53de8b9 1738 if p4PathStartsWith(path, p)]
6326aa58 1739 if not found:
b984733c
SH
1740 fnum = fnum + 1
1741 continue
1742
1743 file = {}
1744 file["path"] = path
1745 file["rev"] = commit["rev%s" % fnum]
1746 file["action"] = commit["action%s" % fnum]
1747 file["type"] = commit["type%s" % fnum]
1748 files.append(file)
1749 fnum = fnum + 1
1750 return files
1751
6326aa58 1752 def stripRepoPath(self, path, prefixes):
3952710b 1753 if self.useClientSpec:
ecb7cf98 1754 return self.clientSpecDirs.map_in_client(path)
3952710b 1755
8b41a97f 1756 if self.keepRepoPath:
6326aa58
HWN
1757 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1758
1759 for p in prefixes:
d53de8b9 1760 if p4PathStartsWith(path, p):
6326aa58 1761 path = path[len(p):]
8b41a97f 1762
6326aa58 1763 return path
6754a299 1764
71b112d4 1765 def splitFilesIntoBranches(self, commit):
d5904674 1766 branches = {}
71b112d4
SH
1767 fnum = 0
1768 while commit.has_key("depotFile%s" % fnum):
1769 path = commit["depotFile%s" % fnum]
6326aa58 1770 found = [p for p in self.depotPaths
d53de8b9 1771 if p4PathStartsWith(path, p)]
6326aa58 1772 if not found:
71b112d4
SH
1773 fnum = fnum + 1
1774 continue
1775
1776 file = {}
1777 file["path"] = path
1778 file["rev"] = commit["rev%s" % fnum]
1779 file["action"] = commit["action%s" % fnum]
1780 file["type"] = commit["type%s" % fnum]
1781 fnum = fnum + 1
1782
6326aa58 1783 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 1784
4b97ffb1 1785 for branch in self.knownBranches.keys():
6754a299
HWN
1786
1787 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1788 if relPath.startswith(branch + "/"):
d5904674
SH
1789 if branch not in branches:
1790 branches[branch] = []
71b112d4 1791 branches[branch].append(file)
6555b2cc 1792 break
b984733c
SH
1793
1794 return branches
1795
b932705b
LD
1796 # output one file from the P4 stream
1797 # - helper for streamP4Files
1798
1799 def streamOneP4File(self, file, contents):
b932705b 1800 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
084f6306 1801 relPath = self.wildcard_decode(relPath)
b932705b
LD
1802 if verbose:
1803 sys.stderr.write("%s\n" % relPath)
1804
9cffb8c8
PW
1805 (type_base, type_mods) = split_p4_type(file["type"])
1806
1807 git_mode = "100644"
1808 if "x" in type_mods:
1809 git_mode = "100755"
1810 if type_base == "symlink":
1811 git_mode = "120000"
1812 # p4 print on a symlink contains "target\n"; remove the newline
b39c3612
EP
1813 data = ''.join(contents)
1814 contents = [data[:-1]]
b932705b 1815
9cffb8c8 1816 if type_base == "utf16":
55aa5714
PW
1817 # p4 delivers different text in the python output to -G
1818 # than it does when using "print -o", or normal p4 client
1819 # operations. utf16 is converted to ascii or utf8, perhaps.
1820 # But ascii text saved as -t utf16 is completely mangled.
1821 # Invoke print -o to get the real contents.
6de040df 1822 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
55aa5714
PW
1823 contents = [ text ]
1824
9f7ef0ea
PW
1825 if type_base == "apple":
1826 # Apple filetype files will be streamed as a concatenation of
1827 # its appledouble header and the contents. This is useless
1828 # on both macs and non-macs. If using "print -q -o xx", it
1829 # will create "xx" with the data, and "%xx" with the header.
1830 # This is also not very useful.
1831 #
1832 # Ideally, someday, this script can learn how to generate
1833 # appledouble files directly and import those to git, but
1834 # non-mac machines can never find a use for apple filetype.
1835 print "\nIgnoring apple filetype file %s" % file['depotFile']
1836 return
1837
9cffb8c8
PW
1838 # Perhaps windows wants unicode, utf16 newlines translated too;
1839 # but this is not doing it.
1840 if self.isWindows and type_base == "text":
b932705b
LD
1841 mangled = []
1842 for data in contents:
1843 data = data.replace("\r\n", "\n")
1844 mangled.append(data)
1845 contents = mangled
1846
55aa5714
PW
1847 # Note that we do not try to de-mangle keywords on utf16 files,
1848 # even though in theory somebody may want that.
60df071c
LD
1849 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1850 if pattern:
1851 regexp = re.compile(pattern, re.VERBOSE)
1852 text = ''.join(contents)
1853 text = regexp.sub(r'$\1$', text)
1854 contents = [ text ]
b932705b 1855
9cffb8c8 1856 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
1857
1858 # total length...
1859 length = 0
1860 for d in contents:
1861 length = length + len(d)
1862
1863 self.gitStream.write("data %d\n" % length)
1864 for d in contents:
1865 self.gitStream.write(d)
1866 self.gitStream.write("\n")
1867
1868 def streamOneP4Deletion(self, file):
1869 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1870 if verbose:
1871 sys.stderr.write("delete %s\n" % relPath)
1872 self.gitStream.write("D %s\n" % relPath)
1873
1874 # handle another chunk of streaming data
1875 def streamP4FilesCb(self, marshalled):
1876
c3f6163b
AG
1877 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1878 # start of a new file - output the old one first
1879 self.streamOneP4File(self.stream_file, self.stream_contents)
1880 self.stream_file = {}
1881 self.stream_contents = []
1882 self.stream_have_file_info = False
b932705b 1883
c3f6163b
AG
1884 # pick up the new file information... for the
1885 # 'data' field we need to append to our array
1886 for k in marshalled.keys():
1887 if k == 'data':
1888 self.stream_contents.append(marshalled['data'])
1889 else:
1890 self.stream_file[k] = marshalled[k]
b932705b 1891
c3f6163b 1892 self.stream_have_file_info = True
b932705b
LD
1893
1894 # Stream directly from "p4 files" into "git fast-import"
1895 def streamP4Files(self, files):
30b5940b
SH
1896 filesForCommit = []
1897 filesToRead = []
b932705b 1898 filesToDelete = []
30b5940b 1899
3a70cdfa 1900 for f in files:
ecb7cf98
PW
1901 # if using a client spec, only add the files that have
1902 # a path in the client
1903 if self.clientSpecDirs:
1904 if self.clientSpecDirs.map_in_client(f['path']) == "":
1905 continue
3a70cdfa 1906
ecb7cf98
PW
1907 filesForCommit.append(f)
1908 if f['action'] in self.delete_actions:
1909 filesToDelete.append(f)
1910 else:
1911 filesToRead.append(f)
6a49f8e2 1912
b932705b
LD
1913 # deleted files...
1914 for f in filesToDelete:
1915 self.streamOneP4Deletion(f)
1b9a4684 1916
b932705b
LD
1917 if len(filesToRead) > 0:
1918 self.stream_file = {}
1919 self.stream_contents = []
1920 self.stream_have_file_info = False
8ff45f2a 1921
c3f6163b
AG
1922 # curry self argument
1923 def streamP4FilesCbSelf(entry):
1924 self.streamP4FilesCb(entry)
6a49f8e2 1925
6de040df
LD
1926 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1927
1928 p4CmdList(["-x", "-", "print"],
1929 stdin=fileArgs,
1930 cb=streamP4FilesCbSelf)
30b5940b 1931
b932705b
LD
1932 # do the last chunk
1933 if self.stream_file.has_key('depotFile'):
1934 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 1935
affb474f
LD
1936 def make_email(self, userid):
1937 if userid in self.users:
1938 return self.users[userid]
1939 else:
1940 return "%s <a@b>" % userid
1941
06804c76
LD
1942 # Stream a p4 tag
1943 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
1944 if verbose:
1945 print "writing tag %s for commit %s" % (labelName, commit)
1946 gitStream.write("tag %s\n" % labelName)
1947 gitStream.write("from %s\n" % commit)
1948
1949 if labelDetails.has_key('Owner'):
1950 owner = labelDetails["Owner"]
1951 else:
1952 owner = None
1953
1954 # Try to use the owner of the p4 label, or failing that,
1955 # the current p4 user id.
1956 if owner:
1957 email = self.make_email(owner)
1958 else:
1959 email = self.make_email(self.p4UserId())
1960 tagger = "%s %s %s" % (email, epoch, self.tz)
1961
1962 gitStream.write("tagger %s\n" % tagger)
1963
1964 print "labelDetails=",labelDetails
1965 if labelDetails.has_key('Description'):
1966 description = labelDetails['Description']
1967 else:
1968 description = 'Label from git p4'
1969
1970 gitStream.write("data %d\n" % len(description))
1971 gitStream.write(description)
1972 gitStream.write("\n")
1973
6326aa58 1974 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
1975 epoch = details["time"]
1976 author = details["user"]
c3f6163b 1977 self.branchPrefixes = branchPrefixes
b984733c 1978
4b97ffb1
SH
1979 if self.verbose:
1980 print "commit into %s" % branch
1981
96e07dd2
HWN
1982 # start with reading files; if that fails, we should not
1983 # create a commit.
1984 new_files = []
1985 for f in files:
d53de8b9 1986 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
1987 new_files.append (f)
1988 else:
afa1dd9a 1989 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 1990
b984733c 1991 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 1992# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
1993 self.committedChanges.add(int(details["change"]))
1994 committer = ""
b607e71e
SH
1995 if author not in self.users:
1996 self.getUserMapFromPerforceServer()
affb474f 1997 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
1998
1999 self.gitStream.write("committer %s\n" % committer)
2000
2001 self.gitStream.write("data <<EOT\n")
2002 self.gitStream.write(details["desc"])
6581de09
SH
2003 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2004 % (','.join (branchPrefixes), details["change"]))
2005 if len(details['options']) > 0:
2006 self.gitStream.write(": options = %s" % details['options'])
2007 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2008
2009 if len(parent) > 0:
4b97ffb1
SH
2010 if self.verbose:
2011 print "parent %s" % parent
b984733c
SH
2012 self.gitStream.write("from %s\n" % parent)
2013
b932705b 2014 self.streamP4Files(new_files)
b984733c
SH
2015 self.gitStream.write("\n")
2016
1f4ba1cb
SH
2017 change = int(details["change"])
2018
9bda3a85 2019 if self.labels.has_key(change):
1f4ba1cb
SH
2020 label = self.labels[change]
2021 labelDetails = label[0]
2022 labelRevisions = label[1]
71b112d4
SH
2023 if self.verbose:
2024 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2025
6de040df
LD
2026 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2027 for p in branchPrefixes])
1f4ba1cb
SH
2028
2029 if len(files) == len(labelRevisions):
2030
2031 cleanedFiles = {}
2032 for info in files:
56c09345 2033 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2034 continue
2035 cleanedFiles[info["depotFile"]] = info["rev"]
2036
2037 if cleanedFiles == labelRevisions:
06804c76 2038 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2039
2040 else:
a46668fa 2041 if not self.silent:
cebdf5af
HWN
2042 print ("Tag %s does not match with change %s: files do not match."
2043 % (labelDetails["label"], change))
1f4ba1cb
SH
2044
2045 else:
a46668fa 2046 if not self.silent:
cebdf5af
HWN
2047 print ("Tag %s does not match with change %s: file count is different."
2048 % (labelDetails["label"], change))
b984733c 2049
06804c76 2050 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2051 def getLabels(self):
2052 self.labels = {}
2053
52a4880b 2054 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2055 if len(l) > 0 and not self.silent:
183f8436 2056 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2057
2058 for output in l:
1f4ba1cb
SH
2059 label = output["label"]
2060 revisions = {}
2061 newestChange = 0
71b112d4
SH
2062 if self.verbose:
2063 print "Querying files for label %s" % label
6de040df
LD
2064 for file in p4CmdList(["files"] +
2065 ["%s...@%s" % (p, label)
2066 for p in self.depotPaths]):
1f4ba1cb
SH
2067 revisions[file["depotFile"]] = file["rev"]
2068 change = int(file["change"])
2069 if change > newestChange:
2070 newestChange = change
2071
9bda3a85
SH
2072 self.labels[newestChange] = [output, revisions]
2073
2074 if self.verbose:
2075 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2076
06804c76
LD
2077 # Import p4 labels as git tags. A direct mapping does not
2078 # exist, so assume that if all the files are at the same revision
2079 # then we can use that, or it's something more complicated we should
2080 # just ignore.
2081 def importP4Labels(self, stream, p4Labels):
2082 if verbose:
2083 print "import p4 labels: " + ' '.join(p4Labels)
2084
2085 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2086 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2087 if len(validLabelRegexp) == 0:
2088 validLabelRegexp = defaultLabelRegexp
2089 m = re.compile(validLabelRegexp)
2090
2091 for name in p4Labels:
2092 commitFound = False
2093
2094 if not m.match(name):
2095 if verbose:
2096 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2097 continue
2098
2099 if name in ignoredP4Labels:
2100 continue
2101
2102 labelDetails = p4CmdList(['label', "-o", name])[0]
2103
2104 # get the most recent changelist for each file in this label
2105 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2106 for p in self.depotPaths])
2107
2108 if change.has_key('change'):
2109 # find the corresponding git commit; take the oldest commit
2110 changelist = int(change['change'])
2111 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2112 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2113 if len(gitCommit) == 0:
2114 print "could not find git commit for changelist %d" % changelist
2115 else:
2116 gitCommit = gitCommit.strip()
2117 commitFound = True
2118 # Convert from p4 time format
2119 try:
2120 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2121 except ValueError:
2122 print "Could not convert label time %s" % labelDetail['Update']
2123 tmwhen = 1
2124
2125 when = int(time.mktime(tmwhen))
2126 self.streamTag(stream, name, labelDetails, gitCommit, when)
2127 if verbose:
2128 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2129 else:
2130 if verbose:
2131 print "Label %s has no changelists - possibly deleted?" % name
2132
2133 if not commitFound:
2134 # We can't import this label; don't try again as it will get very
2135 # expensive repeatedly fetching all the files for labels that will
2136 # never be imported. If the label is moved in the future, the
2137 # ignore will need to be removed manually.
2138 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2139
86dff6b6
HWN
2140 def guessProjectName(self):
2141 for p in self.depotPaths:
6e5295c4
SH
2142 if p.endswith("/"):
2143 p = p[:-1]
2144 p = p[p.strip().rfind("/") + 1:]
2145 if not p.endswith("/"):
2146 p += "/"
2147 return p
86dff6b6 2148
4b97ffb1 2149 def getBranchMapping(self):
6555b2cc
SH
2150 lostAndFoundBranches = set()
2151
8ace74c0
VA
2152 user = gitConfig("git-p4.branchUser")
2153 if len(user) > 0:
2154 command = "branches -u %s" % user
2155 else:
2156 command = "branches"
2157
2158 for info in p4CmdList(command):
52a4880b 2159 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2160 viewIdx = 0
2161 while details.has_key("View%s" % viewIdx):
2162 paths = details["View%s" % viewIdx].split(" ")
2163 viewIdx = viewIdx + 1
2164 # require standard //depot/foo/... //depot/bar/... mapping
2165 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2166 continue
2167 source = paths[0]
2168 destination = paths[1]
6509e19c 2169 ## HACK
d53de8b9 2170 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2171 source = source[len(self.depotPaths[0]):-4]
2172 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2173
1a2edf4e
SH
2174 if destination in self.knownBranches:
2175 if not self.silent:
2176 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2177 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2178 continue
2179
6555b2cc
SH
2180 self.knownBranches[destination] = source
2181
2182 lostAndFoundBranches.discard(destination)
2183
29bdbac1 2184 if source not in self.knownBranches:
6555b2cc
SH
2185 lostAndFoundBranches.add(source)
2186
7199cf13
VA
2187 # Perforce does not strictly require branches to be defined, so we also
2188 # check git config for a branch list.
2189 #
2190 # Example of branch definition in git config file:
2191 # [git-p4]
2192 # branchList=main:branchA
2193 # branchList=main:branchB
2194 # branchList=branchA:branchC
2195 configBranches = gitConfigList("git-p4.branchList")
2196 for branch in configBranches:
2197 if branch:
2198 (source, destination) = branch.split(":")
2199 self.knownBranches[destination] = source
2200
2201 lostAndFoundBranches.discard(destination)
2202
2203 if source not in self.knownBranches:
2204 lostAndFoundBranches.add(source)
2205
6555b2cc
SH
2206
2207 for branch in lostAndFoundBranches:
2208 self.knownBranches[branch] = branch
29bdbac1 2209
38f9f5ec
SH
2210 def getBranchMappingFromGitBranches(self):
2211 branches = p4BranchesInGit(self.importIntoRemotes)
2212 for branch in branches.keys():
2213 if branch == "master":
2214 branch = "main"
2215 else:
2216 branch = branch[len(self.projectName):]
2217 self.knownBranches[branch] = branch
2218
29bdbac1 2219 def listExistingP4GitBranches(self):
144ff46b
SH
2220 # branches holds mapping from name to commit
2221 branches = p4BranchesInGit(self.importIntoRemotes)
2222 self.p4BranchesInGit = branches.keys()
2223 for branch in branches.keys():
2224 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 2225
bb6e09b2
HWN
2226 def updateOptionDict(self, d):
2227 option_keys = {}
2228 if self.keepRepoPath:
2229 option_keys['keepRepoPath'] = 1
2230
2231 d["options"] = ' '.join(sorted(option_keys.keys()))
2232
2233 def readOptions(self, d):
2234 self.keepRepoPath = (d.has_key('options')
2235 and ('keepRepoPath' in d['options']))
6326aa58 2236
8134f69c
SH
2237 def gitRefForBranch(self, branch):
2238 if branch == "main":
2239 return self.refPrefix + "master"
2240
2241 if len(branch) <= 0:
2242 return branch
2243
2244 return self.refPrefix + self.projectName + branch
2245
1ca3d710
SH
2246 def gitCommitByP4Change(self, ref, change):
2247 if self.verbose:
2248 print "looking in ref " + ref + " for change %s using bisect..." % change
2249
2250 earliestCommit = ""
2251 latestCommit = parseRevision(ref)
2252
2253 while True:
2254 if self.verbose:
2255 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2256 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2257 if len(next) == 0:
2258 if self.verbose:
2259 print "argh"
2260 return ""
2261 log = extractLogMessageFromGitCommit(next)
2262 settings = extractSettingsGitLog(log)
2263 currentChange = int(settings['change'])
2264 if self.verbose:
2265 print "current change %s" % currentChange
2266
2267 if currentChange == change:
2268 if self.verbose:
2269 print "found %s" % next
2270 return next
2271
2272 if currentChange < change:
2273 earliestCommit = "^%s" % next
2274 else:
2275 latestCommit = "%s" % next
2276
2277 return ""
2278
2279 def importNewBranch(self, branch, maxChange):
2280 # make fast-import flush all changes to disk and update the refs using the checkpoint
2281 # command so that we can try to find the branch parent in the git history
2282 self.gitStream.write("checkpoint\n\n");
2283 self.gitStream.flush();
2284 branchPrefix = self.depotPaths[0] + branch + "/"
2285 range = "@1,%s" % maxChange
2286 #print "prefix" + branchPrefix
2287 changes = p4ChangesForPaths([branchPrefix], range)
2288 if len(changes) <= 0:
2289 return False
2290 firstChange = changes[0]
2291 #print "first change in branch: %s" % firstChange
2292 sourceBranch = self.knownBranches[branch]
2293 sourceDepotPath = self.depotPaths[0] + sourceBranch
2294 sourceRef = self.gitRefForBranch(sourceBranch)
2295 #print "source " + sourceBranch
2296
52a4880b 2297 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2298 #print "branch parent: %s" % branchParentChange
2299 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2300 if len(gitParent) > 0:
2301 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2302 #print "parent git commit: %s" % gitParent
2303
2304 self.importChanges(changes)
2305 return True
2306
fed23693
VA
2307 def searchParent(self, parent, branch, target):
2308 parentFound = False
2309 for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2310 blob = blob.strip()
2311 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2312 parentFound = True
2313 if self.verbose:
2314 print "Found parent of %s in commit %s" % (branch, blob)
2315 break
2316 if parentFound:
2317 return blob
2318 else:
2319 return None
2320
e87f37ae
SH
2321 def importChanges(self, changes):
2322 cnt = 1
2323 for change in changes:
3558f32f 2324 description = p4Cmd(["describe", str(change)])
e87f37ae
SH
2325 self.updateOptionDict(description)
2326
2327 if not self.silent:
2328 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2329 sys.stdout.flush()
2330 cnt = cnt + 1
2331
2332 try:
2333 if self.detectBranches:
2334 branches = self.splitFilesIntoBranches(description)
2335 for branch in branches.keys():
2336 ## HACK --hwn
2337 branchPrefix = self.depotPaths[0] + branch + "/"
2338
2339 parent = ""
2340
2341 filesForCommit = branches[branch]
2342
2343 if self.verbose:
2344 print "branch is %s" % branch
2345
2346 self.updatedBranches.add(branch)
2347
2348 if branch not in self.createdBranches:
2349 self.createdBranches.add(branch)
2350 parent = self.knownBranches[branch]
2351 if parent == branch:
2352 parent = ""
1ca3d710
SH
2353 else:
2354 fullBranch = self.projectName + branch
2355 if fullBranch not in self.p4BranchesInGit:
2356 if not self.silent:
2357 print("\n Importing new branch %s" % fullBranch);
2358 if self.importNewBranch(branch, change - 1):
2359 parent = ""
2360 self.p4BranchesInGit.append(fullBranch)
2361 if not self.silent:
2362 print("\n Resuming with change %s" % change);
2363
2364 if self.verbose:
2365 print "parent determined through known branches: %s" % parent
e87f37ae 2366
8134f69c
SH
2367 branch = self.gitRefForBranch(branch)
2368 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2369
2370 if self.verbose:
2371 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2372
2373 if len(parent) == 0 and branch in self.initialParents:
2374 parent = self.initialParents[branch]
2375 del self.initialParents[branch]
2376
fed23693
VA
2377 blob = None
2378 if len(parent) > 0:
2379 tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2380 if self.verbose:
2381 print "Creating temporary branch: " + tempBranch
2382 self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2383 self.tempBranches.append(tempBranch)
2384 self.checkpoint()
2385 blob = self.searchParent(parent, branch, tempBranch)
2386 if blob:
2387 self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2388 else:
2389 if self.verbose:
2390 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2391 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
e87f37ae
SH
2392 else:
2393 files = self.extractFilesFromCommit(description)
2394 self.commit(description, files, self.branch, self.depotPaths,
2395 self.initialParent)
2396 self.initialParent = ""
2397 except IOError:
2398 print self.gitError.read()
2399 sys.exit(1)
2400
c208a243
SH
2401 def importHeadRevision(self, revision):
2402 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2403
4e2e6ce4
PW
2404 details = {}
2405 details["user"] = "git perforce import user"
1494fcbb 2406 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2407 % (' '.join(self.depotPaths), revision))
2408 details["change"] = revision
2409 newestRevision = 0
2410
2411 fileCnt = 0
6de040df
LD
2412 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2413
2414 for info in p4CmdList(["files"] + fileArgs):
c208a243 2415
68b28593 2416 if 'code' in info and info['code'] == 'error':
c208a243
SH
2417 sys.stderr.write("p4 returned an error: %s\n"
2418 % info['data'])
d88e707f
PW
2419 if info['data'].find("must refer to client") >= 0:
2420 sys.stderr.write("This particular p4 error is misleading.\n")
2421 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2422 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2423 sys.exit(1)
68b28593
PW
2424 if 'p4ExitCode' in info:
2425 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2426 sys.exit(1)
2427
2428
2429 change = int(info["change"])
2430 if change > newestRevision:
2431 newestRevision = change
2432
56c09345 2433 if info["action"] in self.delete_actions:
c208a243
SH
2434 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2435 #fileCnt = fileCnt + 1
2436 continue
2437
2438 for prop in ["depotFile", "rev", "action", "type" ]:
2439 details["%s%s" % (prop, fileCnt)] = info[prop]
2440
2441 fileCnt = fileCnt + 1
2442
2443 details["change"] = newestRevision
4e2e6ce4 2444
9dcb9f24 2445 # Use time from top-most change so that all git p4 clones of
4e2e6ce4
PW
2446 # the same p4 repo have the same commit SHA1s.
2447 res = p4CmdList("describe -s %d" % newestRevision)
2448 newestTime = None
2449 for r in res:
2450 if r.has_key('time'):
2451 newestTime = int(r['time'])
2452 if newestTime is None:
2453 die("\"describe -s\" on newest change %d did not give a time")
2454 details["time"] = newestTime
2455
c208a243
SH
2456 self.updateOptionDict(details)
2457 try:
2458 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2459 except IOError:
2460 print "IO error with git fast-import. Is your git version recent enough?"
2461 print self.gitError.read()
2462
2463
b984733c 2464 def run(self, args):
6326aa58 2465 self.depotPaths = []
179caebf
SH
2466 self.changeRange = ""
2467 self.initialParent = ""
6326aa58 2468 self.previousDepotPaths = []
ce6f33c8 2469
29bdbac1
SH
2470 # map from branch depot path to parent branch
2471 self.knownBranches = {}
2472 self.initialParents = {}
5ca44617 2473 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
2474 if not self.syncWithOrigin:
2475 self.hasOrigin = False
29bdbac1 2476
a028a98e
SH
2477 if self.importIntoRemotes:
2478 self.refPrefix = "refs/remotes/p4/"
2479 else:
db775559 2480 self.refPrefix = "refs/heads/p4/"
a028a98e 2481
cebdf5af
HWN
2482 if self.syncWithOrigin and self.hasOrigin:
2483 if not self.silent:
2484 print "Syncing with origin first by calling git fetch origin"
2485 system("git fetch origin")
10f880f8 2486
569d1bd4 2487 if len(self.branch) == 0:
db775559 2488 self.branch = self.refPrefix + "master"
a028a98e 2489 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2490 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 2491 system("git branch -D p4");
faf1bd20 2492 # create it /after/ importing, when master exists
0058a33a 2493 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 2494 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 2495
a93d33ee
PW
2496 # accept either the command-line option, or the configuration variable
2497 if self.useClientSpec:
2498 # will use this after clone to set the variable
2499 self.useClientSpec_from_options = True
2500 else:
09fca77b
PW
2501 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2502 self.useClientSpec = True
2503 if self.useClientSpec:
543987bd 2504 self.clientSpecDirs = getClientSpec()
3a70cdfa 2505
6a49f8e2
HWN
2506 # TODO: should always look at previous commits,
2507 # merge with previous imports, if possible.
2508 if args == []:
d414c74a 2509 if self.hasOrigin:
5ca44617 2510 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
2511 self.listExistingP4GitBranches()
2512
2513 if len(self.p4BranchesInGit) > 1:
2514 if not self.silent:
2515 print "Importing from/into multiple branches"
2516 self.detectBranches = True
967f72e2 2517
29bdbac1
SH
2518 if self.verbose:
2519 print "branches: %s" % self.p4BranchesInGit
2520
2521 p4Change = 0
2522 for branch in self.p4BranchesInGit:
cebdf5af 2523 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2524
2525 settings = extractSettingsGitLog(logMsg)
29bdbac1 2526
bb6e09b2
HWN
2527 self.readOptions(settings)
2528 if (settings.has_key('depot-paths')
2529 and settings.has_key ('change')):
2530 change = int(settings['change']) + 1
29bdbac1
SH
2531 p4Change = max(p4Change, change)
2532
bb6e09b2
HWN
2533 depotPaths = sorted(settings['depot-paths'])
2534 if self.previousDepotPaths == []:
6326aa58 2535 self.previousDepotPaths = depotPaths
29bdbac1 2536 else:
6326aa58
HWN
2537 paths = []
2538 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2539 prev_list = prev.split("/")
2540 cur_list = cur.split("/")
2541 for i in range(0, min(len(cur_list), len(prev_list))):
2542 if cur_list[i] <> prev_list[i]:
583e1707 2543 i = i - 1
6326aa58
HWN
2544 break
2545
04d277b3 2546 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2547
2548 self.previousDepotPaths = paths
29bdbac1
SH
2549
2550 if p4Change > 0:
bb6e09b2 2551 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2552 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
2553 if not self.detectBranches:
2554 self.initialParent = parseRevision(self.branch)
341dc1c1 2555 if not self.silent and not self.detectBranches:
967f72e2 2556 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2557
f9162f6a
SH
2558 if not self.branch.startswith("refs/"):
2559 self.branch = "refs/heads/" + self.branch
179caebf 2560
6326aa58 2561 if len(args) == 0 and self.depotPaths:
b984733c 2562 if not self.silent:
6326aa58 2563 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2564 else:
6326aa58 2565 if self.depotPaths and self.depotPaths != args:
cebdf5af 2566 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2567 "This doesn't work!" % (' '.join (self.depotPaths),
2568 ' '.join (args)))
b984733c 2569 sys.exit(1)
6326aa58 2570
bb6e09b2 2571 self.depotPaths = sorted(args)
b984733c 2572
1c49fc19 2573 revision = ""
b984733c 2574 self.users = {}
b984733c 2575
58c8bc7c
PW
2576 # Make sure no revision specifiers are used when --changesfile
2577 # is specified.
2578 bad_changesfile = False
2579 if len(self.changesFile) > 0:
2580 for p in self.depotPaths:
2581 if p.find("@") >= 0 or p.find("#") >= 0:
2582 bad_changesfile = True
2583 break
2584 if bad_changesfile:
2585 die("Option --changesfile is incompatible with revision specifiers")
2586
6326aa58
HWN
2587 newPaths = []
2588 for p in self.depotPaths:
2589 if p.find("@") != -1:
2590 atIdx = p.index("@")
2591 self.changeRange = p[atIdx:]
2592 if self.changeRange == "@all":
2593 self.changeRange = ""
6a49f8e2 2594 elif ',' not in self.changeRange:
1c49fc19 2595 revision = self.changeRange
6326aa58 2596 self.changeRange = ""
7fcff9de 2597 p = p[:atIdx]
6326aa58
HWN
2598 elif p.find("#") != -1:
2599 hashIdx = p.index("#")
1c49fc19 2600 revision = p[hashIdx:]
7fcff9de 2601 p = p[:hashIdx]
6326aa58 2602 elif self.previousDepotPaths == []:
58c8bc7c
PW
2603 # pay attention to changesfile, if given, else import
2604 # the entire p4 tree at the head revision
2605 if len(self.changesFile) == 0:
2606 revision = "#head"
6326aa58
HWN
2607
2608 p = re.sub ("\.\.\.$", "", p)
2609 if not p.endswith("/"):
2610 p += "/"
2611
2612 newPaths.append(p)
2613
2614 self.depotPaths = newPaths
2615
b607e71e 2616 self.loadUserMapFromCache()
cb53e1f8
SH
2617 self.labels = {}
2618 if self.detectLabels:
2619 self.getLabels();
b984733c 2620
4b97ffb1 2621 if self.detectBranches:
df450923
SH
2622 ## FIXME - what's a P4 projectName ?
2623 self.projectName = self.guessProjectName()
2624
38f9f5ec
SH
2625 if self.hasOrigin:
2626 self.getBranchMappingFromGitBranches()
2627 else:
2628 self.getBranchMapping()
29bdbac1
SH
2629 if self.verbose:
2630 print "p4-git branches: %s" % self.p4BranchesInGit
2631 print "initial parents: %s" % self.initialParents
2632 for b in self.p4BranchesInGit:
2633 if b != "master":
6326aa58
HWN
2634
2635 ## FIXME
29bdbac1
SH
2636 b = b[len(self.projectName):]
2637 self.createdBranches.add(b)
4b97ffb1 2638
f291b4e3 2639 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 2640
cebdf5af 2641 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
2642 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2643 stderr=subprocess.PIPE);
08483580
SH
2644 self.gitOutput = importProcess.stdout
2645 self.gitStream = importProcess.stdin
2646 self.gitError = importProcess.stderr
b984733c 2647
1c49fc19 2648 if revision:
c208a243 2649 self.importHeadRevision(revision)
b984733c
SH
2650 else:
2651 changes = []
2652
0828ab14 2653 if len(self.changesFile) > 0:
b984733c 2654 output = open(self.changesFile).readlines()
1d7367dc 2655 changeSet = set()
b984733c
SH
2656 for line in output:
2657 changeSet.add(int(line))
2658
2659 for change in changeSet:
2660 changes.append(change)
2661
2662 changes.sort()
2663 else:
9dcb9f24
PW
2664 # catch "git p4 sync" with no new branches, in a repo that
2665 # does not have any existing p4 branches
accad8e0 2666 if len(args) == 0 and not self.p4BranchesInGit:
e32e00dc 2667 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
29bdbac1 2668 if self.verbose:
86dff6b6 2669 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 2670 self.changeRange)
4f6432d8 2671 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 2672
01a9c9c5 2673 if len(self.maxChanges) > 0:
7fcff9de 2674 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 2675
b984733c 2676 if len(changes) == 0:
0828ab14 2677 if not self.silent:
341dc1c1 2678 print "No changes to import!"
06804c76
LD
2679 else:
2680 if not self.silent and not self.detectBranches:
2681 print "Import destination: %s" % self.branch
2682
2683 self.updatedBranches = set()
b984733c 2684
06804c76 2685 self.importChanges(changes)
a9d1a27a 2686
06804c76
LD
2687 if not self.silent:
2688 print ""
2689 if len(self.updatedBranches) > 0:
2690 sys.stdout.write("Updated branches: ")
2691 for b in self.updatedBranches:
2692 sys.stdout.write("%s " % b)
2693 sys.stdout.write("\n")
341dc1c1 2694
06804c76
LD
2695 if gitConfig("git-p4.importLabels", "--bool") == "true":
2696 self.importLabels = true
b984733c 2697
06804c76
LD
2698 if self.importLabels:
2699 p4Labels = getP4Labels(self.depotPaths)
2700 gitTags = getGitTags()
2701
2702 missingP4Labels = p4Labels - gitTags
2703 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 2704
b984733c 2705 self.gitStream.close()
29bdbac1
SH
2706 if importProcess.wait() != 0:
2707 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
2708 self.gitOutput.close()
2709 self.gitError.close()
2710
fed23693
VA
2711 # Cleanup temporary branches created during import
2712 if self.tempBranches != []:
2713 for branch in self.tempBranches:
2714 read_pipe("git update-ref -d %s" % branch)
2715 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2716
b984733c
SH
2717 return True
2718
01ce1fe9
SH
2719class P4Rebase(Command):
2720 def __init__(self):
2721 Command.__init__(self)
06804c76
LD
2722 self.options = [
2723 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 2724 ]
06804c76 2725 self.importLabels = False
cebdf5af
HWN
2726 self.description = ("Fetches the latest revision from perforce and "
2727 + "rebases the current work (branch) against it")
01ce1fe9
SH
2728
2729 def run(self, args):
2730 sync = P4Sync()
06804c76 2731 sync.importLabels = self.importLabels
01ce1fe9 2732 sync.run([])
d7e3868c 2733
14594f4b
SH
2734 return self.rebase()
2735
2736 def rebase(self):
36ee4ee4
SH
2737 if os.system("git update-index --refresh") != 0:
2738 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2739 if len(read_pipe("git diff-index HEAD --")) > 0:
2740 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2741
d7e3868c
SH
2742 [upstream, settings] = findUpstreamBranchPoint()
2743 if len(upstream) == 0:
2744 die("Cannot find upstream branchpoint for rebase")
2745
2746 # the branchpoint may be p4/foo~3, so strip off the parent
2747 upstream = re.sub("~[0-9]+$", "", upstream)
2748
2749 print "Rebasing the current branch onto %s" % upstream
b25b2065 2750 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 2751 system("git rebase %s" % upstream)
1f52af6c 2752 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
2753 return True
2754
f9a3a4f7
SH
2755class P4Clone(P4Sync):
2756 def __init__(self):
2757 P4Sync.__init__(self)
2758 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 2759 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 2760 self.options += [
bb6e09b2
HWN
2761 optparse.make_option("--destination", dest="cloneDestination",
2762 action='store', default=None,
354081d5
TT
2763 help="where to leave result of the clone"),
2764 optparse.make_option("-/", dest="cloneExclude",
2765 action="append", type="string",
38200076
PW
2766 help="exclude depot path"),
2767 optparse.make_option("--bare", dest="cloneBare",
2768 action="store_true", default=False),
354081d5 2769 ]
bb6e09b2 2770 self.cloneDestination = None
f9a3a4f7 2771 self.needsGit = False
38200076 2772 self.cloneBare = False
f9a3a4f7 2773
354081d5
TT
2774 # This is required for the "append" cloneExclude action
2775 def ensure_value(self, attr, value):
2776 if not hasattr(self, attr) or getattr(self, attr) is None:
2777 setattr(self, attr, value)
2778 return getattr(self, attr)
2779
6a49f8e2
HWN
2780 def defaultDestination(self, args):
2781 ## TODO: use common prefix of args?
2782 depotPath = args[0]
2783 depotDir = re.sub("(@[^@]*)$", "", depotPath)
2784 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 2785 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
2786 depotDir = re.sub(r"/$", "", depotDir)
2787 return os.path.split(depotDir)[1]
2788
f9a3a4f7
SH
2789 def run(self, args):
2790 if len(args) < 1:
2791 return False
bb6e09b2
HWN
2792
2793 if self.keepRepoPath and not self.cloneDestination:
2794 sys.stderr.write("Must specify destination for --keep-path\n")
2795 sys.exit(1)
f9a3a4f7 2796
6326aa58 2797 depotPaths = args
5e100b5c
SH
2798
2799 if not self.cloneDestination and len(depotPaths) > 1:
2800 self.cloneDestination = depotPaths[-1]
2801 depotPaths = depotPaths[:-1]
2802
354081d5 2803 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
2804 for p in depotPaths:
2805 if not p.startswith("//"):
2806 return False
f9a3a4f7 2807
bb6e09b2 2808 if not self.cloneDestination:
98ad4faf 2809 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 2810
86dff6b6 2811 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 2812
c3bf3f13
KG
2813 if not os.path.exists(self.cloneDestination):
2814 os.makedirs(self.cloneDestination)
053fd0c1 2815 chdir(self.cloneDestination)
38200076
PW
2816
2817 init_cmd = [ "git", "init" ]
2818 if self.cloneBare:
2819 init_cmd.append("--bare")
2820 subprocess.check_call(init_cmd)
2821
6326aa58 2822 if not P4Sync.run(self, depotPaths):
f9a3a4f7 2823 return False
f9a3a4f7 2824 if self.branch != "master":
e9905013
TAL
2825 if self.importIntoRemotes:
2826 masterbranch = "refs/remotes/p4/master"
2827 else:
2828 masterbranch = "refs/heads/p4/master"
2829 if gitBranchExists(masterbranch):
2830 system("git branch master %s" % masterbranch)
38200076
PW
2831 if not self.cloneBare:
2832 system("git checkout -f")
8f9b2e08
SH
2833 else:
2834 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 2835
a93d33ee
PW
2836 # auto-set this variable if invoked with --use-client-spec
2837 if self.useClientSpec_from_options:
2838 system("git config --bool git-p4.useclientspec true")
2839
f9a3a4f7
SH
2840 return True
2841
09d89de2
SH
2842class P4Branches(Command):
2843 def __init__(self):
2844 Command.__init__(self)
2845 self.options = [ ]
2846 self.description = ("Shows the git branches that hold imports and their "
2847 + "corresponding perforce depot paths")
2848 self.verbose = False
2849
2850 def run(self, args):
5ca44617
SH
2851 if originP4BranchesExist():
2852 createOrUpdateBranchesFromOrigin()
2853
09d89de2
SH
2854 cmdline = "git rev-parse --symbolic "
2855 cmdline += " --remotes"
2856
2857 for line in read_pipe_lines(cmdline):
2858 line = line.strip()
2859
2860 if not line.startswith('p4/') or line == "p4/HEAD":
2861 continue
2862 branch = line
2863
2864 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2865 settings = extractSettingsGitLog(log)
2866
2867 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2868 return True
2869
b984733c
SH
2870class HelpFormatter(optparse.IndentedHelpFormatter):
2871 def __init__(self):
2872 optparse.IndentedHelpFormatter.__init__(self)
2873
2874 def format_description(self, description):
2875 if description:
2876 return description + "\n"
2877 else:
2878 return ""
4f5cf76a 2879
86949eef
SH
2880def printUsage(commands):
2881 print "usage: %s <command> [options]" % sys.argv[0]
2882 print ""
2883 print "valid commands: %s" % ", ".join(commands)
2884 print ""
2885 print "Try %s <command> --help for command specific help." % sys.argv[0]
2886 print ""
2887
2888commands = {
b86f7378
HWN
2889 "debug" : P4Debug,
2890 "submit" : P4Submit,
a9834f58 2891 "commit" : P4Submit,
b86f7378
HWN
2892 "sync" : P4Sync,
2893 "rebase" : P4Rebase,
2894 "clone" : P4Clone,
09d89de2
SH
2895 "rollback" : P4RollBack,
2896 "branches" : P4Branches
86949eef
SH
2897}
2898
86949eef 2899
bb6e09b2
HWN
2900def main():
2901 if len(sys.argv[1:]) == 0:
2902 printUsage(commands.keys())
2903 sys.exit(2)
4f5cf76a 2904
bb6e09b2
HWN
2905 cmd = ""
2906 cmdName = sys.argv[1]
2907 try:
b86f7378
HWN
2908 klass = commands[cmdName]
2909 cmd = klass()
bb6e09b2
HWN
2910 except KeyError:
2911 print "unknown command %s" % cmdName
2912 print ""
2913 printUsage(commands.keys())
2914 sys.exit(2)
2915
2916 options = cmd.options
b86f7378 2917 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
2918
2919 args = sys.argv[2:]
2920
6a10b6aa
LD
2921 options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))
2922 if cmd.needsGit:
2923 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 2924
6a10b6aa
LD
2925 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2926 options,
2927 description = cmd.description,
2928 formatter = HelpFormatter())
bb6e09b2 2929
6a10b6aa 2930 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
bb6e09b2
HWN
2931 global verbose
2932 verbose = cmd.verbose
2933 if cmd.needsGit:
b86f7378
HWN
2934 if cmd.gitdir == None:
2935 cmd.gitdir = os.path.abspath(".git")
2936 if not isValidGitDir(cmd.gitdir):
2937 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2938 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
2939 cdup = read_pipe("git rev-parse --show-cdup").strip()
2940 if len(cdup) > 0:
053fd0c1 2941 chdir(cdup);
e20a9e53 2942
b86f7378
HWN
2943 if not isValidGitDir(cmd.gitdir):
2944 if isValidGitDir(cmd.gitdir + "/.git"):
2945 cmd.gitdir += "/.git"
bb6e09b2 2946 else:
b86f7378 2947 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 2948
b86f7378 2949 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 2950
bb6e09b2
HWN
2951 if not cmd.run(args):
2952 parser.print_help()
09fca77b 2953 sys.exit(2)
4f5cf76a 2954
4f5cf76a 2955
bb6e09b2
HWN
2956if __name__ == '__main__':
2957 main()