git p4 test: fix badp4dir test
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
10
1d7367dc
RG
11import optparse, sys, os, marshal, subprocess, shelve
12import tempfile, getopt, os.path, time, platform
60df071c 13import re, shutil
8b41a97f 14
4addad22 15verbose = False
86949eef 16
06804c76 17# Only labels/tags matching this will be imported/exported
c8942a22 18defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753
AK
19
20def p4_build_cmd(cmd):
21 """Build a suitable p4 command line.
22
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
26 """
6de040df 27 real_cmd = ["p4"]
abcaf073
AK
28
29 user = gitConfig("git-p4.user")
30 if len(user) > 0:
6de040df 31 real_cmd += ["-u",user]
abcaf073
AK
32
33 password = gitConfig("git-p4.password")
34 if len(password) > 0:
6de040df 35 real_cmd += ["-P", password]
abcaf073
AK
36
37 port = gitConfig("git-p4.port")
38 if len(port) > 0:
6de040df 39 real_cmd += ["-p", port]
abcaf073
AK
40
41 host = gitConfig("git-p4.host")
42 if len(host) > 0:
41799aa2 43 real_cmd += ["-H", host]
abcaf073
AK
44
45 client = gitConfig("git-p4.client")
46 if len(client) > 0:
6de040df 47 real_cmd += ["-c", client]
abcaf073 48
6de040df
LD
49
50 if isinstance(cmd,basestring):
51 real_cmd = ' '.join(real_cmd) + ' ' + cmd
52 else:
53 real_cmd += cmd
21a50753
AK
54 return real_cmd
55
053fd0c1 56def chdir(dir):
6de040df 57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
bf1d68ff
GG
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
053fd0c1 60 os.chdir(dir)
bf1d68ff 61 os.environ['PWD'] = os.getcwd()
053fd0c1 62
86dff6b6
HWN
63def die(msg):
64 if verbose:
65 raise Exception(msg)
66 else:
67 sys.stderr.write(msg + "\n")
68 sys.exit(1)
69
6de040df 70def write_pipe(c, stdin):
4addad22 71 if verbose:
6de040df 72 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 73
6de040df
LD
74 expand = isinstance(c,basestring)
75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
76 pipe = p.stdin
77 val = pipe.write(stdin)
78 pipe.close()
79 if p.wait():
80 die('Command failed: %s' % str(c))
b016d397
HWN
81
82 return val
83
6de040df 84def p4_write_pipe(c, stdin):
d9429194 85 real_cmd = p4_build_cmd(c)
6de040df 86 return write_pipe(real_cmd, stdin)
d9429194 87
4addad22
HWN
88def read_pipe(c, ignore_error=False):
89 if verbose:
6de040df 90 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 91
6de040df
LD
92 expand = isinstance(c,basestring)
93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
94 pipe = p.stdout
b016d397 95 val = pipe.read()
6de040df
LD
96 if p.wait() and not ignore_error:
97 die('Command failed: %s' % str(c))
b016d397
HWN
98
99 return val
100
d9429194
AK
101def p4_read_pipe(c, ignore_error=False):
102 real_cmd = p4_build_cmd(c)
103 return read_pipe(real_cmd, ignore_error)
b016d397 104
bce4c5fc 105def read_pipe_lines(c):
4addad22 106 if verbose:
6de040df
LD
107 sys.stderr.write('Reading pipe: %s\n' % str(c))
108
109 expand = isinstance(c, basestring)
110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
111 pipe = p.stdout
b016d397 112 val = pipe.readlines()
6de040df
LD
113 if pipe.close() or p.wait():
114 die('Command failed: %s' % str(c))
b016d397
HWN
115
116 return val
caace111 117
2318121b
AK
118def p4_read_pipe_lines(c):
119 """Specifically invoke p4 on the command supplied. """
155af834 120 real_cmd = p4_build_cmd(c)
2318121b
AK
121 return read_pipe_lines(real_cmd)
122
6754a299 123def system(cmd):
6de040df 124 expand = isinstance(cmd,basestring)
4addad22 125 if verbose:
6de040df
LD
126 sys.stderr.write("executing %s\n" % str(cmd))
127 subprocess.check_call(cmd, shell=expand)
6754a299 128
bf9320f1
AK
129def p4_system(cmd):
130 """Specifically invoke p4 as the system command. """
155af834 131 real_cmd = p4_build_cmd(cmd)
6de040df
LD
132 expand = isinstance(real_cmd, basestring)
133 subprocess.check_call(real_cmd, shell=expand)
134
135def p4_integrate(src, dest):
9d7d446a 136 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 137
8d7ec362 138def p4_sync(f, *options):
9d7d446a 139 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
140
141def p4_add(f):
9d7d446a
PW
142 # forcibly add file names with wildcards
143 if wildcard_present(f):
144 p4_system(["add", "-f", f])
145 else:
146 p4_system(["add", f])
6de040df
LD
147
148def p4_delete(f):
9d7d446a 149 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
150
151def p4_edit(f):
9d7d446a 152 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
153
154def p4_revert(f):
9d7d446a 155 p4_system(["revert", wildcard_encode(f)])
6de040df 156
9d7d446a
PW
157def p4_reopen(type, f):
158 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 159
9cffb8c8
PW
160#
161# Canonicalize the p4 type and return a tuple of the
162# base type, plus any modifiers. See "p4 help filetypes"
163# for a list and explanation.
164#
165def split_p4_type(p4type):
166
167 p4_filetypes_historical = {
168 "ctempobj": "binary+Sw",
169 "ctext": "text+C",
170 "cxtext": "text+Cx",
171 "ktext": "text+k",
172 "kxtext": "text+kx",
173 "ltext": "text+F",
174 "tempobj": "binary+FSw",
175 "ubinary": "binary+F",
176 "uresource": "resource+F",
177 "uxbinary": "binary+Fx",
178 "xbinary": "binary+x",
179 "xltext": "text+Fx",
180 "xtempobj": "binary+Swx",
181 "xtext": "text+x",
182 "xunicode": "unicode+x",
183 "xutf16": "utf16+x",
184 }
185 if p4type in p4_filetypes_historical:
186 p4type = p4_filetypes_historical[p4type]
187 mods = ""
188 s = p4type.split("+")
189 base = s[0]
190 mods = ""
191 if len(s) > 1:
192 mods = s[1]
193 return (base, mods)
b9fc6ea9 194
60df071c
LD
195#
196# return the raw p4 type of a file (text, text+ko, etc)
197#
198def p4_type(file):
199 results = p4CmdList(["fstat", "-T", "headType", file])
200 return results[0]['headType']
201
202#
203# Given a type base and modifier, return a regexp matching
204# the keywords that can be expanded in the file
205#
206def p4_keywords_regexp_for_type(base, type_mods):
207 if base in ("text", "unicode", "binary"):
208 kwords = None
209 if "ko" in type_mods:
210 kwords = 'Id|Header'
211 elif "k" in type_mods:
212 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
213 else:
214 return None
215 pattern = r"""
216 \$ # Starts with a dollar, followed by...
217 (%s) # one of the keywords, followed by...
218 (:[^$]+)? # possibly an old expansion, followed by...
219 \$ # another dollar
220 """ % kwords
221 return pattern
222 else:
223 return None
224
225#
226# Given a file, return a regexp matching the possible
227# RCS keywords that will be expanded, or None for files
228# with kw expansion turned off.
229#
230def p4_keywords_regexp_for_file(file):
231 if not os.path.exists(file):
232 return None
233 else:
234 (type_base, type_mods) = split_p4_type(p4_type(file))
235 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 236
c65b670e
CP
237def setP4ExecBit(file, mode):
238 # Reopens an already open file and changes the execute bit to match
239 # the execute bit setting in the passed in mode.
240
241 p4Type = "+x"
242
243 if not isModeExec(mode):
244 p4Type = getP4OpenedType(file)
245 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
246 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
247 if p4Type[-1] == "+":
248 p4Type = p4Type[0:-1]
249
6de040df 250 p4_reopen(p4Type, file)
c65b670e
CP
251
252def getP4OpenedType(file):
253 # Returns the perforce file type for the given file.
254
9d7d446a 255 result = p4_read_pipe(["opened", wildcard_encode(file)])
f3e5ae4f 256 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
257 if match:
258 return match.group(1)
259 else:
f3e5ae4f 260 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 261
06804c76
LD
262# Return the set of all p4 labels
263def getP4Labels(depotPaths):
264 labels = set()
265 if isinstance(depotPaths,basestring):
266 depotPaths = [depotPaths]
267
268 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
269 label = l['label']
270 labels.add(label)
271
272 return labels
273
274# Return the set of all git tags
275def getGitTags():
276 gitTags = set()
277 for line in read_pipe_lines(["git", "tag"]):
278 tag = line.strip()
279 gitTags.add(tag)
280 return gitTags
281
b43b0a3c
CP
282def diffTreePattern():
283 # This is a simple generator for the diff tree regex pattern. This could be
284 # a class variable if this and parseDiffTreeEntry were a part of a class.
285 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
286 while True:
287 yield pattern
288
289def parseDiffTreeEntry(entry):
290 """Parses a single diff tree entry into its component elements.
291
292 See git-diff-tree(1) manpage for details about the format of the diff
293 output. This method returns a dictionary with the following elements:
294
295 src_mode - The mode of the source file
296 dst_mode - The mode of the destination file
297 src_sha1 - The sha1 for the source file
298 dst_sha1 - The sha1 fr the destination file
299 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
300 status_score - The score for the status (applicable for 'C' and 'R'
301 statuses). This is None if there is no score.
302 src - The path for the source file.
303 dst - The path for the destination file. This is only present for
304 copy or renames. If it is not present, this is None.
305
306 If the pattern is not matched, None is returned."""
307
308 match = diffTreePattern().next().match(entry)
309 if match:
310 return {
311 'src_mode': match.group(1),
312 'dst_mode': match.group(2),
313 'src_sha1': match.group(3),
314 'dst_sha1': match.group(4),
315 'status': match.group(5),
316 'status_score': match.group(6),
317 'src': match.group(7),
318 'dst': match.group(10)
319 }
320 return None
321
c65b670e
CP
322def isModeExec(mode):
323 # Returns True if the given git mode represents an executable file,
324 # otherwise False.
325 return mode[-3:] == "755"
326
327def isModeExecChanged(src_mode, dst_mode):
328 return isModeExec(src_mode) != isModeExec(dst_mode)
329
b932705b 330def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
331
332 if isinstance(cmd,basestring):
333 cmd = "-G " + cmd
334 expand = True
335 else:
336 cmd = ["-G"] + cmd
337 expand = False
338
339 cmd = p4_build_cmd(cmd)
6a49f8e2 340 if verbose:
6de040df 341 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
342
343 # Use a temporary file to avoid deadlocks without
344 # subprocess.communicate(), which would put another copy
345 # of stdout into memory.
346 stdin_file = None
347 if stdin is not None:
348 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
349 if isinstance(stdin,basestring):
350 stdin_file.write(stdin)
351 else:
352 for i in stdin:
353 stdin_file.write(i + '\n')
9f90c733
SL
354 stdin_file.flush()
355 stdin_file.seek(0)
356
6de040df
LD
357 p4 = subprocess.Popen(cmd,
358 shell=expand,
9f90c733
SL
359 stdin=stdin_file,
360 stdout=subprocess.PIPE)
86949eef
SH
361
362 result = []
363 try:
364 while True:
9f90c733 365 entry = marshal.load(p4.stdout)
c3f6163b
AG
366 if cb is not None:
367 cb(entry)
368 else:
369 result.append(entry)
86949eef
SH
370 except EOFError:
371 pass
9f90c733
SL
372 exitCode = p4.wait()
373 if exitCode != 0:
ac3e0d79
SH
374 entry = {}
375 entry["p4ExitCode"] = exitCode
376 result.append(entry)
86949eef
SH
377
378 return result
379
380def p4Cmd(cmd):
381 list = p4CmdList(cmd)
382 result = {}
383 for entry in list:
384 result.update(entry)
385 return result;
386
cb2c9db5
SH
387def p4Where(depotPath):
388 if not depotPath.endswith("/"):
389 depotPath += "/"
7f705dc3 390 depotPath = depotPath + "..."
6de040df 391 outputList = p4CmdList(["where", depotPath])
7f705dc3
TAL
392 output = None
393 for entry in outputList:
75bc9573
TAL
394 if "depotFile" in entry:
395 if entry["depotFile"] == depotPath:
396 output = entry
397 break
398 elif "data" in entry:
399 data = entry.get("data")
400 space = data.find(" ")
401 if data[:space] == depotPath:
402 output = entry
403 break
7f705dc3
TAL
404 if output == None:
405 return ""
dc524036
SH
406 if output["code"] == "error":
407 return ""
cb2c9db5
SH
408 clientPath = ""
409 if "path" in output:
410 clientPath = output.get("path")
411 elif "data" in output:
412 data = output.get("data")
413 lastSpace = data.rfind(" ")
414 clientPath = data[lastSpace + 1:]
415
416 if clientPath.endswith("..."):
417 clientPath = clientPath[:-3]
418 return clientPath
419
86949eef 420def currentGitBranch():
b25b2065 421 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 422
4f5cf76a 423def isValidGitDir(path):
bb6e09b2
HWN
424 if (os.path.exists(path + "/HEAD")
425 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
426 return True;
427 return False
428
463e8af6 429def parseRevision(ref):
b25b2065 430 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 431
28755dba
PW
432def branchExists(ref):
433 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
434 ignore_error=True)
435 return len(rev) > 0
436
6ae8de88
SH
437def extractLogMessageFromGitCommit(commit):
438 logMessage = ""
b016d397
HWN
439
440 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 441 foundTitle = False
b016d397 442 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
443 if not foundTitle:
444 if len(log) == 1:
1c094184 445 foundTitle = True
6ae8de88
SH
446 continue
447
448 logMessage += log
449 return logMessage
450
bb6e09b2 451def extractSettingsGitLog(log):
6ae8de88
SH
452 values = {}
453 for line in log.split("\n"):
454 line = line.strip()
6326aa58
HWN
455 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
456 if not m:
457 continue
458
459 assignments = m.group(1).split (':')
460 for a in assignments:
461 vals = a.split ('=')
462 key = vals[0].strip()
463 val = ('='.join (vals[1:])).strip()
464 if val.endswith ('\"') and val.startswith('"'):
465 val = val[1:-1]
466
467 values[key] = val
468
845b42cb
SH
469 paths = values.get("depot-paths")
470 if not paths:
471 paths = values.get("depot-path")
a3fdd579
SH
472 if paths:
473 values['depot-paths'] = paths.split(',')
bb6e09b2 474 return values
6ae8de88 475
8136a639 476def gitBranchExists(branch):
bb6e09b2
HWN
477 proc = subprocess.Popen(["git", "rev-parse", branch],
478 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 479 return proc.wait() == 0;
8136a639 480
36bd8446 481_gitConfig = {}
99f790f2 482def gitConfig(key, args = None): # set args to "--bool", for instance
36bd8446 483 if not _gitConfig.has_key(key):
99f790f2
TAL
484 argsFilter = ""
485 if args != None:
486 argsFilter = "%s " % args
487 cmd = "git config %s%s" % (argsFilter, key)
488 _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
36bd8446 489 return _gitConfig[key]
01265103 490
7199cf13
VA
491def gitConfigList(key):
492 if not _gitConfig.has_key(key):
493 _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
494 return _gitConfig[key]
495
062410bb
SH
496def p4BranchesInGit(branchesAreInRemotes = True):
497 branches = {}
498
499 cmdline = "git rev-parse --symbolic "
500 if branchesAreInRemotes:
501 cmdline += " --remotes"
502 else:
503 cmdline += " --branches"
504
505 for line in read_pipe_lines(cmdline):
506 line = line.strip()
507
508 ## only import to p4/
509 if not line.startswith('p4/') or line == "p4/HEAD":
510 continue
511 branch = line
512
513 # strip off p4
514 branch = re.sub ("^p4/", "", line)
515
516 branches[branch] = parseRevision(line)
517 return branches
518
9ceab363 519def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
520 branches = p4BranchesInGit()
521 # map from depot-path to branch name
522 branchByDepotPath = {}
523 for branch in branches.keys():
524 tip = branches[branch]
525 log = extractLogMessageFromGitCommit(tip)
526 settings = extractSettingsGitLog(log)
527 if settings.has_key("depot-paths"):
528 paths = ",".join(settings["depot-paths"])
529 branchByDepotPath[paths] = "remotes/p4/" + branch
530
27d2d811 531 settings = None
27d2d811
SH
532 parent = 0
533 while parent < 65535:
9ceab363 534 commit = head + "~%s" % parent
27d2d811
SH
535 log = extractLogMessageFromGitCommit(commit)
536 settings = extractSettingsGitLog(log)
86506fe5
SH
537 if settings.has_key("depot-paths"):
538 paths = ",".join(settings["depot-paths"])
539 if branchByDepotPath.has_key(paths):
540 return [branchByDepotPath[paths], settings]
27d2d811 541
86506fe5 542 parent = parent + 1
27d2d811 543
86506fe5 544 return ["", settings]
27d2d811 545
5ca44617
SH
546def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
547 if not silent:
548 print ("Creating/updating branch(es) in %s based on origin branch(es)"
549 % localRefPrefix)
550
551 originPrefix = "origin/p4/"
552
553 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
554 line = line.strip()
555 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
556 continue
557
558 headName = line[len(originPrefix):]
559 remoteHead = localRefPrefix + headName
560 originHead = line
561
562 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
563 if (not original.has_key('depot-paths')
564 or not original.has_key('change')):
565 continue
566
567 update = False
568 if not gitBranchExists(remoteHead):
569 if verbose:
570 print "creating %s" % remoteHead
571 update = True
572 else:
573 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
574 if settings.has_key('change') > 0:
575 if settings['depot-paths'] == original['depot-paths']:
576 originP4Change = int(original['change'])
577 p4Change = int(settings['change'])
578 if originP4Change > p4Change:
579 print ("%s (%s) is newer than %s (%s). "
580 "Updating p4 branch from origin."
581 % (originHead, originP4Change,
582 remoteHead, p4Change))
583 update = True
584 else:
585 print ("Ignoring: %s was imported from %s while "
586 "%s was imported from %s"
587 % (originHead, ','.join(original['depot-paths']),
588 remoteHead, ','.join(settings['depot-paths'])))
589
590 if update:
591 system("git update-ref %s %s" % (remoteHead, originHead))
592
593def originP4BranchesExist():
594 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
595
4f6432d8
SH
596def p4ChangesForPaths(depotPaths, changeRange):
597 assert depotPaths
6de040df
LD
598 cmd = ['changes']
599 for p in depotPaths:
600 cmd += ["%s...%s" % (p, changeRange)]
601 output = p4_read_pipe_lines(cmd)
4f6432d8 602
b4b0ba06 603 changes = {}
4f6432d8 604 for line in output:
c3f6163b
AG
605 changeNum = int(line.split(" ")[1])
606 changes[changeNum] = True
4f6432d8 607
b4b0ba06
PW
608 changelist = changes.keys()
609 changelist.sort()
610 return changelist
4f6432d8 611
d53de8b9
TAL
612def p4PathStartsWith(path, prefix):
613 # This method tries to remedy a potential mixed-case issue:
614 #
615 # If UserA adds //depot/DirA/file1
616 # and UserB adds //depot/dira/file2
617 #
618 # we may or may not have a problem. If you have core.ignorecase=true,
619 # we treat DirA and dira as the same directory
620 ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
621 if ignorecase:
622 return path.lower().startswith(prefix.lower())
623 return path.startswith(prefix)
624
543987bd
PW
625def getClientSpec():
626 """Look at the p4 client spec, create a View() object that contains
627 all the mappings, and return it."""
628
629 specList = p4CmdList("client -o")
630 if len(specList) != 1:
631 die('Output from "client -o" is %d lines, expecting 1' %
632 len(specList))
633
634 # dictionary of all client parameters
635 entry = specList[0]
636
637 # just the keys that start with "View"
638 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
639
640 # hold this new View
641 view = View()
642
643 # append the lines, in order, to the view
644 for view_num in range(len(view_keys)):
645 k = "View%d" % view_num
646 if k not in view_keys:
647 die("Expected view key %s missing" % k)
648 view.append(entry[k])
649
650 return view
651
652def getClientRoot():
653 """Grab the client directory."""
654
655 output = p4CmdList("client -o")
656 if len(output) != 1:
657 die('Output from "client -o" is %d lines, expecting 1' % len(output))
658
659 entry = output[0]
660 if "Root" not in entry:
661 die('Client has no "Root"')
662
663 return entry["Root"]
664
9d7d446a
PW
665#
666# P4 wildcards are not allowed in filenames. P4 complains
667# if you simply add them, but you can force it with "-f", in
668# which case it translates them into %xx encoding internally.
669#
670def wildcard_decode(path):
671 # Search for and fix just these four characters. Do % last so
672 # that fixing it does not inadvertently create new %-escapes.
673 # Cannot have * in a filename in windows; untested as to
674 # what p4 would do in such a case.
675 if not platform.system() == "Windows":
676 path = path.replace("%2A", "*")
677 path = path.replace("%23", "#") \
678 .replace("%40", "@") \
679 .replace("%25", "%")
680 return path
681
682def wildcard_encode(path):
683 # do % first to avoid double-encoding the %s introduced here
684 path = path.replace("%", "%25") \
685 .replace("*", "%2A") \
686 .replace("#", "%23") \
687 .replace("@", "%40")
688 return path
689
690def wildcard_present(path):
691 return path.translate(None, "*#@%") != path
692
b984733c
SH
693class Command:
694 def __init__(self):
695 self.usage = "usage: %prog [options]"
8910ac0e 696 self.needsGit = True
6a10b6aa 697 self.verbose = False
b984733c 698
3ea2cfd4
LD
699class P4UserMap:
700 def __init__(self):
701 self.userMapFromPerforceServer = False
affb474f
LD
702 self.myP4UserId = None
703
704 def p4UserId(self):
705 if self.myP4UserId:
706 return self.myP4UserId
707
708 results = p4CmdList("user -o")
709 for r in results:
710 if r.has_key('User'):
711 self.myP4UserId = r['User']
712 return r['User']
713 die("Could not find your p4 user id")
714
715 def p4UserIsMe(self, p4User):
716 # return True if the given p4 user is actually me
717 me = self.p4UserId()
718 if not p4User or p4User != me:
719 return False
720 else:
721 return True
3ea2cfd4
LD
722
723 def getUserCacheFilename(self):
724 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
725 return home + "/.gitp4-usercache.txt"
726
727 def getUserMapFromPerforceServer(self):
728 if self.userMapFromPerforceServer:
729 return
730 self.users = {}
731 self.emails = {}
732
733 for output in p4CmdList("users"):
734 if not output.has_key("User"):
735 continue
736 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
737 self.emails[output["Email"]] = output["User"]
738
739
740 s = ''
741 for (key, val) in self.users.items():
742 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
743
744 open(self.getUserCacheFilename(), "wb").write(s)
745 self.userMapFromPerforceServer = True
746
747 def loadUserMapFromCache(self):
748 self.users = {}
749 self.userMapFromPerforceServer = False
750 try:
751 cache = open(self.getUserCacheFilename(), "rb")
752 lines = cache.readlines()
753 cache.close()
754 for line in lines:
755 entry = line.strip().split("\t")
756 self.users[entry[0]] = entry[1]
757 except IOError:
758 self.getUserMapFromPerforceServer()
759
b984733c 760class P4Debug(Command):
86949eef 761 def __init__(self):
6ae8de88 762 Command.__init__(self)
6a10b6aa 763 self.options = []
c8c39116 764 self.description = "A tool to debug the output of p4 -G."
8910ac0e 765 self.needsGit = False
86949eef
SH
766
767 def run(self, args):
b1ce9447 768 j = 0
6de040df 769 for output in p4CmdList(args):
b1ce9447
HWN
770 print 'Element: %d' % j
771 j += 1
86949eef 772 print output
b984733c 773 return True
86949eef 774
5834684d
SH
775class P4RollBack(Command):
776 def __init__(self):
777 Command.__init__(self)
778 self.options = [
0c66a783 779 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
780 ]
781 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 782 self.rollbackLocalBranches = False
5834684d
SH
783
784 def run(self, args):
785 if len(args) != 1:
786 return False
787 maxChange = int(args[0])
0c66a783 788
ad192f28 789 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
790 die("Problems executing p4");
791
0c66a783
SH
792 if self.rollbackLocalBranches:
793 refPrefix = "refs/heads/"
b016d397 794 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
795 else:
796 refPrefix = "refs/remotes/"
b016d397 797 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
798
799 for line in lines:
800 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
801 line = line.strip()
802 ref = refPrefix + line
5834684d 803 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
804 settings = extractSettingsGitLog(log)
805
806 depotPaths = settings['depot-paths']
807 change = settings['change']
808
5834684d 809 changed = False
52102d47 810
6326aa58
HWN
811 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
812 for p in depotPaths]))) == 0:
52102d47
SH
813 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
814 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
815 continue
816
bb6e09b2 817 while change and int(change) > maxChange:
5834684d 818 changed = True
52102d47
SH
819 if self.verbose:
820 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
821 system("git update-ref %s \"%s^\"" % (ref, ref))
822 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
823 settings = extractSettingsGitLog(log)
824
825
826 depotPaths = settings['depot-paths']
827 change = settings['change']
5834684d
SH
828
829 if changed:
52102d47 830 print "%s rewound to %s" % (ref, change)
5834684d
SH
831
832 return True
833
3ea2cfd4 834class P4Submit(Command, P4UserMap):
4f5cf76a 835 def __init__(self):
b984733c 836 Command.__init__(self)
3ea2cfd4 837 P4UserMap.__init__(self)
4f5cf76a 838 self.options = [
4f5cf76a 839 optparse.make_option("--origin", dest="origin"),
ae901090 840 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
841 # preserve the user, requires relevant p4 permissions
842 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 843 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
4f5cf76a
SH
844 ]
845 self.description = "Submit changes from git to the perforce depot."
c9b50e63 846 self.usage += " [name of git branch to submit into perforce depot]"
4f5cf76a 847 self.interactive = True
9512497b 848 self.origin = ""
ae901090 849 self.detectRenames = False
3ea2cfd4 850 self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
f7baba8b 851 self.isWindows = (platform.system() == "Windows")
06804c76 852 self.exportLabels = False
4f5cf76a 853
4f5cf76a
SH
854 def check(self):
855 if len(p4CmdList("opened ...")) > 0:
856 die("You have files opened with perforce! Close them before starting the sync.")
857
edae1e2f
SH
858 # replaces everything between 'Description:' and the next P4 submit template field with the
859 # commit message
4f5cf76a
SH
860 def prepareLogMessage(self, template, message):
861 result = ""
862
edae1e2f
SH
863 inDescriptionSection = False
864
4f5cf76a
SH
865 for line in template.split("\n"):
866 if line.startswith("#"):
867 result += line + "\n"
868 continue
869
edae1e2f 870 if inDescriptionSection:
c9dbab04 871 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f
SH
872 inDescriptionSection = False
873 else:
874 continue
875 else:
876 if line.startswith("Description:"):
877 inDescriptionSection = True
878 line += "\n"
879 for messageLine in message.split("\n"):
880 line += "\t" + messageLine + "\n"
881
882 result += line + "\n"
4f5cf76a
SH
883
884 return result
885
60df071c
LD
886 def patchRCSKeywords(self, file, pattern):
887 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
888 (handle, outFileName) = tempfile.mkstemp(dir='.')
889 try:
890 outFile = os.fdopen(handle, "w+")
891 inFile = open(file, "r")
892 regexp = re.compile(pattern, re.VERBOSE)
893 for line in inFile.readlines():
894 line = regexp.sub(r'$\1$', line)
895 outFile.write(line)
896 inFile.close()
897 outFile.close()
898 # Forcibly overwrite the original file
899 os.unlink(file)
900 shutil.move(outFileName, file)
901 except:
902 # cleanup our temporary file
903 os.unlink(outFileName)
904 print "Failed to strip RCS keywords in %s" % file
905 raise
906
907 print "Patched up RCS keywords in %s" % file
908
3ea2cfd4
LD
909 def p4UserForCommit(self,id):
910 # Return the tuple (perforce user,git email) for a given git commit id
911 self.getUserMapFromPerforceServer()
912 gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
913 gitEmail = gitEmail.strip()
914 if not self.emails.has_key(gitEmail):
915 return (None,gitEmail)
916 else:
917 return (self.emails[gitEmail],gitEmail)
918
919 def checkValidP4Users(self,commits):
920 # check if any git authors cannot be mapped to p4 users
921 for id in commits:
922 (user,email) = self.p4UserForCommit(id)
923 if not user:
924 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
925 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
926 print "%s" % msg
927 else:
928 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
929
930 def lastP4Changelist(self):
931 # Get back the last changelist number submitted in this client spec. This
932 # then gets used to patch up the username in the change. If the same
933 # client spec is being used by multiple processes then this might go
934 # wrong.
935 results = p4CmdList("client -o") # find the current client
936 client = None
937 for r in results:
938 if r.has_key('Client'):
939 client = r['Client']
940 break
941 if not client:
942 die("could not get client spec")
6de040df 943 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
944 for r in results:
945 if r.has_key('change'):
946 return r['change']
947 die("Could not get changelist number for last submit - cannot patch up user details")
948
949 def modifyChangelistUser(self, changelist, newUser):
950 # fixup the user field of a changelist after it has been submitted.
951 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
952 if len(changes) != 1:
953 die("Bad output from p4 change modifying %s to user %s" %
954 (changelist, newUser))
955
956 c = changes[0]
957 if c['User'] == newUser: return # nothing to do
958 c['User'] = newUser
959 input = marshal.dumps(c)
960
3ea2cfd4
LD
961 result = p4CmdList("change -f -i", stdin=input)
962 for r in result:
963 if r.has_key('code'):
964 if r['code'] == 'error':
965 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
966 if r.has_key('data'):
967 print("Updated user field for changelist %s to %s" % (changelist, newUser))
968 return
969 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
970
971 def canChangeChangelists(self):
972 # check to see if we have p4 admin or super-user permissions, either of
973 # which are required to modify changelists.
52a4880b 974 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
975 for r in results:
976 if r.has_key('perm'):
977 if r['perm'] == 'admin':
978 return 1
979 if r['perm'] == 'super':
980 return 1
981 return 0
982
ea99c3ae
SH
983 def prepareSubmitTemplate(self):
984 # remove lines in the Files section that show changes to files outside the depot path we're committing into
985 template = ""
986 inFilesSection = False
6de040df 987 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
988 if line.endswith("\r\n"):
989 line = line[:-2] + "\n"
ea99c3ae
SH
990 if inFilesSection:
991 if line.startswith("\t"):
992 # path starts and ends with a tab
993 path = line[1:]
994 lastTab = path.rfind("\t")
995 if lastTab != -1:
996 path = path[:lastTab]
d53de8b9 997 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
998 continue
999 else:
1000 inFilesSection = False
1001 else:
1002 if line.startswith("Files:"):
1003 inFilesSection = True
1004
1005 template += line
1006
1007 return template
1008
7c766e57
PW
1009 def edit_template(self, template_file):
1010 """Invoke the editor to let the user change the submission
1011 message. Return true if okay to continue with the submit."""
1012
1013 # if configured to skip the editing part, just submit
1014 if gitConfig("git-p4.skipSubmitEdit") == "true":
1015 return True
1016
1017 # look at the modification time, to check later if the user saved
1018 # the file
1019 mtime = os.stat(template_file).st_mtime
1020
1021 # invoke the editor
f95ceaf0 1022 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1023 editor = os.environ.get("P4EDITOR")
1024 else:
1025 editor = read_pipe("git var GIT_EDITOR").strip()
1026 system(editor + " " + template_file)
1027
1028 # If the file was not saved, prompt to see if this patch should
1029 # be skipped. But skip this verification step if configured so.
1030 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1031 return True
1032
d1652049
PW
1033 # modification time updated means user saved the file
1034 if os.stat(template_file).st_mtime > mtime:
1035 return True
1036
1037 while True:
1038 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1039 if response == 'y':
1040 return True
1041 if response == 'n':
1042 return False
7c766e57 1043
7cb5cbef 1044 def applyCommit(self, id):
0e36f2d7 1045 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
ae901090 1046
848de9c3 1047 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1048
ae901090
VA
1049 if not self.detectRenames:
1050 # If not explicitly set check the config variable
0a9feffc 1051 self.detectRenames = gitConfig("git-p4.detectRenames")
ae901090 1052
0a9feffc
VA
1053 if self.detectRenames.lower() == "false" or self.detectRenames == "":
1054 diffOpts = ""
1055 elif self.detectRenames.lower() == "true":
ae901090
VA
1056 diffOpts = "-M"
1057 else:
0a9feffc 1058 diffOpts = "-M%s" % self.detectRenames
ae901090 1059
0a9feffc
VA
1060 detectCopies = gitConfig("git-p4.detectCopies")
1061 if detectCopies.lower() == "true":
4fddb41b 1062 diffOpts += " -C"
0a9feffc
VA
1063 elif detectCopies != "" and detectCopies.lower() != "false":
1064 diffOpts += " -C%s" % detectCopies
4fddb41b 1065
68cbcf1b 1066 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
4fddb41b
VA
1067 diffOpts += " --find-copies-harder"
1068
0e36f2d7 1069 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
4f5cf76a
SH
1070 filesToAdd = set()
1071 filesToDelete = set()
d336c158 1072 editedFiles = set()
b6ad6dcc 1073 pureRenameCopy = set()
c65b670e 1074 filesToChangeExecBit = {}
60df071c 1075
4f5cf76a 1076 for line in diff:
b43b0a3c
CP
1077 diff = parseDiffTreeEntry(line)
1078 modifier = diff['status']
1079 path = diff['src']
4f5cf76a 1080 if modifier == "M":
6de040df 1081 p4_edit(path)
c65b670e
CP
1082 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1083 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1084 editedFiles.add(path)
4f5cf76a
SH
1085 elif modifier == "A":
1086 filesToAdd.add(path)
c65b670e 1087 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1088 if path in filesToDelete:
1089 filesToDelete.remove(path)
1090 elif modifier == "D":
1091 filesToDelete.add(path)
1092 if path in filesToAdd:
1093 filesToAdd.remove(path)
4fddb41b
VA
1094 elif modifier == "C":
1095 src, dest = diff['src'], diff['dst']
6de040df 1096 p4_integrate(src, dest)
b6ad6dcc 1097 pureRenameCopy.add(dest)
4fddb41b 1098 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1099 p4_edit(dest)
b6ad6dcc 1100 pureRenameCopy.discard(dest)
4fddb41b 1101 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1102 p4_edit(dest)
b6ad6dcc 1103 pureRenameCopy.discard(dest)
4fddb41b
VA
1104 filesToChangeExecBit[dest] = diff['dst_mode']
1105 os.unlink(dest)
1106 editedFiles.add(dest)
d9a5f25b 1107 elif modifier == "R":
b43b0a3c 1108 src, dest = diff['src'], diff['dst']
6de040df 1109 p4_integrate(src, dest)
ae901090 1110 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1111 p4_edit(dest)
b6ad6dcc
PW
1112 else:
1113 pureRenameCopy.add(dest)
c65b670e 1114 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1115 p4_edit(dest)
c65b670e 1116 filesToChangeExecBit[dest] = diff['dst_mode']
d9a5f25b
CP
1117 os.unlink(dest)
1118 editedFiles.add(dest)
1119 filesToDelete.add(src)
4f5cf76a
SH
1120 else:
1121 die("unknown modifier %s for %s" % (modifier, path))
1122
0e36f2d7 1123 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 1124 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1125 tryPatchCmd = patchcmd + "--check -"
1126 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1127 patch_succeeded = True
51a2640a 1128
47a130b7 1129 if os.system(tryPatchCmd) != 0:
60df071c
LD
1130 fixed_rcs_keywords = False
1131 patch_succeeded = False
51a2640a 1132 print "Unfortunately applying the change failed!"
60df071c
LD
1133
1134 # Patch failed, maybe it's just RCS keyword woes. Look through
1135 # the patch to see if that's possible.
1136 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1137 file = None
1138 pattern = None
1139 kwfiles = {}
1140 for file in editedFiles | filesToDelete:
1141 # did this file's delta contain RCS keywords?
1142 pattern = p4_keywords_regexp_for_file(file)
1143
1144 if pattern:
1145 # this file is a possibility...look for RCS keywords.
1146 regexp = re.compile(pattern, re.VERBOSE)
1147 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1148 if regexp.search(line):
1149 if verbose:
1150 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1151 kwfiles[file] = pattern
1152 break
1153
1154 for file in kwfiles:
1155 if verbose:
1156 print "zapping %s with %s" % (line,pattern)
1157 self.patchRCSKeywords(file, kwfiles[file])
1158 fixed_rcs_keywords = True
1159
1160 if fixed_rcs_keywords:
1161 print "Retrying the patch with RCS keywords cleaned up"
1162 if os.system(tryPatchCmd) == 0:
1163 patch_succeeded = True
1164
1165 if not patch_succeeded:
51a2640a
SH
1166 print "What do you want to do?"
1167 response = "x"
1168 while response != "s" and response != "a" and response != "w":
cebdf5af
HWN
1169 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1170 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
51a2640a
SH
1171 if response == "s":
1172 print "Skipping! Good luck with the next patches..."
20947149 1173 for f in editedFiles:
6de040df 1174 p4_revert(f)
20947149 1175 for f in filesToAdd:
6de040df 1176 os.remove(f)
51a2640a
SH
1177 return
1178 elif response == "a":
47a130b7 1179 os.system(applyPatchCmd)
51a2640a
SH
1180 if len(filesToAdd) > 0:
1181 print "You may also want to call p4 add on the following files:"
1182 print " ".join(filesToAdd)
1183 if len(filesToDelete):
1184 print "The following files should be scheduled for deletion with p4 delete:"
1185 print " ".join(filesToDelete)
cebdf5af 1186 die("Please resolve and submit the conflict manually and "
9dcb9f24 1187 + "continue afterwards with git p4 submit --continue")
51a2640a
SH
1188 elif response == "w":
1189 system(diffcmd + " > patch.txt")
1190 print "Patch saved to patch.txt in %s !" % self.clientPath
cebdf5af 1191 die("Please resolve and submit the conflict manually and "
9dcb9f24 1192 "continue afterwards with git p4 submit --continue")
51a2640a 1193
47a130b7 1194 system(applyPatchCmd)
4f5cf76a
SH
1195
1196 for f in filesToAdd:
6de040df 1197 p4_add(f)
4f5cf76a 1198 for f in filesToDelete:
6de040df
LD
1199 p4_revert(f)
1200 p4_delete(f)
4f5cf76a 1201
c65b670e
CP
1202 # Set/clear executable bits
1203 for f in filesToChangeExecBit.keys():
1204 mode = filesToChangeExecBit[f]
1205 setP4ExecBit(f, mode)
1206
0e36f2d7 1207 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1208 logMessage = logMessage.strip()
4f5cf76a 1209
ea99c3ae 1210 template = self.prepareSubmitTemplate()
4f5cf76a
SH
1211
1212 if self.interactive:
1213 submitTemplate = self.prepareLogMessage(template, logMessage)
ecdba36d
LD
1214
1215 if self.preserveUser:
1216 submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1217
67abd417
SB
1218 if os.environ.has_key("P4DIFF"):
1219 del(os.environ["P4DIFF"])
8b130262
AW
1220 diff = ""
1221 for editedFile in editedFiles:
9d7d446a
PW
1222 diff += p4_read_pipe(['diff', '-du',
1223 wildcard_encode(editedFile)])
4f5cf76a 1224
f3e5ae4f 1225 newdiff = ""
4f5cf76a 1226 for newFile in filesToAdd:
f3e5ae4f
MSO
1227 newdiff += "==== new file ====\n"
1228 newdiff += "--- /dev/null\n"
1229 newdiff += "+++ %s\n" % newFile
4f5cf76a
SH
1230 f = open(newFile, "r")
1231 for line in f.readlines():
f3e5ae4f 1232 newdiff += "+" + line
4f5cf76a
SH
1233 f.close()
1234
848de9c3
LD
1235 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1236 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
9dcb9f24
PW
1237 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1238 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
848de9c3 1239
f3e5ae4f 1240 separatorLine = "######## everything below this line is just the diff #######\n"
4f5cf76a 1241
7c766e57 1242 (handle, fileName) = tempfile.mkstemp()
e96e400f 1243 tmpFile = os.fdopen(handle, "w+")
f3e5ae4f
MSO
1244 if self.isWindows:
1245 submitTemplate = submitTemplate.replace("\n", "\r\n")
1246 separatorLine = separatorLine.replace("\n", "\r\n")
1247 newdiff = newdiff.replace("\n", "\r\n")
1248 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
e96e400f 1249 tmpFile.close()
e96e400f 1250
7c766e57
PW
1251 if self.edit_template(fileName):
1252 # read the edited message and submit
cdc7e388
SH
1253 tmpFile = open(fileName, "rb")
1254 message = tmpFile.read()
1255 tmpFile.close()
1256 submitTemplate = message[:message.index(separatorLine)]
1257 if self.isWindows:
1258 submitTemplate = submitTemplate.replace("\r\n", "\n")
6de040df 1259 p4_write_pipe(['submit', '-i'], submitTemplate)
3ea2cfd4
LD
1260
1261 if self.preserveUser:
1262 if p4User:
1263 # Get last changelist number. Cannot easily get it from
7c766e57
PW
1264 # the submit command output as the output is
1265 # unmarshalled.
3ea2cfd4
LD
1266 changelist = self.lastP4Changelist()
1267 self.modifyChangelistUser(changelist, p4User)
b6ad6dcc
PW
1268
1269 # The rename/copy happened by applying a patch that created a
1270 # new file. This leaves it writable, which confuses p4.
1271 for f in pureRenameCopy:
1272 p4_sync(f, "-f")
1273
cdc7e388 1274 else:
7c766e57 1275 # skip this patch
d1652049 1276 print "Submission cancelled, undoing p4 changes."
cdc7e388 1277 for f in editedFiles:
6de040df 1278 p4_revert(f)
cdc7e388 1279 for f in filesToAdd:
6de040df
LD
1280 p4_revert(f)
1281 os.remove(f)
cdc7e388
SH
1282
1283 os.remove(fileName)
4f5cf76a
SH
1284 else:
1285 fileName = "submit.txt"
1286 file = open(fileName, "w+")
1287 file.write(self.prepareLogMessage(template, logMessage))
1288 file.close()
cebdf5af
HWN
1289 print ("Perforce submit template written as %s. "
1290 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1291 % (fileName, fileName))
4f5cf76a 1292
06804c76
LD
1293 # Export git tags as p4 labels. Create a p4 label and then tag
1294 # with that.
1295 def exportGitTags(self, gitTags):
c8942a22
LD
1296 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1297 if len(validLabelRegexp) == 0:
1298 validLabelRegexp = defaultLabelRegexp
1299 m = re.compile(validLabelRegexp)
06804c76
LD
1300
1301 for name in gitTags:
1302
1303 if not m.match(name):
1304 if verbose:
05a3cec5 1305 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1306 continue
1307
1308 # Get the p4 commit this corresponds to
c8942a22
LD
1309 logMessage = extractLogMessageFromGitCommit(name)
1310 values = extractSettingsGitLog(logMessage)
06804c76 1311
c8942a22 1312 if not values.has_key('change'):
06804c76
LD
1313 # a tag pointing to something not sent to p4; ignore
1314 if verbose:
1315 print "git tag %s does not give a p4 commit" % name
1316 continue
c8942a22
LD
1317 else:
1318 changelist = values['change']
06804c76
LD
1319
1320 # Get the tag details.
1321 inHeader = True
1322 isAnnotated = False
1323 body = []
1324 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1325 l = l.strip()
1326 if inHeader:
1327 if re.match(r'tag\s+', l):
1328 isAnnotated = True
1329 elif re.match(r'\s*$', l):
1330 inHeader = False
1331 continue
1332 else:
1333 body.append(l)
1334
1335 if not isAnnotated:
1336 body = ["lightweight tag imported by git p4\n"]
1337
1338 # Create the label - use the same view as the client spec we are using
1339 clientSpec = getClientSpec()
1340
1341 labelTemplate = "Label: %s\n" % name
1342 labelTemplate += "Description:\n"
1343 for b in body:
1344 labelTemplate += "\t" + b + "\n"
1345 labelTemplate += "View:\n"
1346 for mapping in clientSpec.mappings:
1347 labelTemplate += "\t%s\n" % mapping.depot_side.path
1348
1349 p4_write_pipe(["label", "-i"], labelTemplate)
1350
1351 # Use the label
1352 p4_system(["tag", "-l", name] +
1353 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1354
1355 if verbose:
1356 print "created p4 label for tag %s" % name
1357
4f5cf76a 1358 def run(self, args):
c9b50e63
SH
1359 if len(args) == 0:
1360 self.master = currentGitBranch()
4280e533 1361 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1362 die("Detecting current git branch failed!")
1363 elif len(args) == 1:
1364 self.master = args[0]
28755dba
PW
1365 if not branchExists(self.master):
1366 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1367 else:
1368 return False
1369
4c2d5d72
JX
1370 allowSubmit = gitConfig("git-p4.allowSubmit")
1371 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1372 die("%s is not in git-p4.allowSubmit" % self.master)
1373
27d2d811 1374 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1375 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1376 if len(self.origin) == 0:
1377 self.origin = upstream
a3fdd579 1378
3ea2cfd4
LD
1379 if self.preserveUser:
1380 if not self.canChangeChangelists():
1381 die("Cannot preserve user names without p4 super-user or admin permissions")
1382
a3fdd579
SH
1383 if self.verbose:
1384 print "Origin branch is " + self.origin
9512497b 1385
ea99c3ae 1386 if len(self.depotPath) == 0:
9512497b
SH
1387 print "Internal error: cannot locate perforce depot path from existing branches"
1388 sys.exit(128)
1389
543987bd
PW
1390 self.useClientSpec = False
1391 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1392 self.useClientSpec = True
1393 if self.useClientSpec:
1394 self.clientSpecDirs = getClientSpec()
9512497b 1395
543987bd
PW
1396 if self.useClientSpec:
1397 # all files are relative to the client spec
1398 self.clientPath = getClientRoot()
1399 else:
1400 self.clientPath = p4Where(self.depotPath)
9512497b 1401
543987bd
PW
1402 if self.clientPath == "":
1403 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1404
ea99c3ae 1405 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1406 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1407
0591cfa8 1408 # ensure the clientPath exists
8d7ec362 1409 new_client_dir = False
0591cfa8 1410 if not os.path.exists(self.clientPath):
8d7ec362 1411 new_client_dir = True
0591cfa8
GG
1412 os.makedirs(self.clientPath)
1413
053fd0c1 1414 chdir(self.clientPath)
6a01298a 1415 print "Synchronizing p4 checkout..."
8d7ec362
PW
1416 if new_client_dir:
1417 # old one was destroyed, and maybe nobody told p4
1418 p4_sync("...", "-f")
1419 else:
1420 p4_sync("...")
4f5cf76a 1421 self.check()
4f5cf76a 1422
4c750c0d
SH
1423 commits = []
1424 for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1425 commits.append(line.strip())
1426 commits.reverse()
4f5cf76a 1427
848de9c3
LD
1428 if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1429 self.checkAuthorship = False
1430 else:
1431 self.checkAuthorship = True
1432
3ea2cfd4
LD
1433 if self.preserveUser:
1434 self.checkValidP4Users(commits)
1435
4f5cf76a 1436 while len(commits) > 0:
4f5cf76a
SH
1437 commit = commits[0]
1438 commits = commits[1:]
7cb5cbef 1439 self.applyCommit(commit)
4f5cf76a
SH
1440 if not self.interactive:
1441 break
1442
4f5cf76a 1443 if len(commits) == 0:
4c750c0d 1444 print "All changes applied!"
053fd0c1 1445 chdir(self.oldWorkingDirectory)
14594f4b 1446
4c750c0d
SH
1447 sync = P4Sync()
1448 sync.run([])
14594f4b 1449
4c750c0d
SH
1450 rebase = P4Rebase()
1451 rebase.rebase()
4f5cf76a 1452
06804c76 1453 if gitConfig("git-p4.exportLabels", "--bool") == "true":
06dcd152 1454 self.exportLabels = True
06804c76
LD
1455
1456 if self.exportLabels:
1457 p4Labels = getP4Labels(self.depotPath)
1458 gitTags = getGitTags()
1459
1460 missingGitTags = gitTags - p4Labels
1461 self.exportGitTags(missingGitTags)
1462
b984733c
SH
1463 return True
1464
ecb7cf98
PW
1465class View(object):
1466 """Represent a p4 view ("p4 help views"), and map files in a
1467 repo according to the view."""
1468
1469 class Path(object):
1470 """A depot or client path, possibly containing wildcards.
1471 The only one supported is ... at the end, currently.
1472 Initialize with the full path, with //depot or //client."""
1473
1474 def __init__(self, path, is_depot):
1475 self.path = path
1476 self.is_depot = is_depot
1477 self.find_wildcards()
1478 # remember the prefix bit, useful for relative mappings
1479 m = re.match("(//[^/]+/)", self.path)
1480 if not m:
1481 die("Path %s does not start with //prefix/" % self.path)
1482 prefix = m.group(1)
1483 if not self.is_depot:
1484 # strip //client/ on client paths
1485 self.path = self.path[len(prefix):]
1486
1487 def find_wildcards(self):
1488 """Make sure wildcards are valid, and set up internal
1489 variables."""
1490
1491 self.ends_triple_dot = False
1492 # There are three wildcards allowed in p4 views
1493 # (see "p4 help views"). This code knows how to
1494 # handle "..." (only at the end), but cannot deal with
1495 # "%%n" or "*". Only check the depot_side, as p4 should
1496 # validate that the client_side matches too.
1497 if re.search(r'%%[1-9]', self.path):
1498 die("Can't handle %%n wildcards in view: %s" % self.path)
1499 if self.path.find("*") >= 0:
1500 die("Can't handle * wildcards in view: %s" % self.path)
1501 triple_dot_index = self.path.find("...")
1502 if triple_dot_index >= 0:
43b82bd9
PW
1503 if triple_dot_index != len(self.path) - 3:
1504 die("Can handle only single ... wildcard, at end: %s" %
ecb7cf98
PW
1505 self.path)
1506 self.ends_triple_dot = True
1507
1508 def ensure_compatible(self, other_path):
1509 """Make sure the wildcards agree."""
1510 if self.ends_triple_dot != other_path.ends_triple_dot:
1511 die("Both paths must end with ... if either does;\n" +
1512 "paths: %s %s" % (self.path, other_path.path))
1513
1514 def match_wildcards(self, test_path):
1515 """See if this test_path matches us, and fill in the value
1516 of the wildcards if so. Returns a tuple of
1517 (True|False, wildcards[]). For now, only the ... at end
1518 is supported, so at most one wildcard."""
1519 if self.ends_triple_dot:
1520 dotless = self.path[:-3]
1521 if test_path.startswith(dotless):
1522 wildcard = test_path[len(dotless):]
1523 return (True, [ wildcard ])
1524 else:
1525 if test_path == self.path:
1526 return (True, [])
1527 return (False, [])
1528
1529 def match(self, test_path):
1530 """Just return if it matches; don't bother with the wildcards."""
1531 b, _ = self.match_wildcards(test_path)
1532 return b
1533
1534 def fill_in_wildcards(self, wildcards):
1535 """Return the relative path, with the wildcards filled in
1536 if there are any."""
1537 if self.ends_triple_dot:
1538 return self.path[:-3] + wildcards[0]
1539 else:
1540 return self.path
1541
1542 class Mapping(object):
1543 def __init__(self, depot_side, client_side, overlay, exclude):
1544 # depot_side is without the trailing /... if it had one
1545 self.depot_side = View.Path(depot_side, is_depot=True)
1546 self.client_side = View.Path(client_side, is_depot=False)
1547 self.overlay = overlay # started with "+"
1548 self.exclude = exclude # started with "-"
1549 assert not (self.overlay and self.exclude)
1550 self.depot_side.ensure_compatible(self.client_side)
1551
1552 def __str__(self):
1553 c = " "
1554 if self.overlay:
1555 c = "+"
1556 if self.exclude:
1557 c = "-"
1558 return "View.Mapping: %s%s -> %s" % \
329afb8e 1559 (c, self.depot_side.path, self.client_side.path)
ecb7cf98
PW
1560
1561 def map_depot_to_client(self, depot_path):
1562 """Calculate the client path if using this mapping on the
1563 given depot path; does not consider the effect of other
1564 mappings in a view. Even excluded mappings are returned."""
1565 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1566 if not matches:
1567 return ""
1568 client_path = self.client_side.fill_in_wildcards(wildcards)
1569 return client_path
1570
1571 #
1572 # View methods
1573 #
1574 def __init__(self):
1575 self.mappings = []
1576
1577 def append(self, view_line):
1578 """Parse a view line, splitting it into depot and client
1579 sides. Append to self.mappings, preserving order."""
1580
1581 # Split the view line into exactly two words. P4 enforces
1582 # structure on these lines that simplifies this quite a bit.
1583 #
1584 # Either or both words may be double-quoted.
1585 # Single quotes do not matter.
1586 # Double-quote marks cannot occur inside the words.
1587 # A + or - prefix is also inside the quotes.
1588 # There are no quotes unless they contain a space.
1589 # The line is already white-space stripped.
1590 # The two words are separated by a single space.
1591 #
1592 if view_line[0] == '"':
1593 # First word is double quoted. Find its end.
1594 close_quote_index = view_line.find('"', 1)
1595 if close_quote_index <= 0:
1596 die("No first-word closing quote found: %s" % view_line)
1597 depot_side = view_line[1:close_quote_index]
1598 # skip closing quote and space
1599 rhs_index = close_quote_index + 1 + 1
1600 else:
1601 space_index = view_line.find(" ")
1602 if space_index <= 0:
1603 die("No word-splitting space found: %s" % view_line)
1604 depot_side = view_line[0:space_index]
1605 rhs_index = space_index + 1
1606
1607 if view_line[rhs_index] == '"':
1608 # Second word is double quoted. Make sure there is a
1609 # double quote at the end too.
1610 if not view_line.endswith('"'):
1611 die("View line with rhs quote should end with one: %s" %
1612 view_line)
1613 # skip the quotes
1614 client_side = view_line[rhs_index+1:-1]
1615 else:
1616 client_side = view_line[rhs_index:]
1617
1618 # prefix + means overlay on previous mapping
1619 overlay = False
1620 if depot_side.startswith("+"):
1621 overlay = True
1622 depot_side = depot_side[1:]
1623
1624 # prefix - means exclude this path
1625 exclude = False
1626 if depot_side.startswith("-"):
1627 exclude = True
1628 depot_side = depot_side[1:]
1629
1630 m = View.Mapping(depot_side, client_side, overlay, exclude)
1631 self.mappings.append(m)
1632
1633 def map_in_client(self, depot_path):
1634 """Return the relative location in the client where this
1635 depot file should live. Returns "" if the file should
1636 not be mapped in the client."""
1637
1638 paths_filled = []
1639 client_path = ""
1640
1641 # look at later entries first
1642 for m in self.mappings[::-1]:
1643
1644 # see where will this path end up in the client
1645 p = m.map_depot_to_client(depot_path)
1646
1647 if p == "":
1648 # Depot path does not belong in client. Must remember
1649 # this, as previous items should not cause files to
1650 # exist in this path either. Remember that the list is
1651 # being walked from the end, which has higher precedence.
1652 # Overlap mappings do not exclude previous mappings.
1653 if not m.overlay:
1654 paths_filled.append(m.client_side)
1655
1656 else:
1657 # This mapping matched; no need to search any further.
1658 # But, the mapping could be rejected if the client path
6ee9a999
PW
1659 # has already been claimed by an earlier mapping (i.e.
1660 # one later in the list, which we are walking backwards).
ecb7cf98
PW
1661 already_mapped_in_client = False
1662 for f in paths_filled:
1663 # this is View.Path.match
1664 if f.match(p):
1665 already_mapped_in_client = True
1666 break
1667 if not already_mapped_in_client:
1668 # Include this file, unless it is from a line that
1669 # explicitly said to exclude it.
1670 if not m.exclude:
1671 client_path = p
1672
1673 # a match, even if rejected, always stops the search
1674 break
1675
1676 return client_path
1677
3ea2cfd4 1678class P4Sync(Command, P4UserMap):
56c09345
PW
1679 delete_actions = ( "delete", "move/delete", "purge" )
1680
b984733c
SH
1681 def __init__(self):
1682 Command.__init__(self)
3ea2cfd4 1683 P4UserMap.__init__(self)
b984733c
SH
1684 self.options = [
1685 optparse.make_option("--branch", dest="branch"),
1686 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1687 optparse.make_option("--changesfile", dest="changesFile"),
1688 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1689 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 1690 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
1691 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1692 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 1693 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 1694 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1695 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1696 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1697 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
1698 ]
1699 self.description = """Imports from Perforce into a git repository.\n
1700 example:
1701 //depot/my/project/ -- to import the current head
1702 //depot/my/project/@all -- to import everything
1703 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1704
1705 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1706
1707 self.usage += " //depot/path[@revRange]"
b984733c 1708 self.silent = False
1d7367dc
RG
1709 self.createdBranches = set()
1710 self.committedChanges = set()
569d1bd4 1711 self.branch = ""
b984733c 1712 self.detectBranches = False
cb53e1f8 1713 self.detectLabels = False
06804c76 1714 self.importLabels = False
b984733c 1715 self.changesFile = ""
01265103 1716 self.syncWithOrigin = True
a028a98e 1717 self.importIntoRemotes = True
01a9c9c5 1718 self.maxChanges = ""
c1f9197f 1719 self.isWindows = (platform.system() == "Windows")
8b41a97f 1720 self.keepRepoPath = False
6326aa58 1721 self.depotPaths = None
3c699645 1722 self.p4BranchesInGit = []
354081d5 1723 self.cloneExclude = []
3a70cdfa 1724 self.useClientSpec = False
a93d33ee 1725 self.useClientSpec_from_options = False
ecb7cf98 1726 self.clientSpecDirs = None
fed23693
VA
1727 self.tempBranches = []
1728 self.tempBranchLocation = "git-p4-tmp"
b984733c 1729
01265103
SH
1730 if gitConfig("git-p4.syncFromOrigin") == "false":
1731 self.syncWithOrigin = False
1732
fed23693
VA
1733 # Force a checkpoint in fast-import and wait for it to finish
1734 def checkpoint(self):
1735 self.gitStream.write("checkpoint\n\n")
1736 self.gitStream.write("progress checkpoint\n\n")
1737 out = self.gitOutput.readline()
1738 if self.verbose:
1739 print "checkpoint finished: " + out
1740
b984733c 1741 def extractFilesFromCommit(self, commit):
354081d5
TT
1742 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1743 for path in self.cloneExclude]
b984733c
SH
1744 files = []
1745 fnum = 0
1746 while commit.has_key("depotFile%s" % fnum):
1747 path = commit["depotFile%s" % fnum]
6326aa58 1748
354081d5 1749 if [p for p in self.cloneExclude
d53de8b9 1750 if p4PathStartsWith(path, p)]:
354081d5
TT
1751 found = False
1752 else:
1753 found = [p for p in self.depotPaths
d53de8b9 1754 if p4PathStartsWith(path, p)]
6326aa58 1755 if not found:
b984733c
SH
1756 fnum = fnum + 1
1757 continue
1758
1759 file = {}
1760 file["path"] = path
1761 file["rev"] = commit["rev%s" % fnum]
1762 file["action"] = commit["action%s" % fnum]
1763 file["type"] = commit["type%s" % fnum]
1764 files.append(file)
1765 fnum = fnum + 1
1766 return files
1767
6326aa58 1768 def stripRepoPath(self, path, prefixes):
3952710b 1769 if self.useClientSpec:
ecb7cf98 1770 return self.clientSpecDirs.map_in_client(path)
3952710b 1771
8b41a97f 1772 if self.keepRepoPath:
6326aa58
HWN
1773 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1774
1775 for p in prefixes:
d53de8b9 1776 if p4PathStartsWith(path, p):
6326aa58 1777 path = path[len(p):]
8b41a97f 1778
6326aa58 1779 return path
6754a299 1780
71b112d4 1781 def splitFilesIntoBranches(self, commit):
d5904674 1782 branches = {}
71b112d4
SH
1783 fnum = 0
1784 while commit.has_key("depotFile%s" % fnum):
1785 path = commit["depotFile%s" % fnum]
6326aa58 1786 found = [p for p in self.depotPaths
d53de8b9 1787 if p4PathStartsWith(path, p)]
6326aa58 1788 if not found:
71b112d4
SH
1789 fnum = fnum + 1
1790 continue
1791
1792 file = {}
1793 file["path"] = path
1794 file["rev"] = commit["rev%s" % fnum]
1795 file["action"] = commit["action%s" % fnum]
1796 file["type"] = commit["type%s" % fnum]
1797 fnum = fnum + 1
1798
6326aa58 1799 relPath = self.stripRepoPath(path, self.depotPaths)
9d7d446a 1800 relPath = wildcard_decode(relPath)
b984733c 1801
4b97ffb1 1802 for branch in self.knownBranches.keys():
6754a299
HWN
1803
1804 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1805 if relPath.startswith(branch + "/"):
d5904674
SH
1806 if branch not in branches:
1807 branches[branch] = []
71b112d4 1808 branches[branch].append(file)
6555b2cc 1809 break
b984733c
SH
1810
1811 return branches
1812
b932705b
LD
1813 # output one file from the P4 stream
1814 # - helper for streamP4Files
1815
1816 def streamOneP4File(self, file, contents):
b932705b 1817 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
9d7d446a 1818 relPath = wildcard_decode(relPath)
b932705b
LD
1819 if verbose:
1820 sys.stderr.write("%s\n" % relPath)
1821
9cffb8c8
PW
1822 (type_base, type_mods) = split_p4_type(file["type"])
1823
1824 git_mode = "100644"
1825 if "x" in type_mods:
1826 git_mode = "100755"
1827 if type_base == "symlink":
1828 git_mode = "120000"
1829 # p4 print on a symlink contains "target\n"; remove the newline
b39c3612
EP
1830 data = ''.join(contents)
1831 contents = [data[:-1]]
b932705b 1832
9cffb8c8 1833 if type_base == "utf16":
55aa5714
PW
1834 # p4 delivers different text in the python output to -G
1835 # than it does when using "print -o", or normal p4 client
1836 # operations. utf16 is converted to ascii or utf8, perhaps.
1837 # But ascii text saved as -t utf16 is completely mangled.
1838 # Invoke print -o to get the real contents.
6de040df 1839 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
55aa5714
PW
1840 contents = [ text ]
1841
9f7ef0ea
PW
1842 if type_base == "apple":
1843 # Apple filetype files will be streamed as a concatenation of
1844 # its appledouble header and the contents. This is useless
1845 # on both macs and non-macs. If using "print -q -o xx", it
1846 # will create "xx" with the data, and "%xx" with the header.
1847 # This is also not very useful.
1848 #
1849 # Ideally, someday, this script can learn how to generate
1850 # appledouble files directly and import those to git, but
1851 # non-mac machines can never find a use for apple filetype.
1852 print "\nIgnoring apple filetype file %s" % file['depotFile']
1853 return
1854
9cffb8c8
PW
1855 # Perhaps windows wants unicode, utf16 newlines translated too;
1856 # but this is not doing it.
1857 if self.isWindows and type_base == "text":
b932705b
LD
1858 mangled = []
1859 for data in contents:
1860 data = data.replace("\r\n", "\n")
1861 mangled.append(data)
1862 contents = mangled
1863
55aa5714
PW
1864 # Note that we do not try to de-mangle keywords on utf16 files,
1865 # even though in theory somebody may want that.
60df071c
LD
1866 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1867 if pattern:
1868 regexp = re.compile(pattern, re.VERBOSE)
1869 text = ''.join(contents)
1870 text = regexp.sub(r'$\1$', text)
1871 contents = [ text ]
b932705b 1872
9cffb8c8 1873 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
1874
1875 # total length...
1876 length = 0
1877 for d in contents:
1878 length = length + len(d)
1879
1880 self.gitStream.write("data %d\n" % length)
1881 for d in contents:
1882 self.gitStream.write(d)
1883 self.gitStream.write("\n")
1884
1885 def streamOneP4Deletion(self, file):
1886 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
9d7d446a 1887 relPath = wildcard_decode(relPath)
b932705b
LD
1888 if verbose:
1889 sys.stderr.write("delete %s\n" % relPath)
1890 self.gitStream.write("D %s\n" % relPath)
1891
1892 # handle another chunk of streaming data
1893 def streamP4FilesCb(self, marshalled):
1894
c3f6163b
AG
1895 if marshalled.has_key('depotFile') and self.stream_have_file_info:
1896 # start of a new file - output the old one first
1897 self.streamOneP4File(self.stream_file, self.stream_contents)
1898 self.stream_file = {}
1899 self.stream_contents = []
1900 self.stream_have_file_info = False
b932705b 1901
c3f6163b
AG
1902 # pick up the new file information... for the
1903 # 'data' field we need to append to our array
1904 for k in marshalled.keys():
1905 if k == 'data':
1906 self.stream_contents.append(marshalled['data'])
1907 else:
1908 self.stream_file[k] = marshalled[k]
b932705b 1909
c3f6163b 1910 self.stream_have_file_info = True
b932705b
LD
1911
1912 # Stream directly from "p4 files" into "git fast-import"
1913 def streamP4Files(self, files):
30b5940b
SH
1914 filesForCommit = []
1915 filesToRead = []
b932705b 1916 filesToDelete = []
30b5940b 1917
3a70cdfa 1918 for f in files:
ecb7cf98
PW
1919 # if using a client spec, only add the files that have
1920 # a path in the client
1921 if self.clientSpecDirs:
1922 if self.clientSpecDirs.map_in_client(f['path']) == "":
1923 continue
3a70cdfa 1924
ecb7cf98
PW
1925 filesForCommit.append(f)
1926 if f['action'] in self.delete_actions:
1927 filesToDelete.append(f)
1928 else:
1929 filesToRead.append(f)
6a49f8e2 1930
b932705b
LD
1931 # deleted files...
1932 for f in filesToDelete:
1933 self.streamOneP4Deletion(f)
1b9a4684 1934
b932705b
LD
1935 if len(filesToRead) > 0:
1936 self.stream_file = {}
1937 self.stream_contents = []
1938 self.stream_have_file_info = False
8ff45f2a 1939
c3f6163b
AG
1940 # curry self argument
1941 def streamP4FilesCbSelf(entry):
1942 self.streamP4FilesCb(entry)
6a49f8e2 1943
6de040df
LD
1944 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1945
1946 p4CmdList(["-x", "-", "print"],
1947 stdin=fileArgs,
1948 cb=streamP4FilesCbSelf)
30b5940b 1949
b932705b
LD
1950 # do the last chunk
1951 if self.stream_file.has_key('depotFile'):
1952 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 1953
affb474f
LD
1954 def make_email(self, userid):
1955 if userid in self.users:
1956 return self.users[userid]
1957 else:
1958 return "%s <a@b>" % userid
1959
06804c76
LD
1960 # Stream a p4 tag
1961 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
1962 if verbose:
1963 print "writing tag %s for commit %s" % (labelName, commit)
1964 gitStream.write("tag %s\n" % labelName)
1965 gitStream.write("from %s\n" % commit)
1966
1967 if labelDetails.has_key('Owner'):
1968 owner = labelDetails["Owner"]
1969 else:
1970 owner = None
1971
1972 # Try to use the owner of the p4 label, or failing that,
1973 # the current p4 user id.
1974 if owner:
1975 email = self.make_email(owner)
1976 else:
1977 email = self.make_email(self.p4UserId())
1978 tagger = "%s %s %s" % (email, epoch, self.tz)
1979
1980 gitStream.write("tagger %s\n" % tagger)
1981
1982 print "labelDetails=",labelDetails
1983 if labelDetails.has_key('Description'):
1984 description = labelDetails['Description']
1985 else:
1986 description = 'Label from git p4'
1987
1988 gitStream.write("data %d\n" % len(description))
1989 gitStream.write(description)
1990 gitStream.write("\n")
1991
6326aa58 1992 def commit(self, details, files, branch, branchPrefixes, parent = ""):
b984733c
SH
1993 epoch = details["time"]
1994 author = details["user"]
c3f6163b 1995 self.branchPrefixes = branchPrefixes
b984733c 1996
4b97ffb1
SH
1997 if self.verbose:
1998 print "commit into %s" % branch
1999
96e07dd2
HWN
2000 # start with reading files; if that fails, we should not
2001 # create a commit.
2002 new_files = []
2003 for f in files:
d53de8b9 2004 if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2005 new_files.append (f)
2006 else:
afa1dd9a 2007 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2008
b984733c 2009 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 2010# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2011 self.committedChanges.add(int(details["change"]))
2012 committer = ""
b607e71e
SH
2013 if author not in self.users:
2014 self.getUserMapFromPerforceServer()
affb474f 2015 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2016
2017 self.gitStream.write("committer %s\n" % committer)
2018
2019 self.gitStream.write("data <<EOT\n")
2020 self.gitStream.write(details["desc"])
6581de09
SH
2021 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2022 % (','.join (branchPrefixes), details["change"]))
2023 if len(details['options']) > 0:
2024 self.gitStream.write(": options = %s" % details['options'])
2025 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2026
2027 if len(parent) > 0:
4b97ffb1
SH
2028 if self.verbose:
2029 print "parent %s" % parent
b984733c
SH
2030 self.gitStream.write("from %s\n" % parent)
2031
b932705b 2032 self.streamP4Files(new_files)
b984733c
SH
2033 self.gitStream.write("\n")
2034
1f4ba1cb
SH
2035 change = int(details["change"])
2036
9bda3a85 2037 if self.labels.has_key(change):
1f4ba1cb
SH
2038 label = self.labels[change]
2039 labelDetails = label[0]
2040 labelRevisions = label[1]
71b112d4
SH
2041 if self.verbose:
2042 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2043
6de040df
LD
2044 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2045 for p in branchPrefixes])
1f4ba1cb
SH
2046
2047 if len(files) == len(labelRevisions):
2048
2049 cleanedFiles = {}
2050 for info in files:
56c09345 2051 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2052 continue
2053 cleanedFiles[info["depotFile"]] = info["rev"]
2054
2055 if cleanedFiles == labelRevisions:
06804c76 2056 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2057
2058 else:
a46668fa 2059 if not self.silent:
cebdf5af
HWN
2060 print ("Tag %s does not match with change %s: files do not match."
2061 % (labelDetails["label"], change))
1f4ba1cb
SH
2062
2063 else:
a46668fa 2064 if not self.silent:
cebdf5af
HWN
2065 print ("Tag %s does not match with change %s: file count is different."
2066 % (labelDetails["label"], change))
b984733c 2067
06804c76 2068 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2069 def getLabels(self):
2070 self.labels = {}
2071
52a4880b 2072 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2073 if len(l) > 0 and not self.silent:
183f8436 2074 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2075
2076 for output in l:
1f4ba1cb
SH
2077 label = output["label"]
2078 revisions = {}
2079 newestChange = 0
71b112d4
SH
2080 if self.verbose:
2081 print "Querying files for label %s" % label
6de040df
LD
2082 for file in p4CmdList(["files"] +
2083 ["%s...@%s" % (p, label)
2084 for p in self.depotPaths]):
1f4ba1cb
SH
2085 revisions[file["depotFile"]] = file["rev"]
2086 change = int(file["change"])
2087 if change > newestChange:
2088 newestChange = change
2089
9bda3a85
SH
2090 self.labels[newestChange] = [output, revisions]
2091
2092 if self.verbose:
2093 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2094
06804c76
LD
2095 # Import p4 labels as git tags. A direct mapping does not
2096 # exist, so assume that if all the files are at the same revision
2097 # then we can use that, or it's something more complicated we should
2098 # just ignore.
2099 def importP4Labels(self, stream, p4Labels):
2100 if verbose:
2101 print "import p4 labels: " + ' '.join(p4Labels)
2102
2103 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2104 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2105 if len(validLabelRegexp) == 0:
2106 validLabelRegexp = defaultLabelRegexp
2107 m = re.compile(validLabelRegexp)
2108
2109 for name in p4Labels:
2110 commitFound = False
2111
2112 if not m.match(name):
2113 if verbose:
2114 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2115 continue
2116
2117 if name in ignoredP4Labels:
2118 continue
2119
2120 labelDetails = p4CmdList(['label', "-o", name])[0]
2121
2122 # get the most recent changelist for each file in this label
2123 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2124 for p in self.depotPaths])
2125
2126 if change.has_key('change'):
2127 # find the corresponding git commit; take the oldest commit
2128 changelist = int(change['change'])
2129 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2130 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2131 if len(gitCommit) == 0:
2132 print "could not find git commit for changelist %d" % changelist
2133 else:
2134 gitCommit = gitCommit.strip()
2135 commitFound = True
2136 # Convert from p4 time format
2137 try:
2138 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2139 except ValueError:
2140 print "Could not convert label time %s" % labelDetail['Update']
2141 tmwhen = 1
2142
2143 when = int(time.mktime(tmwhen))
2144 self.streamTag(stream, name, labelDetails, gitCommit, when)
2145 if verbose:
2146 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2147 else:
2148 if verbose:
2149 print "Label %s has no changelists - possibly deleted?" % name
2150
2151 if not commitFound:
2152 # We can't import this label; don't try again as it will get very
2153 # expensive repeatedly fetching all the files for labels that will
2154 # never be imported. If the label is moved in the future, the
2155 # ignore will need to be removed manually.
2156 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2157
86dff6b6
HWN
2158 def guessProjectName(self):
2159 for p in self.depotPaths:
6e5295c4
SH
2160 if p.endswith("/"):
2161 p = p[:-1]
2162 p = p[p.strip().rfind("/") + 1:]
2163 if not p.endswith("/"):
2164 p += "/"
2165 return p
86dff6b6 2166
4b97ffb1 2167 def getBranchMapping(self):
6555b2cc
SH
2168 lostAndFoundBranches = set()
2169
8ace74c0
VA
2170 user = gitConfig("git-p4.branchUser")
2171 if len(user) > 0:
2172 command = "branches -u %s" % user
2173 else:
2174 command = "branches"
2175
2176 for info in p4CmdList(command):
52a4880b 2177 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2178 viewIdx = 0
2179 while details.has_key("View%s" % viewIdx):
2180 paths = details["View%s" % viewIdx].split(" ")
2181 viewIdx = viewIdx + 1
2182 # require standard //depot/foo/... //depot/bar/... mapping
2183 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2184 continue
2185 source = paths[0]
2186 destination = paths[1]
6509e19c 2187 ## HACK
d53de8b9 2188 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2189 source = source[len(self.depotPaths[0]):-4]
2190 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2191
1a2edf4e
SH
2192 if destination in self.knownBranches:
2193 if not self.silent:
2194 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2195 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2196 continue
2197
6555b2cc
SH
2198 self.knownBranches[destination] = source
2199
2200 lostAndFoundBranches.discard(destination)
2201
29bdbac1 2202 if source not in self.knownBranches:
6555b2cc
SH
2203 lostAndFoundBranches.add(source)
2204
7199cf13
VA
2205 # Perforce does not strictly require branches to be defined, so we also
2206 # check git config for a branch list.
2207 #
2208 # Example of branch definition in git config file:
2209 # [git-p4]
2210 # branchList=main:branchA
2211 # branchList=main:branchB
2212 # branchList=branchA:branchC
2213 configBranches = gitConfigList("git-p4.branchList")
2214 for branch in configBranches:
2215 if branch:
2216 (source, destination) = branch.split(":")
2217 self.knownBranches[destination] = source
2218
2219 lostAndFoundBranches.discard(destination)
2220
2221 if source not in self.knownBranches:
2222 lostAndFoundBranches.add(source)
2223
6555b2cc
SH
2224
2225 for branch in lostAndFoundBranches:
2226 self.knownBranches[branch] = branch
29bdbac1 2227
38f9f5ec
SH
2228 def getBranchMappingFromGitBranches(self):
2229 branches = p4BranchesInGit(self.importIntoRemotes)
2230 for branch in branches.keys():
2231 if branch == "master":
2232 branch = "main"
2233 else:
2234 branch = branch[len(self.projectName):]
2235 self.knownBranches[branch] = branch
2236
29bdbac1 2237 def listExistingP4GitBranches(self):
144ff46b
SH
2238 # branches holds mapping from name to commit
2239 branches = p4BranchesInGit(self.importIntoRemotes)
2240 self.p4BranchesInGit = branches.keys()
2241 for branch in branches.keys():
2242 self.initialParents[self.refPrefix + branch] = branches[branch]
4b97ffb1 2243
bb6e09b2
HWN
2244 def updateOptionDict(self, d):
2245 option_keys = {}
2246 if self.keepRepoPath:
2247 option_keys['keepRepoPath'] = 1
2248
2249 d["options"] = ' '.join(sorted(option_keys.keys()))
2250
2251 def readOptions(self, d):
2252 self.keepRepoPath = (d.has_key('options')
2253 and ('keepRepoPath' in d['options']))
6326aa58 2254
8134f69c
SH
2255 def gitRefForBranch(self, branch):
2256 if branch == "main":
2257 return self.refPrefix + "master"
2258
2259 if len(branch) <= 0:
2260 return branch
2261
2262 return self.refPrefix + self.projectName + branch
2263
1ca3d710
SH
2264 def gitCommitByP4Change(self, ref, change):
2265 if self.verbose:
2266 print "looking in ref " + ref + " for change %s using bisect..." % change
2267
2268 earliestCommit = ""
2269 latestCommit = parseRevision(ref)
2270
2271 while True:
2272 if self.verbose:
2273 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2274 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2275 if len(next) == 0:
2276 if self.verbose:
2277 print "argh"
2278 return ""
2279 log = extractLogMessageFromGitCommit(next)
2280 settings = extractSettingsGitLog(log)
2281 currentChange = int(settings['change'])
2282 if self.verbose:
2283 print "current change %s" % currentChange
2284
2285 if currentChange == change:
2286 if self.verbose:
2287 print "found %s" % next
2288 return next
2289
2290 if currentChange < change:
2291 earliestCommit = "^%s" % next
2292 else:
2293 latestCommit = "%s" % next
2294
2295 return ""
2296
2297 def importNewBranch(self, branch, maxChange):
2298 # make fast-import flush all changes to disk and update the refs using the checkpoint
2299 # command so that we can try to find the branch parent in the git history
2300 self.gitStream.write("checkpoint\n\n");
2301 self.gitStream.flush();
2302 branchPrefix = self.depotPaths[0] + branch + "/"
2303 range = "@1,%s" % maxChange
2304 #print "prefix" + branchPrefix
2305 changes = p4ChangesForPaths([branchPrefix], range)
2306 if len(changes) <= 0:
2307 return False
2308 firstChange = changes[0]
2309 #print "first change in branch: %s" % firstChange
2310 sourceBranch = self.knownBranches[branch]
2311 sourceDepotPath = self.depotPaths[0] + sourceBranch
2312 sourceRef = self.gitRefForBranch(sourceBranch)
2313 #print "source " + sourceBranch
2314
52a4880b 2315 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2316 #print "branch parent: %s" % branchParentChange
2317 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2318 if len(gitParent) > 0:
2319 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2320 #print "parent git commit: %s" % gitParent
2321
2322 self.importChanges(changes)
2323 return True
2324
fed23693
VA
2325 def searchParent(self, parent, branch, target):
2326 parentFound = False
2327 for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2328 blob = blob.strip()
2329 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2330 parentFound = True
2331 if self.verbose:
2332 print "Found parent of %s in commit %s" % (branch, blob)
2333 break
2334 if parentFound:
2335 return blob
2336 else:
2337 return None
2338
e87f37ae
SH
2339 def importChanges(self, changes):
2340 cnt = 1
2341 for change in changes:
3558f32f 2342 description = p4Cmd(["describe", str(change)])
e87f37ae
SH
2343 self.updateOptionDict(description)
2344
2345 if not self.silent:
2346 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2347 sys.stdout.flush()
2348 cnt = cnt + 1
2349
2350 try:
2351 if self.detectBranches:
2352 branches = self.splitFilesIntoBranches(description)
2353 for branch in branches.keys():
2354 ## HACK --hwn
2355 branchPrefix = self.depotPaths[0] + branch + "/"
2356
2357 parent = ""
2358
2359 filesForCommit = branches[branch]
2360
2361 if self.verbose:
2362 print "branch is %s" % branch
2363
2364 self.updatedBranches.add(branch)
2365
2366 if branch not in self.createdBranches:
2367 self.createdBranches.add(branch)
2368 parent = self.knownBranches[branch]
2369 if parent == branch:
2370 parent = ""
1ca3d710
SH
2371 else:
2372 fullBranch = self.projectName + branch
2373 if fullBranch not in self.p4BranchesInGit:
2374 if not self.silent:
2375 print("\n Importing new branch %s" % fullBranch);
2376 if self.importNewBranch(branch, change - 1):
2377 parent = ""
2378 self.p4BranchesInGit.append(fullBranch)
2379 if not self.silent:
2380 print("\n Resuming with change %s" % change);
2381
2382 if self.verbose:
2383 print "parent determined through known branches: %s" % parent
e87f37ae 2384
8134f69c
SH
2385 branch = self.gitRefForBranch(branch)
2386 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2387
2388 if self.verbose:
2389 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2390
2391 if len(parent) == 0 and branch in self.initialParents:
2392 parent = self.initialParents[branch]
2393 del self.initialParents[branch]
2394
fed23693
VA
2395 blob = None
2396 if len(parent) > 0:
2397 tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2398 if self.verbose:
2399 print "Creating temporary branch: " + tempBranch
2400 self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2401 self.tempBranches.append(tempBranch)
2402 self.checkpoint()
2403 blob = self.searchParent(parent, branch, tempBranch)
2404 if blob:
2405 self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2406 else:
2407 if self.verbose:
2408 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2409 self.commit(description, filesForCommit, branch, [branchPrefix], parent)
e87f37ae
SH
2410 else:
2411 files = self.extractFilesFromCommit(description)
2412 self.commit(description, files, self.branch, self.depotPaths,
2413 self.initialParent)
2414 self.initialParent = ""
2415 except IOError:
2416 print self.gitError.read()
2417 sys.exit(1)
2418
c208a243
SH
2419 def importHeadRevision(self, revision):
2420 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2421
4e2e6ce4
PW
2422 details = {}
2423 details["user"] = "git perforce import user"
1494fcbb 2424 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2425 % (' '.join(self.depotPaths), revision))
2426 details["change"] = revision
2427 newestRevision = 0
2428
2429 fileCnt = 0
6de040df
LD
2430 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2431
2432 for info in p4CmdList(["files"] + fileArgs):
c208a243 2433
68b28593 2434 if 'code' in info and info['code'] == 'error':
c208a243
SH
2435 sys.stderr.write("p4 returned an error: %s\n"
2436 % info['data'])
d88e707f
PW
2437 if info['data'].find("must refer to client") >= 0:
2438 sys.stderr.write("This particular p4 error is misleading.\n")
2439 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2440 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2441 sys.exit(1)
68b28593
PW
2442 if 'p4ExitCode' in info:
2443 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2444 sys.exit(1)
2445
2446
2447 change = int(info["change"])
2448 if change > newestRevision:
2449 newestRevision = change
2450
56c09345 2451 if info["action"] in self.delete_actions:
c208a243
SH
2452 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2453 #fileCnt = fileCnt + 1
2454 continue
2455
2456 for prop in ["depotFile", "rev", "action", "type" ]:
2457 details["%s%s" % (prop, fileCnt)] = info[prop]
2458
2459 fileCnt = fileCnt + 1
2460
2461 details["change"] = newestRevision
4e2e6ce4 2462
9dcb9f24 2463 # Use time from top-most change so that all git p4 clones of
4e2e6ce4
PW
2464 # the same p4 repo have the same commit SHA1s.
2465 res = p4CmdList("describe -s %d" % newestRevision)
2466 newestTime = None
2467 for r in res:
2468 if r.has_key('time'):
2469 newestTime = int(r['time'])
2470 if newestTime is None:
2471 die("\"describe -s\" on newest change %d did not give a time")
2472 details["time"] = newestTime
2473
c208a243
SH
2474 self.updateOptionDict(details)
2475 try:
2476 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2477 except IOError:
2478 print "IO error with git fast-import. Is your git version recent enough?"
2479 print self.gitError.read()
2480
2481
b984733c 2482 def run(self, args):
6326aa58 2483 self.depotPaths = []
179caebf
SH
2484 self.changeRange = ""
2485 self.initialParent = ""
6326aa58 2486 self.previousDepotPaths = []
ce6f33c8 2487
29bdbac1
SH
2488 # map from branch depot path to parent branch
2489 self.knownBranches = {}
2490 self.initialParents = {}
5ca44617 2491 self.hasOrigin = originP4BranchesExist()
a43ff00c
SH
2492 if not self.syncWithOrigin:
2493 self.hasOrigin = False
29bdbac1 2494
a028a98e
SH
2495 if self.importIntoRemotes:
2496 self.refPrefix = "refs/remotes/p4/"
2497 else:
db775559 2498 self.refPrefix = "refs/heads/p4/"
a028a98e 2499
cebdf5af
HWN
2500 if self.syncWithOrigin and self.hasOrigin:
2501 if not self.silent:
2502 print "Syncing with origin first by calling git fetch origin"
2503 system("git fetch origin")
10f880f8 2504
569d1bd4 2505 if len(self.branch) == 0:
db775559 2506 self.branch = self.refPrefix + "master"
a028a98e 2507 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2508 system("git update-ref %s refs/heads/p4" % self.branch)
48df6fd8 2509 system("git branch -D p4");
faf1bd20 2510 # create it /after/ importing, when master exists
0058a33a 2511 if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
a3c55c09 2512 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
967f72e2 2513
a93d33ee
PW
2514 # accept either the command-line option, or the configuration variable
2515 if self.useClientSpec:
2516 # will use this after clone to set the variable
2517 self.useClientSpec_from_options = True
2518 else:
09fca77b
PW
2519 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2520 self.useClientSpec = True
2521 if self.useClientSpec:
543987bd 2522 self.clientSpecDirs = getClientSpec()
3a70cdfa 2523
6a49f8e2
HWN
2524 # TODO: should always look at previous commits,
2525 # merge with previous imports, if possible.
2526 if args == []:
d414c74a 2527 if self.hasOrigin:
5ca44617 2528 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
abcd790f
SH
2529 self.listExistingP4GitBranches()
2530
2531 if len(self.p4BranchesInGit) > 1:
2532 if not self.silent:
2533 print "Importing from/into multiple branches"
2534 self.detectBranches = True
967f72e2 2535
29bdbac1
SH
2536 if self.verbose:
2537 print "branches: %s" % self.p4BranchesInGit
2538
2539 p4Change = 0
2540 for branch in self.p4BranchesInGit:
cebdf5af 2541 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2542
2543 settings = extractSettingsGitLog(logMsg)
29bdbac1 2544
bb6e09b2
HWN
2545 self.readOptions(settings)
2546 if (settings.has_key('depot-paths')
2547 and settings.has_key ('change')):
2548 change = int(settings['change']) + 1
29bdbac1
SH
2549 p4Change = max(p4Change, change)
2550
bb6e09b2
HWN
2551 depotPaths = sorted(settings['depot-paths'])
2552 if self.previousDepotPaths == []:
6326aa58 2553 self.previousDepotPaths = depotPaths
29bdbac1 2554 else:
6326aa58
HWN
2555 paths = []
2556 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2557 prev_list = prev.split("/")
2558 cur_list = cur.split("/")
2559 for i in range(0, min(len(cur_list), len(prev_list))):
2560 if cur_list[i] <> prev_list[i]:
583e1707 2561 i = i - 1
6326aa58
HWN
2562 break
2563
04d277b3 2564 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2565
2566 self.previousDepotPaths = paths
29bdbac1
SH
2567
2568 if p4Change > 0:
bb6e09b2 2569 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2570 self.changeRange = "@%s,#head" % p4Change
330f53b8
SH
2571 if not self.detectBranches:
2572 self.initialParent = parseRevision(self.branch)
341dc1c1 2573 if not self.silent and not self.detectBranches:
967f72e2 2574 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2575
f9162f6a
SH
2576 if not self.branch.startswith("refs/"):
2577 self.branch = "refs/heads/" + self.branch
179caebf 2578
6326aa58 2579 if len(args) == 0 and self.depotPaths:
b984733c 2580 if not self.silent:
6326aa58 2581 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2582 else:
6326aa58 2583 if self.depotPaths and self.depotPaths != args:
cebdf5af 2584 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2585 "This doesn't work!" % (' '.join (self.depotPaths),
2586 ' '.join (args)))
b984733c 2587 sys.exit(1)
6326aa58 2588
bb6e09b2 2589 self.depotPaths = sorted(args)
b984733c 2590
1c49fc19 2591 revision = ""
b984733c 2592 self.users = {}
b984733c 2593
58c8bc7c
PW
2594 # Make sure no revision specifiers are used when --changesfile
2595 # is specified.
2596 bad_changesfile = False
2597 if len(self.changesFile) > 0:
2598 for p in self.depotPaths:
2599 if p.find("@") >= 0 or p.find("#") >= 0:
2600 bad_changesfile = True
2601 break
2602 if bad_changesfile:
2603 die("Option --changesfile is incompatible with revision specifiers")
2604
6326aa58
HWN
2605 newPaths = []
2606 for p in self.depotPaths:
2607 if p.find("@") != -1:
2608 atIdx = p.index("@")
2609 self.changeRange = p[atIdx:]
2610 if self.changeRange == "@all":
2611 self.changeRange = ""
6a49f8e2 2612 elif ',' not in self.changeRange:
1c49fc19 2613 revision = self.changeRange
6326aa58 2614 self.changeRange = ""
7fcff9de 2615 p = p[:atIdx]
6326aa58
HWN
2616 elif p.find("#") != -1:
2617 hashIdx = p.index("#")
1c49fc19 2618 revision = p[hashIdx:]
7fcff9de 2619 p = p[:hashIdx]
6326aa58 2620 elif self.previousDepotPaths == []:
58c8bc7c
PW
2621 # pay attention to changesfile, if given, else import
2622 # the entire p4 tree at the head revision
2623 if len(self.changesFile) == 0:
2624 revision = "#head"
6326aa58
HWN
2625
2626 p = re.sub ("\.\.\.$", "", p)
2627 if not p.endswith("/"):
2628 p += "/"
2629
2630 newPaths.append(p)
2631
2632 self.depotPaths = newPaths
2633
b607e71e 2634 self.loadUserMapFromCache()
cb53e1f8
SH
2635 self.labels = {}
2636 if self.detectLabels:
2637 self.getLabels();
b984733c 2638
4b97ffb1 2639 if self.detectBranches:
df450923
SH
2640 ## FIXME - what's a P4 projectName ?
2641 self.projectName = self.guessProjectName()
2642
38f9f5ec
SH
2643 if self.hasOrigin:
2644 self.getBranchMappingFromGitBranches()
2645 else:
2646 self.getBranchMapping()
29bdbac1
SH
2647 if self.verbose:
2648 print "p4-git branches: %s" % self.p4BranchesInGit
2649 print "initial parents: %s" % self.initialParents
2650 for b in self.p4BranchesInGit:
2651 if b != "master":
6326aa58
HWN
2652
2653 ## FIXME
29bdbac1
SH
2654 b = b[len(self.projectName):]
2655 self.createdBranches.add(b)
4b97ffb1 2656
f291b4e3 2657 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 2658
cebdf5af 2659 importProcess = subprocess.Popen(["git", "fast-import"],
6326aa58
HWN
2660 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2661 stderr=subprocess.PIPE);
08483580
SH
2662 self.gitOutput = importProcess.stdout
2663 self.gitStream = importProcess.stdin
2664 self.gitError = importProcess.stderr
b984733c 2665
1c49fc19 2666 if revision:
c208a243 2667 self.importHeadRevision(revision)
b984733c
SH
2668 else:
2669 changes = []
2670
0828ab14 2671 if len(self.changesFile) > 0:
b984733c 2672 output = open(self.changesFile).readlines()
1d7367dc 2673 changeSet = set()
b984733c
SH
2674 for line in output:
2675 changeSet.add(int(line))
2676
2677 for change in changeSet:
2678 changes.append(change)
2679
2680 changes.sort()
2681 else:
9dcb9f24
PW
2682 # catch "git p4 sync" with no new branches, in a repo that
2683 # does not have any existing p4 branches
accad8e0 2684 if len(args) == 0 and not self.p4BranchesInGit:
e32e00dc 2685 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
29bdbac1 2686 if self.verbose:
86dff6b6 2687 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 2688 self.changeRange)
4f6432d8 2689 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 2690
01a9c9c5 2691 if len(self.maxChanges) > 0:
7fcff9de 2692 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 2693
b984733c 2694 if len(changes) == 0:
0828ab14 2695 if not self.silent:
341dc1c1 2696 print "No changes to import!"
06804c76
LD
2697 else:
2698 if not self.silent and not self.detectBranches:
2699 print "Import destination: %s" % self.branch
2700
2701 self.updatedBranches = set()
b984733c 2702
06804c76 2703 self.importChanges(changes)
a9d1a27a 2704
06804c76
LD
2705 if not self.silent:
2706 print ""
2707 if len(self.updatedBranches) > 0:
2708 sys.stdout.write("Updated branches: ")
2709 for b in self.updatedBranches:
2710 sys.stdout.write("%s " % b)
2711 sys.stdout.write("\n")
341dc1c1 2712
06804c76 2713 if gitConfig("git-p4.importLabels", "--bool") == "true":
06dcd152 2714 self.importLabels = True
b984733c 2715
06804c76
LD
2716 if self.importLabels:
2717 p4Labels = getP4Labels(self.depotPaths)
2718 gitTags = getGitTags()
2719
2720 missingP4Labels = p4Labels - gitTags
2721 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 2722
b984733c 2723 self.gitStream.close()
29bdbac1
SH
2724 if importProcess.wait() != 0:
2725 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
2726 self.gitOutput.close()
2727 self.gitError.close()
2728
fed23693
VA
2729 # Cleanup temporary branches created during import
2730 if self.tempBranches != []:
2731 for branch in self.tempBranches:
2732 read_pipe("git update-ref -d %s" % branch)
2733 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2734
b984733c
SH
2735 return True
2736
01ce1fe9
SH
2737class P4Rebase(Command):
2738 def __init__(self):
2739 Command.__init__(self)
06804c76
LD
2740 self.options = [
2741 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 2742 ]
06804c76 2743 self.importLabels = False
cebdf5af
HWN
2744 self.description = ("Fetches the latest revision from perforce and "
2745 + "rebases the current work (branch) against it")
01ce1fe9
SH
2746
2747 def run(self, args):
2748 sync = P4Sync()
06804c76 2749 sync.importLabels = self.importLabels
01ce1fe9 2750 sync.run([])
d7e3868c 2751
14594f4b
SH
2752 return self.rebase()
2753
2754 def rebase(self):
36ee4ee4
SH
2755 if os.system("git update-index --refresh") != 0:
2756 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2757 if len(read_pipe("git diff-index HEAD --")) > 0:
2758 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2759
d7e3868c
SH
2760 [upstream, settings] = findUpstreamBranchPoint()
2761 if len(upstream) == 0:
2762 die("Cannot find upstream branchpoint for rebase")
2763
2764 # the branchpoint may be p4/foo~3, so strip off the parent
2765 upstream = re.sub("~[0-9]+$", "", upstream)
2766
2767 print "Rebasing the current branch onto %s" % upstream
b25b2065 2768 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 2769 system("git rebase %s" % upstream)
1f52af6c 2770 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
2771 return True
2772
f9a3a4f7
SH
2773class P4Clone(P4Sync):
2774 def __init__(self):
2775 P4Sync.__init__(self)
2776 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 2777 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 2778 self.options += [
bb6e09b2
HWN
2779 optparse.make_option("--destination", dest="cloneDestination",
2780 action='store', default=None,
354081d5
TT
2781 help="where to leave result of the clone"),
2782 optparse.make_option("-/", dest="cloneExclude",
2783 action="append", type="string",
38200076
PW
2784 help="exclude depot path"),
2785 optparse.make_option("--bare", dest="cloneBare",
2786 action="store_true", default=False),
354081d5 2787 ]
bb6e09b2 2788 self.cloneDestination = None
f9a3a4f7 2789 self.needsGit = False
38200076 2790 self.cloneBare = False
f9a3a4f7 2791
354081d5
TT
2792 # This is required for the "append" cloneExclude action
2793 def ensure_value(self, attr, value):
2794 if not hasattr(self, attr) or getattr(self, attr) is None:
2795 setattr(self, attr, value)
2796 return getattr(self, attr)
2797
6a49f8e2
HWN
2798 def defaultDestination(self, args):
2799 ## TODO: use common prefix of args?
2800 depotPath = args[0]
2801 depotDir = re.sub("(@[^@]*)$", "", depotPath)
2802 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 2803 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
2804 depotDir = re.sub(r"/$", "", depotDir)
2805 return os.path.split(depotDir)[1]
2806
f9a3a4f7
SH
2807 def run(self, args):
2808 if len(args) < 1:
2809 return False
bb6e09b2
HWN
2810
2811 if self.keepRepoPath and not self.cloneDestination:
2812 sys.stderr.write("Must specify destination for --keep-path\n")
2813 sys.exit(1)
f9a3a4f7 2814
6326aa58 2815 depotPaths = args
5e100b5c
SH
2816
2817 if not self.cloneDestination and len(depotPaths) > 1:
2818 self.cloneDestination = depotPaths[-1]
2819 depotPaths = depotPaths[:-1]
2820
354081d5 2821 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
2822 for p in depotPaths:
2823 if not p.startswith("//"):
2824 return False
f9a3a4f7 2825
bb6e09b2 2826 if not self.cloneDestination:
98ad4faf 2827 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 2828
86dff6b6 2829 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 2830
c3bf3f13
KG
2831 if not os.path.exists(self.cloneDestination):
2832 os.makedirs(self.cloneDestination)
053fd0c1 2833 chdir(self.cloneDestination)
38200076
PW
2834
2835 init_cmd = [ "git", "init" ]
2836 if self.cloneBare:
2837 init_cmd.append("--bare")
2838 subprocess.check_call(init_cmd)
2839
6326aa58 2840 if not P4Sync.run(self, depotPaths):
f9a3a4f7 2841 return False
f9a3a4f7 2842 if self.branch != "master":
e9905013
TAL
2843 if self.importIntoRemotes:
2844 masterbranch = "refs/remotes/p4/master"
2845 else:
2846 masterbranch = "refs/heads/p4/master"
2847 if gitBranchExists(masterbranch):
2848 system("git branch master %s" % masterbranch)
38200076
PW
2849 if not self.cloneBare:
2850 system("git checkout -f")
8f9b2e08
SH
2851 else:
2852 print "Could not detect main branch. No checkout/master branch created."
86dff6b6 2853
a93d33ee
PW
2854 # auto-set this variable if invoked with --use-client-spec
2855 if self.useClientSpec_from_options:
2856 system("git config --bool git-p4.useclientspec true")
2857
f9a3a4f7
SH
2858 return True
2859
09d89de2
SH
2860class P4Branches(Command):
2861 def __init__(self):
2862 Command.__init__(self)
2863 self.options = [ ]
2864 self.description = ("Shows the git branches that hold imports and their "
2865 + "corresponding perforce depot paths")
2866 self.verbose = False
2867
2868 def run(self, args):
5ca44617
SH
2869 if originP4BranchesExist():
2870 createOrUpdateBranchesFromOrigin()
2871
09d89de2
SH
2872 cmdline = "git rev-parse --symbolic "
2873 cmdline += " --remotes"
2874
2875 for line in read_pipe_lines(cmdline):
2876 line = line.strip()
2877
2878 if not line.startswith('p4/') or line == "p4/HEAD":
2879 continue
2880 branch = line
2881
2882 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2883 settings = extractSettingsGitLog(log)
2884
2885 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2886 return True
2887
b984733c
SH
2888class HelpFormatter(optparse.IndentedHelpFormatter):
2889 def __init__(self):
2890 optparse.IndentedHelpFormatter.__init__(self)
2891
2892 def format_description(self, description):
2893 if description:
2894 return description + "\n"
2895 else:
2896 return ""
4f5cf76a 2897
86949eef
SH
2898def printUsage(commands):
2899 print "usage: %s <command> [options]" % sys.argv[0]
2900 print ""
2901 print "valid commands: %s" % ", ".join(commands)
2902 print ""
2903 print "Try %s <command> --help for command specific help." % sys.argv[0]
2904 print ""
2905
2906commands = {
b86f7378
HWN
2907 "debug" : P4Debug,
2908 "submit" : P4Submit,
a9834f58 2909 "commit" : P4Submit,
b86f7378
HWN
2910 "sync" : P4Sync,
2911 "rebase" : P4Rebase,
2912 "clone" : P4Clone,
09d89de2
SH
2913 "rollback" : P4RollBack,
2914 "branches" : P4Branches
86949eef
SH
2915}
2916
86949eef 2917
bb6e09b2
HWN
2918def main():
2919 if len(sys.argv[1:]) == 0:
2920 printUsage(commands.keys())
2921 sys.exit(2)
4f5cf76a 2922
bb6e09b2
HWN
2923 cmd = ""
2924 cmdName = sys.argv[1]
2925 try:
b86f7378
HWN
2926 klass = commands[cmdName]
2927 cmd = klass()
bb6e09b2
HWN
2928 except KeyError:
2929 print "unknown command %s" % cmdName
2930 print ""
2931 printUsage(commands.keys())
2932 sys.exit(2)
2933
2934 options = cmd.options
b86f7378 2935 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
2936
2937 args = sys.argv[2:]
2938
6a10b6aa
LD
2939 options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))
2940 if cmd.needsGit:
2941 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 2942
6a10b6aa
LD
2943 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2944 options,
2945 description = cmd.description,
2946 formatter = HelpFormatter())
bb6e09b2 2947
6a10b6aa 2948 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
bb6e09b2
HWN
2949 global verbose
2950 verbose = cmd.verbose
2951 if cmd.needsGit:
b86f7378
HWN
2952 if cmd.gitdir == None:
2953 cmd.gitdir = os.path.abspath(".git")
2954 if not isValidGitDir(cmd.gitdir):
2955 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2956 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
2957 cdup = read_pipe("git rev-parse --show-cdup").strip()
2958 if len(cdup) > 0:
053fd0c1 2959 chdir(cdup);
e20a9e53 2960
b86f7378
HWN
2961 if not isValidGitDir(cmd.gitdir):
2962 if isValidGitDir(cmd.gitdir + "/.git"):
2963 cmd.gitdir += "/.git"
bb6e09b2 2964 else:
b86f7378 2965 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 2966
b86f7378 2967 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 2968
bb6e09b2
HWN
2969 if not cmd.run(args):
2970 parser.print_help()
09fca77b 2971 sys.exit(2)
4f5cf76a 2972
4f5cf76a 2973
bb6e09b2
HWN
2974if __name__ == '__main__':
2975 main()