git-p4: add support for large file systems
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
8b41a97f 27
a235e85c
BC
28try:
29 from subprocess import CalledProcessError
30except ImportError:
31 # from python2.7:subprocess.py
32 # Exception classes used by this module.
33 class CalledProcessError(Exception):
34 """This exception is raised when a process run by check_call() returns
35 a non-zero exit status. The exit status will be stored in the
36 returncode attribute."""
37 def __init__(self, returncode, cmd):
38 self.returncode = returncode
39 self.cmd = cmd
40 def __str__(self):
41 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
42
4addad22 43verbose = False
86949eef 44
06804c76 45# Only labels/tags matching this will be imported/exported
c8942a22 46defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 47
1051ef00
LD
48# Grab changes in blocks of this many revisions, unless otherwise requested
49defaultBlockSize = 512
50
21a50753
AK
51def p4_build_cmd(cmd):
52 """Build a suitable p4 command line.
53
54 This consolidates building and returning a p4 command line into one
55 location. It means that hooking into the environment, or other configuration
56 can be done more easily.
57 """
6de040df 58 real_cmd = ["p4"]
abcaf073
AK
59
60 user = gitConfig("git-p4.user")
61 if len(user) > 0:
6de040df 62 real_cmd += ["-u",user]
abcaf073
AK
63
64 password = gitConfig("git-p4.password")
65 if len(password) > 0:
6de040df 66 real_cmd += ["-P", password]
abcaf073
AK
67
68 port = gitConfig("git-p4.port")
69 if len(port) > 0:
6de040df 70 real_cmd += ["-p", port]
abcaf073
AK
71
72 host = gitConfig("git-p4.host")
73 if len(host) > 0:
41799aa2 74 real_cmd += ["-H", host]
abcaf073
AK
75
76 client = gitConfig("git-p4.client")
77 if len(client) > 0:
6de040df 78 real_cmd += ["-c", client]
abcaf073 79
6de040df
LD
80
81 if isinstance(cmd,basestring):
82 real_cmd = ' '.join(real_cmd) + ' ' + cmd
83 else:
84 real_cmd += cmd
21a50753
AK
85 return real_cmd
86
bbd84863
MF
87def chdir(path, is_client_path=False):
88 """Do chdir to the given path, and set the PWD environment
89 variable for use by P4. It does not look at getcwd() output.
90 Since we're not using the shell, it is necessary to set the
91 PWD environment variable explicitly.
92
93 Normally, expand the path to force it to be absolute. This
94 addresses the use of relative path names inside P4 settings,
95 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
96 as given; it looks for .p4config using PWD.
97
98 If is_client_path, the path was handed to us directly by p4,
99 and may be a symbolic link. Do not call os.getcwd() in this
100 case, because it will cause p4 to think that PWD is not inside
101 the client path.
102 """
103
104 os.chdir(path)
105 if not is_client_path:
106 path = os.getcwd()
107 os.environ['PWD'] = path
053fd0c1 108
4d25dc44
LS
109def calcDiskFree():
110 """Return free space in bytes on the disk of the given dirname."""
111 if platform.system() == 'Windows':
112 free_bytes = ctypes.c_ulonglong(0)
113 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
114 return free_bytes.value
115 else:
116 st = os.statvfs(os.getcwd())
117 return st.f_bavail * st.f_frsize
118
86dff6b6
HWN
119def die(msg):
120 if verbose:
121 raise Exception(msg)
122 else:
123 sys.stderr.write(msg + "\n")
124 sys.exit(1)
125
6de040df 126def write_pipe(c, stdin):
4addad22 127 if verbose:
6de040df 128 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 129
6de040df
LD
130 expand = isinstance(c,basestring)
131 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
132 pipe = p.stdin
133 val = pipe.write(stdin)
134 pipe.close()
135 if p.wait():
136 die('Command failed: %s' % str(c))
b016d397
HWN
137
138 return val
139
6de040df 140def p4_write_pipe(c, stdin):
d9429194 141 real_cmd = p4_build_cmd(c)
6de040df 142 return write_pipe(real_cmd, stdin)
d9429194 143
4addad22
HWN
144def read_pipe(c, ignore_error=False):
145 if verbose:
6de040df 146 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 147
6de040df
LD
148 expand = isinstance(c,basestring)
149 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
150 pipe = p.stdout
b016d397 151 val = pipe.read()
6de040df
LD
152 if p.wait() and not ignore_error:
153 die('Command failed: %s' % str(c))
b016d397
HWN
154
155 return val
156
d9429194
AK
157def p4_read_pipe(c, ignore_error=False):
158 real_cmd = p4_build_cmd(c)
159 return read_pipe(real_cmd, ignore_error)
b016d397 160
bce4c5fc 161def read_pipe_lines(c):
4addad22 162 if verbose:
6de040df
LD
163 sys.stderr.write('Reading pipe: %s\n' % str(c))
164
165 expand = isinstance(c, basestring)
166 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
167 pipe = p.stdout
b016d397 168 val = pipe.readlines()
6de040df
LD
169 if pipe.close() or p.wait():
170 die('Command failed: %s' % str(c))
b016d397
HWN
171
172 return val
caace111 173
2318121b
AK
174def p4_read_pipe_lines(c):
175 """Specifically invoke p4 on the command supplied. """
155af834 176 real_cmd = p4_build_cmd(c)
2318121b
AK
177 return read_pipe_lines(real_cmd)
178
8e9497c2
GG
179def p4_has_command(cmd):
180 """Ask p4 for help on this command. If it returns an error, the
181 command does not exist in this version of p4."""
182 real_cmd = p4_build_cmd(["help", cmd])
183 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
184 stderr=subprocess.PIPE)
185 p.communicate()
186 return p.returncode == 0
187
249da4c0
PW
188def p4_has_move_command():
189 """See if the move command exists, that it supports -k, and that
190 it has not been administratively disabled. The arguments
191 must be correct, but the filenames do not have to exist. Use
192 ones with wildcards so even if they exist, it will fail."""
193
194 if not p4_has_command("move"):
195 return False
196 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
197 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
198 (out, err) = p.communicate()
199 # return code will be 1 in either case
200 if err.find("Invalid option") >= 0:
201 return False
202 if err.find("disabled") >= 0:
203 return False
204 # assume it failed because @... was invalid changelist
205 return True
206
6754a299 207def system(cmd):
6de040df 208 expand = isinstance(cmd,basestring)
4addad22 209 if verbose:
6de040df 210 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c
BC
211 retcode = subprocess.call(cmd, shell=expand)
212 if retcode:
213 raise CalledProcessError(retcode, cmd)
6754a299 214
bf9320f1
AK
215def p4_system(cmd):
216 """Specifically invoke p4 as the system command. """
155af834 217 real_cmd = p4_build_cmd(cmd)
6de040df 218 expand = isinstance(real_cmd, basestring)
a235e85c
BC
219 retcode = subprocess.call(real_cmd, shell=expand)
220 if retcode:
221 raise CalledProcessError(retcode, real_cmd)
6de040df 222
7f0e5962
PW
223_p4_version_string = None
224def p4_version_string():
225 """Read the version string, showing just the last line, which
226 hopefully is the interesting version bit.
227
228 $ p4 -V
229 Perforce - The Fast Software Configuration Management System.
230 Copyright 1995-2011 Perforce Software. All rights reserved.
231 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
232 """
233 global _p4_version_string
234 if not _p4_version_string:
235 a = p4_read_pipe_lines(["-V"])
236 _p4_version_string = a[-1].rstrip()
237 return _p4_version_string
238
6de040df 239def p4_integrate(src, dest):
9d7d446a 240 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 241
8d7ec362 242def p4_sync(f, *options):
9d7d446a 243 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
244
245def p4_add(f):
9d7d446a
PW
246 # forcibly add file names with wildcards
247 if wildcard_present(f):
248 p4_system(["add", "-f", f])
249 else:
250 p4_system(["add", f])
6de040df
LD
251
252def p4_delete(f):
9d7d446a 253 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
254
255def p4_edit(f):
9d7d446a 256 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
257
258def p4_revert(f):
9d7d446a 259 p4_system(["revert", wildcard_encode(f)])
6de040df 260
9d7d446a
PW
261def p4_reopen(type, f):
262 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 263
8e9497c2
GG
264def p4_move(src, dest):
265 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
266
1051ef00
LD
267def p4_last_change():
268 results = p4CmdList(["changes", "-m", "1"])
269 return int(results[0]['change'])
270
18fa13d0
PW
271def p4_describe(change):
272 """Make sure it returns a valid result by checking for
273 the presence of field "time". Return a dict of the
274 results."""
275
276 ds = p4CmdList(["describe", "-s", str(change)])
277 if len(ds) != 1:
278 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
279
280 d = ds[0]
281
282 if "p4ExitCode" in d:
283 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
284 str(d)))
285 if "code" in d:
286 if d["code"] == "error":
287 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
288
289 if "time" not in d:
290 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
291
292 return d
293
9cffb8c8
PW
294#
295# Canonicalize the p4 type and return a tuple of the
296# base type, plus any modifiers. See "p4 help filetypes"
297# for a list and explanation.
298#
299def split_p4_type(p4type):
300
301 p4_filetypes_historical = {
302 "ctempobj": "binary+Sw",
303 "ctext": "text+C",
304 "cxtext": "text+Cx",
305 "ktext": "text+k",
306 "kxtext": "text+kx",
307 "ltext": "text+F",
308 "tempobj": "binary+FSw",
309 "ubinary": "binary+F",
310 "uresource": "resource+F",
311 "uxbinary": "binary+Fx",
312 "xbinary": "binary+x",
313 "xltext": "text+Fx",
314 "xtempobj": "binary+Swx",
315 "xtext": "text+x",
316 "xunicode": "unicode+x",
317 "xutf16": "utf16+x",
318 }
319 if p4type in p4_filetypes_historical:
320 p4type = p4_filetypes_historical[p4type]
321 mods = ""
322 s = p4type.split("+")
323 base = s[0]
324 mods = ""
325 if len(s) > 1:
326 mods = s[1]
327 return (base, mods)
b9fc6ea9 328
60df071c
LD
329#
330# return the raw p4 type of a file (text, text+ko, etc)
331#
79467e61
PW
332def p4_type(f):
333 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
334 return results[0]['headType']
335
336#
337# Given a type base and modifier, return a regexp matching
338# the keywords that can be expanded in the file
339#
340def p4_keywords_regexp_for_type(base, type_mods):
341 if base in ("text", "unicode", "binary"):
342 kwords = None
343 if "ko" in type_mods:
344 kwords = 'Id|Header'
345 elif "k" in type_mods:
346 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
347 else:
348 return None
349 pattern = r"""
350 \$ # Starts with a dollar, followed by...
351 (%s) # one of the keywords, followed by...
6b2bf41e 352 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
353 \$ # another dollar
354 """ % kwords
355 return pattern
356 else:
357 return None
358
359#
360# Given a file, return a regexp matching the possible
361# RCS keywords that will be expanded, or None for files
362# with kw expansion turned off.
363#
364def p4_keywords_regexp_for_file(file):
365 if not os.path.exists(file):
366 return None
367 else:
368 (type_base, type_mods) = split_p4_type(p4_type(file))
369 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 370
c65b670e
CP
371def setP4ExecBit(file, mode):
372 # Reopens an already open file and changes the execute bit to match
373 # the execute bit setting in the passed in mode.
374
375 p4Type = "+x"
376
377 if not isModeExec(mode):
378 p4Type = getP4OpenedType(file)
379 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
380 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
381 if p4Type[-1] == "+":
382 p4Type = p4Type[0:-1]
383
6de040df 384 p4_reopen(p4Type, file)
c65b670e
CP
385
386def getP4OpenedType(file):
387 # Returns the perforce file type for the given file.
388
9d7d446a 389 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 390 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
391 if match:
392 return match.group(1)
393 else:
f3e5ae4f 394 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 395
06804c76
LD
396# Return the set of all p4 labels
397def getP4Labels(depotPaths):
398 labels = set()
399 if isinstance(depotPaths,basestring):
400 depotPaths = [depotPaths]
401
402 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
403 label = l['label']
404 labels.add(label)
405
406 return labels
407
408# Return the set of all git tags
409def getGitTags():
410 gitTags = set()
411 for line in read_pipe_lines(["git", "tag"]):
412 tag = line.strip()
413 gitTags.add(tag)
414 return gitTags
415
b43b0a3c
CP
416def diffTreePattern():
417 # This is a simple generator for the diff tree regex pattern. This could be
418 # a class variable if this and parseDiffTreeEntry were a part of a class.
419 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
420 while True:
421 yield pattern
422
423def parseDiffTreeEntry(entry):
424 """Parses a single diff tree entry into its component elements.
425
426 See git-diff-tree(1) manpage for details about the format of the diff
427 output. This method returns a dictionary with the following elements:
428
429 src_mode - The mode of the source file
430 dst_mode - The mode of the destination file
431 src_sha1 - The sha1 for the source file
432 dst_sha1 - The sha1 fr the destination file
433 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
434 status_score - The score for the status (applicable for 'C' and 'R'
435 statuses). This is None if there is no score.
436 src - The path for the source file.
437 dst - The path for the destination file. This is only present for
438 copy or renames. If it is not present, this is None.
439
440 If the pattern is not matched, None is returned."""
441
442 match = diffTreePattern().next().match(entry)
443 if match:
444 return {
445 'src_mode': match.group(1),
446 'dst_mode': match.group(2),
447 'src_sha1': match.group(3),
448 'dst_sha1': match.group(4),
449 'status': match.group(5),
450 'status_score': match.group(6),
451 'src': match.group(7),
452 'dst': match.group(10)
453 }
454 return None
455
c65b670e
CP
456def isModeExec(mode):
457 # Returns True if the given git mode represents an executable file,
458 # otherwise False.
459 return mode[-3:] == "755"
460
461def isModeExecChanged(src_mode, dst_mode):
462 return isModeExec(src_mode) != isModeExec(dst_mode)
463
b932705b 464def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
465
466 if isinstance(cmd,basestring):
467 cmd = "-G " + cmd
468 expand = True
469 else:
470 cmd = ["-G"] + cmd
471 expand = False
472
473 cmd = p4_build_cmd(cmd)
6a49f8e2 474 if verbose:
6de040df 475 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
476
477 # Use a temporary file to avoid deadlocks without
478 # subprocess.communicate(), which would put another copy
479 # of stdout into memory.
480 stdin_file = None
481 if stdin is not None:
482 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
483 if isinstance(stdin,basestring):
484 stdin_file.write(stdin)
485 else:
486 for i in stdin:
487 stdin_file.write(i + '\n')
9f90c733
SL
488 stdin_file.flush()
489 stdin_file.seek(0)
490
6de040df
LD
491 p4 = subprocess.Popen(cmd,
492 shell=expand,
9f90c733
SL
493 stdin=stdin_file,
494 stdout=subprocess.PIPE)
86949eef
SH
495
496 result = []
497 try:
498 while True:
9f90c733 499 entry = marshal.load(p4.stdout)
c3f6163b
AG
500 if cb is not None:
501 cb(entry)
502 else:
503 result.append(entry)
86949eef
SH
504 except EOFError:
505 pass
9f90c733
SL
506 exitCode = p4.wait()
507 if exitCode != 0:
ac3e0d79
SH
508 entry = {}
509 entry["p4ExitCode"] = exitCode
510 result.append(entry)
86949eef
SH
511
512 return result
513
514def p4Cmd(cmd):
515 list = p4CmdList(cmd)
516 result = {}
517 for entry in list:
518 result.update(entry)
519 return result;
520
cb2c9db5
SH
521def p4Where(depotPath):
522 if not depotPath.endswith("/"):
523 depotPath += "/"
cd884106
VA
524 depotPathLong = depotPath + "..."
525 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
526 output = None
527 for entry in outputList:
75bc9573 528 if "depotFile" in entry:
cd884106
VA
529 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
530 # The base path always ends with "/...".
531 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
532 output = entry
533 break
534 elif "data" in entry:
535 data = entry.get("data")
536 space = data.find(" ")
537 if data[:space] == depotPath:
538 output = entry
539 break
7f705dc3
TAL
540 if output == None:
541 return ""
dc524036
SH
542 if output["code"] == "error":
543 return ""
cb2c9db5
SH
544 clientPath = ""
545 if "path" in output:
546 clientPath = output.get("path")
547 elif "data" in output:
548 data = output.get("data")
549 lastSpace = data.rfind(" ")
550 clientPath = data[lastSpace + 1:]
551
552 if clientPath.endswith("..."):
553 clientPath = clientPath[:-3]
554 return clientPath
555
86949eef 556def currentGitBranch():
b25b2065 557 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 558
4f5cf76a 559def isValidGitDir(path):
bb6e09b2
HWN
560 if (os.path.exists(path + "/HEAD")
561 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
562 return True;
563 return False
564
463e8af6 565def parseRevision(ref):
b25b2065 566 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 567
28755dba
PW
568def branchExists(ref):
569 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
570 ignore_error=True)
571 return len(rev) > 0
572
6ae8de88
SH
573def extractLogMessageFromGitCommit(commit):
574 logMessage = ""
b016d397
HWN
575
576 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 577 foundTitle = False
b016d397 578 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
579 if not foundTitle:
580 if len(log) == 1:
1c094184 581 foundTitle = True
6ae8de88
SH
582 continue
583
584 logMessage += log
585 return logMessage
586
bb6e09b2 587def extractSettingsGitLog(log):
6ae8de88
SH
588 values = {}
589 for line in log.split("\n"):
590 line = line.strip()
6326aa58
HWN
591 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
592 if not m:
593 continue
594
595 assignments = m.group(1).split (':')
596 for a in assignments:
597 vals = a.split ('=')
598 key = vals[0].strip()
599 val = ('='.join (vals[1:])).strip()
600 if val.endswith ('\"') and val.startswith('"'):
601 val = val[1:-1]
602
603 values[key] = val
604
845b42cb
SH
605 paths = values.get("depot-paths")
606 if not paths:
607 paths = values.get("depot-path")
a3fdd579
SH
608 if paths:
609 values['depot-paths'] = paths.split(',')
bb6e09b2 610 return values
6ae8de88 611
8136a639 612def gitBranchExists(branch):
bb6e09b2
HWN
613 proc = subprocess.Popen(["git", "rev-parse", branch],
614 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 615 return proc.wait() == 0;
8136a639 616
36bd8446 617_gitConfig = {}
b345d6c3 618
692e1796 619def gitConfig(key, typeSpecifier=None):
36bd8446 620 if not _gitConfig.has_key(key):
692e1796
LS
621 cmd = [ "git", "config" ]
622 if typeSpecifier:
623 cmd += [ typeSpecifier ]
624 cmd += [ key ]
b345d6c3
PW
625 s = read_pipe(cmd, ignore_error=True)
626 _gitConfig[key] = s.strip()
36bd8446 627 return _gitConfig[key]
01265103 628
0d609032
PW
629def gitConfigBool(key):
630 """Return a bool, using git config --bool. It is True only if the
631 variable is set to true, and False if set to false or not present
632 in the config."""
633
36bd8446 634 if not _gitConfig.has_key(key):
692e1796 635 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 636 return _gitConfig[key]
01265103 637
cb1dafdf
LS
638def gitConfigInt(key):
639 if not _gitConfig.has_key(key):
640 cmd = [ "git", "config", "--int", key ]
641 s = read_pipe(cmd, ignore_error=True)
642 v = s.strip()
643 try:
644 _gitConfig[key] = int(gitConfig(key, '--int'))
645 except ValueError:
646 _gitConfig[key] = None
647 return _gitConfig[key]
648
7199cf13
VA
649def gitConfigList(key):
650 if not _gitConfig.has_key(key):
2abba301
PW
651 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
652 _gitConfig[key] = s.strip().split(os.linesep)
7960e707
LS
653 if _gitConfig[key] == ['']:
654 _gitConfig[key] = []
7199cf13
VA
655 return _gitConfig[key]
656
2c8037ed
PW
657def p4BranchesInGit(branchesAreInRemotes=True):
658 """Find all the branches whose names start with "p4/", looking
659 in remotes or heads as specified by the argument. Return
660 a dictionary of { branch: revision } for each one found.
661 The branch names are the short names, without any
662 "p4/" prefix."""
663
062410bb
SH
664 branches = {}
665
666 cmdline = "git rev-parse --symbolic "
667 if branchesAreInRemotes:
2c8037ed 668 cmdline += "--remotes"
062410bb 669 else:
2c8037ed 670 cmdline += "--branches"
062410bb
SH
671
672 for line in read_pipe_lines(cmdline):
673 line = line.strip()
674
2c8037ed
PW
675 # only import to p4/
676 if not line.startswith('p4/'):
677 continue
678 # special symbolic ref to p4/master
679 if line == "p4/HEAD":
062410bb 680 continue
062410bb 681
2c8037ed
PW
682 # strip off p4/ prefix
683 branch = line[len("p4/"):]
062410bb
SH
684
685 branches[branch] = parseRevision(line)
2c8037ed 686
062410bb
SH
687 return branches
688
5a8e84cd
PW
689def branch_exists(branch):
690 """Make sure that the given ref name really exists."""
691
692 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
693 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
694 out, _ = p.communicate()
695 if p.returncode:
696 return False
697 # expect exactly one line of output: the branch name
698 return out.rstrip() == branch
699
9ceab363 700def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
701 branches = p4BranchesInGit()
702 # map from depot-path to branch name
703 branchByDepotPath = {}
704 for branch in branches.keys():
705 tip = branches[branch]
706 log = extractLogMessageFromGitCommit(tip)
707 settings = extractSettingsGitLog(log)
708 if settings.has_key("depot-paths"):
709 paths = ",".join(settings["depot-paths"])
710 branchByDepotPath[paths] = "remotes/p4/" + branch
711
27d2d811 712 settings = None
27d2d811
SH
713 parent = 0
714 while parent < 65535:
9ceab363 715 commit = head + "~%s" % parent
27d2d811
SH
716 log = extractLogMessageFromGitCommit(commit)
717 settings = extractSettingsGitLog(log)
86506fe5
SH
718 if settings.has_key("depot-paths"):
719 paths = ",".join(settings["depot-paths"])
720 if branchByDepotPath.has_key(paths):
721 return [branchByDepotPath[paths], settings]
27d2d811 722
86506fe5 723 parent = parent + 1
27d2d811 724
86506fe5 725 return ["", settings]
27d2d811 726
5ca44617
SH
727def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
728 if not silent:
729 print ("Creating/updating branch(es) in %s based on origin branch(es)"
730 % localRefPrefix)
731
732 originPrefix = "origin/p4/"
733
734 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
735 line = line.strip()
736 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
737 continue
738
739 headName = line[len(originPrefix):]
740 remoteHead = localRefPrefix + headName
741 originHead = line
742
743 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
744 if (not original.has_key('depot-paths')
745 or not original.has_key('change')):
746 continue
747
748 update = False
749 if not gitBranchExists(remoteHead):
750 if verbose:
751 print "creating %s" % remoteHead
752 update = True
753 else:
754 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
755 if settings.has_key('change') > 0:
756 if settings['depot-paths'] == original['depot-paths']:
757 originP4Change = int(original['change'])
758 p4Change = int(settings['change'])
759 if originP4Change > p4Change:
760 print ("%s (%s) is newer than %s (%s). "
761 "Updating p4 branch from origin."
762 % (originHead, originP4Change,
763 remoteHead, p4Change))
764 update = True
765 else:
766 print ("Ignoring: %s was imported from %s while "
767 "%s was imported from %s"
768 % (originHead, ','.join(original['depot-paths']),
769 remoteHead, ','.join(settings['depot-paths'])))
770
771 if update:
772 system("git update-ref %s %s" % (remoteHead, originHead))
773
774def originP4BranchesExist():
775 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
776
1051ef00
LD
777
778def p4ParseNumericChangeRange(parts):
779 changeStart = int(parts[0][1:])
780 if parts[1] == '#head':
781 changeEnd = p4_last_change()
782 else:
783 changeEnd = int(parts[1])
784
785 return (changeStart, changeEnd)
786
787def chooseBlockSize(blockSize):
788 if blockSize:
789 return blockSize
790 else:
791 return defaultBlockSize
792
793def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 794 assert depotPaths
96b2d54a 795
1051ef00
LD
796 # Parse the change range into start and end. Try to find integer
797 # revision ranges as these can be broken up into blocks to avoid
798 # hitting server-side limits (maxrows, maxscanresults). But if
799 # that doesn't work, fall back to using the raw revision specifier
800 # strings, without using block mode.
801
96b2d54a 802 if changeRange is None or changeRange == '':
1051ef00
LD
803 changeStart = 1
804 changeEnd = p4_last_change()
805 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
806 else:
807 parts = changeRange.split(',')
808 assert len(parts) == 2
1051ef00
LD
809 try:
810 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
811 block_size = chooseBlockSize(requestedBlockSize)
812 except:
813 changeStart = parts[0][1:]
814 changeEnd = parts[1]
815 if requestedBlockSize:
816 die("cannot use --changes-block-size with non-numeric revisions")
817 block_size = None
4f6432d8 818
96b2d54a 819 # Accumulate change numbers in a dictionary to avoid duplicates
b4b0ba06 820 changes = {}
96b2d54a
LS
821
822 for p in depotPaths:
823 # Retrieve changes a block at a time, to prevent running
1051ef00
LD
824 # into a MaxResults/MaxScanRows error from the server.
825
826 while True:
96b2d54a 827 cmd = ['changes']
1051ef00
LD
828
829 if block_size:
830 end = min(changeEnd, changeStart + block_size)
831 revisionRange = "%d,%d" % (changeStart, end)
832 else:
833 revisionRange = "%s,%s" % (changeStart, changeEnd)
834
835 cmd += ["%s...@%s" % (p, revisionRange)]
836
96b2d54a
LS
837 for line in p4_read_pipe_lines(cmd):
838 changeNum = int(line.split(" ")[1])
96b2d54a 839 changes[changeNum] = True
1051ef00
LD
840
841 if not block_size:
842 break
843
844 if end >= changeEnd:
845 break
846
847 changeStart = end + 1
4f6432d8 848
b4b0ba06
PW
849 changelist = changes.keys()
850 changelist.sort()
851 return changelist
4f6432d8 852
d53de8b9
TAL
853def p4PathStartsWith(path, prefix):
854 # This method tries to remedy a potential mixed-case issue:
855 #
856 # If UserA adds //depot/DirA/file1
857 # and UserB adds //depot/dira/file2
858 #
859 # we may or may not have a problem. If you have core.ignorecase=true,
860 # we treat DirA and dira as the same directory
0d609032 861 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
862 return path.lower().startswith(prefix.lower())
863 return path.startswith(prefix)
864
543987bd
PW
865def getClientSpec():
866 """Look at the p4 client spec, create a View() object that contains
867 all the mappings, and return it."""
868
869 specList = p4CmdList("client -o")
870 if len(specList) != 1:
871 die('Output from "client -o" is %d lines, expecting 1' %
872 len(specList))
873
874 # dictionary of all client parameters
875 entry = specList[0]
876
9d57c4a6
KS
877 # the //client/ name
878 client_name = entry["Client"]
879
543987bd
PW
880 # just the keys that start with "View"
881 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
882
883 # hold this new View
9d57c4a6 884 view = View(client_name)
543987bd
PW
885
886 # append the lines, in order, to the view
887 for view_num in range(len(view_keys)):
888 k = "View%d" % view_num
889 if k not in view_keys:
890 die("Expected view key %s missing" % k)
891 view.append(entry[k])
892
893 return view
894
895def getClientRoot():
896 """Grab the client directory."""
897
898 output = p4CmdList("client -o")
899 if len(output) != 1:
900 die('Output from "client -o" is %d lines, expecting 1' % len(output))
901
902 entry = output[0]
903 if "Root" not in entry:
904 die('Client has no "Root"')
905
906 return entry["Root"]
907
9d7d446a
PW
908#
909# P4 wildcards are not allowed in filenames. P4 complains
910# if you simply add them, but you can force it with "-f", in
911# which case it translates them into %xx encoding internally.
912#
913def wildcard_decode(path):
914 # Search for and fix just these four characters. Do % last so
915 # that fixing it does not inadvertently create new %-escapes.
916 # Cannot have * in a filename in windows; untested as to
917 # what p4 would do in such a case.
918 if not platform.system() == "Windows":
919 path = path.replace("%2A", "*")
920 path = path.replace("%23", "#") \
921 .replace("%40", "@") \
922 .replace("%25", "%")
923 return path
924
925def wildcard_encode(path):
926 # do % first to avoid double-encoding the %s introduced here
927 path = path.replace("%", "%25") \
928 .replace("*", "%2A") \
929 .replace("#", "%23") \
930 .replace("@", "%40")
931 return path
932
933def wildcard_present(path):
598354c0
BC
934 m = re.search("[*#@%]", path)
935 return m is not None
9d7d446a 936
a5db4b12
LS
937class LargeFileSystem(object):
938 """Base class for large file system support."""
939
940 def __init__(self, writeToGitStream):
941 self.largeFiles = set()
942 self.writeToGitStream = writeToGitStream
943
944 def generatePointer(self, cloneDestination, contentFile):
945 """Return the content of a pointer file that is stored in Git instead of
946 the actual content."""
947 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
948
949 def pushFile(self, localLargeFile):
950 """Push the actual content which is not stored in the Git repository to
951 a server."""
952 assert False, "Method 'pushFile' required in " + self.__class__.__name__
953
954 def hasLargeFileExtension(self, relPath):
955 return reduce(
956 lambda a, b: a or b,
957 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
958 False
959 )
960
961 def generateTempFile(self, contents):
962 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
963 for d in contents:
964 contentFile.write(d)
965 contentFile.close()
966 return contentFile.name
967
968 def exceedsLargeFileThreshold(self, relPath, contents):
969 if gitConfigInt('git-p4.largeFileThreshold'):
970 contentsSize = sum(len(d) for d in contents)
971 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
972 return True
973 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
974 contentsSize = sum(len(d) for d in contents)
975 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
976 return False
977 contentTempFile = self.generateTempFile(contents)
978 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
979 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
980 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
981 zf.close()
982 compressedContentsSize = zf.infolist()[0].compress_size
983 os.remove(contentTempFile)
984 os.remove(compressedContentFile.name)
985 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
986 return True
987 return False
988
989 def addLargeFile(self, relPath):
990 self.largeFiles.add(relPath)
991
992 def removeLargeFile(self, relPath):
993 self.largeFiles.remove(relPath)
994
995 def isLargeFile(self, relPath):
996 return relPath in self.largeFiles
997
998 def processContent(self, git_mode, relPath, contents):
999 """Processes the content of git fast import. This method decides if a
1000 file is stored in the large file system and handles all necessary
1001 steps."""
1002 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1003 contentTempFile = self.generateTempFile(contents)
1004 (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1005
1006 # Move temp file to final location in large file system
1007 largeFileDir = os.path.dirname(localLargeFile)
1008 if not os.path.isdir(largeFileDir):
1009 os.makedirs(largeFileDir)
1010 shutil.move(contentTempFile, localLargeFile)
1011 self.addLargeFile(relPath)
1012 if gitConfigBool('git-p4.largeFilePush'):
1013 self.pushFile(localLargeFile)
1014 if verbose:
1015 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1016 return (git_mode, contents)
1017
1018class MockLFS(LargeFileSystem):
1019 """Mock large file system for testing."""
1020
1021 def generatePointer(self, contentFile):
1022 """The pointer content is the original content prefixed with "pointer-".
1023 The local filename of the large file storage is derived from the file content.
1024 """
1025 with open(contentFile, 'r') as f:
1026 content = next(f)
1027 gitMode = '100644'
1028 pointerContents = 'pointer-' + content
1029 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1030 return (gitMode, pointerContents, localLargeFile)
1031
1032 def pushFile(self, localLargeFile):
1033 """The remote filename of the large file storage is the same as the local
1034 one but in a different directory.
1035 """
1036 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1037 if not os.path.exists(remotePath):
1038 os.makedirs(remotePath)
1039 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1040
b984733c
SH
1041class Command:
1042 def __init__(self):
1043 self.usage = "usage: %prog [options]"
8910ac0e 1044 self.needsGit = True
6a10b6aa 1045 self.verbose = False
b984733c 1046
3ea2cfd4
LD
1047class P4UserMap:
1048 def __init__(self):
1049 self.userMapFromPerforceServer = False
affb474f
LD
1050 self.myP4UserId = None
1051
1052 def p4UserId(self):
1053 if self.myP4UserId:
1054 return self.myP4UserId
1055
1056 results = p4CmdList("user -o")
1057 for r in results:
1058 if r.has_key('User'):
1059 self.myP4UserId = r['User']
1060 return r['User']
1061 die("Could not find your p4 user id")
1062
1063 def p4UserIsMe(self, p4User):
1064 # return True if the given p4 user is actually me
1065 me = self.p4UserId()
1066 if not p4User or p4User != me:
1067 return False
1068 else:
1069 return True
3ea2cfd4
LD
1070
1071 def getUserCacheFilename(self):
1072 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1073 return home + "/.gitp4-usercache.txt"
1074
1075 def getUserMapFromPerforceServer(self):
1076 if self.userMapFromPerforceServer:
1077 return
1078 self.users = {}
1079 self.emails = {}
1080
1081 for output in p4CmdList("users"):
1082 if not output.has_key("User"):
1083 continue
1084 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1085 self.emails[output["Email"]] = output["User"]
1086
1087
1088 s = ''
1089 for (key, val) in self.users.items():
1090 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1091
1092 open(self.getUserCacheFilename(), "wb").write(s)
1093 self.userMapFromPerforceServer = True
1094
1095 def loadUserMapFromCache(self):
1096 self.users = {}
1097 self.userMapFromPerforceServer = False
1098 try:
1099 cache = open(self.getUserCacheFilename(), "rb")
1100 lines = cache.readlines()
1101 cache.close()
1102 for line in lines:
1103 entry = line.strip().split("\t")
1104 self.users[entry[0]] = entry[1]
1105 except IOError:
1106 self.getUserMapFromPerforceServer()
1107
b984733c 1108class P4Debug(Command):
86949eef 1109 def __init__(self):
6ae8de88 1110 Command.__init__(self)
6a10b6aa 1111 self.options = []
c8c39116 1112 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1113 self.needsGit = False
86949eef
SH
1114
1115 def run(self, args):
b1ce9447 1116 j = 0
6de040df 1117 for output in p4CmdList(args):
b1ce9447
HWN
1118 print 'Element: %d' % j
1119 j += 1
86949eef 1120 print output
b984733c 1121 return True
86949eef 1122
5834684d
SH
1123class P4RollBack(Command):
1124 def __init__(self):
1125 Command.__init__(self)
1126 self.options = [
0c66a783 1127 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1128 ]
1129 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1130 self.rollbackLocalBranches = False
5834684d
SH
1131
1132 def run(self, args):
1133 if len(args) != 1:
1134 return False
1135 maxChange = int(args[0])
0c66a783 1136
ad192f28 1137 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1138 die("Problems executing p4");
1139
0c66a783
SH
1140 if self.rollbackLocalBranches:
1141 refPrefix = "refs/heads/"
b016d397 1142 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1143 else:
1144 refPrefix = "refs/remotes/"
b016d397 1145 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1146
1147 for line in lines:
1148 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1149 line = line.strip()
1150 ref = refPrefix + line
5834684d 1151 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1152 settings = extractSettingsGitLog(log)
1153
1154 depotPaths = settings['depot-paths']
1155 change = settings['change']
1156
5834684d 1157 changed = False
52102d47 1158
6326aa58
HWN
1159 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1160 for p in depotPaths]))) == 0:
52102d47
SH
1161 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1162 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1163 continue
1164
bb6e09b2 1165 while change and int(change) > maxChange:
5834684d 1166 changed = True
52102d47
SH
1167 if self.verbose:
1168 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1169 system("git update-ref %s \"%s^\"" % (ref, ref))
1170 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1171 settings = extractSettingsGitLog(log)
1172
1173
1174 depotPaths = settings['depot-paths']
1175 change = settings['change']
5834684d
SH
1176
1177 if changed:
52102d47 1178 print "%s rewound to %s" % (ref, change)
5834684d
SH
1179
1180 return True
1181
3ea2cfd4 1182class P4Submit(Command, P4UserMap):
6bbfd137
PW
1183
1184 conflict_behavior_choices = ("ask", "skip", "quit")
1185
4f5cf76a 1186 def __init__(self):
b984733c 1187 Command.__init__(self)
3ea2cfd4 1188 P4UserMap.__init__(self)
4f5cf76a 1189 self.options = [
4f5cf76a 1190 optparse.make_option("--origin", dest="origin"),
ae901090 1191 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1192 # preserve the user, requires relevant p4 permissions
1193 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1194 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1195 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1196 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1197 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1198 choices=self.conflict_behavior_choices),
1199 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
1200 ]
1201 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1202 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1203 self.origin = ""
ae901090 1204 self.detectRenames = False
0d609032 1205 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1206 self.dry_run = False
728b7ad8 1207 self.prepare_p4_only = False
6bbfd137 1208 self.conflict_behavior = None
f7baba8b 1209 self.isWindows = (platform.system() == "Windows")
06804c76 1210 self.exportLabels = False
249da4c0 1211 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1212 self.branch = None
4f5cf76a 1213
a5db4b12
LS
1214 if gitConfig('git-p4.largeFileSystem'):
1215 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1216
4f5cf76a
SH
1217 def check(self):
1218 if len(p4CmdList("opened ...")) > 0:
1219 die("You have files opened with perforce! Close them before starting the sync.")
1220
f19cb0a0
PW
1221 def separate_jobs_from_description(self, message):
1222 """Extract and return a possible Jobs field in the commit
1223 message. It goes into a separate section in the p4 change
1224 specification.
1225
1226 A jobs line starts with "Jobs:" and looks like a new field
1227 in a form. Values are white-space separated on the same
1228 line or on following lines that start with a tab.
1229
1230 This does not parse and extract the full git commit message
1231 like a p4 form. It just sees the Jobs: line as a marker
1232 to pass everything from then on directly into the p4 form,
1233 but outside the description section.
1234
1235 Return a tuple (stripped log message, jobs string)."""
1236
1237 m = re.search(r'^Jobs:', message, re.MULTILINE)
1238 if m is None:
1239 return (message, None)
1240
1241 jobtext = message[m.start():]
1242 stripped_message = message[:m.start()].rstrip()
1243 return (stripped_message, jobtext)
1244
1245 def prepareLogMessage(self, template, message, jobs):
1246 """Edits the template returned from "p4 change -o" to insert
1247 the message in the Description field, and the jobs text in
1248 the Jobs field."""
4f5cf76a
SH
1249 result = ""
1250
edae1e2f
SH
1251 inDescriptionSection = False
1252
4f5cf76a
SH
1253 for line in template.split("\n"):
1254 if line.startswith("#"):
1255 result += line + "\n"
1256 continue
1257
edae1e2f 1258 if inDescriptionSection:
c9dbab04 1259 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1260 inDescriptionSection = False
f19cb0a0
PW
1261 # insert Jobs section
1262 if jobs:
1263 result += jobs + "\n"
edae1e2f
SH
1264 else:
1265 continue
1266 else:
1267 if line.startswith("Description:"):
1268 inDescriptionSection = True
1269 line += "\n"
1270 for messageLine in message.split("\n"):
1271 line += "\t" + messageLine + "\n"
1272
1273 result += line + "\n"
4f5cf76a
SH
1274
1275 return result
1276
60df071c
LD
1277 def patchRCSKeywords(self, file, pattern):
1278 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1279 (handle, outFileName) = tempfile.mkstemp(dir='.')
1280 try:
1281 outFile = os.fdopen(handle, "w+")
1282 inFile = open(file, "r")
1283 regexp = re.compile(pattern, re.VERBOSE)
1284 for line in inFile.readlines():
1285 line = regexp.sub(r'$\1$', line)
1286 outFile.write(line)
1287 inFile.close()
1288 outFile.close()
1289 # Forcibly overwrite the original file
1290 os.unlink(file)
1291 shutil.move(outFileName, file)
1292 except:
1293 # cleanup our temporary file
1294 os.unlink(outFileName)
1295 print "Failed to strip RCS keywords in %s" % file
1296 raise
1297
1298 print "Patched up RCS keywords in %s" % file
1299
3ea2cfd4
LD
1300 def p4UserForCommit(self,id):
1301 # Return the tuple (perforce user,git email) for a given git commit id
1302 self.getUserMapFromPerforceServer()
9bf28855
PW
1303 gitEmail = read_pipe(["git", "log", "--max-count=1",
1304 "--format=%ae", id])
3ea2cfd4
LD
1305 gitEmail = gitEmail.strip()
1306 if not self.emails.has_key(gitEmail):
1307 return (None,gitEmail)
1308 else:
1309 return (self.emails[gitEmail],gitEmail)
1310
1311 def checkValidP4Users(self,commits):
1312 # check if any git authors cannot be mapped to p4 users
1313 for id in commits:
1314 (user,email) = self.p4UserForCommit(id)
1315 if not user:
1316 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1317 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1318 print "%s" % msg
1319 else:
1320 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1321
1322 def lastP4Changelist(self):
1323 # Get back the last changelist number submitted in this client spec. This
1324 # then gets used to patch up the username in the change. If the same
1325 # client spec is being used by multiple processes then this might go
1326 # wrong.
1327 results = p4CmdList("client -o") # find the current client
1328 client = None
1329 for r in results:
1330 if r.has_key('Client'):
1331 client = r['Client']
1332 break
1333 if not client:
1334 die("could not get client spec")
6de040df 1335 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1336 for r in results:
1337 if r.has_key('change'):
1338 return r['change']
1339 die("Could not get changelist number for last submit - cannot patch up user details")
1340
1341 def modifyChangelistUser(self, changelist, newUser):
1342 # fixup the user field of a changelist after it has been submitted.
1343 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1344 if len(changes) != 1:
1345 die("Bad output from p4 change modifying %s to user %s" %
1346 (changelist, newUser))
1347
1348 c = changes[0]
1349 if c['User'] == newUser: return # nothing to do
1350 c['User'] = newUser
1351 input = marshal.dumps(c)
1352
3ea2cfd4
LD
1353 result = p4CmdList("change -f -i", stdin=input)
1354 for r in result:
1355 if r.has_key('code'):
1356 if r['code'] == 'error':
1357 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1358 if r.has_key('data'):
1359 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1360 return
1361 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1362
1363 def canChangeChangelists(self):
1364 # check to see if we have p4 admin or super-user permissions, either of
1365 # which are required to modify changelists.
52a4880b 1366 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1367 for r in results:
1368 if r.has_key('perm'):
1369 if r['perm'] == 'admin':
1370 return 1
1371 if r['perm'] == 'super':
1372 return 1
1373 return 0
1374
ea99c3ae 1375 def prepareSubmitTemplate(self):
f19cb0a0
PW
1376 """Run "p4 change -o" to grab a change specification template.
1377 This does not use "p4 -G", as it is nice to keep the submission
1378 template in original order, since a human might edit it.
1379
1380 Remove lines in the Files section that show changes to files
1381 outside the depot path we're committing into."""
1382
ea99c3ae
SH
1383 template = ""
1384 inFilesSection = False
6de040df 1385 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1386 if line.endswith("\r\n"):
1387 line = line[:-2] + "\n"
ea99c3ae
SH
1388 if inFilesSection:
1389 if line.startswith("\t"):
1390 # path starts and ends with a tab
1391 path = line[1:]
1392 lastTab = path.rfind("\t")
1393 if lastTab != -1:
1394 path = path[:lastTab]
d53de8b9 1395 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
1396 continue
1397 else:
1398 inFilesSection = False
1399 else:
1400 if line.startswith("Files:"):
1401 inFilesSection = True
1402
1403 template += line
1404
1405 return template
1406
7c766e57
PW
1407 def edit_template(self, template_file):
1408 """Invoke the editor to let the user change the submission
1409 message. Return true if okay to continue with the submit."""
1410
1411 # if configured to skip the editing part, just submit
0d609032 1412 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1413 return True
1414
1415 # look at the modification time, to check later if the user saved
1416 # the file
1417 mtime = os.stat(template_file).st_mtime
1418
1419 # invoke the editor
f95ceaf0 1420 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1421 editor = os.environ.get("P4EDITOR")
1422 else:
1423 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1424 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1425
1426 # If the file was not saved, prompt to see if this patch should
1427 # be skipped. But skip this verification step if configured so.
0d609032 1428 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1429 return True
1430
d1652049
PW
1431 # modification time updated means user saved the file
1432 if os.stat(template_file).st_mtime > mtime:
1433 return True
1434
1435 while True:
1436 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1437 if response == 'y':
1438 return True
1439 if response == 'n':
1440 return False
7c766e57 1441
e2a892ee 1442 def get_diff_description(self, editedFiles, filesToAdd):
b4073bb3
MC
1443 # diff
1444 if os.environ.has_key("P4DIFF"):
1445 del(os.environ["P4DIFF"])
1446 diff = ""
1447 for editedFile in editedFiles:
1448 diff += p4_read_pipe(['diff', '-du',
1449 wildcard_encode(editedFile)])
1450
1451 # new file diff
1452 newdiff = ""
1453 for newFile in filesToAdd:
1454 newdiff += "==== new file ====\n"
1455 newdiff += "--- /dev/null\n"
1456 newdiff += "+++ %s\n" % newFile
1457 f = open(newFile, "r")
1458 for line in f.readlines():
1459 newdiff += "+" + line
1460 f.close()
1461
e2a892ee 1462 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1463
7cb5cbef 1464 def applyCommit(self, id):
67b0fe2e
PW
1465 """Apply one commit, return True if it succeeded."""
1466
1467 print "Applying", read_pipe(["git", "show", "-s",
1468 "--format=format:%h %s", id])
ae901090 1469
848de9c3 1470 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1471
84cb0003 1472 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1473 filesToAdd = set()
1474 filesToDelete = set()
d336c158 1475 editedFiles = set()
b6ad6dcc 1476 pureRenameCopy = set()
c65b670e 1477 filesToChangeExecBit = {}
60df071c 1478
4f5cf76a 1479 for line in diff:
b43b0a3c
CP
1480 diff = parseDiffTreeEntry(line)
1481 modifier = diff['status']
1482 path = diff['src']
4f5cf76a 1483 if modifier == "M":
6de040df 1484 p4_edit(path)
c65b670e
CP
1485 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1486 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1487 editedFiles.add(path)
4f5cf76a
SH
1488 elif modifier == "A":
1489 filesToAdd.add(path)
c65b670e 1490 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1491 if path in filesToDelete:
1492 filesToDelete.remove(path)
1493 elif modifier == "D":
1494 filesToDelete.add(path)
1495 if path in filesToAdd:
1496 filesToAdd.remove(path)
4fddb41b
VA
1497 elif modifier == "C":
1498 src, dest = diff['src'], diff['dst']
6de040df 1499 p4_integrate(src, dest)
b6ad6dcc 1500 pureRenameCopy.add(dest)
4fddb41b 1501 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1502 p4_edit(dest)
b6ad6dcc 1503 pureRenameCopy.discard(dest)
4fddb41b 1504 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1505 p4_edit(dest)
b6ad6dcc 1506 pureRenameCopy.discard(dest)
4fddb41b 1507 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1508 if self.isWindows:
1509 # turn off read-only attribute
1510 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1511 os.unlink(dest)
1512 editedFiles.add(dest)
d9a5f25b 1513 elif modifier == "R":
b43b0a3c 1514 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1515 if self.p4HasMoveCommand:
1516 p4_edit(src) # src must be open before move
1517 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1518 else:
8e9497c2
GG
1519 p4_integrate(src, dest)
1520 if diff['src_sha1'] != diff['dst_sha1']:
1521 p4_edit(dest)
1522 else:
1523 pureRenameCopy.add(dest)
c65b670e 1524 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1525 if not self.p4HasMoveCommand:
1526 p4_edit(dest) # with move: already open, writable
c65b670e 1527 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1528 if not self.p4HasMoveCommand:
d20f0f8e
PW
1529 if self.isWindows:
1530 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1531 os.unlink(dest)
1532 filesToDelete.add(src)
d9a5f25b 1533 editedFiles.add(dest)
4f5cf76a
SH
1534 else:
1535 die("unknown modifier %s for %s" % (modifier, path))
1536
749b668c 1537 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1538 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1539 tryPatchCmd = patchcmd + "--check -"
1540 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1541 patch_succeeded = True
51a2640a 1542
47a130b7 1543 if os.system(tryPatchCmd) != 0:
60df071c
LD
1544 fixed_rcs_keywords = False
1545 patch_succeeded = False
51a2640a 1546 print "Unfortunately applying the change failed!"
60df071c
LD
1547
1548 # Patch failed, maybe it's just RCS keyword woes. Look through
1549 # the patch to see if that's possible.
0d609032 1550 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1551 file = None
1552 pattern = None
1553 kwfiles = {}
1554 for file in editedFiles | filesToDelete:
1555 # did this file's delta contain RCS keywords?
1556 pattern = p4_keywords_regexp_for_file(file)
1557
1558 if pattern:
1559 # this file is a possibility...look for RCS keywords.
1560 regexp = re.compile(pattern, re.VERBOSE)
1561 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1562 if regexp.search(line):
1563 if verbose:
1564 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1565 kwfiles[file] = pattern
1566 break
1567
1568 for file in kwfiles:
1569 if verbose:
1570 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1571 # File is being deleted, so not open in p4. Must
1572 # disable the read-only bit on windows.
1573 if self.isWindows and file not in editedFiles:
1574 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1575 self.patchRCSKeywords(file, kwfiles[file])
1576 fixed_rcs_keywords = True
1577
1578 if fixed_rcs_keywords:
1579 print "Retrying the patch with RCS keywords cleaned up"
1580 if os.system(tryPatchCmd) == 0:
1581 patch_succeeded = True
1582
1583 if not patch_succeeded:
7e5dd9f2
PW
1584 for f in editedFiles:
1585 p4_revert(f)
7e5dd9f2 1586 return False
51a2640a 1587
55ac2ed6
PW
1588 #
1589 # Apply the patch for real, and do add/delete/+x handling.
1590 #
47a130b7 1591 system(applyPatchCmd)
4f5cf76a
SH
1592
1593 for f in filesToAdd:
6de040df 1594 p4_add(f)
4f5cf76a 1595 for f in filesToDelete:
6de040df
LD
1596 p4_revert(f)
1597 p4_delete(f)
4f5cf76a 1598
c65b670e
CP
1599 # Set/clear executable bits
1600 for f in filesToChangeExecBit.keys():
1601 mode = filesToChangeExecBit[f]
1602 setP4ExecBit(f, mode)
1603
55ac2ed6
PW
1604 #
1605 # Build p4 change description, starting with the contents
1606 # of the git commit message.
1607 #
0e36f2d7 1608 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1609 logMessage = logMessage.strip()
f19cb0a0 1610 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1611
ea99c3ae 1612 template = self.prepareSubmitTemplate()
f19cb0a0 1613 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1614
c47178d4 1615 if self.preserveUser:
55ac2ed6 1616 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1617
55ac2ed6
PW
1618 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1619 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1620 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1621 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1622
55ac2ed6 1623 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1624 if not self.prepare_p4_only:
1625 submitTemplate += separatorLine
e2a892ee 1626 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
55ac2ed6 1627
c47178d4 1628 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1629 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1630 if self.isWindows:
1631 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1632 tmpFile.write(submitTemplate)
c47178d4
PW
1633 tmpFile.close()
1634
728b7ad8
PW
1635 if self.prepare_p4_only:
1636 #
1637 # Leave the p4 tree prepared, and the submit template around
1638 # and let the user decide what to do next
1639 #
1640 print
1641 print "P4 workspace prepared for submission."
1642 print "To submit or revert, go to client workspace"
1643 print " " + self.clientPath
1644 print
1645 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1646 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1647 " \"git p4\"." % fileName
1648 print "You can delete the file \"%s\" when finished." % fileName
1649
1650 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1651 print "To preserve change ownership by user %s, you must\n" \
1652 "do \"p4 change -f <change>\" after submitting and\n" \
1653 "edit the User field."
1654 if pureRenameCopy:
1655 print "After submitting, renamed files must be re-synced."
1656 print "Invoke \"p4 sync -f\" on each of these files:"
1657 for f in pureRenameCopy:
1658 print " " + f
1659
1660 print
1661 print "To revert the changes, use \"p4 revert ...\", and delete"
1662 print "the submit template file \"%s\"" % fileName
1663 if filesToAdd:
1664 print "Since the commit adds new files, they must be deleted:"
1665 for f in filesToAdd:
1666 print " " + f
1667 print
1668 return True
1669
55ac2ed6
PW
1670 #
1671 # Let the user edit the change description, then submit it.
1672 #
c47178d4
PW
1673 if self.edit_template(fileName):
1674 # read the edited message and submit
5a41c16a 1675 ret = True
c47178d4
PW
1676 tmpFile = open(fileName, "rb")
1677 message = tmpFile.read()
e96e400f 1678 tmpFile.close()
c47178d4 1679 if self.isWindows:
e2a892ee
MC
1680 message = message.replace("\r\n", "\n")
1681 submitTemplate = message[:message.index(separatorLine)]
c47178d4 1682 p4_write_pipe(['submit', '-i'], submitTemplate)
e96e400f 1683
c47178d4
PW
1684 if self.preserveUser:
1685 if p4User:
1686 # Get last changelist number. Cannot easily get it from
1687 # the submit command output as the output is
1688 # unmarshalled.
1689 changelist = self.lastP4Changelist()
1690 self.modifyChangelistUser(changelist, p4User)
1691
1692 # The rename/copy happened by applying a patch that created a
1693 # new file. This leaves it writable, which confuses p4.
1694 for f in pureRenameCopy:
1695 p4_sync(f, "-f")
cdc7e388 1696
4f5cf76a 1697 else:
c47178d4 1698 # skip this patch
5a41c16a 1699 ret = False
c47178d4
PW
1700 print "Submission cancelled, undoing p4 changes."
1701 for f in editedFiles:
1702 p4_revert(f)
1703 for f in filesToAdd:
1704 p4_revert(f)
1705 os.remove(f)
df9c5453
PW
1706 for f in filesToDelete:
1707 p4_revert(f)
c47178d4
PW
1708
1709 os.remove(fileName)
5a41c16a 1710 return ret
4f5cf76a 1711
06804c76
LD
1712 # Export git tags as p4 labels. Create a p4 label and then tag
1713 # with that.
1714 def exportGitTags(self, gitTags):
c8942a22
LD
1715 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1716 if len(validLabelRegexp) == 0:
1717 validLabelRegexp = defaultLabelRegexp
1718 m = re.compile(validLabelRegexp)
06804c76
LD
1719
1720 for name in gitTags:
1721
1722 if not m.match(name):
1723 if verbose:
05a3cec5 1724 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1725 continue
1726
1727 # Get the p4 commit this corresponds to
c8942a22
LD
1728 logMessage = extractLogMessageFromGitCommit(name)
1729 values = extractSettingsGitLog(logMessage)
06804c76 1730
c8942a22 1731 if not values.has_key('change'):
06804c76
LD
1732 # a tag pointing to something not sent to p4; ignore
1733 if verbose:
1734 print "git tag %s does not give a p4 commit" % name
1735 continue
c8942a22
LD
1736 else:
1737 changelist = values['change']
06804c76
LD
1738
1739 # Get the tag details.
1740 inHeader = True
1741 isAnnotated = False
1742 body = []
1743 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1744 l = l.strip()
1745 if inHeader:
1746 if re.match(r'tag\s+', l):
1747 isAnnotated = True
1748 elif re.match(r'\s*$', l):
1749 inHeader = False
1750 continue
1751 else:
1752 body.append(l)
1753
1754 if not isAnnotated:
1755 body = ["lightweight tag imported by git p4\n"]
1756
1757 # Create the label - use the same view as the client spec we are using
1758 clientSpec = getClientSpec()
1759
1760 labelTemplate = "Label: %s\n" % name
1761 labelTemplate += "Description:\n"
1762 for b in body:
1763 labelTemplate += "\t" + b + "\n"
1764 labelTemplate += "View:\n"
9d57c4a6
KS
1765 for depot_side in clientSpec.mappings:
1766 labelTemplate += "\t%s\n" % depot_side
06804c76 1767
ef739f08
PW
1768 if self.dry_run:
1769 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1770 elif self.prepare_p4_only:
1771 print "Not creating p4 label %s for tag due to option" \
1772 " --prepare-p4-only" % name
ef739f08
PW
1773 else:
1774 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1775
ef739f08
PW
1776 # Use the label
1777 p4_system(["tag", "-l", name] +
9d57c4a6 1778 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1779
ef739f08
PW
1780 if verbose:
1781 print "created p4 label for tag %s" % name
06804c76 1782
4f5cf76a 1783 def run(self, args):
c9b50e63
SH
1784 if len(args) == 0:
1785 self.master = currentGitBranch()
4280e533 1786 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1787 die("Detecting current git branch failed!")
1788 elif len(args) == 1:
1789 self.master = args[0]
28755dba
PW
1790 if not branchExists(self.master):
1791 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1792 else:
1793 return False
1794
4c2d5d72
JX
1795 allowSubmit = gitConfig("git-p4.allowSubmit")
1796 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1797 die("%s is not in git-p4.allowSubmit" % self.master)
1798
27d2d811 1799 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1800 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1801 if len(self.origin) == 0:
1802 self.origin = upstream
a3fdd579 1803
3ea2cfd4
LD
1804 if self.preserveUser:
1805 if not self.canChangeChangelists():
1806 die("Cannot preserve user names without p4 super-user or admin permissions")
1807
6bbfd137
PW
1808 # if not set from the command line, try the config file
1809 if self.conflict_behavior is None:
1810 val = gitConfig("git-p4.conflict")
1811 if val:
1812 if val not in self.conflict_behavior_choices:
1813 die("Invalid value '%s' for config git-p4.conflict" % val)
1814 else:
1815 val = "ask"
1816 self.conflict_behavior = val
1817
a3fdd579
SH
1818 if self.verbose:
1819 print "Origin branch is " + self.origin
9512497b 1820
ea99c3ae 1821 if len(self.depotPath) == 0:
9512497b
SH
1822 print "Internal error: cannot locate perforce depot path from existing branches"
1823 sys.exit(128)
1824
543987bd 1825 self.useClientSpec = False
0d609032 1826 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1827 self.useClientSpec = True
1828 if self.useClientSpec:
1829 self.clientSpecDirs = getClientSpec()
9512497b 1830
cd884106
VA
1831 # Check for the existance of P4 branches
1832 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1833
1834 if self.useClientSpec and not branchesDetected:
543987bd
PW
1835 # all files are relative to the client spec
1836 self.clientPath = getClientRoot()
1837 else:
1838 self.clientPath = p4Where(self.depotPath)
9512497b 1839
543987bd
PW
1840 if self.clientPath == "":
1841 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1842
ea99c3ae 1843 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1844 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1845
0591cfa8 1846 # ensure the clientPath exists
8d7ec362 1847 new_client_dir = False
0591cfa8 1848 if not os.path.exists(self.clientPath):
8d7ec362 1849 new_client_dir = True
0591cfa8
GG
1850 os.makedirs(self.clientPath)
1851
bbd84863 1852 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1853 if self.dry_run:
1854 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1855 else:
ef739f08
PW
1856 print "Synchronizing p4 checkout..."
1857 if new_client_dir:
1858 # old one was destroyed, and maybe nobody told p4
1859 p4_sync("...", "-f")
1860 else:
1861 p4_sync("...")
4f5cf76a 1862 self.check()
4f5cf76a 1863
4c750c0d 1864 commits = []
c7d34884 1865 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
4c750c0d
SH
1866 commits.append(line.strip())
1867 commits.reverse()
4f5cf76a 1868
0d609032 1869 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1870 self.checkAuthorship = False
1871 else:
1872 self.checkAuthorship = True
1873
3ea2cfd4
LD
1874 if self.preserveUser:
1875 self.checkValidP4Users(commits)
1876
84cb0003
GG
1877 #
1878 # Build up a set of options to be passed to diff when
1879 # submitting each commit to p4.
1880 #
1881 if self.detectRenames:
1882 # command-line -M arg
1883 self.diffOpts = "-M"
1884 else:
1885 # If not explicitly set check the config variable
1886 detectRenames = gitConfig("git-p4.detectRenames")
1887
1888 if detectRenames.lower() == "false" or detectRenames == "":
1889 self.diffOpts = ""
1890 elif detectRenames.lower() == "true":
1891 self.diffOpts = "-M"
1892 else:
1893 self.diffOpts = "-M%s" % detectRenames
1894
1895 # no command-line arg for -C or --find-copies-harder, just
1896 # config variables
1897 detectCopies = gitConfig("git-p4.detectCopies")
1898 if detectCopies.lower() == "false" or detectCopies == "":
1899 pass
1900 elif detectCopies.lower() == "true":
1901 self.diffOpts += " -C"
1902 else:
1903 self.diffOpts += " -C%s" % detectCopies
1904
0d609032 1905 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1906 self.diffOpts += " --find-copies-harder"
1907
7e5dd9f2
PW
1908 #
1909 # Apply the commits, one at a time. On failure, ask if should
1910 # continue to try the rest of the patches, or quit.
1911 #
ef739f08
PW
1912 if self.dry_run:
1913 print "Would apply"
67b0fe2e 1914 applied = []
7e5dd9f2
PW
1915 last = len(commits) - 1
1916 for i, commit in enumerate(commits):
ef739f08
PW
1917 if self.dry_run:
1918 print " ", read_pipe(["git", "show", "-s",
1919 "--format=format:%h %s", commit])
1920 ok = True
1921 else:
1922 ok = self.applyCommit(commit)
67b0fe2e
PW
1923 if ok:
1924 applied.append(commit)
7e5dd9f2 1925 else:
728b7ad8
PW
1926 if self.prepare_p4_only and i < last:
1927 print "Processing only the first commit due to option" \
1928 " --prepare-p4-only"
1929 break
7e5dd9f2
PW
1930 if i < last:
1931 quit = False
1932 while True:
6bbfd137
PW
1933 # prompt for what to do, or use the option/variable
1934 if self.conflict_behavior == "ask":
1935 print "What do you want to do?"
1936 response = raw_input("[s]kip this commit but apply"
1937 " the rest, or [q]uit? ")
1938 if not response:
1939 continue
1940 elif self.conflict_behavior == "skip":
1941 response = "s"
1942 elif self.conflict_behavior == "quit":
1943 response = "q"
1944 else:
1945 die("Unknown conflict_behavior '%s'" %
1946 self.conflict_behavior)
1947
7e5dd9f2
PW
1948 if response[0] == "s":
1949 print "Skipping this commit, but applying the rest"
1950 break
1951 if response[0] == "q":
1952 print "Quitting"
1953 quit = True
1954 break
1955 if quit:
1956 break
4f5cf76a 1957
67b0fe2e 1958 chdir(self.oldWorkingDirectory)
4f5cf76a 1959
ef739f08
PW
1960 if self.dry_run:
1961 pass
728b7ad8
PW
1962 elif self.prepare_p4_only:
1963 pass
ef739f08 1964 elif len(commits) == len(applied):
67b0fe2e 1965 print "All commits applied!"
14594f4b 1966
4c750c0d 1967 sync = P4Sync()
44e8d26c
PW
1968 if self.branch:
1969 sync.branch = self.branch
4c750c0d 1970 sync.run([])
14594f4b 1971
4c750c0d
SH
1972 rebase = P4Rebase()
1973 rebase.rebase()
4f5cf76a 1974
67b0fe2e
PW
1975 else:
1976 if len(applied) == 0:
1977 print "No commits applied."
1978 else:
1979 print "Applied only the commits marked with '*':"
1980 for c in commits:
1981 if c in applied:
1982 star = "*"
1983 else:
1984 star = " "
1985 print star, read_pipe(["git", "show", "-s",
1986 "--format=format:%h %s", c])
1987 print "You will have to do 'git p4 sync' and rebase."
1988
0d609032 1989 if gitConfigBool("git-p4.exportLabels"):
06dcd152 1990 self.exportLabels = True
06804c76
LD
1991
1992 if self.exportLabels:
1993 p4Labels = getP4Labels(self.depotPath)
1994 gitTags = getGitTags()
1995
1996 missingGitTags = gitTags - p4Labels
1997 self.exportGitTags(missingGitTags)
1998
98e023de 1999 # exit with error unless everything applied perfectly
67b0fe2e
PW
2000 if len(commits) != len(applied):
2001 sys.exit(1)
2002
b984733c
SH
2003 return True
2004
ecb7cf98
PW
2005class View(object):
2006 """Represent a p4 view ("p4 help views"), and map files in a
2007 repo according to the view."""
2008
9d57c4a6 2009 def __init__(self, client_name):
ecb7cf98 2010 self.mappings = []
9d57c4a6
KS
2011 self.client_prefix = "//%s/" % client_name
2012 # cache results of "p4 where" to lookup client file locations
2013 self.client_spec_path_cache = {}
ecb7cf98
PW
2014
2015 def append(self, view_line):
2016 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2017 sides. Append to self.mappings, preserving order. This
2018 is only needed for tag creation."""
ecb7cf98
PW
2019
2020 # Split the view line into exactly two words. P4 enforces
2021 # structure on these lines that simplifies this quite a bit.
2022 #
2023 # Either or both words may be double-quoted.
2024 # Single quotes do not matter.
2025 # Double-quote marks cannot occur inside the words.
2026 # A + or - prefix is also inside the quotes.
2027 # There are no quotes unless they contain a space.
2028 # The line is already white-space stripped.
2029 # The two words are separated by a single space.
2030 #
2031 if view_line[0] == '"':
2032 # First word is double quoted. Find its end.
2033 close_quote_index = view_line.find('"', 1)
2034 if close_quote_index <= 0:
2035 die("No first-word closing quote found: %s" % view_line)
2036 depot_side = view_line[1:close_quote_index]
2037 # skip closing quote and space
2038 rhs_index = close_quote_index + 1 + 1
2039 else:
2040 space_index = view_line.find(" ")
2041 if space_index <= 0:
2042 die("No word-splitting space found: %s" % view_line)
2043 depot_side = view_line[0:space_index]
2044 rhs_index = space_index + 1
2045
ecb7cf98 2046 # prefix + means overlay on previous mapping
ecb7cf98 2047 if depot_side.startswith("+"):
ecb7cf98
PW
2048 depot_side = depot_side[1:]
2049
9d57c4a6 2050 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2051 exclude = False
2052 if depot_side.startswith("-"):
2053 exclude = True
2054 depot_side = depot_side[1:]
2055
9d57c4a6
KS
2056 if not exclude:
2057 self.mappings.append(depot_side)
ecb7cf98 2058
9d57c4a6
KS
2059 def convert_client_path(self, clientFile):
2060 # chop off //client/ part to make it relative
2061 if not clientFile.startswith(self.client_prefix):
2062 die("No prefix '%s' on clientFile '%s'" %
2063 (self.client_prefix, clientFile))
2064 return clientFile[len(self.client_prefix):]
ecb7cf98 2065
9d57c4a6
KS
2066 def update_client_spec_path_cache(self, files):
2067 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2068
9d57c4a6
KS
2069 # List depot file paths exclude that already cached
2070 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2071
9d57c4a6
KS
2072 if len(fileArgs) == 0:
2073 return # All files in cache
ecb7cf98 2074
9d57c4a6
KS
2075 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2076 for res in where_result:
2077 if "code" in res and res["code"] == "error":
2078 # assume error is "... file(s) not in client view"
2079 continue
2080 if "clientFile" not in res:
20005443 2081 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2082 if "unmap" in res:
2083 # it will list all of them, but only one not unmap-ped
2084 continue
a0a50d87
LS
2085 if gitConfigBool("core.ignorecase"):
2086 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2087 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2088
9d57c4a6
KS
2089 # not found files or unmap files set to ""
2090 for depotFile in fileArgs:
a0a50d87
LS
2091 if gitConfigBool("core.ignorecase"):
2092 depotFile = depotFile.lower()
9d57c4a6
KS
2093 if depotFile not in self.client_spec_path_cache:
2094 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2095
9d57c4a6
KS
2096 def map_in_client(self, depot_path):
2097 """Return the relative location in the client where this
2098 depot file should live. Returns "" if the file should
2099 not be mapped in the client."""
ecb7cf98 2100
a0a50d87
LS
2101 if gitConfigBool("core.ignorecase"):
2102 depot_path = depot_path.lower()
2103
9d57c4a6
KS
2104 if depot_path in self.client_spec_path_cache:
2105 return self.client_spec_path_cache[depot_path]
2106
2107 die( "Error: %s is not found in client spec path" % depot_path )
2108 return ""
ecb7cf98 2109
3ea2cfd4 2110class P4Sync(Command, P4UserMap):
56c09345
PW
2111 delete_actions = ( "delete", "move/delete", "purge" )
2112
b984733c
SH
2113 def __init__(self):
2114 Command.__init__(self)
3ea2cfd4 2115 P4UserMap.__init__(self)
b984733c
SH
2116 self.options = [
2117 optparse.make_option("--branch", dest="branch"),
2118 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2119 optparse.make_option("--changesfile", dest="changesFile"),
2120 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2121 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2122 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2123 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2124 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2125 optparse.make_option("--max-changes", dest="maxChanges",
2126 help="Maximum number of changes to import"),
2127 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2128 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2129 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2130 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2131 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2132 help="Only sync files that are included in the Perforce Client Spec"),
2133 optparse.make_option("-/", dest="cloneExclude",
2134 action="append", type="string",
2135 help="exclude depot path"),
b984733c
SH
2136 ]
2137 self.description = """Imports from Perforce into a git repository.\n
2138 example:
2139 //depot/my/project/ -- to import the current head
2140 //depot/my/project/@all -- to import everything
2141 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2142
2143 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2144
2145 self.usage += " //depot/path[@revRange]"
b984733c 2146 self.silent = False
1d7367dc
RG
2147 self.createdBranches = set()
2148 self.committedChanges = set()
569d1bd4 2149 self.branch = ""
b984733c 2150 self.detectBranches = False
cb53e1f8 2151 self.detectLabels = False
06804c76 2152 self.importLabels = False
b984733c 2153 self.changesFile = ""
01265103 2154 self.syncWithOrigin = True
a028a98e 2155 self.importIntoRemotes = True
01a9c9c5 2156 self.maxChanges = ""
1051ef00 2157 self.changes_block_size = None
8b41a97f 2158 self.keepRepoPath = False
6326aa58 2159 self.depotPaths = None
3c699645 2160 self.p4BranchesInGit = []
354081d5 2161 self.cloneExclude = []
3a70cdfa 2162 self.useClientSpec = False
a93d33ee 2163 self.useClientSpec_from_options = False
ecb7cf98 2164 self.clientSpecDirs = None
fed23693
VA
2165 self.tempBranches = []
2166 self.tempBranchLocation = "git-p4-tmp"
a5db4b12
LS
2167 self.largeFileSystem = None
2168
2169 if gitConfig('git-p4.largeFileSystem'):
2170 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2171 self.largeFileSystem = largeFileSystemConstructor(
2172 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2173 )
b984733c 2174
01265103
SH
2175 if gitConfig("git-p4.syncFromOrigin") == "false":
2176 self.syncWithOrigin = False
2177
51334bb0
LD
2178 # This is required for the "append" cloneExclude action
2179 def ensure_value(self, attr, value):
2180 if not hasattr(self, attr) or getattr(self, attr) is None:
2181 setattr(self, attr, value)
2182 return getattr(self, attr)
2183
fed23693
VA
2184 # Force a checkpoint in fast-import and wait for it to finish
2185 def checkpoint(self):
2186 self.gitStream.write("checkpoint\n\n")
2187 self.gitStream.write("progress checkpoint\n\n")
2188 out = self.gitOutput.readline()
2189 if self.verbose:
2190 print "checkpoint finished: " + out
2191
b984733c 2192 def extractFilesFromCommit(self, commit):
354081d5
TT
2193 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2194 for path in self.cloneExclude]
b984733c
SH
2195 files = []
2196 fnum = 0
2197 while commit.has_key("depotFile%s" % fnum):
2198 path = commit["depotFile%s" % fnum]
6326aa58 2199
354081d5 2200 if [p for p in self.cloneExclude
d53de8b9 2201 if p4PathStartsWith(path, p)]:
354081d5
TT
2202 found = False
2203 else:
2204 found = [p for p in self.depotPaths
d53de8b9 2205 if p4PathStartsWith(path, p)]
6326aa58 2206 if not found:
b984733c
SH
2207 fnum = fnum + 1
2208 continue
2209
2210 file = {}
2211 file["path"] = path
2212 file["rev"] = commit["rev%s" % fnum]
2213 file["action"] = commit["action%s" % fnum]
2214 file["type"] = commit["type%s" % fnum]
2215 files.append(file)
2216 fnum = fnum + 1
2217 return files
2218
6326aa58 2219 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2220 """When streaming files, this is called to map a p4 depot path
2221 to where it should go in git. The prefixes are either
2222 self.depotPaths, or self.branchPrefixes in the case of
2223 branch detection."""
2224
3952710b 2225 if self.useClientSpec:
21ef5df4
PW
2226 # branch detection moves files up a level (the branch name)
2227 # from what client spec interpretation gives
0d1696ef 2228 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2229 if self.detectBranches:
2230 for b in self.knownBranches:
2231 if path.startswith(b + "/"):
2232 path = path[len(b)+1:]
2233
2234 elif self.keepRepoPath:
2235 # Preserve everything in relative path name except leading
2236 # //depot/; just look at first prefix as they all should
2237 # be in the same depot.
2238 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2239 if p4PathStartsWith(path, depot):
2240 path = path[len(depot):]
3952710b 2241
0d1696ef 2242 else:
0d1696ef
PW
2243 for p in prefixes:
2244 if p4PathStartsWith(path, p):
2245 path = path[len(p):]
21ef5df4 2246 break
8b41a97f 2247
0d1696ef 2248 path = wildcard_decode(path)
6326aa58 2249 return path
6754a299 2250
71b112d4 2251 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2252 """Look at each depotFile in the commit to figure out to what
2253 branch it belongs."""
2254
9d57c4a6
KS
2255 if self.clientSpecDirs:
2256 files = self.extractFilesFromCommit(commit)
2257 self.clientSpecDirs.update_client_spec_path_cache(files)
2258
d5904674 2259 branches = {}
71b112d4
SH
2260 fnum = 0
2261 while commit.has_key("depotFile%s" % fnum):
2262 path = commit["depotFile%s" % fnum]
6326aa58 2263 found = [p for p in self.depotPaths
d53de8b9 2264 if p4PathStartsWith(path, p)]
6326aa58 2265 if not found:
71b112d4
SH
2266 fnum = fnum + 1
2267 continue
2268
2269 file = {}
2270 file["path"] = path
2271 file["rev"] = commit["rev%s" % fnum]
2272 file["action"] = commit["action%s" % fnum]
2273 file["type"] = commit["type%s" % fnum]
2274 fnum = fnum + 1
2275
21ef5df4
PW
2276 # start with the full relative path where this file would
2277 # go in a p4 client
2278 if self.useClientSpec:
2279 relPath = self.clientSpecDirs.map_in_client(path)
2280 else:
2281 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2282
4b97ffb1 2283 for branch in self.knownBranches.keys():
21ef5df4
PW
2284 # add a trailing slash so that a commit into qt/4.2foo
2285 # doesn't end up in qt/4.2, e.g.
6754a299 2286 if relPath.startswith(branch + "/"):
d5904674
SH
2287 if branch not in branches:
2288 branches[branch] = []
71b112d4 2289 branches[branch].append(file)
6555b2cc 2290 break
b984733c
SH
2291
2292 return branches
2293
a5db4b12
LS
2294 def writeToGitStream(self, gitMode, relPath, contents):
2295 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2296 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2297 for d in contents:
2298 self.gitStream.write(d)
2299 self.gitStream.write('\n')
2300
b932705b
LD
2301 # output one file from the P4 stream
2302 # - helper for streamP4Files
2303
2304 def streamOneP4File(self, file, contents):
b932705b
LD
2305 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2306 if verbose:
d2176a50
LS
2307 size = int(self.stream_file['fileSize'])
2308 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2309 sys.stdout.flush()
b932705b 2310
9cffb8c8
PW
2311 (type_base, type_mods) = split_p4_type(file["type"])
2312
2313 git_mode = "100644"
2314 if "x" in type_mods:
2315 git_mode = "100755"
2316 if type_base == "symlink":
2317 git_mode = "120000"
1292df11
AJ
2318 # p4 print on a symlink sometimes contains "target\n";
2319 # if it does, remove the newline
b39c3612 2320 data = ''.join(contents)
40f846c3
PW
2321 if not data:
2322 # Some version of p4 allowed creating a symlink that pointed
2323 # to nothing. This causes p4 errors when checking out such
2324 # a change, and errors here too. Work around it by ignoring
2325 # the bad symlink; hopefully a future change fixes it.
2326 print "\nIgnoring empty symlink in %s" % file['depotFile']
2327 return
2328 elif data[-1] == '\n':
1292df11
AJ
2329 contents = [data[:-1]]
2330 else:
2331 contents = [data]
b932705b 2332
9cffb8c8 2333 if type_base == "utf16":
55aa5714
PW
2334 # p4 delivers different text in the python output to -G
2335 # than it does when using "print -o", or normal p4 client
2336 # operations. utf16 is converted to ascii or utf8, perhaps.
2337 # But ascii text saved as -t utf16 is completely mangled.
2338 # Invoke print -o to get the real contents.
7f0e5962
PW
2339 #
2340 # On windows, the newlines will always be mangled by print, so put
2341 # them back too. This is not needed to the cygwin windows version,
2342 # just the native "NT" type.
2343 #
f5f53f14 2344 text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
7f0e5962
PW
2345 if p4_version_string().find("/NT") >= 0:
2346 text = text.replace("\r\n", "\n")
55aa5714
PW
2347 contents = [ text ]
2348
9f7ef0ea
PW
2349 if type_base == "apple":
2350 # Apple filetype files will be streamed as a concatenation of
2351 # its appledouble header and the contents. This is useless
2352 # on both macs and non-macs. If using "print -q -o xx", it
2353 # will create "xx" with the data, and "%xx" with the header.
2354 # This is also not very useful.
2355 #
2356 # Ideally, someday, this script can learn how to generate
2357 # appledouble files directly and import those to git, but
2358 # non-mac machines can never find a use for apple filetype.
2359 print "\nIgnoring apple filetype file %s" % file['depotFile']
2360 return
2361
55aa5714
PW
2362 # Note that we do not try to de-mangle keywords on utf16 files,
2363 # even though in theory somebody may want that.
60df071c
LD
2364 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2365 if pattern:
2366 regexp = re.compile(pattern, re.VERBOSE)
2367 text = ''.join(contents)
2368 text = regexp.sub(r'$\1$', text)
2369 contents = [ text ]
b932705b 2370
a5db4b12
LS
2371 if self.largeFileSystem:
2372 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2373
a5db4b12 2374 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2375
2376 def streamOneP4Deletion(self, file):
2377 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2378 if verbose:
d2176a50
LS
2379 sys.stdout.write("delete %s\n" % relPath)
2380 sys.stdout.flush()
b932705b
LD
2381 self.gitStream.write("D %s\n" % relPath)
2382
a5db4b12
LS
2383 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2384 self.largeFileSystem.removeLargeFile(relPath)
2385
b932705b
LD
2386 # handle another chunk of streaming data
2387 def streamP4FilesCb(self, marshalled):
2388
78189bea
PW
2389 # catch p4 errors and complain
2390 err = None
2391 if "code" in marshalled:
2392 if marshalled["code"] == "error":
2393 if "data" in marshalled:
2394 err = marshalled["data"].rstrip()
4d25dc44
LS
2395
2396 if not err and 'fileSize' in self.stream_file:
2397 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2398 if required_bytes > 0:
2399 err = 'Not enough space left on %s! Free at least %i MB.' % (
2400 os.getcwd(), required_bytes/1024/1024
2401 )
2402
78189bea
PW
2403 if err:
2404 f = None
2405 if self.stream_have_file_info:
2406 if "depotFile" in self.stream_file:
2407 f = self.stream_file["depotFile"]
2408 # force a failure in fast-import, else an empty
2409 # commit will be made
2410 self.gitStream.write("\n")
2411 self.gitStream.write("die-now\n")
2412 self.gitStream.close()
2413 # ignore errors, but make sure it exits first
2414 self.importProcess.wait()
2415 if f:
2416 die("Error from p4 print for %s: %s" % (f, err))
2417 else:
2418 die("Error from p4 print: %s" % err)
2419
c3f6163b
AG
2420 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2421 # start of a new file - output the old one first
2422 self.streamOneP4File(self.stream_file, self.stream_contents)
2423 self.stream_file = {}
2424 self.stream_contents = []
2425 self.stream_have_file_info = False
b932705b 2426
c3f6163b
AG
2427 # pick up the new file information... for the
2428 # 'data' field we need to append to our array
2429 for k in marshalled.keys():
2430 if k == 'data':
d2176a50
LS
2431 if 'streamContentSize' not in self.stream_file:
2432 self.stream_file['streamContentSize'] = 0
2433 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2434 self.stream_contents.append(marshalled['data'])
2435 else:
2436 self.stream_file[k] = marshalled[k]
b932705b 2437
d2176a50
LS
2438 if (verbose and
2439 'streamContentSize' in self.stream_file and
2440 'fileSize' in self.stream_file and
2441 'depotFile' in self.stream_file):
2442 size = int(self.stream_file["fileSize"])
2443 if size > 0:
2444 progress = 100*self.stream_file['streamContentSize']/size
2445 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2446 sys.stdout.flush()
2447
c3f6163b 2448 self.stream_have_file_info = True
b932705b
LD
2449
2450 # Stream directly from "p4 files" into "git fast-import"
2451 def streamP4Files(self, files):
30b5940b
SH
2452 filesForCommit = []
2453 filesToRead = []
b932705b 2454 filesToDelete = []
30b5940b 2455
3a70cdfa 2456 for f in files:
ecb7cf98
PW
2457 # if using a client spec, only add the files that have
2458 # a path in the client
2459 if self.clientSpecDirs:
2460 if self.clientSpecDirs.map_in_client(f['path']) == "":
2461 continue
3a70cdfa 2462
ecb7cf98
PW
2463 filesForCommit.append(f)
2464 if f['action'] in self.delete_actions:
2465 filesToDelete.append(f)
2466 else:
2467 filesToRead.append(f)
6a49f8e2 2468
b932705b
LD
2469 # deleted files...
2470 for f in filesToDelete:
2471 self.streamOneP4Deletion(f)
1b9a4684 2472
b932705b
LD
2473 if len(filesToRead) > 0:
2474 self.stream_file = {}
2475 self.stream_contents = []
2476 self.stream_have_file_info = False
8ff45f2a 2477
c3f6163b
AG
2478 # curry self argument
2479 def streamP4FilesCbSelf(entry):
2480 self.streamP4FilesCb(entry)
6a49f8e2 2481
6de040df
LD
2482 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2483
2484 p4CmdList(["-x", "-", "print"],
2485 stdin=fileArgs,
2486 cb=streamP4FilesCbSelf)
30b5940b 2487
b932705b
LD
2488 # do the last chunk
2489 if self.stream_file.has_key('depotFile'):
2490 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2491
affb474f
LD
2492 def make_email(self, userid):
2493 if userid in self.users:
2494 return self.users[userid]
2495 else:
2496 return "%s <a@b>" % userid
2497
06804c76
LD
2498 # Stream a p4 tag
2499 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2500 if verbose:
2501 print "writing tag %s for commit %s" % (labelName, commit)
2502 gitStream.write("tag %s\n" % labelName)
2503 gitStream.write("from %s\n" % commit)
2504
2505 if labelDetails.has_key('Owner'):
2506 owner = labelDetails["Owner"]
2507 else:
2508 owner = None
2509
2510 # Try to use the owner of the p4 label, or failing that,
2511 # the current p4 user id.
2512 if owner:
2513 email = self.make_email(owner)
2514 else:
2515 email = self.make_email(self.p4UserId())
2516 tagger = "%s %s %s" % (email, epoch, self.tz)
2517
2518 gitStream.write("tagger %s\n" % tagger)
2519
2520 print "labelDetails=",labelDetails
2521 if labelDetails.has_key('Description'):
2522 description = labelDetails['Description']
2523 else:
2524 description = 'Label from git p4'
2525
2526 gitStream.write("data %d\n" % len(description))
2527 gitStream.write(description)
2528 gitStream.write("\n")
2529
e63231e5 2530 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2531 epoch = details["time"]
2532 author = details["user"]
2533
4b97ffb1
SH
2534 if self.verbose:
2535 print "commit into %s" % branch
2536
96e07dd2
HWN
2537 # start with reading files; if that fails, we should not
2538 # create a commit.
2539 new_files = []
2540 for f in files:
e63231e5 2541 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2542 new_files.append (f)
2543 else:
afa1dd9a 2544 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2545
9d57c4a6
KS
2546 if self.clientSpecDirs:
2547 self.clientSpecDirs.update_client_spec_path_cache(files)
2548
b984733c 2549 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 2550# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2551 self.committedChanges.add(int(details["change"]))
2552 committer = ""
b607e71e
SH
2553 if author not in self.users:
2554 self.getUserMapFromPerforceServer()
affb474f 2555 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2556
2557 self.gitStream.write("committer %s\n" % committer)
2558
2559 self.gitStream.write("data <<EOT\n")
2560 self.gitStream.write(details["desc"])
e63231e5
PW
2561 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2562 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2563 if len(details['options']) > 0:
2564 self.gitStream.write(": options = %s" % details['options'])
2565 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2566
2567 if len(parent) > 0:
4b97ffb1
SH
2568 if self.verbose:
2569 print "parent %s" % parent
b984733c
SH
2570 self.gitStream.write("from %s\n" % parent)
2571
b932705b 2572 self.streamP4Files(new_files)
b984733c
SH
2573 self.gitStream.write("\n")
2574
1f4ba1cb
SH
2575 change = int(details["change"])
2576
9bda3a85 2577 if self.labels.has_key(change):
1f4ba1cb
SH
2578 label = self.labels[change]
2579 labelDetails = label[0]
2580 labelRevisions = label[1]
71b112d4
SH
2581 if self.verbose:
2582 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2583
6de040df 2584 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2585 for p in self.branchPrefixes])
1f4ba1cb
SH
2586
2587 if len(files) == len(labelRevisions):
2588
2589 cleanedFiles = {}
2590 for info in files:
56c09345 2591 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2592 continue
2593 cleanedFiles[info["depotFile"]] = info["rev"]
2594
2595 if cleanedFiles == labelRevisions:
06804c76 2596 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2597
2598 else:
a46668fa 2599 if not self.silent:
cebdf5af
HWN
2600 print ("Tag %s does not match with change %s: files do not match."
2601 % (labelDetails["label"], change))
1f4ba1cb
SH
2602
2603 else:
a46668fa 2604 if not self.silent:
cebdf5af
HWN
2605 print ("Tag %s does not match with change %s: file count is different."
2606 % (labelDetails["label"], change))
b984733c 2607
06804c76 2608 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2609 def getLabels(self):
2610 self.labels = {}
2611
52a4880b 2612 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2613 if len(l) > 0 and not self.silent:
183f8436 2614 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2615
2616 for output in l:
1f4ba1cb
SH
2617 label = output["label"]
2618 revisions = {}
2619 newestChange = 0
71b112d4
SH
2620 if self.verbose:
2621 print "Querying files for label %s" % label
6de040df
LD
2622 for file in p4CmdList(["files"] +
2623 ["%s...@%s" % (p, label)
2624 for p in self.depotPaths]):
1f4ba1cb
SH
2625 revisions[file["depotFile"]] = file["rev"]
2626 change = int(file["change"])
2627 if change > newestChange:
2628 newestChange = change
2629
9bda3a85
SH
2630 self.labels[newestChange] = [output, revisions]
2631
2632 if self.verbose:
2633 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2634
06804c76
LD
2635 # Import p4 labels as git tags. A direct mapping does not
2636 # exist, so assume that if all the files are at the same revision
2637 # then we can use that, or it's something more complicated we should
2638 # just ignore.
2639 def importP4Labels(self, stream, p4Labels):
2640 if verbose:
2641 print "import p4 labels: " + ' '.join(p4Labels)
2642
2643 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2644 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2645 if len(validLabelRegexp) == 0:
2646 validLabelRegexp = defaultLabelRegexp
2647 m = re.compile(validLabelRegexp)
2648
2649 for name in p4Labels:
2650 commitFound = False
2651
2652 if not m.match(name):
2653 if verbose:
2654 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2655 continue
2656
2657 if name in ignoredP4Labels:
2658 continue
2659
2660 labelDetails = p4CmdList(['label', "-o", name])[0]
2661
2662 # get the most recent changelist for each file in this label
2663 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2664 for p in self.depotPaths])
2665
2666 if change.has_key('change'):
2667 # find the corresponding git commit; take the oldest commit
2668 changelist = int(change['change'])
2669 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2670 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2671 if len(gitCommit) == 0:
2672 print "could not find git commit for changelist %d" % changelist
2673 else:
2674 gitCommit = gitCommit.strip()
2675 commitFound = True
2676 # Convert from p4 time format
2677 try:
2678 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2679 except ValueError:
a4e9054c 2680 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2681 tmwhen = 1
2682
2683 when = int(time.mktime(tmwhen))
2684 self.streamTag(stream, name, labelDetails, gitCommit, when)
2685 if verbose:
2686 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2687 else:
2688 if verbose:
2689 print "Label %s has no changelists - possibly deleted?" % name
2690
2691 if not commitFound:
2692 # We can't import this label; don't try again as it will get very
2693 # expensive repeatedly fetching all the files for labels that will
2694 # never be imported. If the label is moved in the future, the
2695 # ignore will need to be removed manually.
2696 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2697
86dff6b6
HWN
2698 def guessProjectName(self):
2699 for p in self.depotPaths:
6e5295c4
SH
2700 if p.endswith("/"):
2701 p = p[:-1]
2702 p = p[p.strip().rfind("/") + 1:]
2703 if not p.endswith("/"):
2704 p += "/"
2705 return p
86dff6b6 2706
4b97ffb1 2707 def getBranchMapping(self):
6555b2cc
SH
2708 lostAndFoundBranches = set()
2709
8ace74c0
VA
2710 user = gitConfig("git-p4.branchUser")
2711 if len(user) > 0:
2712 command = "branches -u %s" % user
2713 else:
2714 command = "branches"
2715
2716 for info in p4CmdList(command):
52a4880b 2717 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2718 viewIdx = 0
2719 while details.has_key("View%s" % viewIdx):
2720 paths = details["View%s" % viewIdx].split(" ")
2721 viewIdx = viewIdx + 1
2722 # require standard //depot/foo/... //depot/bar/... mapping
2723 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2724 continue
2725 source = paths[0]
2726 destination = paths[1]
6509e19c 2727 ## HACK
d53de8b9 2728 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2729 source = source[len(self.depotPaths[0]):-4]
2730 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2731
1a2edf4e
SH
2732 if destination in self.knownBranches:
2733 if not self.silent:
2734 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2735 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2736 continue
2737
6555b2cc
SH
2738 self.knownBranches[destination] = source
2739
2740 lostAndFoundBranches.discard(destination)
2741
29bdbac1 2742 if source not in self.knownBranches:
6555b2cc
SH
2743 lostAndFoundBranches.add(source)
2744
7199cf13
VA
2745 # Perforce does not strictly require branches to be defined, so we also
2746 # check git config for a branch list.
2747 #
2748 # Example of branch definition in git config file:
2749 # [git-p4]
2750 # branchList=main:branchA
2751 # branchList=main:branchB
2752 # branchList=branchA:branchC
2753 configBranches = gitConfigList("git-p4.branchList")
2754 for branch in configBranches:
2755 if branch:
2756 (source, destination) = branch.split(":")
2757 self.knownBranches[destination] = source
2758
2759 lostAndFoundBranches.discard(destination)
2760
2761 if source not in self.knownBranches:
2762 lostAndFoundBranches.add(source)
2763
6555b2cc
SH
2764
2765 for branch in lostAndFoundBranches:
2766 self.knownBranches[branch] = branch
29bdbac1 2767
38f9f5ec
SH
2768 def getBranchMappingFromGitBranches(self):
2769 branches = p4BranchesInGit(self.importIntoRemotes)
2770 for branch in branches.keys():
2771 if branch == "master":
2772 branch = "main"
2773 else:
2774 branch = branch[len(self.projectName):]
2775 self.knownBranches[branch] = branch
2776
bb6e09b2
HWN
2777 def updateOptionDict(self, d):
2778 option_keys = {}
2779 if self.keepRepoPath:
2780 option_keys['keepRepoPath'] = 1
2781
2782 d["options"] = ' '.join(sorted(option_keys.keys()))
2783
2784 def readOptions(self, d):
2785 self.keepRepoPath = (d.has_key('options')
2786 and ('keepRepoPath' in d['options']))
6326aa58 2787
8134f69c
SH
2788 def gitRefForBranch(self, branch):
2789 if branch == "main":
2790 return self.refPrefix + "master"
2791
2792 if len(branch) <= 0:
2793 return branch
2794
2795 return self.refPrefix + self.projectName + branch
2796
1ca3d710
SH
2797 def gitCommitByP4Change(self, ref, change):
2798 if self.verbose:
2799 print "looking in ref " + ref + " for change %s using bisect..." % change
2800
2801 earliestCommit = ""
2802 latestCommit = parseRevision(ref)
2803
2804 while True:
2805 if self.verbose:
2806 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2807 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2808 if len(next) == 0:
2809 if self.verbose:
2810 print "argh"
2811 return ""
2812 log = extractLogMessageFromGitCommit(next)
2813 settings = extractSettingsGitLog(log)
2814 currentChange = int(settings['change'])
2815 if self.verbose:
2816 print "current change %s" % currentChange
2817
2818 if currentChange == change:
2819 if self.verbose:
2820 print "found %s" % next
2821 return next
2822
2823 if currentChange < change:
2824 earliestCommit = "^%s" % next
2825 else:
2826 latestCommit = "%s" % next
2827
2828 return ""
2829
2830 def importNewBranch(self, branch, maxChange):
2831 # make fast-import flush all changes to disk and update the refs using the checkpoint
2832 # command so that we can try to find the branch parent in the git history
2833 self.gitStream.write("checkpoint\n\n");
2834 self.gitStream.flush();
2835 branchPrefix = self.depotPaths[0] + branch + "/"
2836 range = "@1,%s" % maxChange
2837 #print "prefix" + branchPrefix
96b2d54a 2838 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
2839 if len(changes) <= 0:
2840 return False
2841 firstChange = changes[0]
2842 #print "first change in branch: %s" % firstChange
2843 sourceBranch = self.knownBranches[branch]
2844 sourceDepotPath = self.depotPaths[0] + sourceBranch
2845 sourceRef = self.gitRefForBranch(sourceBranch)
2846 #print "source " + sourceBranch
2847
52a4880b 2848 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2849 #print "branch parent: %s" % branchParentChange
2850 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2851 if len(gitParent) > 0:
2852 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2853 #print "parent git commit: %s" % gitParent
2854
2855 self.importChanges(changes)
2856 return True
2857
fed23693
VA
2858 def searchParent(self, parent, branch, target):
2859 parentFound = False
c7d34884
PW
2860 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2861 "--no-merges", parent]):
fed23693
VA
2862 blob = blob.strip()
2863 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2864 parentFound = True
2865 if self.verbose:
2866 print "Found parent of %s in commit %s" % (branch, blob)
2867 break
2868 if parentFound:
2869 return blob
2870 else:
2871 return None
2872
e87f37ae
SH
2873 def importChanges(self, changes):
2874 cnt = 1
2875 for change in changes:
18fa13d0 2876 description = p4_describe(change)
e87f37ae
SH
2877 self.updateOptionDict(description)
2878
2879 if not self.silent:
2880 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2881 sys.stdout.flush()
2882 cnt = cnt + 1
2883
2884 try:
2885 if self.detectBranches:
2886 branches = self.splitFilesIntoBranches(description)
2887 for branch in branches.keys():
2888 ## HACK --hwn
2889 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 2890 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
2891
2892 parent = ""
2893
2894 filesForCommit = branches[branch]
2895
2896 if self.verbose:
2897 print "branch is %s" % branch
2898
2899 self.updatedBranches.add(branch)
2900
2901 if branch not in self.createdBranches:
2902 self.createdBranches.add(branch)
2903 parent = self.knownBranches[branch]
2904 if parent == branch:
2905 parent = ""
1ca3d710
SH
2906 else:
2907 fullBranch = self.projectName + branch
2908 if fullBranch not in self.p4BranchesInGit:
2909 if not self.silent:
2910 print("\n Importing new branch %s" % fullBranch);
2911 if self.importNewBranch(branch, change - 1):
2912 parent = ""
2913 self.p4BranchesInGit.append(fullBranch)
2914 if not self.silent:
2915 print("\n Resuming with change %s" % change);
2916
2917 if self.verbose:
2918 print "parent determined through known branches: %s" % parent
e87f37ae 2919
8134f69c
SH
2920 branch = self.gitRefForBranch(branch)
2921 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2922
2923 if self.verbose:
2924 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2925
2926 if len(parent) == 0 and branch in self.initialParents:
2927 parent = self.initialParents[branch]
2928 del self.initialParents[branch]
2929
fed23693
VA
2930 blob = None
2931 if len(parent) > 0:
4f9273d2 2932 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
2933 if self.verbose:
2934 print "Creating temporary branch: " + tempBranch
e63231e5 2935 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
2936 self.tempBranches.append(tempBranch)
2937 self.checkpoint()
2938 blob = self.searchParent(parent, branch, tempBranch)
2939 if blob:
e63231e5 2940 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
2941 else:
2942 if self.verbose:
2943 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 2944 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
2945 else:
2946 files = self.extractFilesFromCommit(description)
e63231e5 2947 self.commit(description, files, self.branch,
e87f37ae 2948 self.initialParent)
47497844 2949 # only needed once, to connect to the previous commit
e87f37ae
SH
2950 self.initialParent = ""
2951 except IOError:
2952 print self.gitError.read()
2953 sys.exit(1)
2954
c208a243
SH
2955 def importHeadRevision(self, revision):
2956 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2957
4e2e6ce4
PW
2958 details = {}
2959 details["user"] = "git perforce import user"
1494fcbb 2960 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2961 % (' '.join(self.depotPaths), revision))
2962 details["change"] = revision
2963 newestRevision = 0
2964
2965 fileCnt = 0
6de040df
LD
2966 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2967
2968 for info in p4CmdList(["files"] + fileArgs):
c208a243 2969
68b28593 2970 if 'code' in info and info['code'] == 'error':
c208a243
SH
2971 sys.stderr.write("p4 returned an error: %s\n"
2972 % info['data'])
d88e707f
PW
2973 if info['data'].find("must refer to client") >= 0:
2974 sys.stderr.write("This particular p4 error is misleading.\n")
2975 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2976 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2977 sys.exit(1)
68b28593
PW
2978 if 'p4ExitCode' in info:
2979 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2980 sys.exit(1)
2981
2982
2983 change = int(info["change"])
2984 if change > newestRevision:
2985 newestRevision = change
2986
56c09345 2987 if info["action"] in self.delete_actions:
c208a243
SH
2988 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2989 #fileCnt = fileCnt + 1
2990 continue
2991
2992 for prop in ["depotFile", "rev", "action", "type" ]:
2993 details["%s%s" % (prop, fileCnt)] = info[prop]
2994
2995 fileCnt = fileCnt + 1
2996
2997 details["change"] = newestRevision
4e2e6ce4 2998
9dcb9f24 2999 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3000 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3001 res = p4_describe(newestRevision)
3002 details["time"] = res["time"]
4e2e6ce4 3003
c208a243
SH
3004 self.updateOptionDict(details)
3005 try:
e63231e5 3006 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3007 except IOError:
3008 print "IO error with git fast-import. Is your git version recent enough?"
3009 print self.gitError.read()
3010
3011
b984733c 3012 def run(self, args):
6326aa58 3013 self.depotPaths = []
179caebf 3014 self.changeRange = ""
6326aa58 3015 self.previousDepotPaths = []
991a2de4 3016 self.hasOrigin = False
ce6f33c8 3017
29bdbac1
SH
3018 # map from branch depot path to parent branch
3019 self.knownBranches = {}
3020 self.initialParents = {}
3021
a028a98e
SH
3022 if self.importIntoRemotes:
3023 self.refPrefix = "refs/remotes/p4/"
3024 else:
db775559 3025 self.refPrefix = "refs/heads/p4/"
a028a98e 3026
991a2de4
PW
3027 if self.syncWithOrigin:
3028 self.hasOrigin = originP4BranchesExist()
3029 if self.hasOrigin:
3030 if not self.silent:
3031 print 'Syncing with origin first, using "git fetch origin"'
3032 system("git fetch origin")
10f880f8 3033
5a8e84cd 3034 branch_arg_given = bool(self.branch)
569d1bd4 3035 if len(self.branch) == 0:
db775559 3036 self.branch = self.refPrefix + "master"
a028a98e 3037 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3038 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3039 system("git branch -D p4")
967f72e2 3040
a93d33ee
PW
3041 # accept either the command-line option, or the configuration variable
3042 if self.useClientSpec:
3043 # will use this after clone to set the variable
3044 self.useClientSpec_from_options = True
3045 else:
0d609032 3046 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3047 self.useClientSpec = True
3048 if self.useClientSpec:
543987bd 3049 self.clientSpecDirs = getClientSpec()
3a70cdfa 3050
6a49f8e2
HWN
3051 # TODO: should always look at previous commits,
3052 # merge with previous imports, if possible.
3053 if args == []:
d414c74a 3054 if self.hasOrigin:
5ca44617 3055 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3056
3057 # branches holds mapping from branch name to sha1
3058 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3059
3060 # restrict to just this one, disabling detect-branches
3061 if branch_arg_given:
3062 short = self.branch.split("/")[-1]
3063 if short in branches:
3064 self.p4BranchesInGit = [ short ]
3065 else:
3066 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3067
3068 if len(self.p4BranchesInGit) > 1:
3069 if not self.silent:
3070 print "Importing from/into multiple branches"
3071 self.detectBranches = True
8c9e8b6e
PW
3072 for branch in branches.keys():
3073 self.initialParents[self.refPrefix + branch] = \
3074 branches[branch]
967f72e2 3075
29bdbac1
SH
3076 if self.verbose:
3077 print "branches: %s" % self.p4BranchesInGit
3078
3079 p4Change = 0
3080 for branch in self.p4BranchesInGit:
cebdf5af 3081 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3082
3083 settings = extractSettingsGitLog(logMsg)
29bdbac1 3084
bb6e09b2
HWN
3085 self.readOptions(settings)
3086 if (settings.has_key('depot-paths')
3087 and settings.has_key ('change')):
3088 change = int(settings['change']) + 1
29bdbac1
SH
3089 p4Change = max(p4Change, change)
3090
bb6e09b2
HWN
3091 depotPaths = sorted(settings['depot-paths'])
3092 if self.previousDepotPaths == []:
6326aa58 3093 self.previousDepotPaths = depotPaths
29bdbac1 3094 else:
6326aa58
HWN
3095 paths = []
3096 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3097 prev_list = prev.split("/")
3098 cur_list = cur.split("/")
3099 for i in range(0, min(len(cur_list), len(prev_list))):
3100 if cur_list[i] <> prev_list[i]:
583e1707 3101 i = i - 1
6326aa58
HWN
3102 break
3103
04d277b3 3104 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3105
3106 self.previousDepotPaths = paths
29bdbac1
SH
3107
3108 if p4Change > 0:
bb6e09b2 3109 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3110 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3111 if not self.silent and not self.detectBranches:
967f72e2 3112 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3113
40d69ac3
PW
3114 # accept multiple ref name abbreviations:
3115 # refs/foo/bar/branch -> use it exactly
3116 # p4/branch -> prepend refs/remotes/ or refs/heads/
3117 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3118 if not self.branch.startswith("refs/"):
40d69ac3
PW
3119 if self.importIntoRemotes:
3120 prepend = "refs/remotes/"
3121 else:
3122 prepend = "refs/heads/"
3123 if not self.branch.startswith("p4/"):
3124 prepend += "p4/"
3125 self.branch = prepend + self.branch
179caebf 3126
6326aa58 3127 if len(args) == 0 and self.depotPaths:
b984733c 3128 if not self.silent:
6326aa58 3129 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3130 else:
6326aa58 3131 if self.depotPaths and self.depotPaths != args:
cebdf5af 3132 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3133 "This doesn't work!" % (' '.join (self.depotPaths),
3134 ' '.join (args)))
b984733c 3135 sys.exit(1)
6326aa58 3136
bb6e09b2 3137 self.depotPaths = sorted(args)
b984733c 3138
1c49fc19 3139 revision = ""
b984733c 3140 self.users = {}
b984733c 3141
58c8bc7c
PW
3142 # Make sure no revision specifiers are used when --changesfile
3143 # is specified.
3144 bad_changesfile = False
3145 if len(self.changesFile) > 0:
3146 for p in self.depotPaths:
3147 if p.find("@") >= 0 or p.find("#") >= 0:
3148 bad_changesfile = True
3149 break
3150 if bad_changesfile:
3151 die("Option --changesfile is incompatible with revision specifiers")
3152
6326aa58
HWN
3153 newPaths = []
3154 for p in self.depotPaths:
3155 if p.find("@") != -1:
3156 atIdx = p.index("@")
3157 self.changeRange = p[atIdx:]
3158 if self.changeRange == "@all":
3159 self.changeRange = ""
6a49f8e2 3160 elif ',' not in self.changeRange:
1c49fc19 3161 revision = self.changeRange
6326aa58 3162 self.changeRange = ""
7fcff9de 3163 p = p[:atIdx]
6326aa58
HWN
3164 elif p.find("#") != -1:
3165 hashIdx = p.index("#")
1c49fc19 3166 revision = p[hashIdx:]
7fcff9de 3167 p = p[:hashIdx]
6326aa58 3168 elif self.previousDepotPaths == []:
58c8bc7c
PW
3169 # pay attention to changesfile, if given, else import
3170 # the entire p4 tree at the head revision
3171 if len(self.changesFile) == 0:
3172 revision = "#head"
6326aa58
HWN
3173
3174 p = re.sub ("\.\.\.$", "", p)
3175 if not p.endswith("/"):
3176 p += "/"
3177
3178 newPaths.append(p)
3179
3180 self.depotPaths = newPaths
3181
e63231e5
PW
3182 # --detect-branches may change this for each branch
3183 self.branchPrefixes = self.depotPaths
3184
b607e71e 3185 self.loadUserMapFromCache()
cb53e1f8
SH
3186 self.labels = {}
3187 if self.detectLabels:
3188 self.getLabels();
b984733c 3189
4b97ffb1 3190 if self.detectBranches:
df450923
SH
3191 ## FIXME - what's a P4 projectName ?
3192 self.projectName = self.guessProjectName()
3193
38f9f5ec
SH
3194 if self.hasOrigin:
3195 self.getBranchMappingFromGitBranches()
3196 else:
3197 self.getBranchMapping()
29bdbac1
SH
3198 if self.verbose:
3199 print "p4-git branches: %s" % self.p4BranchesInGit
3200 print "initial parents: %s" % self.initialParents
3201 for b in self.p4BranchesInGit:
3202 if b != "master":
6326aa58
HWN
3203
3204 ## FIXME
29bdbac1
SH
3205 b = b[len(self.projectName):]
3206 self.createdBranches.add(b)
4b97ffb1 3207
f291b4e3 3208 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3209
78189bea
PW
3210 self.importProcess = subprocess.Popen(["git", "fast-import"],
3211 stdin=subprocess.PIPE,
3212 stdout=subprocess.PIPE,
3213 stderr=subprocess.PIPE);
3214 self.gitOutput = self.importProcess.stdout
3215 self.gitStream = self.importProcess.stdin
3216 self.gitError = self.importProcess.stderr
b984733c 3217
1c49fc19 3218 if revision:
c208a243 3219 self.importHeadRevision(revision)
b984733c
SH
3220 else:
3221 changes = []
3222
0828ab14 3223 if len(self.changesFile) > 0:
b984733c 3224 output = open(self.changesFile).readlines()
1d7367dc 3225 changeSet = set()
b984733c
SH
3226 for line in output:
3227 changeSet.add(int(line))
3228
3229 for change in changeSet:
3230 changes.append(change)
3231
3232 changes.sort()
3233 else:
9dcb9f24
PW
3234 # catch "git p4 sync" with no new branches, in a repo that
3235 # does not have any existing p4 branches
5a8e84cd
PW
3236 if len(args) == 0:
3237 if not self.p4BranchesInGit:
3238 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3239
3240 # The default branch is master, unless --branch is used to
3241 # specify something else. Make sure it exists, or complain
3242 # nicely about how to use --branch.
3243 if not self.detectBranches:
3244 if not branch_exists(self.branch):
3245 if branch_arg_given:
3246 die("Error: branch %s does not exist." % self.branch)
3247 else:
3248 die("Error: no branch %s; perhaps specify one with --branch." %
3249 self.branch)
3250
29bdbac1 3251 if self.verbose:
86dff6b6 3252 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3253 self.changeRange)
96b2d54a 3254 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
b984733c 3255
01a9c9c5 3256 if len(self.maxChanges) > 0:
7fcff9de 3257 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3258
b984733c 3259 if len(changes) == 0:
0828ab14 3260 if not self.silent:
341dc1c1 3261 print "No changes to import!"
06804c76
LD
3262 else:
3263 if not self.silent and not self.detectBranches:
3264 print "Import destination: %s" % self.branch
3265
3266 self.updatedBranches = set()
b984733c 3267
47497844
PW
3268 if not self.detectBranches:
3269 if args:
3270 # start a new branch
3271 self.initialParent = ""
3272 else:
3273 # build on a previous revision
3274 self.initialParent = parseRevision(self.branch)
3275
06804c76 3276 self.importChanges(changes)
a9d1a27a 3277
06804c76
LD
3278 if not self.silent:
3279 print ""
3280 if len(self.updatedBranches) > 0:
3281 sys.stdout.write("Updated branches: ")
3282 for b in self.updatedBranches:
3283 sys.stdout.write("%s " % b)
3284 sys.stdout.write("\n")
341dc1c1 3285
0d609032 3286 if gitConfigBool("git-p4.importLabels"):
06dcd152 3287 self.importLabels = True
b984733c 3288
06804c76
LD
3289 if self.importLabels:
3290 p4Labels = getP4Labels(self.depotPaths)
3291 gitTags = getGitTags()
3292
3293 missingP4Labels = p4Labels - gitTags
3294 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3295
b984733c 3296 self.gitStream.close()
78189bea 3297 if self.importProcess.wait() != 0:
29bdbac1 3298 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3299 self.gitOutput.close()
3300 self.gitError.close()
3301
fed23693
VA
3302 # Cleanup temporary branches created during import
3303 if self.tempBranches != []:
3304 for branch in self.tempBranches:
3305 read_pipe("git update-ref -d %s" % branch)
3306 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3307
55d12437
PW
3308 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3309 # a convenient shortcut refname "p4".
3310 if self.importIntoRemotes:
3311 head_ref = self.refPrefix + "HEAD"
3312 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3313 system(["git", "symbolic-ref", head_ref, self.branch])
3314
b984733c
SH
3315 return True
3316
01ce1fe9
SH
3317class P4Rebase(Command):
3318 def __init__(self):
3319 Command.__init__(self)
06804c76
LD
3320 self.options = [
3321 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3322 ]
06804c76 3323 self.importLabels = False
cebdf5af
HWN
3324 self.description = ("Fetches the latest revision from perforce and "
3325 + "rebases the current work (branch) against it")
01ce1fe9
SH
3326
3327 def run(self, args):
3328 sync = P4Sync()
06804c76 3329 sync.importLabels = self.importLabels
01ce1fe9 3330 sync.run([])
d7e3868c 3331
14594f4b
SH
3332 return self.rebase()
3333
3334 def rebase(self):
36ee4ee4
SH
3335 if os.system("git update-index --refresh") != 0:
3336 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3337 if len(read_pipe("git diff-index HEAD --")) > 0:
f7e604ed 3338 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
36ee4ee4 3339
d7e3868c
SH
3340 [upstream, settings] = findUpstreamBranchPoint()
3341 if len(upstream) == 0:
3342 die("Cannot find upstream branchpoint for rebase")
3343
3344 # the branchpoint may be p4/foo~3, so strip off the parent
3345 upstream = re.sub("~[0-9]+$", "", upstream)
3346
3347 print "Rebasing the current branch onto %s" % upstream
b25b2065 3348 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 3349 system("git rebase %s" % upstream)
4e49d95e 3350 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
01ce1fe9
SH
3351 return True
3352
f9a3a4f7
SH
3353class P4Clone(P4Sync):
3354 def __init__(self):
3355 P4Sync.__init__(self)
3356 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 3357 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 3358 self.options += [
bb6e09b2
HWN
3359 optparse.make_option("--destination", dest="cloneDestination",
3360 action='store', default=None,
354081d5 3361 help="where to leave result of the clone"),
38200076
PW
3362 optparse.make_option("--bare", dest="cloneBare",
3363 action="store_true", default=False),
354081d5 3364 ]
bb6e09b2 3365 self.cloneDestination = None
f9a3a4f7 3366 self.needsGit = False
38200076 3367 self.cloneBare = False
f9a3a4f7 3368
6a49f8e2
HWN
3369 def defaultDestination(self, args):
3370 ## TODO: use common prefix of args?
3371 depotPath = args[0]
3372 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3373 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 3374 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
3375 depotDir = re.sub(r"/$", "", depotDir)
3376 return os.path.split(depotDir)[1]
3377
f9a3a4f7
SH
3378 def run(self, args):
3379 if len(args) < 1:
3380 return False
bb6e09b2
HWN
3381
3382 if self.keepRepoPath and not self.cloneDestination:
3383 sys.stderr.write("Must specify destination for --keep-path\n")
3384 sys.exit(1)
f9a3a4f7 3385
6326aa58 3386 depotPaths = args
5e100b5c
SH
3387
3388 if not self.cloneDestination and len(depotPaths) > 1:
3389 self.cloneDestination = depotPaths[-1]
3390 depotPaths = depotPaths[:-1]
3391
354081d5 3392 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
3393 for p in depotPaths:
3394 if not p.startswith("//"):
0f487d30 3395 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
6326aa58 3396 return False
f9a3a4f7 3397
bb6e09b2 3398 if not self.cloneDestination:
98ad4faf 3399 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 3400
86dff6b6 3401 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 3402
c3bf3f13
KG
3403 if not os.path.exists(self.cloneDestination):
3404 os.makedirs(self.cloneDestination)
053fd0c1 3405 chdir(self.cloneDestination)
38200076
PW
3406
3407 init_cmd = [ "git", "init" ]
3408 if self.cloneBare:
3409 init_cmd.append("--bare")
a235e85c
BC
3410 retcode = subprocess.call(init_cmd)
3411 if retcode:
3412 raise CalledProcessError(retcode, init_cmd)
38200076 3413
6326aa58 3414 if not P4Sync.run(self, depotPaths):
f9a3a4f7 3415 return False
c595956d
PW
3416