git-p4: add failing test for name-rev rather than symbolic-ref
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
a235e85c
BC
30try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
4addad22 45verbose = False
86949eef 46
06804c76 47# Only labels/tags matching this will be imported/exported
c8942a22 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 49
1051ef00
LD
50# Grab changes in blocks of this many revisions, unless otherwise requested
51defaultBlockSize = 512
52
21a50753
AK
53def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
6de040df 60 real_cmd = ["p4"]
abcaf073
AK
61
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
6de040df 64 real_cmd += ["-u",user]
abcaf073
AK
65
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
6de040df 68 real_cmd += ["-P", password]
abcaf073
AK
69
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
6de040df 72 real_cmd += ["-p", port]
abcaf073
AK
73
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
41799aa2 76 real_cmd += ["-H", host]
abcaf073
AK
77
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
6de040df 80 real_cmd += ["-c", client]
abcaf073 81
89a6ecc5
LS
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
bc233524
IK
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
6de040df
LD
89
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
21a50753
AK
94 return real_cmd
95
378f7be1
LD
96def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
bbd84863
MF
106def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
053fd0c1 127
4d25dc44
LS
128def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
86dff6b6
HWN
138def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
6de040df 145def write_pipe(c, stdin):
4addad22 146 if verbose:
6de040df 147 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 148
6de040df
LD
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
b016d397
HWN
156
157 return val
158
6de040df 159def p4_write_pipe(c, stdin):
d9429194 160 real_cmd = p4_build_cmd(c)
6de040df 161 return write_pipe(real_cmd, stdin)
d9429194 162
4addad22
HWN
163def read_pipe(c, ignore_error=False):
164 if verbose:
6de040df 165 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 166
6de040df 167 expand = isinstance(c,basestring)
1f5f3907
LS
168 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
169 (out, err) = p.communicate()
170 if p.returncode != 0 and not ignore_error:
171 die('Command failed: %s\nError: %s' % (str(c), err))
172 return out
b016d397 173
d9429194
AK
174def p4_read_pipe(c, ignore_error=False):
175 real_cmd = p4_build_cmd(c)
176 return read_pipe(real_cmd, ignore_error)
b016d397 177
bce4c5fc 178def read_pipe_lines(c):
4addad22 179 if verbose:
6de040df
LD
180 sys.stderr.write('Reading pipe: %s\n' % str(c))
181
182 expand = isinstance(c, basestring)
183 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
184 pipe = p.stdout
b016d397 185 val = pipe.readlines()
6de040df
LD
186 if pipe.close() or p.wait():
187 die('Command failed: %s' % str(c))
b016d397
HWN
188
189 return val
caace111 190
2318121b
AK
191def p4_read_pipe_lines(c):
192 """Specifically invoke p4 on the command supplied. """
155af834 193 real_cmd = p4_build_cmd(c)
2318121b
AK
194 return read_pipe_lines(real_cmd)
195
8e9497c2
GG
196def p4_has_command(cmd):
197 """Ask p4 for help on this command. If it returns an error, the
198 command does not exist in this version of p4."""
199 real_cmd = p4_build_cmd(["help", cmd])
200 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
201 stderr=subprocess.PIPE)
202 p.communicate()
203 return p.returncode == 0
204
249da4c0
PW
205def p4_has_move_command():
206 """See if the move command exists, that it supports -k, and that
207 it has not been administratively disabled. The arguments
208 must be correct, but the filenames do not have to exist. Use
209 ones with wildcards so even if they exist, it will fail."""
210
211 if not p4_has_command("move"):
212 return False
213 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
214 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
215 (out, err) = p.communicate()
216 # return code will be 1 in either case
217 if err.find("Invalid option") >= 0:
218 return False
219 if err.find("disabled") >= 0:
220 return False
221 # assume it failed because @... was invalid changelist
222 return True
223
cbff4b25 224def system(cmd, ignore_error=False):
6de040df 225 expand = isinstance(cmd,basestring)
4addad22 226 if verbose:
6de040df 227 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 228 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 229 if retcode and not ignore_error:
a235e85c 230 raise CalledProcessError(retcode, cmd)
6754a299 231
cbff4b25
LD
232 return retcode
233
bf9320f1
AK
234def p4_system(cmd):
235 """Specifically invoke p4 as the system command. """
155af834 236 real_cmd = p4_build_cmd(cmd)
6de040df 237 expand = isinstance(real_cmd, basestring)
a235e85c
BC
238 retcode = subprocess.call(real_cmd, shell=expand)
239 if retcode:
240 raise CalledProcessError(retcode, real_cmd)
6de040df 241
7f0e5962
PW
242_p4_version_string = None
243def p4_version_string():
244 """Read the version string, showing just the last line, which
245 hopefully is the interesting version bit.
246
247 $ p4 -V
248 Perforce - The Fast Software Configuration Management System.
249 Copyright 1995-2011 Perforce Software. All rights reserved.
250 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
251 """
252 global _p4_version_string
253 if not _p4_version_string:
254 a = p4_read_pipe_lines(["-V"])
255 _p4_version_string = a[-1].rstrip()
256 return _p4_version_string
257
6de040df 258def p4_integrate(src, dest):
9d7d446a 259 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 260
8d7ec362 261def p4_sync(f, *options):
9d7d446a 262 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
263
264def p4_add(f):
9d7d446a
PW
265 # forcibly add file names with wildcards
266 if wildcard_present(f):
267 p4_system(["add", "-f", f])
268 else:
269 p4_system(["add", f])
6de040df
LD
270
271def p4_delete(f):
9d7d446a 272 p4_system(["delete", wildcard_encode(f)])
6de040df 273
a02b8bc4
RP
274def p4_edit(f, *options):
275 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
276
277def p4_revert(f):
9d7d446a 278 p4_system(["revert", wildcard_encode(f)])
6de040df 279
9d7d446a
PW
280def p4_reopen(type, f):
281 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 282
46c609e9
LD
283def p4_reopen_in_change(changelist, files):
284 cmd = ["reopen", "-c", str(changelist)] + files
285 p4_system(cmd)
286
8e9497c2
GG
287def p4_move(src, dest):
288 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
289
1051ef00
LD
290def p4_last_change():
291 results = p4CmdList(["changes", "-m", "1"])
292 return int(results[0]['change'])
293
18fa13d0
PW
294def p4_describe(change):
295 """Make sure it returns a valid result by checking for
296 the presence of field "time". Return a dict of the
297 results."""
298
299 ds = p4CmdList(["describe", "-s", str(change)])
300 if len(ds) != 1:
301 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
302
303 d = ds[0]
304
305 if "p4ExitCode" in d:
306 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
307 str(d)))
308 if "code" in d:
309 if d["code"] == "error":
310 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
311
312 if "time" not in d:
313 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
314
315 return d
316
9cffb8c8
PW
317#
318# Canonicalize the p4 type and return a tuple of the
319# base type, plus any modifiers. See "p4 help filetypes"
320# for a list and explanation.
321#
322def split_p4_type(p4type):
323
324 p4_filetypes_historical = {
325 "ctempobj": "binary+Sw",
326 "ctext": "text+C",
327 "cxtext": "text+Cx",
328 "ktext": "text+k",
329 "kxtext": "text+kx",
330 "ltext": "text+F",
331 "tempobj": "binary+FSw",
332 "ubinary": "binary+F",
333 "uresource": "resource+F",
334 "uxbinary": "binary+Fx",
335 "xbinary": "binary+x",
336 "xltext": "text+Fx",
337 "xtempobj": "binary+Swx",
338 "xtext": "text+x",
339 "xunicode": "unicode+x",
340 "xutf16": "utf16+x",
341 }
342 if p4type in p4_filetypes_historical:
343 p4type = p4_filetypes_historical[p4type]
344 mods = ""
345 s = p4type.split("+")
346 base = s[0]
347 mods = ""
348 if len(s) > 1:
349 mods = s[1]
350 return (base, mods)
b9fc6ea9 351
60df071c
LD
352#
353# return the raw p4 type of a file (text, text+ko, etc)
354#
79467e61
PW
355def p4_type(f):
356 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
357 return results[0]['headType']
358
359#
360# Given a type base and modifier, return a regexp matching
361# the keywords that can be expanded in the file
362#
363def p4_keywords_regexp_for_type(base, type_mods):
364 if base in ("text", "unicode", "binary"):
365 kwords = None
366 if "ko" in type_mods:
367 kwords = 'Id|Header'
368 elif "k" in type_mods:
369 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
370 else:
371 return None
372 pattern = r"""
373 \$ # Starts with a dollar, followed by...
374 (%s) # one of the keywords, followed by...
6b2bf41e 375 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
376 \$ # another dollar
377 """ % kwords
378 return pattern
379 else:
380 return None
381
382#
383# Given a file, return a regexp matching the possible
384# RCS keywords that will be expanded, or None for files
385# with kw expansion turned off.
386#
387def p4_keywords_regexp_for_file(file):
388 if not os.path.exists(file):
389 return None
390 else:
391 (type_base, type_mods) = split_p4_type(p4_type(file))
392 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 393
c65b670e
CP
394def setP4ExecBit(file, mode):
395 # Reopens an already open file and changes the execute bit to match
396 # the execute bit setting in the passed in mode.
397
398 p4Type = "+x"
399
400 if not isModeExec(mode):
401 p4Type = getP4OpenedType(file)
402 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
403 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
404 if p4Type[-1] == "+":
405 p4Type = p4Type[0:-1]
406
6de040df 407 p4_reopen(p4Type, file)
c65b670e
CP
408
409def getP4OpenedType(file):
410 # Returns the perforce file type for the given file.
411
9d7d446a 412 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 413 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
414 if match:
415 return match.group(1)
416 else:
f3e5ae4f 417 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 418
06804c76
LD
419# Return the set of all p4 labels
420def getP4Labels(depotPaths):
421 labels = set()
422 if isinstance(depotPaths,basestring):
423 depotPaths = [depotPaths]
424
425 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
426 label = l['label']
427 labels.add(label)
428
429 return labels
430
431# Return the set of all git tags
432def getGitTags():
433 gitTags = set()
434 for line in read_pipe_lines(["git", "tag"]):
435 tag = line.strip()
436 gitTags.add(tag)
437 return gitTags
438
b43b0a3c
CP
439def diffTreePattern():
440 # This is a simple generator for the diff tree regex pattern. This could be
441 # a class variable if this and parseDiffTreeEntry were a part of a class.
442 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
443 while True:
444 yield pattern
445
446def parseDiffTreeEntry(entry):
447 """Parses a single diff tree entry into its component elements.
448
449 See git-diff-tree(1) manpage for details about the format of the diff
450 output. This method returns a dictionary with the following elements:
451
452 src_mode - The mode of the source file
453 dst_mode - The mode of the destination file
454 src_sha1 - The sha1 for the source file
455 dst_sha1 - The sha1 fr the destination file
456 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
457 status_score - The score for the status (applicable for 'C' and 'R'
458 statuses). This is None if there is no score.
459 src - The path for the source file.
460 dst - The path for the destination file. This is only present for
461 copy or renames. If it is not present, this is None.
462
463 If the pattern is not matched, None is returned."""
464
465 match = diffTreePattern().next().match(entry)
466 if match:
467 return {
468 'src_mode': match.group(1),
469 'dst_mode': match.group(2),
470 'src_sha1': match.group(3),
471 'dst_sha1': match.group(4),
472 'status': match.group(5),
473 'status_score': match.group(6),
474 'src': match.group(7),
475 'dst': match.group(10)
476 }
477 return None
478
c65b670e
CP
479def isModeExec(mode):
480 # Returns True if the given git mode represents an executable file,
481 # otherwise False.
482 return mode[-3:] == "755"
483
484def isModeExecChanged(src_mode, dst_mode):
485 return isModeExec(src_mode) != isModeExec(dst_mode)
486
b932705b 487def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
488
489 if isinstance(cmd,basestring):
490 cmd = "-G " + cmd
491 expand = True
492 else:
493 cmd = ["-G"] + cmd
494 expand = False
495
496 cmd = p4_build_cmd(cmd)
6a49f8e2 497 if verbose:
6de040df 498 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
499
500 # Use a temporary file to avoid deadlocks without
501 # subprocess.communicate(), which would put another copy
502 # of stdout into memory.
503 stdin_file = None
504 if stdin is not None:
505 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
506 if isinstance(stdin,basestring):
507 stdin_file.write(stdin)
508 else:
509 for i in stdin:
510 stdin_file.write(i + '\n')
9f90c733
SL
511 stdin_file.flush()
512 stdin_file.seek(0)
513
6de040df
LD
514 p4 = subprocess.Popen(cmd,
515 shell=expand,
9f90c733
SL
516 stdin=stdin_file,
517 stdout=subprocess.PIPE)
86949eef
SH
518
519 result = []
520 try:
521 while True:
9f90c733 522 entry = marshal.load(p4.stdout)
c3f6163b
AG
523 if cb is not None:
524 cb(entry)
525 else:
526 result.append(entry)
86949eef
SH
527 except EOFError:
528 pass
9f90c733
SL
529 exitCode = p4.wait()
530 if exitCode != 0:
ac3e0d79
SH
531 entry = {}
532 entry["p4ExitCode"] = exitCode
533 result.append(entry)
86949eef
SH
534
535 return result
536
537def p4Cmd(cmd):
538 list = p4CmdList(cmd)
539 result = {}
540 for entry in list:
541 result.update(entry)
542 return result;
543
cb2c9db5
SH
544def p4Where(depotPath):
545 if not depotPath.endswith("/"):
546 depotPath += "/"
cd884106
VA
547 depotPathLong = depotPath + "..."
548 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
549 output = None
550 for entry in outputList:
75bc9573 551 if "depotFile" in entry:
cd884106
VA
552 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
553 # The base path always ends with "/...".
554 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
555 output = entry
556 break
557 elif "data" in entry:
558 data = entry.get("data")
559 space = data.find(" ")
560 if data[:space] == depotPath:
561 output = entry
562 break
7f705dc3
TAL
563 if output == None:
564 return ""
dc524036
SH
565 if output["code"] == "error":
566 return ""
cb2c9db5
SH
567 clientPath = ""
568 if "path" in output:
569 clientPath = output.get("path")
570 elif "data" in output:
571 data = output.get("data")
572 lastSpace = data.rfind(" ")
573 clientPath = data[lastSpace + 1:]
574
575 if clientPath.endswith("..."):
576 clientPath = clientPath[:-3]
577 return clientPath
578
86949eef 579def currentGitBranch():
00ad6e31
LD
580 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
581 if retcode != 0:
582 # on a detached head
583 return None
584 else:
585 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
86949eef 586
4f5cf76a 587def isValidGitDir(path):
378f7be1 588 return git_dir(path) != None
4f5cf76a 589
463e8af6 590def parseRevision(ref):
b25b2065 591 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 592
28755dba
PW
593def branchExists(ref):
594 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
595 ignore_error=True)
596 return len(rev) > 0
597
6ae8de88
SH
598def extractLogMessageFromGitCommit(commit):
599 logMessage = ""
b016d397
HWN
600
601 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 602 foundTitle = False
b016d397 603 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
604 if not foundTitle:
605 if len(log) == 1:
1c094184 606 foundTitle = True
6ae8de88
SH
607 continue
608
609 logMessage += log
610 return logMessage
611
bb6e09b2 612def extractSettingsGitLog(log):
6ae8de88
SH
613 values = {}
614 for line in log.split("\n"):
615 line = line.strip()
6326aa58
HWN
616 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
617 if not m:
618 continue
619
620 assignments = m.group(1).split (':')
621 for a in assignments:
622 vals = a.split ('=')
623 key = vals[0].strip()
624 val = ('='.join (vals[1:])).strip()
625 if val.endswith ('\"') and val.startswith('"'):
626 val = val[1:-1]
627
628 values[key] = val
629
845b42cb
SH
630 paths = values.get("depot-paths")
631 if not paths:
632 paths = values.get("depot-path")
a3fdd579
SH
633 if paths:
634 values['depot-paths'] = paths.split(',')
bb6e09b2 635 return values
6ae8de88 636
8136a639 637def gitBranchExists(branch):
bb6e09b2
HWN
638 proc = subprocess.Popen(["git", "rev-parse", branch],
639 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 640 return proc.wait() == 0;
8136a639 641
36bd8446 642_gitConfig = {}
b345d6c3 643
692e1796 644def gitConfig(key, typeSpecifier=None):
36bd8446 645 if not _gitConfig.has_key(key):
692e1796
LS
646 cmd = [ "git", "config" ]
647 if typeSpecifier:
648 cmd += [ typeSpecifier ]
649 cmd += [ key ]
b345d6c3
PW
650 s = read_pipe(cmd, ignore_error=True)
651 _gitConfig[key] = s.strip()
36bd8446 652 return _gitConfig[key]
01265103 653
0d609032
PW
654def gitConfigBool(key):
655 """Return a bool, using git config --bool. It is True only if the
656 variable is set to true, and False if set to false or not present
657 in the config."""
658
36bd8446 659 if not _gitConfig.has_key(key):
692e1796 660 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 661 return _gitConfig[key]
01265103 662
cb1dafdf
LS
663def gitConfigInt(key):
664 if not _gitConfig.has_key(key):
665 cmd = [ "git", "config", "--int", key ]
0d609032
PW
666 s = read_pipe(cmd, ignore_error=True)
667 v = s.strip()
cb1dafdf
LS
668 try:
669 _gitConfig[key] = int(gitConfig(key, '--int'))
670 except ValueError:
671 _gitConfig[key] = None
36bd8446 672 return _gitConfig[key]
01265103 673
7199cf13
VA
674def gitConfigList(key):
675 if not _gitConfig.has_key(key):
2abba301 676 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 677 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
678 if _gitConfig[key] == ['']:
679 _gitConfig[key] = []
7199cf13
VA
680 return _gitConfig[key]
681
2c8037ed
PW
682def p4BranchesInGit(branchesAreInRemotes=True):
683 """Find all the branches whose names start with "p4/", looking
684 in remotes or heads as specified by the argument. Return
685 a dictionary of { branch: revision } for each one found.
686 The branch names are the short names, without any
687 "p4/" prefix."""
688
062410bb
SH
689 branches = {}
690
691 cmdline = "git rev-parse --symbolic "
692 if branchesAreInRemotes:
2c8037ed 693 cmdline += "--remotes"
062410bb 694 else:
2c8037ed 695 cmdline += "--branches"
062410bb
SH
696
697 for line in read_pipe_lines(cmdline):
698 line = line.strip()
699
2c8037ed
PW
700 # only import to p4/
701 if not line.startswith('p4/'):
702 continue
703 # special symbolic ref to p4/master
704 if line == "p4/HEAD":
062410bb 705 continue
062410bb 706
2c8037ed
PW
707 # strip off p4/ prefix
708 branch = line[len("p4/"):]
062410bb
SH
709
710 branches[branch] = parseRevision(line)
2c8037ed 711
062410bb
SH
712 return branches
713
5a8e84cd
PW
714def branch_exists(branch):
715 """Make sure that the given ref name really exists."""
716
717 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
718 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
719 out, _ = p.communicate()
720 if p.returncode:
721 return False
722 # expect exactly one line of output: the branch name
723 return out.rstrip() == branch
724
9ceab363 725def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
726 branches = p4BranchesInGit()
727 # map from depot-path to branch name
728 branchByDepotPath = {}
729 for branch in branches.keys():
730 tip = branches[branch]
731 log = extractLogMessageFromGitCommit(tip)
732 settings = extractSettingsGitLog(log)
733 if settings.has_key("depot-paths"):
734 paths = ",".join(settings["depot-paths"])
735 branchByDepotPath[paths] = "remotes/p4/" + branch
736
27d2d811 737 settings = None
27d2d811
SH
738 parent = 0
739 while parent < 65535:
9ceab363 740 commit = head + "~%s" % parent
27d2d811
SH
741 log = extractLogMessageFromGitCommit(commit)
742 settings = extractSettingsGitLog(log)
86506fe5
SH
743 if settings.has_key("depot-paths"):
744 paths = ",".join(settings["depot-paths"])
745 if branchByDepotPath.has_key(paths):
746 return [branchByDepotPath[paths], settings]
27d2d811 747
86506fe5 748 parent = parent + 1
27d2d811 749
86506fe5 750 return ["", settings]
27d2d811 751
5ca44617
SH
752def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
753 if not silent:
754 print ("Creating/updating branch(es) in %s based on origin branch(es)"
755 % localRefPrefix)
756
757 originPrefix = "origin/p4/"
758
759 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
760 line = line.strip()
761 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
762 continue
763
764 headName = line[len(originPrefix):]
765 remoteHead = localRefPrefix + headName
766 originHead = line
767
768 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
769 if (not original.has_key('depot-paths')
770 or not original.has_key('change')):
771 continue
772
773 update = False
774 if not gitBranchExists(remoteHead):
775 if verbose:
776 print "creating %s" % remoteHead
777 update = True
778 else:
779 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
780 if settings.has_key('change') > 0:
781 if settings['depot-paths'] == original['depot-paths']:
782 originP4Change = int(original['change'])
783 p4Change = int(settings['change'])
784 if originP4Change > p4Change:
785 print ("%s (%s) is newer than %s (%s). "
786 "Updating p4 branch from origin."
787 % (originHead, originP4Change,
788 remoteHead, p4Change))
789 update = True
790 else:
791 print ("Ignoring: %s was imported from %s while "
792 "%s was imported from %s"
793 % (originHead, ','.join(original['depot-paths']),
794 remoteHead, ','.join(settings['depot-paths'])))
795
796 if update:
797 system("git update-ref %s %s" % (remoteHead, originHead))
798
799def originP4BranchesExist():
800 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
801
1051ef00
LD
802
803def p4ParseNumericChangeRange(parts):
804 changeStart = int(parts[0][1:])
805 if parts[1] == '#head':
806 changeEnd = p4_last_change()
807 else:
808 changeEnd = int(parts[1])
809
810 return (changeStart, changeEnd)
811
812def chooseBlockSize(blockSize):
813 if blockSize:
814 return blockSize
815 else:
816 return defaultBlockSize
817
818def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 819 assert depotPaths
96b2d54a 820
1051ef00
LD
821 # Parse the change range into start and end. Try to find integer
822 # revision ranges as these can be broken up into blocks to avoid
823 # hitting server-side limits (maxrows, maxscanresults). But if
824 # that doesn't work, fall back to using the raw revision specifier
825 # strings, without using block mode.
826
96b2d54a 827 if changeRange is None or changeRange == '':
1051ef00
LD
828 changeStart = 1
829 changeEnd = p4_last_change()
830 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
831 else:
832 parts = changeRange.split(',')
833 assert len(parts) == 2
1051ef00
LD
834 try:
835 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
836 block_size = chooseBlockSize(requestedBlockSize)
837 except:
838 changeStart = parts[0][1:]
839 changeEnd = parts[1]
840 if requestedBlockSize:
841 die("cannot use --changes-block-size with non-numeric revisions")
842 block_size = None
4f6432d8 843
9943e5b9 844 changes = set()
96b2d54a 845
1f90a648
SH
846 # Retrieve changes a block at a time, to prevent running
847 # into a MaxResults/MaxScanRows error from the server.
1051ef00 848
1f90a648
SH
849 while True:
850 cmd = ['changes']
1051ef00 851
1f90a648
SH
852 if block_size:
853 end = min(changeEnd, changeStart + block_size)
854 revisionRange = "%d,%d" % (changeStart, end)
855 else:
856 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 857
1f90a648 858 for p in depotPaths:
1051ef00
LD
859 cmd += ["%s...@%s" % (p, revisionRange)]
860
1f90a648
SH
861 # Insert changes in chronological order
862 for line in reversed(p4_read_pipe_lines(cmd)):
9943e5b9 863 changes.add(int(line.split(" ")[1]))
1051ef00 864
1f90a648
SH
865 if not block_size:
866 break
1051ef00 867
1f90a648
SH
868 if end >= changeEnd:
869 break
1051ef00 870
1f90a648 871 changeStart = end + 1
4f6432d8 872
1f90a648
SH
873 changes = sorted(changes)
874 return changes
4f6432d8 875
d53de8b9
TAL
876def p4PathStartsWith(path, prefix):
877 # This method tries to remedy a potential mixed-case issue:
878 #
879 # If UserA adds //depot/DirA/file1
880 # and UserB adds //depot/dira/file2
881 #
882 # we may or may not have a problem. If you have core.ignorecase=true,
883 # we treat DirA and dira as the same directory
0d609032 884 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
885 return path.lower().startswith(prefix.lower())
886 return path.startswith(prefix)
887
543987bd
PW
888def getClientSpec():
889 """Look at the p4 client spec, create a View() object that contains
890 all the mappings, and return it."""
891
892 specList = p4CmdList("client -o")
893 if len(specList) != 1:
894 die('Output from "client -o" is %d lines, expecting 1' %
895 len(specList))
896
897 # dictionary of all client parameters
898 entry = specList[0]
899
9d57c4a6
KS
900 # the //client/ name
901 client_name = entry["Client"]
902
543987bd
PW
903 # just the keys that start with "View"
904 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
905
906 # hold this new View
9d57c4a6 907 view = View(client_name)
543987bd
PW
908
909 # append the lines, in order, to the view
910 for view_num in range(len(view_keys)):
911 k = "View%d" % view_num
912 if k not in view_keys:
913 die("Expected view key %s missing" % k)
914 view.append(entry[k])
915
916 return view
917
918def getClientRoot():
919 """Grab the client directory."""
920
921 output = p4CmdList("client -o")
922 if len(output) != 1:
923 die('Output from "client -o" is %d lines, expecting 1' % len(output))
924
925 entry = output[0]
926 if "Root" not in entry:
927 die('Client has no "Root"')
928
929 return entry["Root"]
930
9d7d446a
PW
931#
932# P4 wildcards are not allowed in filenames. P4 complains
933# if you simply add them, but you can force it with "-f", in
934# which case it translates them into %xx encoding internally.
935#
936def wildcard_decode(path):
937 # Search for and fix just these four characters. Do % last so
938 # that fixing it does not inadvertently create new %-escapes.
939 # Cannot have * in a filename in windows; untested as to
940 # what p4 would do in such a case.
941 if not platform.system() == "Windows":
942 path = path.replace("%2A", "*")
943 path = path.replace("%23", "#") \
944 .replace("%40", "@") \
945 .replace("%25", "%")
946 return path
947
948def wildcard_encode(path):
949 # do % first to avoid double-encoding the %s introduced here
950 path = path.replace("%", "%25") \
951 .replace("*", "%2A") \
952 .replace("#", "%23") \
953 .replace("@", "%40")
954 return path
955
956def wildcard_present(path):
598354c0
BC
957 m = re.search("[*#@%]", path)
958 return m is not None
9d7d446a 959
a5db4b12
LS
960class LargeFileSystem(object):
961 """Base class for large file system support."""
962
963 def __init__(self, writeToGitStream):
964 self.largeFiles = set()
965 self.writeToGitStream = writeToGitStream
966
967 def generatePointer(self, cloneDestination, contentFile):
968 """Return the content of a pointer file that is stored in Git instead of
969 the actual content."""
970 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
971
972 def pushFile(self, localLargeFile):
973 """Push the actual content which is not stored in the Git repository to
974 a server."""
975 assert False, "Method 'pushFile' required in " + self.__class__.__name__
976
977 def hasLargeFileExtension(self, relPath):
978 return reduce(
979 lambda a, b: a or b,
980 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
981 False
982 )
983
984 def generateTempFile(self, contents):
985 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
986 for d in contents:
987 contentFile.write(d)
988 contentFile.close()
989 return contentFile.name
990
991 def exceedsLargeFileThreshold(self, relPath, contents):
992 if gitConfigInt('git-p4.largeFileThreshold'):
993 contentsSize = sum(len(d) for d in contents)
994 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
995 return True
996 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
997 contentsSize = sum(len(d) for d in contents)
998 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
999 return False
1000 contentTempFile = self.generateTempFile(contents)
1001 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1002 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1003 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1004 zf.close()
1005 compressedContentsSize = zf.infolist()[0].compress_size
1006 os.remove(contentTempFile)
1007 os.remove(compressedContentFile.name)
1008 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1009 return True
1010 return False
1011
1012 def addLargeFile(self, relPath):
1013 self.largeFiles.add(relPath)
1014
1015 def removeLargeFile(self, relPath):
1016 self.largeFiles.remove(relPath)
1017
1018 def isLargeFile(self, relPath):
1019 return relPath in self.largeFiles
1020
1021 def processContent(self, git_mode, relPath, contents):
1022 """Processes the content of git fast import. This method decides if a
1023 file is stored in the large file system and handles all necessary
1024 steps."""
1025 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1026 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1027 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1028 if pointer_git_mode:
1029 git_mode = pointer_git_mode
1030 if localLargeFile:
1031 # Move temp file to final location in large file system
1032 largeFileDir = os.path.dirname(localLargeFile)
1033 if not os.path.isdir(largeFileDir):
1034 os.makedirs(largeFileDir)
1035 shutil.move(contentTempFile, localLargeFile)
1036 self.addLargeFile(relPath)
1037 if gitConfigBool('git-p4.largeFilePush'):
1038 self.pushFile(localLargeFile)
1039 if verbose:
1040 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1041 return (git_mode, contents)
1042
1043class MockLFS(LargeFileSystem):
1044 """Mock large file system for testing."""
1045
1046 def generatePointer(self, contentFile):
1047 """The pointer content is the original content prefixed with "pointer-".
1048 The local filename of the large file storage is derived from the file content.
1049 """
1050 with open(contentFile, 'r') as f:
1051 content = next(f)
1052 gitMode = '100644'
1053 pointerContents = 'pointer-' + content
1054 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1055 return (gitMode, pointerContents, localLargeFile)
1056
1057 def pushFile(self, localLargeFile):
1058 """The remote filename of the large file storage is the same as the local
1059 one but in a different directory.
1060 """
1061 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1062 if not os.path.exists(remotePath):
1063 os.makedirs(remotePath)
1064 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1065
b47d807d
LS
1066class GitLFS(LargeFileSystem):
1067 """Git LFS as backend for the git-p4 large file system.
1068 See https://git-lfs.github.com/ for details."""
1069
1070 def __init__(self, *args):
1071 LargeFileSystem.__init__(self, *args)
1072 self.baseGitAttributes = []
1073
1074 def generatePointer(self, contentFile):
1075 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1076 mode and content which is stored in the Git repository instead of
1077 the actual content. Return also the new location of the actual
1078 content.
1079 """
d5eb3cf5
LS
1080 if os.path.getsize(contentFile) == 0:
1081 return (None, '', None)
1082
b47d807d
LS
1083 pointerProcess = subprocess.Popen(
1084 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1085 stdout=subprocess.PIPE
1086 )
1087 pointerFile = pointerProcess.stdout.read()
1088 if pointerProcess.wait():
1089 os.remove(contentFile)
1090 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1091
1092 # Git LFS removed the preamble in the output of the 'pointer' command
1093 # starting from version 1.2.0. Check for the preamble here to support
1094 # earlier versions.
1095 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1096 if pointerFile.startswith('Git LFS pointer for'):
1097 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1098
1099 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1100 localLargeFile = os.path.join(
1101 os.getcwd(),
1102 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1103 oid,
1104 )
1105 # LFS Spec states that pointer files should not have the executable bit set.
1106 gitMode = '100644'
82f2567e 1107 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1108
1109 def pushFile(self, localLargeFile):
1110 uploadProcess = subprocess.Popen(
1111 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1112 )
1113 if uploadProcess.wait():
1114 die('git-lfs push command failed. Did you define a remote?')
1115
1116 def generateGitAttributes(self):
1117 return (
1118 self.baseGitAttributes +
1119 [
1120 '\n',
1121 '#\n',
1122 '# Git LFS (see https://git-lfs.github.com/)\n',
1123 '#\n',
1124 ] +
862f9312 1125 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1126 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1127 ] +
862f9312 1128 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1129 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1130 ]
1131 )
1132
1133 def addLargeFile(self, relPath):
1134 LargeFileSystem.addLargeFile(self, relPath)
1135 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1136
1137 def removeLargeFile(self, relPath):
1138 LargeFileSystem.removeLargeFile(self, relPath)
1139 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1140
1141 def processContent(self, git_mode, relPath, contents):
1142 if relPath == '.gitattributes':
1143 self.baseGitAttributes = contents
1144 return (git_mode, self.generateGitAttributes())
1145 else:
1146 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1147
b984733c
SH
1148class Command:
1149 def __init__(self):
1150 self.usage = "usage: %prog [options]"
8910ac0e 1151 self.needsGit = True
6a10b6aa 1152 self.verbose = False
b984733c 1153
3ea2cfd4
LD
1154class P4UserMap:
1155 def __init__(self):
1156 self.userMapFromPerforceServer = False
affb474f
LD
1157 self.myP4UserId = None
1158
1159 def p4UserId(self):
1160 if self.myP4UserId:
1161 return self.myP4UserId
1162
1163 results = p4CmdList("user -o")
1164 for r in results:
1165 if r.has_key('User'):
1166 self.myP4UserId = r['User']
1167 return r['User']
1168 die("Could not find your p4 user id")
1169
1170 def p4UserIsMe(self, p4User):
1171 # return True if the given p4 user is actually me
1172 me = self.p4UserId()
1173 if not p4User or p4User != me:
1174 return False
1175 else:
1176 return True
3ea2cfd4
LD
1177
1178 def getUserCacheFilename(self):
1179 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1180 return home + "/.gitp4-usercache.txt"
1181
1182 def getUserMapFromPerforceServer(self):
1183 if self.userMapFromPerforceServer:
1184 return
1185 self.users = {}
1186 self.emails = {}
1187
1188 for output in p4CmdList("users"):
1189 if not output.has_key("User"):
1190 continue
1191 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1192 self.emails[output["Email"]] = output["User"]
1193
10d08a14
LS
1194 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1195 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1196 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1197 if mapUser and len(mapUser[0]) == 3:
1198 user = mapUser[0][0]
1199 fullname = mapUser[0][1]
1200 email = mapUser[0][2]
1201 self.users[user] = fullname + " <" + email + ">"
1202 self.emails[email] = user
3ea2cfd4
LD
1203
1204 s = ''
1205 for (key, val) in self.users.items():
1206 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1207
1208 open(self.getUserCacheFilename(), "wb").write(s)
1209 self.userMapFromPerforceServer = True
1210
1211 def loadUserMapFromCache(self):
1212 self.users = {}
1213 self.userMapFromPerforceServer = False
1214 try:
1215 cache = open(self.getUserCacheFilename(), "rb")
1216 lines = cache.readlines()
1217 cache.close()
1218 for line in lines:
1219 entry = line.strip().split("\t")
1220 self.users[entry[0]] = entry[1]
1221 except IOError:
1222 self.getUserMapFromPerforceServer()
1223
b984733c 1224class P4Debug(Command):
86949eef 1225 def __init__(self):
6ae8de88 1226 Command.__init__(self)
6a10b6aa 1227 self.options = []
c8c39116 1228 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1229 self.needsGit = False
86949eef
SH
1230
1231 def run(self, args):
b1ce9447 1232 j = 0
6de040df 1233 for output in p4CmdList(args):
b1ce9447
HWN
1234 print 'Element: %d' % j
1235 j += 1
86949eef 1236 print output
b984733c 1237 return True
86949eef 1238
5834684d
SH
1239class P4RollBack(Command):
1240 def __init__(self):
1241 Command.__init__(self)
1242 self.options = [
0c66a783 1243 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1244 ]
1245 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1246 self.rollbackLocalBranches = False
5834684d
SH
1247
1248 def run(self, args):
1249 if len(args) != 1:
1250 return False
1251 maxChange = int(args[0])
0c66a783 1252
ad192f28 1253 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1254 die("Problems executing p4");
1255
0c66a783
SH
1256 if self.rollbackLocalBranches:
1257 refPrefix = "refs/heads/"
b016d397 1258 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1259 else:
1260 refPrefix = "refs/remotes/"
b016d397 1261 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1262
1263 for line in lines:
1264 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1265 line = line.strip()
1266 ref = refPrefix + line
5834684d 1267 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1268 settings = extractSettingsGitLog(log)
1269
1270 depotPaths = settings['depot-paths']
1271 change = settings['change']
1272
5834684d 1273 changed = False
52102d47 1274
6326aa58
HWN
1275 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1276 for p in depotPaths]))) == 0:
52102d47
SH
1277 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1278 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1279 continue
1280
bb6e09b2 1281 while change and int(change) > maxChange:
5834684d 1282 changed = True
52102d47
SH
1283 if self.verbose:
1284 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1285 system("git update-ref %s \"%s^\"" % (ref, ref))
1286 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1287 settings = extractSettingsGitLog(log)
1288
1289
1290 depotPaths = settings['depot-paths']
1291 change = settings['change']
5834684d
SH
1292
1293 if changed:
52102d47 1294 print "%s rewound to %s" % (ref, change)
5834684d
SH
1295
1296 return True
1297
3ea2cfd4 1298class P4Submit(Command, P4UserMap):
6bbfd137
PW
1299
1300 conflict_behavior_choices = ("ask", "skip", "quit")
1301
4f5cf76a 1302 def __init__(self):
b984733c 1303 Command.__init__(self)
3ea2cfd4 1304 P4UserMap.__init__(self)
4f5cf76a 1305 self.options = [
4f5cf76a 1306 optparse.make_option("--origin", dest="origin"),
ae901090 1307 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1308 # preserve the user, requires relevant p4 permissions
1309 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1310 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1311 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1312 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1313 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1314 choices=self.conflict_behavior_choices),
1315 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1316 optparse.make_option("--shelve", dest="shelve", action="store_true",
1317 help="Shelve instead of submit. Shelved files are reverted, "
1318 "restoring the workspace to the state before the shelve"),
46c609e9
LD
1319 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1320 metavar="CHANGELIST",
1321 help="update an existing shelved changelist, implies --shelve")
4f5cf76a
SH
1322 ]
1323 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1324 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1325 self.origin = ""
ae901090 1326 self.detectRenames = False
0d609032 1327 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1328 self.dry_run = False
b34fa577 1329 self.shelve = False
46c609e9 1330 self.update_shelve = None
728b7ad8 1331 self.prepare_p4_only = False
6bbfd137 1332 self.conflict_behavior = None
f7baba8b 1333 self.isWindows = (platform.system() == "Windows")
06804c76 1334 self.exportLabels = False
249da4c0 1335 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1336 self.branch = None
4f5cf76a 1337
a5db4b12
LS
1338 if gitConfig('git-p4.largeFileSystem'):
1339 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1340
4f5cf76a
SH
1341 def check(self):
1342 if len(p4CmdList("opened ...")) > 0:
1343 die("You have files opened with perforce! Close them before starting the sync.")
1344
f19cb0a0
PW
1345 def separate_jobs_from_description(self, message):
1346 """Extract and return a possible Jobs field in the commit
1347 message. It goes into a separate section in the p4 change
1348 specification.
1349
1350 A jobs line starts with "Jobs:" and looks like a new field
1351 in a form. Values are white-space separated on the same
1352 line or on following lines that start with a tab.
1353
1354 This does not parse and extract the full git commit message
1355 like a p4 form. It just sees the Jobs: line as a marker
1356 to pass everything from then on directly into the p4 form,
1357 but outside the description section.
1358
1359 Return a tuple (stripped log message, jobs string)."""
1360
1361 m = re.search(r'^Jobs:', message, re.MULTILINE)
1362 if m is None:
1363 return (message, None)
1364
1365 jobtext = message[m.start():]
1366 stripped_message = message[:m.start()].rstrip()
1367 return (stripped_message, jobtext)
1368
1369 def prepareLogMessage(self, template, message, jobs):
1370 """Edits the template returned from "p4 change -o" to insert
1371 the message in the Description field, and the jobs text in
1372 the Jobs field."""
4f5cf76a
SH
1373 result = ""
1374
edae1e2f
SH
1375 inDescriptionSection = False
1376
4f5cf76a
SH
1377 for line in template.split("\n"):
1378 if line.startswith("#"):
1379 result += line + "\n"
1380 continue
1381
edae1e2f 1382 if inDescriptionSection:
c9dbab04 1383 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1384 inDescriptionSection = False
f19cb0a0
PW
1385 # insert Jobs section
1386 if jobs:
1387 result += jobs + "\n"
edae1e2f
SH
1388 else:
1389 continue
1390 else:
1391 if line.startswith("Description:"):
1392 inDescriptionSection = True
1393 line += "\n"
1394 for messageLine in message.split("\n"):
1395 line += "\t" + messageLine + "\n"
1396
1397 result += line + "\n"
4f5cf76a
SH
1398
1399 return result
1400
60df071c
LD
1401 def patchRCSKeywords(self, file, pattern):
1402 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1403 (handle, outFileName) = tempfile.mkstemp(dir='.')
1404 try:
1405 outFile = os.fdopen(handle, "w+")
1406 inFile = open(file, "r")
1407 regexp = re.compile(pattern, re.VERBOSE)
1408 for line in inFile.readlines():
1409 line = regexp.sub(r'$\1$', line)
1410 outFile.write(line)
1411 inFile.close()
1412 outFile.close()
1413 # Forcibly overwrite the original file
1414 os.unlink(file)
1415 shutil.move(outFileName, file)
1416 except:
1417 # cleanup our temporary file
1418 os.unlink(outFileName)
1419 print "Failed to strip RCS keywords in %s" % file
1420 raise
1421
1422 print "Patched up RCS keywords in %s" % file
1423
3ea2cfd4
LD
1424 def p4UserForCommit(self,id):
1425 # Return the tuple (perforce user,git email) for a given git commit id
1426 self.getUserMapFromPerforceServer()
9bf28855
PW
1427 gitEmail = read_pipe(["git", "log", "--max-count=1",
1428 "--format=%ae", id])
3ea2cfd4
LD
1429 gitEmail = gitEmail.strip()
1430 if not self.emails.has_key(gitEmail):
1431 return (None,gitEmail)
1432 else:
1433 return (self.emails[gitEmail],gitEmail)
1434
1435 def checkValidP4Users(self,commits):
1436 # check if any git authors cannot be mapped to p4 users
1437 for id in commits:
1438 (user,email) = self.p4UserForCommit(id)
1439 if not user:
1440 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1441 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1442 print "%s" % msg
1443 else:
1444 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1445
1446 def lastP4Changelist(self):
1447 # Get back the last changelist number submitted in this client spec. This
1448 # then gets used to patch up the username in the change. If the same
1449 # client spec is being used by multiple processes then this might go
1450 # wrong.
1451 results = p4CmdList("client -o") # find the current client
1452 client = None
1453 for r in results:
1454 if r.has_key('Client'):
1455 client = r['Client']
1456 break
1457 if not client:
1458 die("could not get client spec")
6de040df 1459 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1460 for r in results:
1461 if r.has_key('change'):
1462 return r['change']
1463 die("Could not get changelist number for last submit - cannot patch up user details")
1464
1465 def modifyChangelistUser(self, changelist, newUser):
1466 # fixup the user field of a changelist after it has been submitted.
1467 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1468 if len(changes) != 1:
1469 die("Bad output from p4 change modifying %s to user %s" %
1470 (changelist, newUser))
1471
1472 c = changes[0]
1473 if c['User'] == newUser: return # nothing to do
1474 c['User'] = newUser
1475 input = marshal.dumps(c)
1476
3ea2cfd4
LD
1477 result = p4CmdList("change -f -i", stdin=input)
1478 for r in result:
1479 if r.has_key('code'):
1480 if r['code'] == 'error':
1481 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1482 if r.has_key('data'):
1483 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1484 return
1485 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1486
1487 def canChangeChangelists(self):
1488 # check to see if we have p4 admin or super-user permissions, either of
1489 # which are required to modify changelists.
52a4880b 1490 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1491 for r in results:
1492 if r.has_key('perm'):
1493 if r['perm'] == 'admin':
1494 return 1
1495 if r['perm'] == 'super':
1496 return 1
1497 return 0
1498
46c609e9 1499 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1500 """Run "p4 change -o" to grab a change specification template.
1501 This does not use "p4 -G", as it is nice to keep the submission
1502 template in original order, since a human might edit it.
1503
1504 Remove lines in the Files section that show changes to files
1505 outside the depot path we're committing into."""
1506
cbc69242
SH
1507 [upstream, settings] = findUpstreamBranchPoint()
1508
ea99c3ae
SH
1509 template = ""
1510 inFilesSection = False
46c609e9
LD
1511 args = ['change', '-o']
1512 if changelist:
1513 args.append(str(changelist))
1514
1515 for line in p4_read_pipe_lines(args):
f3e5ae4f
MSO
1516 if line.endswith("\r\n"):
1517 line = line[:-2] + "\n"
ea99c3ae
SH
1518 if inFilesSection:
1519 if line.startswith("\t"):
1520 # path starts and ends with a tab
1521 path = line[1:]
1522 lastTab = path.rfind("\t")
1523 if lastTab != -1:
1524 path = path[:lastTab]
cbc69242
SH
1525 if settings.has_key('depot-paths'):
1526 if not [p for p in settings['depot-paths']
1527 if p4PathStartsWith(path, p)]:
1528 continue
1529 else:
1530 if not p4PathStartsWith(path, self.depotPath):
1531 continue
ea99c3ae
SH
1532 else:
1533 inFilesSection = False
1534 else:
1535 if line.startswith("Files:"):
1536 inFilesSection = True
1537
1538 template += line
1539
1540 return template
1541
7c766e57
PW
1542 def edit_template(self, template_file):
1543 """Invoke the editor to let the user change the submission
1544 message. Return true if okay to continue with the submit."""
1545
1546 # if configured to skip the editing part, just submit
0d609032 1547 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1548 return True
1549
1550 # look at the modification time, to check later if the user saved
1551 # the file
1552 mtime = os.stat(template_file).st_mtime
1553
1554 # invoke the editor
f95ceaf0 1555 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1556 editor = os.environ.get("P4EDITOR")
1557 else:
1558 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1559 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1560
1561 # If the file was not saved, prompt to see if this patch should
1562 # be skipped. But skip this verification step if configured so.
0d609032 1563 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1564 return True
1565
d1652049
PW
1566 # modification time updated means user saved the file
1567 if os.stat(template_file).st_mtime > mtime:
1568 return True
1569
1570 while True:
1571 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1572 if response == 'y':
1573 return True
1574 if response == 'n':
1575 return False
7c766e57 1576
df8a9e86 1577 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3
MC
1578 # diff
1579 if os.environ.has_key("P4DIFF"):
1580 del(os.environ["P4DIFF"])
1581 diff = ""
1582 for editedFile in editedFiles:
1583 diff += p4_read_pipe(['diff', '-du',
1584 wildcard_encode(editedFile)])
1585
1586 # new file diff
1587 newdiff = ""
1588 for newFile in filesToAdd:
1589 newdiff += "==== new file ====\n"
1590 newdiff += "--- /dev/null\n"
1591 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1592
1593 is_link = os.path.islink(newFile)
1594 expect_link = newFile in symlinks
1595
1596 if is_link and expect_link:
1597 newdiff += "+%s\n" % os.readlink(newFile)
1598 else:
1599 f = open(newFile, "r")
1600 for line in f.readlines():
1601 newdiff += "+" + line
1602 f.close()
b4073bb3 1603
e2a892ee 1604 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1605
7cb5cbef 1606 def applyCommit(self, id):
67b0fe2e
PW
1607 """Apply one commit, return True if it succeeded."""
1608
1609 print "Applying", read_pipe(["git", "show", "-s",
1610 "--format=format:%h %s", id])
ae901090 1611
848de9c3 1612 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1613
84cb0003 1614 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1615 filesToAdd = set()
a02b8bc4 1616 filesToChangeType = set()
4f5cf76a 1617 filesToDelete = set()
d336c158 1618 editedFiles = set()
b6ad6dcc 1619 pureRenameCopy = set()
df8a9e86 1620 symlinks = set()
c65b670e 1621 filesToChangeExecBit = {}
46c609e9 1622 all_files = list()
60df071c 1623
4f5cf76a 1624 for line in diff:
b43b0a3c
CP
1625 diff = parseDiffTreeEntry(line)
1626 modifier = diff['status']
1627 path = diff['src']
46c609e9
LD
1628 all_files.append(path)
1629
4f5cf76a 1630 if modifier == "M":
6de040df 1631 p4_edit(path)
c65b670e
CP
1632 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1633 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1634 editedFiles.add(path)
4f5cf76a
SH
1635 elif modifier == "A":
1636 filesToAdd.add(path)
c65b670e 1637 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1638 if path in filesToDelete:
1639 filesToDelete.remove(path)
df8a9e86
LD
1640
1641 dst_mode = int(diff['dst_mode'], 8)
1642 if dst_mode == 0120000:
1643 symlinks.add(path)
1644
4f5cf76a
SH
1645 elif modifier == "D":
1646 filesToDelete.add(path)
1647 if path in filesToAdd:
1648 filesToAdd.remove(path)
4fddb41b
VA
1649 elif modifier == "C":
1650 src, dest = diff['src'], diff['dst']
6de040df 1651 p4_integrate(src, dest)
b6ad6dcc 1652 pureRenameCopy.add(dest)
4fddb41b 1653 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1654 p4_edit(dest)
b6ad6dcc 1655 pureRenameCopy.discard(dest)
4fddb41b 1656 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1657 p4_edit(dest)
b6ad6dcc 1658 pureRenameCopy.discard(dest)
4fddb41b 1659 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1660 if self.isWindows:
1661 # turn off read-only attribute
1662 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1663 os.unlink(dest)
1664 editedFiles.add(dest)
d9a5f25b 1665 elif modifier == "R":
b43b0a3c 1666 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1667 if self.p4HasMoveCommand:
1668 p4_edit(src) # src must be open before move
1669 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1670 else:
8e9497c2
GG
1671 p4_integrate(src, dest)
1672 if diff['src_sha1'] != diff['dst_sha1']:
1673 p4_edit(dest)
1674 else:
1675 pureRenameCopy.add(dest)
c65b670e 1676 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1677 if not self.p4HasMoveCommand:
1678 p4_edit(dest) # with move: already open, writable
c65b670e 1679 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1680 if not self.p4HasMoveCommand:
d20f0f8e
PW
1681 if self.isWindows:
1682 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1683 os.unlink(dest)
1684 filesToDelete.add(src)
d9a5f25b 1685 editedFiles.add(dest)
a02b8bc4
RP
1686 elif modifier == "T":
1687 filesToChangeType.add(path)
4f5cf76a
SH
1688 else:
1689 die("unknown modifier %s for %s" % (modifier, path))
1690
749b668c 1691 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1692 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1693 tryPatchCmd = patchcmd + "--check -"
1694 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1695 patch_succeeded = True
51a2640a 1696
47a130b7 1697 if os.system(tryPatchCmd) != 0:
60df071c
LD
1698 fixed_rcs_keywords = False
1699 patch_succeeded = False
51a2640a 1700 print "Unfortunately applying the change failed!"
60df071c
LD
1701
1702 # Patch failed, maybe it's just RCS keyword woes. Look through
1703 # the patch to see if that's possible.
0d609032 1704 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1705 file = None
1706 pattern = None
1707 kwfiles = {}
1708 for file in editedFiles | filesToDelete:
1709 # did this file's delta contain RCS keywords?
1710 pattern = p4_keywords_regexp_for_file(file)
1711
1712 if pattern:
1713 # this file is a possibility...look for RCS keywords.
1714 regexp = re.compile(pattern, re.VERBOSE)
1715 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1716 if regexp.search(line):
1717 if verbose:
1718 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1719 kwfiles[file] = pattern
1720 break
1721
1722 for file in kwfiles:
1723 if verbose:
1724 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1725 # File is being deleted, so not open in p4. Must
1726 # disable the read-only bit on windows.
1727 if self.isWindows and file not in editedFiles:
1728 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1729 self.patchRCSKeywords(file, kwfiles[file])
1730 fixed_rcs_keywords = True
1731
1732 if fixed_rcs_keywords:
1733 print "Retrying the patch with RCS keywords cleaned up"
1734 if os.system(tryPatchCmd) == 0:
1735 patch_succeeded = True
1736
1737 if not patch_succeeded:
7e5dd9f2
PW
1738 for f in editedFiles:
1739 p4_revert(f)
7e5dd9f2 1740 return False
51a2640a 1741
55ac2ed6
PW
1742 #
1743 # Apply the patch for real, and do add/delete/+x handling.
1744 #
47a130b7 1745 system(applyPatchCmd)
4f5cf76a 1746
a02b8bc4
RP
1747 for f in filesToChangeType:
1748 p4_edit(f, "-t", "auto")
4f5cf76a 1749 for f in filesToAdd:
6de040df 1750 p4_add(f)
4f5cf76a 1751 for f in filesToDelete:
6de040df
LD
1752 p4_revert(f)
1753 p4_delete(f)
4f5cf76a 1754
c65b670e
CP
1755 # Set/clear executable bits
1756 for f in filesToChangeExecBit.keys():
1757 mode = filesToChangeExecBit[f]
1758 setP4ExecBit(f, mode)
1759
46c609e9
LD
1760 if self.update_shelve:
1761 print("all_files = %s" % str(all_files))
1762 p4_reopen_in_change(self.update_shelve, all_files)
1763
55ac2ed6
PW
1764 #
1765 # Build p4 change description, starting with the contents
1766 # of the git commit message.
1767 #
0e36f2d7 1768 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1769 logMessage = logMessage.strip()
f19cb0a0 1770 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1771
46c609e9 1772 template = self.prepareSubmitTemplate(self.update_shelve)
f19cb0a0 1773 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1774
c47178d4 1775 if self.preserveUser:
55ac2ed6 1776 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1777
55ac2ed6
PW
1778 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1779 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1780 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1781 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1782
55ac2ed6 1783 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1784 if not self.prepare_p4_only:
1785 submitTemplate += separatorLine
df8a9e86 1786 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1787
c47178d4 1788 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1789 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1790 if self.isWindows:
1791 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1792 tmpFile.write(submitTemplate)
c47178d4
PW
1793 tmpFile.close()
1794
728b7ad8
PW
1795 if self.prepare_p4_only:
1796 #
1797 # Leave the p4 tree prepared, and the submit template around
1798 # and let the user decide what to do next
1799 #
1800 print
1801 print "P4 workspace prepared for submission."
1802 print "To submit or revert, go to client workspace"
1803 print " " + self.clientPath
1804 print
1805 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1806 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1807 " \"git p4\"." % fileName
1808 print "You can delete the file \"%s\" when finished." % fileName
1809
1810 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1811 print "To preserve change ownership by user %s, you must\n" \
1812 "do \"p4 change -f <change>\" after submitting and\n" \
1813 "edit the User field."
1814 if pureRenameCopy:
1815 print "After submitting, renamed files must be re-synced."
1816 print "Invoke \"p4 sync -f\" on each of these files:"
1817 for f in pureRenameCopy:
1818 print " " + f
1819
1820 print
1821 print "To revert the changes, use \"p4 revert ...\", and delete"
1822 print "the submit template file \"%s\"" % fileName
1823 if filesToAdd:
1824 print "Since the commit adds new files, they must be deleted:"
1825 for f in filesToAdd:
1826 print " " + f
1827 print
1828 return True
1829
55ac2ed6
PW
1830 #
1831 # Let the user edit the change description, then submit it.
1832 #
b7638fed 1833 submitted = False
cdc7e388 1834
b7638fed
GE
1835 try:
1836 if self.edit_template(fileName):
1837 # read the edited message and submit
1838 tmpFile = open(fileName, "rb")
1839 message = tmpFile.read()
1840 tmpFile.close()
1841 if self.isWindows:
1842 message = message.replace("\r\n", "\n")
1843 submitTemplate = message[:message.index(separatorLine)]
46c609e9
LD
1844
1845 if self.update_shelve:
1846 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1847 elif self.shelve:
b34fa577
VK
1848 p4_write_pipe(['shelve', '-i'], submitTemplate)
1849 else:
1850 p4_write_pipe(['submit', '-i'], submitTemplate)
1851 # The rename/copy happened by applying a patch that created a
1852 # new file. This leaves it writable, which confuses p4.
1853 for f in pureRenameCopy:
1854 p4_sync(f, "-f")
b7638fed
GE
1855
1856 if self.preserveUser:
1857 if p4User:
1858 # Get last changelist number. Cannot easily get it from
1859 # the submit command output as the output is
1860 # unmarshalled.
1861 changelist = self.lastP4Changelist()
1862 self.modifyChangelistUser(changelist, p4User)
1863
b7638fed
GE
1864 submitted = True
1865
1866 finally:
c47178d4 1867 # skip this patch
b34fa577
VK
1868 if not submitted or self.shelve:
1869 if self.shelve:
1870 print ("Reverting shelved files.")
1871 else:
1872 print ("Submission cancelled, undoing p4 changes.")
1873 for f in editedFiles | filesToDelete:
b7638fed
GE
1874 p4_revert(f)
1875 for f in filesToAdd:
1876 p4_revert(f)
1877 os.remove(f)
c47178d4
PW
1878
1879 os.remove(fileName)
b7638fed 1880 return submitted
4f5cf76a 1881
06804c76
LD
1882 # Export git tags as p4 labels. Create a p4 label and then tag
1883 # with that.
1884 def exportGitTags(self, gitTags):
c8942a22
LD
1885 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1886 if len(validLabelRegexp) == 0:
1887 validLabelRegexp = defaultLabelRegexp
1888 m = re.compile(validLabelRegexp)
06804c76
LD
1889
1890 for name in gitTags:
1891
1892 if not m.match(name):
1893 if verbose:
05a3cec5 1894 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1895 continue
1896
1897 # Get the p4 commit this corresponds to
c8942a22
LD
1898 logMessage = extractLogMessageFromGitCommit(name)
1899 values = extractSettingsGitLog(logMessage)
06804c76 1900
c8942a22 1901 if not values.has_key('change'):
06804c76
LD
1902 # a tag pointing to something not sent to p4; ignore
1903 if verbose:
1904 print "git tag %s does not give a p4 commit" % name
1905 continue
c8942a22
LD
1906 else:
1907 changelist = values['change']
06804c76
LD
1908
1909 # Get the tag details.
1910 inHeader = True
1911 isAnnotated = False
1912 body = []
1913 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1914 l = l.strip()
1915 if inHeader:
1916 if re.match(r'tag\s+', l):
1917 isAnnotated = True
1918 elif re.match(r'\s*$', l):
1919 inHeader = False
1920 continue
1921 else:
1922 body.append(l)
1923
1924 if not isAnnotated:
1925 body = ["lightweight tag imported by git p4\n"]
1926
1927 # Create the label - use the same view as the client spec we are using
1928 clientSpec = getClientSpec()
1929
1930 labelTemplate = "Label: %s\n" % name
1931 labelTemplate += "Description:\n"
1932 for b in body:
1933 labelTemplate += "\t" + b + "\n"
1934 labelTemplate += "View:\n"
9d57c4a6
KS
1935 for depot_side in clientSpec.mappings:
1936 labelTemplate += "\t%s\n" % depot_side
06804c76 1937
ef739f08
PW
1938 if self.dry_run:
1939 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1940 elif self.prepare_p4_only:
1941 print "Not creating p4 label %s for tag due to option" \
1942 " --prepare-p4-only" % name
ef739f08
PW
1943 else:
1944 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1945
ef739f08
PW
1946 # Use the label
1947 p4_system(["tag", "-l", name] +
9d57c4a6 1948 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1949
ef739f08
PW
1950 if verbose:
1951 print "created p4 label for tag %s" % name
06804c76 1952
4f5cf76a 1953 def run(self, args):
c9b50e63
SH
1954 if len(args) == 0:
1955 self.master = currentGitBranch()
c9b50e63
SH
1956 elif len(args) == 1:
1957 self.master = args[0]
28755dba
PW
1958 if not branchExists(self.master):
1959 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1960 else:
1961 return False
1962
00ad6e31
LD
1963 if self.master:
1964 allowSubmit = gitConfig("git-p4.allowSubmit")
1965 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1966 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 1967
27d2d811 1968 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1969 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1970 if len(self.origin) == 0:
1971 self.origin = upstream
a3fdd579 1972
46c609e9
LD
1973 if self.update_shelve:
1974 self.shelve = True
1975
3ea2cfd4
LD
1976 if self.preserveUser:
1977 if not self.canChangeChangelists():
1978 die("Cannot preserve user names without p4 super-user or admin permissions")
1979
6bbfd137
PW
1980 # if not set from the command line, try the config file
1981 if self.conflict_behavior is None:
1982 val = gitConfig("git-p4.conflict")
1983 if val:
1984 if val not in self.conflict_behavior_choices:
1985 die("Invalid value '%s' for config git-p4.conflict" % val)
1986 else:
1987 val = "ask"
1988 self.conflict_behavior = val
1989
a3fdd579
SH
1990 if self.verbose:
1991 print "Origin branch is " + self.origin
9512497b 1992
ea99c3ae 1993 if len(self.depotPath) == 0:
9512497b
SH
1994 print "Internal error: cannot locate perforce depot path from existing branches"
1995 sys.exit(128)
1996
543987bd 1997 self.useClientSpec = False
0d609032 1998 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1999 self.useClientSpec = True
2000 if self.useClientSpec:
2001 self.clientSpecDirs = getClientSpec()
9512497b 2002
2e3a16b2 2003 # Check for the existence of P4 branches
cd884106
VA
2004 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2005
2006 if self.useClientSpec and not branchesDetected:
543987bd
PW
2007 # all files are relative to the client spec
2008 self.clientPath = getClientRoot()
2009 else:
2010 self.clientPath = p4Where(self.depotPath)
9512497b 2011
543987bd
PW
2012 if self.clientPath == "":
2013 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2014
ea99c3ae 2015 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2016 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2017
0591cfa8 2018 # ensure the clientPath exists
8d7ec362 2019 new_client_dir = False
0591cfa8 2020 if not os.path.exists(self.clientPath):
8d7ec362 2021 new_client_dir = True
0591cfa8
GG
2022 os.makedirs(self.clientPath)
2023
bbd84863 2024 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2025 if self.dry_run:
2026 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2027 else:
ef739f08
PW
2028 print "Synchronizing p4 checkout..."
2029 if new_client_dir:
2030 # old one was destroyed, and maybe nobody told p4
2031 p4_sync("...", "-f")
2032 else:
2033 p4_sync("...")
4f5cf76a 2034 self.check()
4f5cf76a 2035
4c750c0d 2036 commits = []
00ad6e31
LD
2037 if self.master:
2038 commitish = self.master
2039 else:
2040 commitish = 'HEAD'
2041
2042 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
4c750c0d
SH
2043 commits.append(line.strip())
2044 commits.reverse()
4f5cf76a 2045
0d609032 2046 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2047 self.checkAuthorship = False
2048 else:
2049 self.checkAuthorship = True
2050
3ea2cfd4
LD
2051 if self.preserveUser:
2052 self.checkValidP4Users(commits)
2053
84cb0003
GG
2054 #
2055 # Build up a set of options to be passed to diff when
2056 # submitting each commit to p4.
2057 #
2058 if self.detectRenames:
2059 # command-line -M arg
2060 self.diffOpts = "-M"
2061 else:
2062 # If not explicitly set check the config variable
2063 detectRenames = gitConfig("git-p4.detectRenames")
2064
2065 if detectRenames.lower() == "false" or detectRenames == "":
2066 self.diffOpts = ""
2067 elif detectRenames.lower() == "true":
2068 self.diffOpts = "-M"
2069 else:
2070 self.diffOpts = "-M%s" % detectRenames
2071
2072 # no command-line arg for -C or --find-copies-harder, just
2073 # config variables
2074 detectCopies = gitConfig("git-p4.detectCopies")
2075 if detectCopies.lower() == "false" or detectCopies == "":
2076 pass
2077 elif detectCopies.lower() == "true":
2078 self.diffOpts += " -C"
2079 else:
2080 self.diffOpts += " -C%s" % detectCopies
2081
0d609032 2082 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2083 self.diffOpts += " --find-copies-harder"
2084
7e5dd9f2
PW
2085 #
2086 # Apply the commits, one at a time. On failure, ask if should
2087 # continue to try the rest of the patches, or quit.
2088 #
ef739f08
PW
2089 if self.dry_run:
2090 print "Would apply"
67b0fe2e 2091 applied = []
7e5dd9f2
PW
2092 last = len(commits) - 1
2093 for i, commit in enumerate(commits):
ef739f08
PW
2094 if self.dry_run:
2095 print " ", read_pipe(["git", "show", "-s",
2096 "--format=format:%h %s", commit])
2097 ok = True
2098 else:
2099 ok = self.applyCommit(commit)
67b0fe2e
PW
2100 if ok:
2101 applied.append(commit)
7e5dd9f2 2102 else:
728b7ad8
PW
2103 if self.prepare_p4_only and i < last:
2104 print "Processing only the first commit due to option" \
2105 " --prepare-p4-only"
2106 break
7e5dd9f2
PW
2107 if i < last:
2108 quit = False
2109 while True:
6bbfd137
PW
2110 # prompt for what to do, or use the option/variable
2111 if self.conflict_behavior == "ask":
2112 print "What do you want to do?"
2113 response = raw_input("[s]kip this commit but apply"
2114 " the rest, or [q]uit? ")
2115 if not response:
2116 continue
2117 elif self.conflict_behavior == "skip":
2118 response = "s"
2119 elif self.conflict_behavior == "quit":
2120 response = "q"
2121 else:
2122 die("Unknown conflict_behavior '%s'" %
2123 self.conflict_behavior)
2124
7e5dd9f2
PW
2125 if response[0] == "s":
2126 print "Skipping this commit, but applying the rest"
2127 break
2128 if response[0] == "q":
2129 print "Quitting"
2130 quit = True
2131 break
2132 if quit:
2133 break
4f5cf76a 2134
67b0fe2e 2135 chdir(self.oldWorkingDirectory)
b34fa577 2136 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2137 if self.dry_run:
2138 pass
728b7ad8
PW
2139 elif self.prepare_p4_only:
2140 pass
ef739f08 2141 elif len(commits) == len(applied):
b34fa577 2142 print ("All commits {0}!".format(shelved_applied))
14594f4b 2143
4c750c0d 2144 sync = P4Sync()
44e8d26c
PW
2145 if self.branch:
2146 sync.branch = self.branch
4c750c0d 2147 sync.run([])
14594f4b 2148
4c750c0d
SH
2149 rebase = P4Rebase()
2150 rebase.rebase()
4f5cf76a 2151
67b0fe2e
PW
2152 else:
2153 if len(applied) == 0:
b34fa577 2154 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2155 else:
b34fa577 2156 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2157 for c in commits:
2158 if c in applied:
2159 star = "*"
2160 else:
2161 star = " "
2162 print star, read_pipe(["git", "show", "-s",
2163 "--format=format:%h %s", c])
2164 print "You will have to do 'git p4 sync' and rebase."
2165
0d609032 2166 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2167 self.exportLabels = True
06804c76
LD
2168
2169 if self.exportLabels:
2170 p4Labels = getP4Labels(self.depotPath)
2171 gitTags = getGitTags()
2172
2173 missingGitTags = gitTags - p4Labels
2174 self.exportGitTags(missingGitTags)
2175
98e023de 2176 # exit with error unless everything applied perfectly
67b0fe2e
PW
2177 if len(commits) != len(applied):
2178 sys.exit(1)
2179
b984733c
SH
2180 return True
2181
ecb7cf98
PW
2182class View(object):
2183 """Represent a p4 view ("p4 help views"), and map files in a
2184 repo according to the view."""
2185
9d57c4a6 2186 def __init__(self, client_name):
ecb7cf98 2187 self.mappings = []
9d57c4a6
KS
2188 self.client_prefix = "//%s/" % client_name
2189 # cache results of "p4 where" to lookup client file locations
2190 self.client_spec_path_cache = {}
ecb7cf98
PW
2191
2192 def append(self, view_line):
2193 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2194 sides. Append to self.mappings, preserving order. This
2195 is only needed for tag creation."""
ecb7cf98
PW
2196
2197 # Split the view line into exactly two words. P4 enforces
2198 # structure on these lines that simplifies this quite a bit.
2199 #
2200 # Either or both words may be double-quoted.
2201 # Single quotes do not matter.
2202 # Double-quote marks cannot occur inside the words.
2203 # A + or - prefix is also inside the quotes.
2204 # There are no quotes unless they contain a space.
2205 # The line is already white-space stripped.
2206 # The two words are separated by a single space.
2207 #
2208 if view_line[0] == '"':
2209 # First word is double quoted. Find its end.
2210 close_quote_index = view_line.find('"', 1)
2211 if close_quote_index <= 0:
2212 die("No first-word closing quote found: %s" % view_line)
2213 depot_side = view_line[1:close_quote_index]
2214 # skip closing quote and space
2215 rhs_index = close_quote_index + 1 + 1
2216 else:
2217 space_index = view_line.find(" ")
2218 if space_index <= 0:
2219 die("No word-splitting space found: %s" % view_line)
2220 depot_side = view_line[0:space_index]
2221 rhs_index = space_index + 1
2222
ecb7cf98 2223 # prefix + means overlay on previous mapping
ecb7cf98 2224 if depot_side.startswith("+"):
ecb7cf98
PW
2225 depot_side = depot_side[1:]
2226
9d57c4a6 2227 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2228 exclude = False
2229 if depot_side.startswith("-"):
2230 exclude = True
2231 depot_side = depot_side[1:]
2232
9d57c4a6
KS
2233 if not exclude:
2234 self.mappings.append(depot_side)
ecb7cf98 2235
9d57c4a6
KS
2236 def convert_client_path(self, clientFile):
2237 # chop off //client/ part to make it relative
2238 if not clientFile.startswith(self.client_prefix):
2239 die("No prefix '%s' on clientFile '%s'" %
2240 (self.client_prefix, clientFile))
2241 return clientFile[len(self.client_prefix):]
ecb7cf98 2242
9d57c4a6
KS
2243 def update_client_spec_path_cache(self, files):
2244 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2245
9d57c4a6
KS
2246 # List depot file paths exclude that already cached
2247 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2248
9d57c4a6
KS
2249 if len(fileArgs) == 0:
2250 return # All files in cache
ecb7cf98 2251
9d57c4a6
KS
2252 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2253 for res in where_result:
2254 if "code" in res and res["code"] == "error":
2255 # assume error is "... file(s) not in client view"
2256 continue
2257 if "clientFile" not in res:
20005443 2258 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2259 if "unmap" in res:
2260 # it will list all of them, but only one not unmap-ped
2261 continue
a0a50d87
LS
2262 if gitConfigBool("core.ignorecase"):
2263 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2264 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2265
9d57c4a6
KS
2266 # not found files or unmap files set to ""
2267 for depotFile in fileArgs:
a0a50d87
LS
2268 if gitConfigBool("core.ignorecase"):
2269 depotFile = depotFile.lower()
9d57c4a6
KS
2270 if depotFile not in self.client_spec_path_cache:
2271 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2272
9d57c4a6
KS
2273 def map_in_client(self, depot_path):
2274 """Return the relative location in the client where this
2275 depot file should live. Returns "" if the file should
2276 not be mapped in the client."""
ecb7cf98 2277
a0a50d87
LS
2278 if gitConfigBool("core.ignorecase"):
2279 depot_path = depot_path.lower()
2280
9d57c4a6
KS
2281 if depot_path in self.client_spec_path_cache:
2282 return self.client_spec_path_cache[depot_path]
2283
2284 die( "Error: %s is not found in client spec path" % depot_path )
2285 return ""
ecb7cf98 2286
3ea2cfd4 2287class P4Sync(Command, P4UserMap):
56c09345
PW
2288 delete_actions = ( "delete", "move/delete", "purge" )
2289
b984733c
SH
2290 def __init__(self):
2291 Command.__init__(self)
3ea2cfd4 2292 P4UserMap.__init__(self)
b984733c
SH
2293 self.options = [
2294 optparse.make_option("--branch", dest="branch"),
2295 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2296 optparse.make_option("--changesfile", dest="changesFile"),
2297 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2298 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2299 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2300 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2301 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2302 optparse.make_option("--max-changes", dest="maxChanges",
2303 help="Maximum number of changes to import"),
2304 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2305 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2306 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2307 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2308 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2309 help="Only sync files that are included in the Perforce Client Spec"),
2310 optparse.make_option("-/", dest="cloneExclude",
2311 action="append", type="string",
2312 help="exclude depot path"),
b984733c
SH
2313 ]
2314 self.description = """Imports from Perforce into a git repository.\n
2315 example:
2316 //depot/my/project/ -- to import the current head
2317 //depot/my/project/@all -- to import everything
2318 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2319
2320 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2321
2322 self.usage += " //depot/path[@revRange]"
b984733c 2323 self.silent = False
1d7367dc
RG
2324 self.createdBranches = set()
2325 self.committedChanges = set()
569d1bd4 2326 self.branch = ""
b984733c 2327 self.detectBranches = False
cb53e1f8 2328 self.detectLabels = False
06804c76 2329 self.importLabels = False
b984733c 2330 self.changesFile = ""
01265103 2331 self.syncWithOrigin = True
a028a98e 2332 self.importIntoRemotes = True
01a9c9c5 2333 self.maxChanges = ""
1051ef00 2334 self.changes_block_size = None
8b41a97f 2335 self.keepRepoPath = False
6326aa58 2336 self.depotPaths = None
3c699645 2337 self.p4BranchesInGit = []
354081d5 2338 self.cloneExclude = []
3a70cdfa 2339 self.useClientSpec = False
a93d33ee 2340 self.useClientSpec_from_options = False
ecb7cf98 2341 self.clientSpecDirs = None
fed23693 2342 self.tempBranches = []
d604176d 2343 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12
LS
2344 self.largeFileSystem = None
2345
2346 if gitConfig('git-p4.largeFileSystem'):
2347 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2348 self.largeFileSystem = largeFileSystemConstructor(
2349 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2350 )
b984733c 2351
01265103
SH
2352 if gitConfig("git-p4.syncFromOrigin") == "false":
2353 self.syncWithOrigin = False
2354
51334bb0
LD
2355 # This is required for the "append" cloneExclude action
2356 def ensure_value(self, attr, value):
2357 if not hasattr(self, attr) or getattr(self, attr) is None:
2358 setattr(self, attr, value)
2359 return getattr(self, attr)
2360
fed23693
VA
2361 # Force a checkpoint in fast-import and wait for it to finish
2362 def checkpoint(self):
2363 self.gitStream.write("checkpoint\n\n")
2364 self.gitStream.write("progress checkpoint\n\n")
2365 out = self.gitOutput.readline()
2366 if self.verbose:
2367 print "checkpoint finished: " + out
2368
b984733c 2369 def extractFilesFromCommit(self, commit):
354081d5
TT
2370 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2371 for path in self.cloneExclude]
b984733c
SH
2372 files = []
2373 fnum = 0
2374 while commit.has_key("depotFile%s" % fnum):
2375 path = commit["depotFile%s" % fnum]
6326aa58 2376
354081d5 2377 if [p for p in self.cloneExclude
d53de8b9 2378 if p4PathStartsWith(path, p)]:
354081d5
TT
2379 found = False
2380 else:
2381 found = [p for p in self.depotPaths
d53de8b9 2382 if p4PathStartsWith(path, p)]
6326aa58 2383 if not found:
b984733c
SH
2384 fnum = fnum + 1
2385 continue
2386
2387 file = {}
2388 file["path"] = path
2389 file["rev"] = commit["rev%s" % fnum]
2390 file["action"] = commit["action%s" % fnum]
2391 file["type"] = commit["type%s" % fnum]
2392 files.append(file)
2393 fnum = fnum + 1
2394 return files
2395
26e6a27d
JD
2396 def extractJobsFromCommit(self, commit):
2397 jobs = []
2398 jnum = 0
2399 while commit.has_key("job%s" % jnum):
2400 job = commit["job%s" % jnum]
2401 jobs.append(job)
2402 jnum = jnum + 1
2403 return jobs
2404
6326aa58 2405 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2406 """When streaming files, this is called to map a p4 depot path
2407 to where it should go in git. The prefixes are either
2408 self.depotPaths, or self.branchPrefixes in the case of
2409 branch detection."""
2410
3952710b 2411 if self.useClientSpec:
21ef5df4
PW
2412 # branch detection moves files up a level (the branch name)
2413 # from what client spec interpretation gives
0d1696ef 2414 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2415 if self.detectBranches:
2416 for b in self.knownBranches:
2417 if path.startswith(b + "/"):
2418 path = path[len(b)+1:]
2419
2420 elif self.keepRepoPath:
2421 # Preserve everything in relative path name except leading
2422 # //depot/; just look at first prefix as they all should
2423 # be in the same depot.
2424 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2425 if p4PathStartsWith(path, depot):
2426 path = path[len(depot):]
3952710b 2427
0d1696ef 2428 else:
0d1696ef
PW
2429 for p in prefixes:
2430 if p4PathStartsWith(path, p):
2431 path = path[len(p):]
21ef5df4 2432 break
8b41a97f 2433
0d1696ef 2434 path = wildcard_decode(path)
6326aa58 2435 return path
6754a299 2436
71b112d4 2437 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2438 """Look at each depotFile in the commit to figure out to what
2439 branch it belongs."""
2440
9d57c4a6
KS
2441 if self.clientSpecDirs:
2442 files = self.extractFilesFromCommit(commit)
2443 self.clientSpecDirs.update_client_spec_path_cache(files)
2444
d5904674 2445 branches = {}
71b112d4
SH
2446 fnum = 0
2447 while commit.has_key("depotFile%s" % fnum):
2448 path = commit["depotFile%s" % fnum]
6326aa58 2449 found = [p for p in self.depotPaths
d53de8b9 2450 if p4PathStartsWith(path, p)]
6326aa58 2451 if not found:
71b112d4
SH
2452 fnum = fnum + 1
2453 continue
2454
2455 file = {}
2456 file["path"] = path
2457 file["rev"] = commit["rev%s" % fnum]
2458 file["action"] = commit["action%s" % fnum]
2459 file["type"] = commit["type%s" % fnum]
2460 fnum = fnum + 1
2461
21ef5df4
PW
2462 # start with the full relative path where this file would
2463 # go in a p4 client
2464 if self.useClientSpec:
2465 relPath = self.clientSpecDirs.map_in_client(path)
2466 else:
2467 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2468
4b97ffb1 2469 for branch in self.knownBranches.keys():
21ef5df4
PW
2470 # add a trailing slash so that a commit into qt/4.2foo
2471 # doesn't end up in qt/4.2, e.g.
6754a299 2472 if relPath.startswith(branch + "/"):
d5904674
SH
2473 if branch not in branches:
2474 branches[branch] = []
71b112d4 2475 branches[branch].append(file)
6555b2cc 2476 break
b984733c
SH
2477
2478 return branches
2479
a5db4b12
LS
2480 def writeToGitStream(self, gitMode, relPath, contents):
2481 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2482 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2483 for d in contents:
2484 self.gitStream.write(d)
2485 self.gitStream.write('\n')
2486
a8b05162
LS
2487 def encodeWithUTF8(self, path):
2488 try:
2489 path.decode('ascii')
2490 except:
2491 encoding = 'utf8'
2492 if gitConfig('git-p4.pathEncoding'):
2493 encoding = gitConfig('git-p4.pathEncoding')
2494 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2495 if self.verbose:
2496 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2497 return path
2498
b932705b
LD
2499 # output one file from the P4 stream
2500 # - helper for streamP4Files
2501
2502 def streamOneP4File(self, file, contents):
b932705b 2503 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2504 relPath = self.encodeWithUTF8(relPath)
b932705b 2505 if verbose:
d2176a50
LS
2506 size = int(self.stream_file['fileSize'])
2507 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2508 sys.stdout.flush()
b932705b 2509
9cffb8c8
PW
2510 (type_base, type_mods) = split_p4_type(file["type"])
2511
2512 git_mode = "100644"
2513 if "x" in type_mods:
2514 git_mode = "100755"
2515 if type_base == "symlink":
2516 git_mode = "120000"
1292df11
AJ
2517 # p4 print on a symlink sometimes contains "target\n";
2518 # if it does, remove the newline
b39c3612 2519 data = ''.join(contents)
40f846c3
PW
2520 if not data:
2521 # Some version of p4 allowed creating a symlink that pointed
2522 # to nothing. This causes p4 errors when checking out such
2523 # a change, and errors here too. Work around it by ignoring
2524 # the bad symlink; hopefully a future change fixes it.
2525 print "\nIgnoring empty symlink in %s" % file['depotFile']
2526 return
2527 elif data[-1] == '\n':
1292df11
AJ
2528 contents = [data[:-1]]
2529 else:
2530 contents = [data]
b932705b 2531
9cffb8c8 2532 if type_base == "utf16":
55aa5714
PW
2533 # p4 delivers different text in the python output to -G
2534 # than it does when using "print -o", or normal p4 client
2535 # operations. utf16 is converted to ascii or utf8, perhaps.
2536 # But ascii text saved as -t utf16 is completely mangled.
2537 # Invoke print -o to get the real contents.
7f0e5962
PW
2538 #
2539 # On windows, the newlines will always be mangled by print, so put
2540 # them back too. This is not needed to the cygwin windows version,
2541 # just the native "NT" type.
2542 #
1f5f3907
LS
2543 try:
2544 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2545 except Exception as e:
2546 if 'Translation of file content failed' in str(e):
2547 type_base = 'binary'
2548 else:
2549 raise e
2550 else:
2551 if p4_version_string().find('/NT') >= 0:
2552 text = text.replace('\r\n', '\n')
2553 contents = [ text ]
55aa5714 2554
9f7ef0ea
PW
2555 if type_base == "apple":
2556 # Apple filetype files will be streamed as a concatenation of
2557 # its appledouble header and the contents. This is useless
2558 # on both macs and non-macs. If using "print -q -o xx", it
2559 # will create "xx" with the data, and "%xx" with the header.
2560 # This is also not very useful.
2561 #
2562 # Ideally, someday, this script can learn how to generate
2563 # appledouble files directly and import those to git, but
2564 # non-mac machines can never find a use for apple filetype.
2565 print "\nIgnoring apple filetype file %s" % file['depotFile']
2566 return
2567
55aa5714
PW
2568 # Note that we do not try to de-mangle keywords on utf16 files,
2569 # even though in theory somebody may want that.
60df071c
LD
2570 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2571 if pattern:
2572 regexp = re.compile(pattern, re.VERBOSE)
2573 text = ''.join(contents)
2574 text = regexp.sub(r'$\1$', text)
2575 contents = [ text ]
b932705b 2576
a5db4b12
LS
2577 if self.largeFileSystem:
2578 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2579
a5db4b12 2580 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2581
2582 def streamOneP4Deletion(self, file):
2583 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2584 relPath = self.encodeWithUTF8(relPath)
b932705b 2585 if verbose:
d2176a50
LS
2586 sys.stdout.write("delete %s\n" % relPath)
2587 sys.stdout.flush()
b932705b
LD
2588 self.gitStream.write("D %s\n" % relPath)
2589
a5db4b12
LS
2590 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2591 self.largeFileSystem.removeLargeFile(relPath)
2592
b932705b
LD
2593 # handle another chunk of streaming data
2594 def streamP4FilesCb(self, marshalled):
2595
78189bea
PW
2596 # catch p4 errors and complain
2597 err = None
2598 if "code" in marshalled:
2599 if marshalled["code"] == "error":
2600 if "data" in marshalled:
2601 err = marshalled["data"].rstrip()
4d25dc44
LS
2602
2603 if not err and 'fileSize' in self.stream_file:
2604 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2605 if required_bytes > 0:
2606 err = 'Not enough space left on %s! Free at least %i MB.' % (
2607 os.getcwd(), required_bytes/1024/1024
2608 )
2609
78189bea
PW
2610 if err:
2611 f = None
2612 if self.stream_have_file_info:
2613 if "depotFile" in self.stream_file:
2614 f = self.stream_file["depotFile"]
2615 # force a failure in fast-import, else an empty
2616 # commit will be made
2617 self.gitStream.write("\n")
2618 self.gitStream.write("die-now\n")
2619 self.gitStream.close()
2620 # ignore errors, but make sure it exits first
2621 self.importProcess.wait()
2622 if f:
2623 die("Error from p4 print for %s: %s" % (f, err))
2624 else:
2625 die("Error from p4 print: %s" % err)
2626
c3f6163b
AG
2627 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2628 # start of a new file - output the old one first
2629 self.streamOneP4File(self.stream_file, self.stream_contents)
2630 self.stream_file = {}
2631 self.stream_contents = []
2632 self.stream_have_file_info = False
b932705b 2633
c3f6163b
AG
2634 # pick up the new file information... for the
2635 # 'data' field we need to append to our array
2636 for k in marshalled.keys():
2637 if k == 'data':
d2176a50
LS
2638 if 'streamContentSize' not in self.stream_file:
2639 self.stream_file['streamContentSize'] = 0
2640 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2641 self.stream_contents.append(marshalled['data'])
2642 else:
2643 self.stream_file[k] = marshalled[k]
b932705b 2644
d2176a50
LS
2645 if (verbose and
2646 'streamContentSize' in self.stream_file and
2647 'fileSize' in self.stream_file and
2648 'depotFile' in self.stream_file):
2649 size = int(self.stream_file["fileSize"])
2650 if size > 0:
2651 progress = 100*self.stream_file['streamContentSize']/size
2652 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2653 sys.stdout.flush()
2654
c3f6163b 2655 self.stream_have_file_info = True
b932705b
LD
2656
2657 # Stream directly from "p4 files" into "git fast-import"
2658 def streamP4Files(self, files):
30b5940b
SH
2659 filesForCommit = []
2660 filesToRead = []
b932705b 2661 filesToDelete = []
30b5940b 2662
3a70cdfa 2663 for f in files:
ecb7cf98
PW
2664 filesForCommit.append(f)
2665 if f['action'] in self.delete_actions:
2666 filesToDelete.append(f)
2667 else:
2668 filesToRead.append(f)
6a49f8e2 2669
b932705b
LD
2670 # deleted files...
2671 for f in filesToDelete:
2672 self.streamOneP4Deletion(f)
1b9a4684 2673
b932705b
LD
2674 if len(filesToRead) > 0:
2675 self.stream_file = {}
2676 self.stream_contents = []
2677 self.stream_have_file_info = False
8ff45f2a 2678
c3f6163b
AG
2679 # curry self argument
2680 def streamP4FilesCbSelf(entry):
2681 self.streamP4FilesCb(entry)
6a49f8e2 2682
6de040df
LD
2683 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2684
2685 p4CmdList(["-x", "-", "print"],
2686 stdin=fileArgs,
2687 cb=streamP4FilesCbSelf)
30b5940b 2688
b932705b
LD
2689 # do the last chunk
2690 if self.stream_file.has_key('depotFile'):
2691 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2692
affb474f
LD
2693 def make_email(self, userid):
2694 if userid in self.users:
2695 return self.users[userid]
2696 else:
2697 return "%s <a@b>" % userid
2698
06804c76 2699 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2700 """ Stream a p4 tag.
2701 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2702 """
2703
06804c76
LD
2704 if verbose:
2705 print "writing tag %s for commit %s" % (labelName, commit)
2706 gitStream.write("tag %s\n" % labelName)
2707 gitStream.write("from %s\n" % commit)
2708
2709 if labelDetails.has_key('Owner'):
2710 owner = labelDetails["Owner"]
2711 else:
2712 owner = None
2713
2714 # Try to use the owner of the p4 label, or failing that,
2715 # the current p4 user id.
2716 if owner:
2717 email = self.make_email(owner)
2718 else:
2719 email = self.make_email(self.p4UserId())
2720 tagger = "%s %s %s" % (email, epoch, self.tz)
2721
2722 gitStream.write("tagger %s\n" % tagger)
2723
2724 print "labelDetails=",labelDetails
2725 if labelDetails.has_key('Description'):
2726 description = labelDetails['Description']
2727 else:
2728 description = 'Label from git p4'
2729
2730 gitStream.write("data %d\n" % len(description))
2731 gitStream.write(description)
2732 gitStream.write("\n")
2733
4ae048e6
LS
2734 def inClientSpec(self, path):
2735 if not self.clientSpecDirs:
2736 return True
2737 inClientSpec = self.clientSpecDirs.map_in_client(path)
2738 if not inClientSpec and self.verbose:
2739 print('Ignoring file outside of client spec: {0}'.format(path))
2740 return inClientSpec
2741
2742 def hasBranchPrefix(self, path):
2743 if not self.branchPrefixes:
2744 return True
2745 hasPrefix = [p for p in self.branchPrefixes
2746 if p4PathStartsWith(path, p)]
09667d01 2747 if not hasPrefix and self.verbose:
4ae048e6
LS
2748 print('Ignoring file outside of prefix: {0}'.format(path))
2749 return hasPrefix
2750
e63231e5 2751 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2752 epoch = details["time"]
2753 author = details["user"]
26e6a27d 2754 jobs = self.extractJobsFromCommit(details)
b984733c 2755
4b97ffb1 2756 if self.verbose:
4ae048e6 2757 print('commit into {0}'.format(branch))
96e07dd2 2758
9d57c4a6
KS
2759 if self.clientSpecDirs:
2760 self.clientSpecDirs.update_client_spec_path_cache(files)
2761
4ae048e6
LS
2762 files = [f for f in files
2763 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2764
2765 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2766 print('Ignoring revision {0} as it would produce an empty commit.'
2767 .format(details['change']))
2768 return
2769
b984733c 2770 self.gitStream.write("commit %s\n" % branch)
b43702ac 2771 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2772 self.committedChanges.add(int(details["change"]))
2773 committer = ""
b607e71e
SH
2774 if author not in self.users:
2775 self.getUserMapFromPerforceServer()
affb474f 2776 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2777
2778 self.gitStream.write("committer %s\n" % committer)
2779
2780 self.gitStream.write("data <<EOT\n")
2781 self.gitStream.write(details["desc"])
26e6a27d
JD
2782 if len(jobs) > 0:
2783 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
e63231e5
PW
2784 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2785 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2786 if len(details['options']) > 0:
2787 self.gitStream.write(": options = %s" % details['options'])
2788 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2789
2790 if len(parent) > 0:
4b97ffb1
SH
2791 if self.verbose:
2792 print "parent %s" % parent
b984733c
SH
2793 self.gitStream.write("from %s\n" % parent)
2794
4ae048e6 2795 self.streamP4Files(files)
b984733c
SH
2796 self.gitStream.write("\n")
2797
1f4ba1cb
SH
2798 change = int(details["change"])
2799
9bda3a85 2800 if self.labels.has_key(change):
1f4ba1cb
SH
2801 label = self.labels[change]
2802 labelDetails = label[0]
2803 labelRevisions = label[1]
71b112d4
SH
2804 if self.verbose:
2805 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2806
6de040df 2807 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2808 for p in self.branchPrefixes])
1f4ba1cb
SH
2809
2810 if len(files) == len(labelRevisions):
2811
2812 cleanedFiles = {}
2813 for info in files:
56c09345 2814 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2815 continue
2816 cleanedFiles[info["depotFile"]] = info["rev"]
2817
2818 if cleanedFiles == labelRevisions:
06804c76 2819 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2820
2821 else:
a46668fa 2822 if not self.silent:
cebdf5af
HWN
2823 print ("Tag %s does not match with change %s: files do not match."
2824 % (labelDetails["label"], change))
1f4ba1cb
SH
2825
2826 else:
a46668fa 2827 if not self.silent:
cebdf5af
HWN
2828 print ("Tag %s does not match with change %s: file count is different."
2829 % (labelDetails["label"], change))
b984733c 2830
06804c76 2831 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2832 def getLabels(self):
2833 self.labels = {}
2834
52a4880b 2835 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2836 if len(l) > 0 and not self.silent:
183f8436 2837 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2838
2839 for output in l:
1f4ba1cb
SH
2840 label = output["label"]
2841 revisions = {}
2842 newestChange = 0
71b112d4
SH
2843 if self.verbose:
2844 print "Querying files for label %s" % label
6de040df
LD
2845 for file in p4CmdList(["files"] +
2846 ["%s...@%s" % (p, label)
2847 for p in self.depotPaths]):
1f4ba1cb
SH
2848 revisions[file["depotFile"]] = file["rev"]
2849 change = int(file["change"])
2850 if change > newestChange:
2851 newestChange = change
2852
9bda3a85
SH
2853 self.labels[newestChange] = [output, revisions]
2854
2855 if self.verbose:
2856 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2857
06804c76
LD
2858 # Import p4 labels as git tags. A direct mapping does not
2859 # exist, so assume that if all the files are at the same revision
2860 # then we can use that, or it's something more complicated we should
2861 # just ignore.
2862 def importP4Labels(self, stream, p4Labels):
2863 if verbose:
2864 print "import p4 labels: " + ' '.join(p4Labels)
2865
2866 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2867 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2868 if len(validLabelRegexp) == 0:
2869 validLabelRegexp = defaultLabelRegexp
2870 m = re.compile(validLabelRegexp)
2871
2872 for name in p4Labels:
2873 commitFound = False
2874
2875 if not m.match(name):
2876 if verbose:
2877 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2878 continue
2879
2880 if name in ignoredP4Labels:
2881 continue
2882
2883 labelDetails = p4CmdList(['label', "-o", name])[0]
2884
2885 # get the most recent changelist for each file in this label
2886 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2887 for p in self.depotPaths])
2888
2889 if change.has_key('change'):
2890 # find the corresponding git commit; take the oldest commit
2891 changelist = int(change['change'])
b43702ac
LD
2892 if changelist in self.committedChanges:
2893 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2894 commitFound = True
b43702ac
LD
2895 else:
2896 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2897 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2898 if len(gitCommit) == 0:
2899 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2900 else:
2901 commitFound = True
2902 gitCommit = gitCommit.strip()
2903
2904 if commitFound:
06804c76
LD
2905 # Convert from p4 time format
2906 try:
2907 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2908 except ValueError:
a4e9054c 2909 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2910 tmwhen = 1
2911
2912 when = int(time.mktime(tmwhen))
2913 self.streamTag(stream, name, labelDetails, gitCommit, when)
2914 if verbose:
2915 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2916 else:
2917 if verbose:
2918 print "Label %s has no changelists - possibly deleted?" % name
2919
2920 if not commitFound:
2921 # We can't import this label; don't try again as it will get very
2922 # expensive repeatedly fetching all the files for labels that will
2923 # never be imported. If the label is moved in the future, the
2924 # ignore will need to be removed manually.
2925 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2926
86dff6b6
HWN
2927 def guessProjectName(self):
2928 for p in self.depotPaths:
6e5295c4
SH
2929 if p.endswith("/"):
2930 p = p[:-1]
2931 p = p[p.strip().rfind("/") + 1:]
2932 if not p.endswith("/"):
2933 p += "/"
2934 return p
86dff6b6 2935
4b97ffb1 2936 def getBranchMapping(self):
6555b2cc
SH
2937 lostAndFoundBranches = set()
2938
8ace74c0
VA
2939 user = gitConfig("git-p4.branchUser")
2940 if len(user) > 0:
2941 command = "branches -u %s" % user
2942 else:
2943 command = "branches"
2944
2945 for info in p4CmdList(command):
52a4880b 2946 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2947 viewIdx = 0
2948 while details.has_key("View%s" % viewIdx):
2949 paths = details["View%s" % viewIdx].split(" ")
2950 viewIdx = viewIdx + 1
2951 # require standard //depot/foo/... //depot/bar/... mapping
2952 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2953 continue
2954 source = paths[0]
2955 destination = paths[1]
6509e19c 2956 ## HACK
d53de8b9 2957 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2958 source = source[len(self.depotPaths[0]):-4]
2959 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2960
1a2edf4e
SH
2961 if destination in self.knownBranches:
2962 if not self.silent:
2963 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2964 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2965 continue
2966
6555b2cc
SH
2967 self.knownBranches[destination] = source
2968
2969 lostAndFoundBranches.discard(destination)
2970
29bdbac1 2971 if source not in self.knownBranches:
6555b2cc
SH
2972 lostAndFoundBranches.add(source)
2973
7199cf13
VA
2974 # Perforce does not strictly require branches to be defined, so we also
2975 # check git config for a branch list.
2976 #
2977 # Example of branch definition in git config file:
2978 # [git-p4]
2979 # branchList=main:branchA
2980 # branchList=main:branchB
2981 # branchList=branchA:branchC
2982 configBranches = gitConfigList("git-p4.branchList")
2983 for branch in configBranches:
2984 if branch:
2985 (source, destination) = branch.split(":")
2986 self.knownBranches[destination] = source
2987
2988 lostAndFoundBranches.discard(destination)
2989
2990 if source not in self.knownBranches:
2991 lostAndFoundBranches.add(source)
2992
6555b2cc
SH
2993
2994 for branch in lostAndFoundBranches:
2995 self.knownBranches[branch] = branch
29bdbac1 2996
38f9f5ec
SH
2997 def getBranchMappingFromGitBranches(self):
2998 branches = p4BranchesInGit(self.importIntoRemotes)
2999 for branch in branches.keys():
3000 if branch == "master":
3001 branch = "main"
3002 else:
3003 branch = branch[len(self.projectName):]
3004 self.knownBranches[branch] = branch
3005
bb6e09b2
HWN
3006 def updateOptionDict(self, d):
3007 option_keys = {}
3008 if self.keepRepoPath:
3009 option_keys['keepRepoPath'] = 1
3010
3011 d["options"] = ' '.join(sorted(option_keys.keys()))
3012
3013 def readOptions(self, d):
3014 self.keepRepoPath = (d.has_key('options')
3015 and ('keepRepoPath' in d['options']))
6326aa58 3016
8134f69c
SH
3017 def gitRefForBranch(self, branch):
3018 if branch == "main":
3019 return self.refPrefix + "master"
3020
3021 if len(branch) <= 0:
3022 return branch
3023
3024 return self.refPrefix + self.projectName + branch
3025
1ca3d710
SH
3026 def gitCommitByP4Change(self, ref, change):
3027 if self.verbose:
3028 print "looking in ref " + ref + " for change %s using bisect..." % change
3029
3030 earliestCommit = ""
3031 latestCommit = parseRevision(ref)
3032
3033 while True:
3034 if self.verbose:
3035 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3036 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3037 if len(next) == 0:
3038 if self.verbose:
3039 print "argh"
3040 return ""
3041 log = extractLogMessageFromGitCommit(next)
3042 settings = extractSettingsGitLog(log)
3043 currentChange = int(settings['change'])
3044 if self.verbose:
3045 print "current change %s" % currentChange
3046
3047 if currentChange == change:
3048 if self.verbose:
3049 print "found %s" % next
3050 return next
3051
3052 if currentChange < change:
3053 earliestCommit = "^%s" % next
3054 else:
3055 latestCommit = "%s" % next
3056
3057 return ""
3058
3059 def importNewBranch(self, branch, maxChange):
3060 # make fast-import flush all changes to disk and update the refs using the checkpoint
3061 # command so that we can try to find the branch parent in the git history
3062 self.gitStream.write("checkpoint\n\n");
3063 self.gitStream.flush();
3064 branchPrefix = self.depotPaths[0] + branch + "/"
3065 range = "@1,%s" % maxChange
3066 #print "prefix" + branchPrefix
96b2d54a 3067 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3068 if len(changes) <= 0:
3069 return False
3070 firstChange = changes[0]
3071 #print "first change in branch: %s" % firstChange
3072 sourceBranch = self.knownBranches[branch]
3073 sourceDepotPath = self.depotPaths[0] + sourceBranch
3074 sourceRef = self.gitRefForBranch(sourceBranch)
3075 #print "source " + sourceBranch
3076
52a4880b 3077 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3078 #print "branch parent: %s" % branchParentChange
3079 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3080 if len(gitParent) > 0:
3081 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3082 #print "parent git commit: %s" % gitParent
3083
3084 self.importChanges(changes)
3085 return True
3086
fed23693
VA
3087 def searchParent(self, parent, branch, target):
3088 parentFound = False
c7d34884
PW
3089 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3090 "--no-merges", parent]):
fed23693
VA
3091 blob = blob.strip()
3092 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3093 parentFound = True
3094 if self.verbose:
3095 print "Found parent of %s in commit %s" % (branch, blob)
3096 break
3097 if parentFound:
3098 return blob
3099 else:
3100 return None
3101
e87f37ae
SH
3102 def importChanges(self, changes):
3103 cnt = 1
3104 for change in changes:
18fa13d0 3105 description = p4_describe(change)
e87f37ae
SH
3106 self.updateOptionDict(description)
3107
3108 if not self.silent:
3109 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3110 sys.stdout.flush()
3111 cnt = cnt + 1
3112
3113 try:
3114 if self.detectBranches:
3115 branches = self.splitFilesIntoBranches(description)
3116 for branch in branches.keys():
3117 ## HACK --hwn
3118 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3119 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3120
3121 parent = ""
3122
3123 filesForCommit = branches[branch]
3124
3125 if self.verbose:
3126 print "branch is %s" % branch
3127
3128 self.updatedBranches.add(branch)
3129
3130 if branch not in self.createdBranches:
3131 self.createdBranches.add(branch)
3132 parent = self.knownBranches[branch]
3133 if parent == branch:
3134 parent = ""
1ca3d710
SH
3135 else:
3136 fullBranch = self.projectName + branch
3137 if fullBranch not in self.p4BranchesInGit:
3138 if not self.silent:
3139 print("\n Importing new branch %s" % fullBranch);
3140 if self.importNewBranch(branch, change - 1):
3141 parent = ""
3142 self.p4BranchesInGit.append(fullBranch)
3143 if not self.silent:
3144 print("\n Resuming with change %s" % change);
3145
3146 if self.verbose:
3147 print "parent determined through known branches: %s" % parent
e87f37ae 3148
8134f69c
SH
3149 branch = self.gitRefForBranch(branch)
3150 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3151
3152 if self.verbose:
3153 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3154
3155 if len(parent) == 0 and branch in self.initialParents:
3156 parent = self.initialParents[branch]
3157 del self.initialParents[branch]
3158
fed23693
VA
3159 blob = None
3160 if len(parent) > 0:
4f9273d2 3161 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3162 if self.verbose:
3163 print "Creating temporary branch: " + tempBranch
e63231e5 3164 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3165 self.tempBranches.append(tempBranch)
3166 self.checkpoint()
3167 blob = self.searchParent(parent, branch, tempBranch)
3168 if blob:
e63231e5 3169 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3170 else:
3171 if self.verbose:
3172 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3173 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3174 else:
3175 files = self.extractFilesFromCommit(description)
e63231e5 3176 self.commit(description, files, self.branch,
e87f37ae 3177 self.initialParent)
47497844 3178 # only needed once, to connect to the previous commit
e87f37ae
SH
3179 self.initialParent = ""
3180 except IOError:
3181 print self.gitError.read()
3182 sys.exit(1)
3183
c208a243
SH
3184 def importHeadRevision(self, revision):
3185 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3186
4e2e6ce4
PW
3187 details = {}
3188 details["user"] = "git perforce import user"
1494fcbb 3189 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3190 % (' '.join(self.depotPaths), revision))
3191 details["change"] = revision
3192 newestRevision = 0
3193
3194 fileCnt = 0
6de040df
LD
3195 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3196
3197 for info in p4CmdList(["files"] + fileArgs):
c208a243 3198
68b28593 3199 if 'code' in info and info['code'] == 'error':
c208a243
SH
3200 sys.stderr.write("p4 returned an error: %s\n"
3201 % info['data'])
d88e707f
PW
3202 if info['data'].find("must refer to client") >= 0:
3203 sys.stderr.write("This particular p4 error is misleading.\n")
3204 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3205 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3206 sys.exit(1)
68b28593
PW
3207 if 'p4ExitCode' in info:
3208 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3209 sys.exit(1)
3210
3211
3212 change = int(info["change"])
3213 if change > newestRevision:
3214 newestRevision = change
3215
56c09345 3216 if info["action"] in self.delete_actions:
c208a243
SH
3217 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3218 #fileCnt = fileCnt + 1
3219 continue
3220
3221 for prop in ["depotFile", "rev", "action", "type" ]:
3222 details["%s%s" % (prop, fileCnt)] = info[prop]
3223
3224 fileCnt = fileCnt + 1
3225
3226 details["change"] = newestRevision
4e2e6ce4 3227
9dcb9f24 3228 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3229 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3230 res = p4_describe(newestRevision)
3231 details["time"] = res["time"]
4e2e6ce4 3232
c208a243
SH
3233 self.updateOptionDict(details)
3234 try:
e63231e5 3235 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3236 except IOError:
3237 print "IO error with git fast-import. Is your git version recent enough?"
3238 print self.gitError.read()
3239
3240
b984733c 3241 def run(self, args):
6326aa58 3242 self.depotPaths = []
179caebf 3243 self.changeRange = ""
6326aa58 3244 self.previousDepotPaths = []
991a2de4 3245 self.hasOrigin = False
ce6f33c8 3246
29bdbac1
SH
3247 # map from branch depot path to parent branch
3248 self.knownBranches = {}
3249 self.initialParents = {}
3250
a028a98e
SH
3251 if self.importIntoRemotes:
3252 self.refPrefix = "refs/remotes/p4/"
3253 else:
db775559 3254 self.refPrefix = "refs/heads/p4/"
a028a98e 3255
991a2de4
PW
3256 if self.syncWithOrigin:
3257 self.hasOrigin = originP4BranchesExist()
3258 if self.hasOrigin:
3259 if not self.silent:
3260 print 'Syncing with origin first, using "git fetch origin"'
3261 system("git fetch origin")
10f880f8 3262
5a8e84cd 3263 branch_arg_given = bool(self.branch)
569d1bd4 3264 if len(self.branch) == 0:
db775559 3265 self.branch = self.refPrefix + "master"
a028a98e 3266 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3267 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3268 system("git branch -D p4")
967f72e2 3269
a93d33ee
PW
3270 # accept either the command-line option, or the configuration variable
3271 if self.useClientSpec:
3272 # will use this after clone to set the variable
3273 self.useClientSpec_from_options = True
3274 else:
0d609032 3275 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3276 self.useClientSpec = True
3277 if self.useClientSpec:
543987bd 3278 self.clientSpecDirs = getClientSpec()
3a70cdfa 3279
6a49f8e2
HWN
3280 # TODO: should always look at previous commits,
3281 # merge with previous imports, if possible.
3282 if args == []:
d414c74a 3283 if self.hasOrigin:
5ca44617 3284 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3285
3286 # branches holds mapping from branch name to sha1
3287 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3288
3289 # restrict to just this one, disabling detect-branches
3290 if branch_arg_given:
3291 short = self.branch.split("/")[-1]
3292 if short in branches:
3293 self.p4BranchesInGit = [ short ]
3294 else:
3295 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3296
3297 if len(self.p4BranchesInGit) > 1:
3298 if not self.silent:
3299 print "Importing from/into multiple branches"
3300 self.detectBranches = True
8c9e8b6e
PW
3301 for branch in branches.keys():
3302 self.initialParents[self.refPrefix + branch] = \
3303 branches[branch]
967f72e2 3304
29bdbac1
SH
3305 if self.verbose:
3306 print "branches: %s" % self.p4BranchesInGit
3307
3308 p4Change = 0
3309 for branch in self.p4BranchesInGit:
cebdf5af 3310 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3311
3312 settings = extractSettingsGitLog(logMsg)
29bdbac1 3313
bb6e09b2
HWN
3314 self.readOptions(settings)
3315 if (settings.has_key('depot-paths')
3316 and settings.has_key ('change')):
3317 change = int(settings['change']) + 1
29bdbac1
SH
3318 p4Change = max(p4Change, change)
3319
bb6e09b2
HWN
3320 depotPaths = sorted(settings['depot-paths'])
3321 if self.previousDepotPaths == []:
6326aa58 3322 self.previousDepotPaths = depotPaths
29bdbac1 3323 else:
6326aa58
HWN
3324 paths = []
3325 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3326 prev_list = prev.split("/")
3327 cur_list = cur.split("/")
3328 for i in range(0, min(len(cur_list), len(prev_list))):
3329 if cur_list[i] <> prev_list[i]:
583e1707 3330 i = i - 1
6326aa58
HWN
3331 break
3332
04d277b3 3333 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3334
3335 self.previousDepotPaths = paths
29bdbac1
SH
3336
3337 if p4Change > 0:
bb6e09b2 3338 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3339 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3340 if not self.silent and not self.detectBranches:
967f72e2 3341 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3342
40d69ac3
PW
3343 # accept multiple ref name abbreviations:
3344 # refs/foo/bar/branch -> use it exactly
3345 # p4/branch -> prepend refs/remotes/ or refs/heads/
3346 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3347 if not self.branch.startswith("refs/"):
40d69ac3
PW
3348 if self.importIntoRemotes:
3349 prepend = "refs/remotes/"
3350 else:
3351 prepend = "refs/heads/"
3352 if not self.branch.startswith("p4/"):
3353 prepend += "p4/"
3354 self.branch = prepend + self.branch
179caebf 3355
6326aa58 3356 if len(args) == 0 and self.depotPaths:
b984733c 3357 if not self.silent:
6326aa58 3358 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3359 else:
6326aa58 3360 if self.depotPaths and self.depotPaths != args:
cebdf5af 3361 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3362 "This doesn't work!" % (' '.join (self.depotPaths),
3363 ' '.join (args)))
b984733c 3364 sys.exit(1)
6326aa58 3365
bb6e09b2 3366 self.depotPaths = sorted(args)
b984733c 3367
1c49fc19 3368 revision = ""
b984733c 3369 self.users = {}
b984733c 3370
58c8bc7c
PW
3371 # Make sure no revision specifiers are used when --changesfile
3372 # is specified.
3373 bad_changesfile = False
3374 if len(self.changesFile) > 0:
3375 for p in self.depotPaths:
3376 if p.find("@") >= 0 or p.find("#") >= 0:
3377 bad_changesfile = True
3378 break
3379 if bad_changesfile:
3380 die("Option --changesfile is incompatible with revision specifiers")
3381
6326aa58
HWN
3382 newPaths = []
3383 for p in self.depotPaths:
3384 if p.find("@") != -1:
3385 atIdx = p.index("@")
3386 self.changeRange = p[atIdx:]
3387 if self.changeRange == "@all":
3388 self.changeRange = ""
6a49f8e2 3389 elif ',' not in self.changeRange:
1c49fc19 3390 revision = self.changeRange
6326aa58 3391 self.changeRange = ""
7fcff9de 3392 p = p[:atIdx]
6326aa58
HWN
3393 elif p.find("#") != -1:
3394 hashIdx = p.index("#")
1c49fc19 3395 revision = p[hashIdx:]
7fcff9de 3396 p = p[:hashIdx]
6326aa58 3397 elif self.previousDepotPaths == []:
58c8bc7c
PW
3398 # pay attention to changesfile, if given, else import
3399 # the entire p4 tree at the head revision
3400 if len(self.changesFile) == 0:
3401 revision = "#head"
6326aa58
HWN
3402
3403 p = re.sub ("\.\.\.$", "", p)
3404 if not p.endswith("/"):
3405 p += "/"
3406
3407 newPaths.append(p)
3408
3409 self.depotPaths = newPaths
3410
e63231e5
PW
3411 # --detect-branches may change this for each branch
3412 self.branchPrefixes = self.depotPaths
3413
b607e71e 3414 self.loadUserMapFromCache()
cb53e1f8
SH
3415 self.labels = {}
3416 if self.detectLabels:
3417 self.getLabels();
b984733c 3418
4b97ffb1 3419 if self.detectBranches:
df450923
SH
3420 ## FIXME - what's a P4 projectName ?
3421 self.projectName = self.guessProjectName()
3422
38f9f5ec
SH
3423 if self.hasOrigin:
3424 self.getBranchMappingFromGitBranches()
3425 else:
3426 self.getBranchMapping()
29bdbac1
SH
3427 if self.verbose:
3428 print "p4-git branches: %s" % self.p4BranchesInGit
3429 print "initial parents: %s" % self.initialParents
3430 for b in self.p4BranchesInGit:
3431 if b != "master":
6326aa58
HWN
3432
3433 ## FIXME
29bdbac1
SH
3434 b = b[len(self.projectName):]
3435 self.createdBranches.add(b)
4b97ffb1 3436
f291b4e3 3437 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3438
78189bea
PW
3439 self.importProcess = subprocess.Popen(["git", "fast-import"],
3440 stdin=subprocess.PIPE,
3441 stdout=subprocess.PIPE,
3442 stderr=subprocess.PIPE);
3443 self.gitOutput = self.importProcess.stdout
3444 self.gitStream = self.importProcess.stdin
3445 self.gitError = self.importProcess.stderr