Git 2.10.2
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
8b41a97f 28
a235e85c
BC
29try:
30 from subprocess import CalledProcessError
31except ImportError:
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self, returncode, cmd):
39 self.returncode = returncode
40 self.cmd = cmd
41 def __str__(self):
42 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
4addad22 44verbose = False
86949eef 45
06804c76 46# Only labels/tags matching this will be imported/exported
c8942a22 47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 48
1051ef00
LD
49# Grab changes in blocks of this many revisions, unless otherwise requested
50defaultBlockSize = 512
51
21a50753
AK
52def p4_build_cmd(cmd):
53 """Build a suitable p4 command line.
54
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
58 """
6de040df 59 real_cmd = ["p4"]
abcaf073
AK
60
61 user = gitConfig("git-p4.user")
62 if len(user) > 0:
6de040df 63 real_cmd += ["-u",user]
abcaf073
AK
64
65 password = gitConfig("git-p4.password")
66 if len(password) > 0:
6de040df 67 real_cmd += ["-P", password]
abcaf073
AK
68
69 port = gitConfig("git-p4.port")
70 if len(port) > 0:
6de040df 71 real_cmd += ["-p", port]
abcaf073
AK
72
73 host = gitConfig("git-p4.host")
74 if len(host) > 0:
41799aa2 75 real_cmd += ["-H", host]
abcaf073
AK
76
77 client = gitConfig("git-p4.client")
78 if len(client) > 0:
6de040df 79 real_cmd += ["-c", client]
abcaf073 80
6de040df
LD
81
82 if isinstance(cmd,basestring):
83 real_cmd = ' '.join(real_cmd) + ' ' + cmd
84 else:
85 real_cmd += cmd
21a50753
AK
86 return real_cmd
87
bbd84863
MF
88def chdir(path, is_client_path=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
93
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
98
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
102 the client path.
103 """
104
105 os.chdir(path)
106 if not is_client_path:
107 path = os.getcwd()
108 os.environ['PWD'] = path
053fd0c1 109
4d25dc44
LS
110def calcDiskFree():
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform.system() == 'Windows':
113 free_bytes = ctypes.c_ulonglong(0)
114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115 return free_bytes.value
116 else:
117 st = os.statvfs(os.getcwd())
118 return st.f_bavail * st.f_frsize
119
86dff6b6
HWN
120def die(msg):
121 if verbose:
122 raise Exception(msg)
123 else:
124 sys.stderr.write(msg + "\n")
125 sys.exit(1)
126
6de040df 127def write_pipe(c, stdin):
4addad22 128 if verbose:
6de040df 129 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 130
6de040df
LD
131 expand = isinstance(c,basestring)
132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133 pipe = p.stdin
134 val = pipe.write(stdin)
135 pipe.close()
136 if p.wait():
137 die('Command failed: %s' % str(c))
b016d397
HWN
138
139 return val
140
6de040df 141def p4_write_pipe(c, stdin):
d9429194 142 real_cmd = p4_build_cmd(c)
6de040df 143 return write_pipe(real_cmd, stdin)
d9429194 144
4addad22
HWN
145def read_pipe(c, ignore_error=False):
146 if verbose:
6de040df 147 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 148
6de040df 149 expand = isinstance(c,basestring)
1f5f3907
LS
150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151 (out, err) = p.communicate()
152 if p.returncode != 0 and not ignore_error:
153 die('Command failed: %s\nError: %s' % (str(c), err))
154 return out
b016d397 155
d9429194
AK
156def p4_read_pipe(c, ignore_error=False):
157 real_cmd = p4_build_cmd(c)
158 return read_pipe(real_cmd, ignore_error)
b016d397 159
bce4c5fc 160def read_pipe_lines(c):
4addad22 161 if verbose:
6de040df
LD
162 sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164 expand = isinstance(c, basestring)
165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166 pipe = p.stdout
b016d397 167 val = pipe.readlines()
6de040df
LD
168 if pipe.close() or p.wait():
169 die('Command failed: %s' % str(c))
b016d397
HWN
170
171 return val
caace111 172
2318121b
AK
173def p4_read_pipe_lines(c):
174 """Specifically invoke p4 on the command supplied. """
155af834 175 real_cmd = p4_build_cmd(c)
2318121b
AK
176 return read_pipe_lines(real_cmd)
177
8e9497c2
GG
178def p4_has_command(cmd):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd = p4_build_cmd(["help", cmd])
182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183 stderr=subprocess.PIPE)
184 p.communicate()
185 return p.returncode == 0
186
249da4c0
PW
187def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
192
193 if not p4_has_command("move"):
194 return False
195 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197 (out, err) = p.communicate()
198 # return code will be 1 in either case
199 if err.find("Invalid option") >= 0:
200 return False
201 if err.find("disabled") >= 0:
202 return False
203 # assume it failed because @... was invalid changelist
204 return True
205
cbff4b25 206def system(cmd, ignore_error=False):
6de040df 207 expand = isinstance(cmd,basestring)
4addad22 208 if verbose:
6de040df 209 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 210 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 211 if retcode and not ignore_error:
a235e85c 212 raise CalledProcessError(retcode, cmd)
6754a299 213
cbff4b25
LD
214 return retcode
215
bf9320f1
AK
216def p4_system(cmd):
217 """Specifically invoke p4 as the system command. """
155af834 218 real_cmd = p4_build_cmd(cmd)
6de040df 219 expand = isinstance(real_cmd, basestring)
a235e85c
BC
220 retcode = subprocess.call(real_cmd, shell=expand)
221 if retcode:
222 raise CalledProcessError(retcode, real_cmd)
6de040df 223
7f0e5962
PW
224_p4_version_string = None
225def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
228
229 $ p4 -V
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233 """
234 global _p4_version_string
235 if not _p4_version_string:
236 a = p4_read_pipe_lines(["-V"])
237 _p4_version_string = a[-1].rstrip()
238 return _p4_version_string
239
6de040df 240def p4_integrate(src, dest):
9d7d446a 241 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 242
8d7ec362 243def p4_sync(f, *options):
9d7d446a 244 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
245
246def p4_add(f):
9d7d446a
PW
247 # forcibly add file names with wildcards
248 if wildcard_present(f):
249 p4_system(["add", "-f", f])
250 else:
251 p4_system(["add", f])
6de040df
LD
252
253def p4_delete(f):
9d7d446a 254 p4_system(["delete", wildcard_encode(f)])
6de040df 255
a02b8bc4
RP
256def p4_edit(f, *options):
257 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
258
259def p4_revert(f):
9d7d446a 260 p4_system(["revert", wildcard_encode(f)])
6de040df 261
9d7d446a
PW
262def p4_reopen(type, f):
263 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 264
8e9497c2
GG
265def p4_move(src, dest):
266 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
267
1051ef00
LD
268def p4_last_change():
269 results = p4CmdList(["changes", "-m", "1"])
270 return int(results[0]['change'])
271
18fa13d0
PW
272def p4_describe(change):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
275 results."""
276
277 ds = p4CmdList(["describe", "-s", str(change)])
278 if len(ds) != 1:
279 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280
281 d = ds[0]
282
283 if "p4ExitCode" in d:
284 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
285 str(d)))
286 if "code" in d:
287 if d["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
289
290 if "time" not in d:
291 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
292
293 return d
294
9cffb8c8
PW
295#
296# Canonicalize the p4 type and return a tuple of the
297# base type, plus any modifiers. See "p4 help filetypes"
298# for a list and explanation.
299#
300def split_p4_type(p4type):
301
302 p4_filetypes_historical = {
303 "ctempobj": "binary+Sw",
304 "ctext": "text+C",
305 "cxtext": "text+Cx",
306 "ktext": "text+k",
307 "kxtext": "text+kx",
308 "ltext": "text+F",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
314 "xltext": "text+Fx",
315 "xtempobj": "binary+Swx",
316 "xtext": "text+x",
317 "xunicode": "unicode+x",
318 "xutf16": "utf16+x",
319 }
320 if p4type in p4_filetypes_historical:
321 p4type = p4_filetypes_historical[p4type]
322 mods = ""
323 s = p4type.split("+")
324 base = s[0]
325 mods = ""
326 if len(s) > 1:
327 mods = s[1]
328 return (base, mods)
b9fc6ea9 329
60df071c
LD
330#
331# return the raw p4 type of a file (text, text+ko, etc)
332#
79467e61
PW
333def p4_type(f):
334 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
335 return results[0]['headType']
336
337#
338# Given a type base and modifier, return a regexp matching
339# the keywords that can be expanded in the file
340#
341def p4_keywords_regexp_for_type(base, type_mods):
342 if base in ("text", "unicode", "binary"):
343 kwords = None
344 if "ko" in type_mods:
345 kwords = 'Id|Header'
346 elif "k" in type_mods:
347 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348 else:
349 return None
350 pattern = r"""
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
6b2bf41e 353 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
354 \$ # another dollar
355 """ % kwords
356 return pattern
357 else:
358 return None
359
360#
361# Given a file, return a regexp matching the possible
362# RCS keywords that will be expanded, or None for files
363# with kw expansion turned off.
364#
365def p4_keywords_regexp_for_file(file):
366 if not os.path.exists(file):
367 return None
368 else:
369 (type_base, type_mods) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 371
c65b670e
CP
372def setP4ExecBit(file, mode):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
375
376 p4Type = "+x"
377
378 if not isModeExec(mode):
379 p4Type = getP4OpenedType(file)
380 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382 if p4Type[-1] == "+":
383 p4Type = p4Type[0:-1]
384
6de040df 385 p4_reopen(p4Type, file)
c65b670e
CP
386
387def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
389
9d7d446a 390 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 391 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
392 if match:
393 return match.group(1)
394 else:
f3e5ae4f 395 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 396
06804c76
LD
397# Return the set of all p4 labels
398def getP4Labels(depotPaths):
399 labels = set()
400 if isinstance(depotPaths,basestring):
401 depotPaths = [depotPaths]
402
403 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
404 label = l['label']
405 labels.add(label)
406
407 return labels
408
409# Return the set of all git tags
410def getGitTags():
411 gitTags = set()
412 for line in read_pipe_lines(["git", "tag"]):
413 tag = line.strip()
414 gitTags.add(tag)
415 return gitTags
416
b43b0a3c
CP
417def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421 while True:
422 yield pattern
423
424def parseDiffTreeEntry(entry):
425 """Parses a single diff tree entry into its component elements.
426
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
429
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
440
441 If the pattern is not matched, None is returned."""
442
443 match = diffTreePattern().next().match(entry)
444 if match:
445 return {
446 'src_mode': match.group(1),
447 'dst_mode': match.group(2),
448 'src_sha1': match.group(3),
449 'dst_sha1': match.group(4),
450 'status': match.group(5),
451 'status_score': match.group(6),
452 'src': match.group(7),
453 'dst': match.group(10)
454 }
455 return None
456
c65b670e
CP
457def isModeExec(mode):
458 # Returns True if the given git mode represents an executable file,
459 # otherwise False.
460 return mode[-3:] == "755"
461
462def isModeExecChanged(src_mode, dst_mode):
463 return isModeExec(src_mode) != isModeExec(dst_mode)
464
b932705b 465def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
466
467 if isinstance(cmd,basestring):
468 cmd = "-G " + cmd
469 expand = True
470 else:
471 cmd = ["-G"] + cmd
472 expand = False
473
474 cmd = p4_build_cmd(cmd)
6a49f8e2 475 if verbose:
6de040df 476 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
477
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
481 stdin_file = None
482 if stdin is not None:
483 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
484 if isinstance(stdin,basestring):
485 stdin_file.write(stdin)
486 else:
487 for i in stdin:
488 stdin_file.write(i + '\n')
9f90c733
SL
489 stdin_file.flush()
490 stdin_file.seek(0)
491
6de040df
LD
492 p4 = subprocess.Popen(cmd,
493 shell=expand,
9f90c733
SL
494 stdin=stdin_file,
495 stdout=subprocess.PIPE)
86949eef
SH
496
497 result = []
498 try:
499 while True:
9f90c733 500 entry = marshal.load(p4.stdout)
c3f6163b
AG
501 if cb is not None:
502 cb(entry)
503 else:
504 result.append(entry)
86949eef
SH
505 except EOFError:
506 pass
9f90c733
SL
507 exitCode = p4.wait()
508 if exitCode != 0:
ac3e0d79
SH
509 entry = {}
510 entry["p4ExitCode"] = exitCode
511 result.append(entry)
86949eef
SH
512
513 return result
514
515def p4Cmd(cmd):
516 list = p4CmdList(cmd)
517 result = {}
518 for entry in list:
519 result.update(entry)
520 return result;
521
cb2c9db5
SH
522def p4Where(depotPath):
523 if not depotPath.endswith("/"):
524 depotPath += "/"
cd884106
VA
525 depotPathLong = depotPath + "..."
526 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
527 output = None
528 for entry in outputList:
75bc9573 529 if "depotFile" in entry:
cd884106
VA
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
533 output = entry
534 break
535 elif "data" in entry:
536 data = entry.get("data")
537 space = data.find(" ")
538 if data[:space] == depotPath:
539 output = entry
540 break
7f705dc3
TAL
541 if output == None:
542 return ""
dc524036
SH
543 if output["code"] == "error":
544 return ""
cb2c9db5
SH
545 clientPath = ""
546 if "path" in output:
547 clientPath = output.get("path")
548 elif "data" in output:
549 data = output.get("data")
550 lastSpace = data.rfind(" ")
551 clientPath = data[lastSpace + 1:]
552
553 if clientPath.endswith("..."):
554 clientPath = clientPath[:-3]
555 return clientPath
556
86949eef 557def currentGitBranch():
00ad6e31
LD
558 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
559 if retcode != 0:
560 # on a detached head
561 return None
562 else:
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
86949eef 564
4f5cf76a 565def isValidGitDir(path):
bb6e09b2
HWN
566 if (os.path.exists(path + "/HEAD")
567 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
568 return True;
569 return False
570
463e8af6 571def parseRevision(ref):
b25b2065 572 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 573
28755dba
PW
574def branchExists(ref):
575 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
576 ignore_error=True)
577 return len(rev) > 0
578
6ae8de88
SH
579def extractLogMessageFromGitCommit(commit):
580 logMessage = ""
b016d397
HWN
581
582 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 583 foundTitle = False
b016d397 584 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
585 if not foundTitle:
586 if len(log) == 1:
1c094184 587 foundTitle = True
6ae8de88
SH
588 continue
589
590 logMessage += log
591 return logMessage
592
bb6e09b2 593def extractSettingsGitLog(log):
6ae8de88
SH
594 values = {}
595 for line in log.split("\n"):
596 line = line.strip()
6326aa58
HWN
597 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
598 if not m:
599 continue
600
601 assignments = m.group(1).split (':')
602 for a in assignments:
603 vals = a.split ('=')
604 key = vals[0].strip()
605 val = ('='.join (vals[1:])).strip()
606 if val.endswith ('\"') and val.startswith('"'):
607 val = val[1:-1]
608
609 values[key] = val
610
845b42cb
SH
611 paths = values.get("depot-paths")
612 if not paths:
613 paths = values.get("depot-path")
a3fdd579
SH
614 if paths:
615 values['depot-paths'] = paths.split(',')
bb6e09b2 616 return values
6ae8de88 617
8136a639 618def gitBranchExists(branch):
bb6e09b2
HWN
619 proc = subprocess.Popen(["git", "rev-parse", branch],
620 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 621 return proc.wait() == 0;
8136a639 622
36bd8446 623_gitConfig = {}
b345d6c3 624
692e1796 625def gitConfig(key, typeSpecifier=None):
36bd8446 626 if not _gitConfig.has_key(key):
692e1796
LS
627 cmd = [ "git", "config" ]
628 if typeSpecifier:
629 cmd += [ typeSpecifier ]
630 cmd += [ key ]
b345d6c3
PW
631 s = read_pipe(cmd, ignore_error=True)
632 _gitConfig[key] = s.strip()
36bd8446 633 return _gitConfig[key]
01265103 634
0d609032
PW
635def gitConfigBool(key):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
638 in the config."""
639
36bd8446 640 if not _gitConfig.has_key(key):
692e1796 641 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 642 return _gitConfig[key]
01265103 643
cb1dafdf
LS
644def gitConfigInt(key):
645 if not _gitConfig.has_key(key):
646 cmd = [ "git", "config", "--int", key ]
0d609032
PW
647 s = read_pipe(cmd, ignore_error=True)
648 v = s.strip()
cb1dafdf
LS
649 try:
650 _gitConfig[key] = int(gitConfig(key, '--int'))
651 except ValueError:
652 _gitConfig[key] = None
36bd8446 653 return _gitConfig[key]
01265103 654
7199cf13
VA
655def gitConfigList(key):
656 if not _gitConfig.has_key(key):
2abba301
PW
657 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658 _gitConfig[key] = s.strip().split(os.linesep)
7960e707
LS
659 if _gitConfig[key] == ['']:
660 _gitConfig[key] = []
7199cf13
VA
661 return _gitConfig[key]
662
2c8037ed
PW
663def p4BranchesInGit(branchesAreInRemotes=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
668 "p4/" prefix."""
669
062410bb
SH
670 branches = {}
671
672 cmdline = "git rev-parse --symbolic "
673 if branchesAreInRemotes:
2c8037ed 674 cmdline += "--remotes"
062410bb 675 else:
2c8037ed 676 cmdline += "--branches"
062410bb
SH
677
678 for line in read_pipe_lines(cmdline):
679 line = line.strip()
680
2c8037ed
PW
681 # only import to p4/
682 if not line.startswith('p4/'):
683 continue
684 # special symbolic ref to p4/master
685 if line == "p4/HEAD":
062410bb 686 continue
062410bb 687
2c8037ed
PW
688 # strip off p4/ prefix
689 branch = line[len("p4/"):]
062410bb
SH
690
691 branches[branch] = parseRevision(line)
2c8037ed 692
062410bb
SH
693 return branches
694
5a8e84cd
PW
695def branch_exists(branch):
696 """Make sure that the given ref name really exists."""
697
698 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700 out, _ = p.communicate()
701 if p.returncode:
702 return False
703 # expect exactly one line of output: the branch name
704 return out.rstrip() == branch
705
9ceab363 706def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
707 branches = p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath = {}
710 for branch in branches.keys():
711 tip = branches[branch]
712 log = extractLogMessageFromGitCommit(tip)
713 settings = extractSettingsGitLog(log)
714 if settings.has_key("depot-paths"):
715 paths = ",".join(settings["depot-paths"])
716 branchByDepotPath[paths] = "remotes/p4/" + branch
717
27d2d811 718 settings = None
27d2d811
SH
719 parent = 0
720 while parent < 65535:
9ceab363 721 commit = head + "~%s" % parent
27d2d811
SH
722 log = extractLogMessageFromGitCommit(commit)
723 settings = extractSettingsGitLog(log)
86506fe5
SH
724 if settings.has_key("depot-paths"):
725 paths = ",".join(settings["depot-paths"])
726 if branchByDepotPath.has_key(paths):
727 return [branchByDepotPath[paths], settings]
27d2d811 728
86506fe5 729 parent = parent + 1
27d2d811 730
86506fe5 731 return ["", settings]
27d2d811 732
5ca44617
SH
733def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
734 if not silent:
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
736 % localRefPrefix)
737
738 originPrefix = "origin/p4/"
739
740 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
741 line = line.strip()
742 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
743 continue
744
745 headName = line[len(originPrefix):]
746 remoteHead = localRefPrefix + headName
747 originHead = line
748
749 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750 if (not original.has_key('depot-paths')
751 or not original.has_key('change')):
752 continue
753
754 update = False
755 if not gitBranchExists(remoteHead):
756 if verbose:
757 print "creating %s" % remoteHead
758 update = True
759 else:
760 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761 if settings.has_key('change') > 0:
762 if settings['depot-paths'] == original['depot-paths']:
763 originP4Change = int(original['change'])
764 p4Change = int(settings['change'])
765 if originP4Change > p4Change:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead, originP4Change,
769 remoteHead, p4Change))
770 update = True
771 else:
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead, ','.join(original['depot-paths']),
775 remoteHead, ','.join(settings['depot-paths'])))
776
777 if update:
778 system("git update-ref %s %s" % (remoteHead, originHead))
779
780def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
782
1051ef00
LD
783
784def p4ParseNumericChangeRange(parts):
785 changeStart = int(parts[0][1:])
786 if parts[1] == '#head':
787 changeEnd = p4_last_change()
788 else:
789 changeEnd = int(parts[1])
790
791 return (changeStart, changeEnd)
792
793def chooseBlockSize(blockSize):
794 if blockSize:
795 return blockSize
796 else:
797 return defaultBlockSize
798
799def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 800 assert depotPaths
96b2d54a 801
1051ef00
LD
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
807
96b2d54a 808 if changeRange is None or changeRange == '':
1051ef00
LD
809 changeStart = 1
810 changeEnd = p4_last_change()
811 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
812 else:
813 parts = changeRange.split(',')
814 assert len(parts) == 2
1051ef00
LD
815 try:
816 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817 block_size = chooseBlockSize(requestedBlockSize)
818 except:
819 changeStart = parts[0][1:]
820 changeEnd = parts[1]
821 if requestedBlockSize:
822 die("cannot use --changes-block-size with non-numeric revisions")
823 block_size = None
4f6432d8 824
1f90a648 825 changes = []
96b2d54a 826
1f90a648
SH
827 # Retrieve changes a block at a time, to prevent running
828 # into a MaxResults/MaxScanRows error from the server.
1051ef00 829
1f90a648
SH
830 while True:
831 cmd = ['changes']
1051ef00 832
1f90a648
SH
833 if block_size:
834 end = min(changeEnd, changeStart + block_size)
835 revisionRange = "%d,%d" % (changeStart, end)
836 else:
837 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 838
1f90a648 839 for p in depotPaths:
1051ef00
LD
840 cmd += ["%s...@%s" % (p, revisionRange)]
841
1f90a648
SH
842 # Insert changes in chronological order
843 for line in reversed(p4_read_pipe_lines(cmd)):
844 changes.append(int(line.split(" ")[1]))
1051ef00 845
1f90a648
SH
846 if not block_size:
847 break
1051ef00 848
1f90a648
SH
849 if end >= changeEnd:
850 break
1051ef00 851
1f90a648 852 changeStart = end + 1
4f6432d8 853
1f90a648
SH
854 changes = sorted(changes)
855 return changes
4f6432d8 856
d53de8b9
TAL
857def p4PathStartsWith(path, prefix):
858 # This method tries to remedy a potential mixed-case issue:
859 #
860 # If UserA adds //depot/DirA/file1
861 # and UserB adds //depot/dira/file2
862 #
863 # we may or may not have a problem. If you have core.ignorecase=true,
864 # we treat DirA and dira as the same directory
0d609032 865 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
866 return path.lower().startswith(prefix.lower())
867 return path.startswith(prefix)
868
543987bd
PW
869def getClientSpec():
870 """Look at the p4 client spec, create a View() object that contains
871 all the mappings, and return it."""
872
873 specList = p4CmdList("client -o")
874 if len(specList) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' %
876 len(specList))
877
878 # dictionary of all client parameters
879 entry = specList[0]
880
9d57c4a6
KS
881 # the //client/ name
882 client_name = entry["Client"]
883
543987bd
PW
884 # just the keys that start with "View"
885 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
886
887 # hold this new View
9d57c4a6 888 view = View(client_name)
543987bd
PW
889
890 # append the lines, in order, to the view
891 for view_num in range(len(view_keys)):
892 k = "View%d" % view_num
893 if k not in view_keys:
894 die("Expected view key %s missing" % k)
895 view.append(entry[k])
896
897 return view
898
899def getClientRoot():
900 """Grab the client directory."""
901
902 output = p4CmdList("client -o")
903 if len(output) != 1:
904 die('Output from "client -o" is %d lines, expecting 1' % len(output))
905
906 entry = output[0]
907 if "Root" not in entry:
908 die('Client has no "Root"')
909
910 return entry["Root"]
911
9d7d446a
PW
912#
913# P4 wildcards are not allowed in filenames. P4 complains
914# if you simply add them, but you can force it with "-f", in
915# which case it translates them into %xx encoding internally.
916#
917def wildcard_decode(path):
918 # Search for and fix just these four characters. Do % last so
919 # that fixing it does not inadvertently create new %-escapes.
920 # Cannot have * in a filename in windows; untested as to
921 # what p4 would do in such a case.
922 if not platform.system() == "Windows":
923 path = path.replace("%2A", "*")
924 path = path.replace("%23", "#") \
925 .replace("%40", "@") \
926 .replace("%25", "%")
927 return path
928
929def wildcard_encode(path):
930 # do % first to avoid double-encoding the %s introduced here
931 path = path.replace("%", "%25") \
932 .replace("*", "%2A") \
933 .replace("#", "%23") \
934 .replace("@", "%40")
935 return path
936
937def wildcard_present(path):
598354c0
BC
938 m = re.search("[*#@%]", path)
939 return m is not None
9d7d446a 940
a5db4b12
LS
941class LargeFileSystem(object):
942 """Base class for large file system support."""
943
944 def __init__(self, writeToGitStream):
945 self.largeFiles = set()
946 self.writeToGitStream = writeToGitStream
947
948 def generatePointer(self, cloneDestination, contentFile):
949 """Return the content of a pointer file that is stored in Git instead of
950 the actual content."""
951 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
952
953 def pushFile(self, localLargeFile):
954 """Push the actual content which is not stored in the Git repository to
955 a server."""
956 assert False, "Method 'pushFile' required in " + self.__class__.__name__
957
958 def hasLargeFileExtension(self, relPath):
959 return reduce(
960 lambda a, b: a or b,
961 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
962 False
963 )
964
965 def generateTempFile(self, contents):
966 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
967 for d in contents:
968 contentFile.write(d)
969 contentFile.close()
970 return contentFile.name
971
972 def exceedsLargeFileThreshold(self, relPath, contents):
973 if gitConfigInt('git-p4.largeFileThreshold'):
974 contentsSize = sum(len(d) for d in contents)
975 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
976 return True
977 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978 contentsSize = sum(len(d) for d in contents)
979 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
980 return False
981 contentTempFile = self.generateTempFile(contents)
982 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
985 zf.close()
986 compressedContentsSize = zf.infolist()[0].compress_size
987 os.remove(contentTempFile)
988 os.remove(compressedContentFile.name)
989 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
990 return True
991 return False
992
993 def addLargeFile(self, relPath):
994 self.largeFiles.add(relPath)
995
996 def removeLargeFile(self, relPath):
997 self.largeFiles.remove(relPath)
998
999 def isLargeFile(self, relPath):
1000 return relPath in self.largeFiles
1001
1002 def processContent(self, git_mode, relPath, contents):
1003 """Processes the content of git fast import. This method decides if a
1004 file is stored in the large file system and handles all necessary
1005 steps."""
1006 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007 contentTempFile = self.generateTempFile(contents)
1008 (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1009
1010 # Move temp file to final location in large file system
1011 largeFileDir = os.path.dirname(localLargeFile)
1012 if not os.path.isdir(largeFileDir):
1013 os.makedirs(largeFileDir)
1014 shutil.move(contentTempFile, localLargeFile)
1015 self.addLargeFile(relPath)
1016 if gitConfigBool('git-p4.largeFilePush'):
1017 self.pushFile(localLargeFile)
1018 if verbose:
1019 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020 return (git_mode, contents)
1021
1022class MockLFS(LargeFileSystem):
1023 """Mock large file system for testing."""
1024
1025 def generatePointer(self, contentFile):
1026 """The pointer content is the original content prefixed with "pointer-".
1027 The local filename of the large file storage is derived from the file content.
1028 """
1029 with open(contentFile, 'r') as f:
1030 content = next(f)
1031 gitMode = '100644'
1032 pointerContents = 'pointer-' + content
1033 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034 return (gitMode, pointerContents, localLargeFile)
1035
1036 def pushFile(self, localLargeFile):
1037 """The remote filename of the large file storage is the same as the local
1038 one but in a different directory.
1039 """
1040 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041 if not os.path.exists(remotePath):
1042 os.makedirs(remotePath)
1043 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1044
b47d807d
LS
1045class GitLFS(LargeFileSystem):
1046 """Git LFS as backend for the git-p4 large file system.
1047 See https://git-lfs.github.com/ for details."""
1048
1049 def __init__(self, *args):
1050 LargeFileSystem.__init__(self, *args)
1051 self.baseGitAttributes = []
1052
1053 def generatePointer(self, contentFile):
1054 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055 mode and content which is stored in the Git repository instead of
1056 the actual content. Return also the new location of the actual
1057 content.
1058 """
1059 pointerProcess = subprocess.Popen(
1060 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061 stdout=subprocess.PIPE
1062 )
1063 pointerFile = pointerProcess.stdout.read()
1064 if pointerProcess.wait():
1065 os.remove(contentFile)
1066 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1067
1068 # Git LFS removed the preamble in the output of the 'pointer' command
1069 # starting from version 1.2.0. Check for the preamble here to support
1070 # earlier versions.
1071 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1072 if pointerFile.startswith('Git LFS pointer for'):
1073 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1074
1075 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1076 localLargeFile = os.path.join(
1077 os.getcwd(),
1078 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1079 oid,
1080 )
1081 # LFS Spec states that pointer files should not have the executable bit set.
1082 gitMode = '100644'
82f2567e 1083 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1084
1085 def pushFile(self, localLargeFile):
1086 uploadProcess = subprocess.Popen(
1087 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1088 )
1089 if uploadProcess.wait():
1090 die('git-lfs push command failed. Did you define a remote?')
1091
1092 def generateGitAttributes(self):
1093 return (
1094 self.baseGitAttributes +
1095 [
1096 '\n',
1097 '#\n',
1098 '# Git LFS (see https://git-lfs.github.com/)\n',
1099 '#\n',
1100 ] +
1101 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1102 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1103 ] +
1104 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1105 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1106 ]
1107 )
1108
1109 def addLargeFile(self, relPath):
1110 LargeFileSystem.addLargeFile(self, relPath)
1111 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1112
1113 def removeLargeFile(self, relPath):
1114 LargeFileSystem.removeLargeFile(self, relPath)
1115 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1116
1117 def processContent(self, git_mode, relPath, contents):
1118 if relPath == '.gitattributes':
1119 self.baseGitAttributes = contents
1120 return (git_mode, self.generateGitAttributes())
1121 else:
1122 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1123
b984733c
SH
1124class Command:
1125 def __init__(self):
1126 self.usage = "usage: %prog [options]"
8910ac0e 1127 self.needsGit = True
6a10b6aa 1128 self.verbose = False
b984733c 1129
3ea2cfd4
LD
1130class P4UserMap:
1131 def __init__(self):
1132 self.userMapFromPerforceServer = False
affb474f
LD
1133 self.myP4UserId = None
1134
1135 def p4UserId(self):
1136 if self.myP4UserId:
1137 return self.myP4UserId
1138
1139 results = p4CmdList("user -o")
1140 for r in results:
1141 if r.has_key('User'):
1142 self.myP4UserId = r['User']
1143 return r['User']
1144 die("Could not find your p4 user id")
1145
1146 def p4UserIsMe(self, p4User):
1147 # return True if the given p4 user is actually me
1148 me = self.p4UserId()
1149 if not p4User or p4User != me:
1150 return False
1151 else:
1152 return True
3ea2cfd4
LD
1153
1154 def getUserCacheFilename(self):
1155 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1156 return home + "/.gitp4-usercache.txt"
1157
1158 def getUserMapFromPerforceServer(self):
1159 if self.userMapFromPerforceServer:
1160 return
1161 self.users = {}
1162 self.emails = {}
1163
1164 for output in p4CmdList("users"):
1165 if not output.has_key("User"):
1166 continue
1167 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1168 self.emails[output["Email"]] = output["User"]
1169
10d08a14
LS
1170 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1171 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1172 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1173 if mapUser and len(mapUser[0]) == 3:
1174 user = mapUser[0][0]
1175 fullname = mapUser[0][1]
1176 email = mapUser[0][2]
1177 self.users[user] = fullname + " <" + email + ">"
1178 self.emails[email] = user
3ea2cfd4
LD
1179
1180 s = ''
1181 for (key, val) in self.users.items():
1182 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1183
1184 open(self.getUserCacheFilename(), "wb").write(s)
1185 self.userMapFromPerforceServer = True
1186
1187 def loadUserMapFromCache(self):
1188 self.users = {}
1189 self.userMapFromPerforceServer = False
1190 try:
1191 cache = open(self.getUserCacheFilename(), "rb")
1192 lines = cache.readlines()
1193 cache.close()
1194 for line in lines:
1195 entry = line.strip().split("\t")
1196 self.users[entry[0]] = entry[1]
1197 except IOError:
1198 self.getUserMapFromPerforceServer()
1199
b984733c 1200class P4Debug(Command):
86949eef 1201 def __init__(self):
6ae8de88 1202 Command.__init__(self)
6a10b6aa 1203 self.options = []
c8c39116 1204 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1205 self.needsGit = False
86949eef
SH
1206
1207 def run(self, args):
b1ce9447 1208 j = 0
6de040df 1209 for output in p4CmdList(args):
b1ce9447
HWN
1210 print 'Element: %d' % j
1211 j += 1
86949eef 1212 print output
b984733c 1213 return True
86949eef 1214
5834684d
SH
1215class P4RollBack(Command):
1216 def __init__(self):
1217 Command.__init__(self)
1218 self.options = [
0c66a783 1219 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1220 ]
1221 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1222 self.rollbackLocalBranches = False
5834684d
SH
1223
1224 def run(self, args):
1225 if len(args) != 1:
1226 return False
1227 maxChange = int(args[0])
0c66a783 1228
ad192f28 1229 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1230 die("Problems executing p4");
1231
0c66a783
SH
1232 if self.rollbackLocalBranches:
1233 refPrefix = "refs/heads/"
b016d397 1234 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1235 else:
1236 refPrefix = "refs/remotes/"
b016d397 1237 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1238
1239 for line in lines:
1240 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1241 line = line.strip()
1242 ref = refPrefix + line
5834684d 1243 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1244 settings = extractSettingsGitLog(log)
1245
1246 depotPaths = settings['depot-paths']
1247 change = settings['change']
1248
5834684d 1249 changed = False
52102d47 1250
6326aa58
HWN
1251 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1252 for p in depotPaths]))) == 0:
52102d47
SH
1253 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1254 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1255 continue
1256
bb6e09b2 1257 while change and int(change) > maxChange:
5834684d 1258 changed = True
52102d47
SH
1259 if self.verbose:
1260 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1261 system("git update-ref %s \"%s^\"" % (ref, ref))
1262 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1263 settings = extractSettingsGitLog(log)
1264
1265
1266 depotPaths = settings['depot-paths']
1267 change = settings['change']
5834684d
SH
1268
1269 if changed:
52102d47 1270 print "%s rewound to %s" % (ref, change)
5834684d
SH
1271
1272 return True
1273
3ea2cfd4 1274class P4Submit(Command, P4UserMap):
6bbfd137
PW
1275
1276 conflict_behavior_choices = ("ask", "skip", "quit")
1277
4f5cf76a 1278 def __init__(self):
b984733c 1279 Command.__init__(self)
3ea2cfd4 1280 P4UserMap.__init__(self)
4f5cf76a 1281 self.options = [
4f5cf76a 1282 optparse.make_option("--origin", dest="origin"),
ae901090 1283 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1284 # preserve the user, requires relevant p4 permissions
1285 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1286 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1287 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1288 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1289 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1290 choices=self.conflict_behavior_choices),
1291 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
1292 ]
1293 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1294 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1295 self.origin = ""
ae901090 1296 self.detectRenames = False
0d609032 1297 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1298 self.dry_run = False
728b7ad8 1299 self.prepare_p4_only = False
6bbfd137 1300 self.conflict_behavior = None
f7baba8b 1301 self.isWindows = (platform.system() == "Windows")
06804c76 1302 self.exportLabels = False
249da4c0 1303 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1304 self.branch = None
4f5cf76a 1305
a5db4b12
LS
1306 if gitConfig('git-p4.largeFileSystem'):
1307 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1308
4f5cf76a
SH
1309 def check(self):
1310 if len(p4CmdList("opened ...")) > 0:
1311 die("You have files opened with perforce! Close them before starting the sync.")
1312
f19cb0a0
PW
1313 def separate_jobs_from_description(self, message):
1314 """Extract and return a possible Jobs field in the commit
1315 message. It goes into a separate section in the p4 change
1316 specification.
1317
1318 A jobs line starts with "Jobs:" and looks like a new field
1319 in a form. Values are white-space separated on the same
1320 line or on following lines that start with a tab.
1321
1322 This does not parse and extract the full git commit message
1323 like a p4 form. It just sees the Jobs: line as a marker
1324 to pass everything from then on directly into the p4 form,
1325 but outside the description section.
1326
1327 Return a tuple (stripped log message, jobs string)."""
1328
1329 m = re.search(r'^Jobs:', message, re.MULTILINE)
1330 if m is None:
1331 return (message, None)
1332
1333 jobtext = message[m.start():]
1334 stripped_message = message[:m.start()].rstrip()
1335 return (stripped_message, jobtext)
1336
1337 def prepareLogMessage(self, template, message, jobs):
1338 """Edits the template returned from "p4 change -o" to insert
1339 the message in the Description field, and the jobs text in
1340 the Jobs field."""
4f5cf76a
SH
1341 result = ""
1342
edae1e2f
SH
1343 inDescriptionSection = False
1344
4f5cf76a
SH
1345 for line in template.split("\n"):
1346 if line.startswith("#"):
1347 result += line + "\n"
1348 continue
1349
edae1e2f 1350 if inDescriptionSection:
c9dbab04 1351 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1352 inDescriptionSection = False
f19cb0a0
PW
1353 # insert Jobs section
1354 if jobs:
1355 result += jobs + "\n"
edae1e2f
SH
1356 else:
1357 continue
1358 else:
1359 if line.startswith("Description:"):
1360 inDescriptionSection = True
1361 line += "\n"
1362 for messageLine in message.split("\n"):
1363 line += "\t" + messageLine + "\n"
1364
1365 result += line + "\n"
4f5cf76a
SH
1366
1367 return result
1368
60df071c
LD
1369 def patchRCSKeywords(self, file, pattern):
1370 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1371 (handle, outFileName) = tempfile.mkstemp(dir='.')
1372 try:
1373 outFile = os.fdopen(handle, "w+")
1374 inFile = open(file, "r")
1375 regexp = re.compile(pattern, re.VERBOSE)
1376 for line in inFile.readlines():
1377 line = regexp.sub(r'$\1$', line)
1378 outFile.write(line)
1379 inFile.close()
1380 outFile.close()
1381 # Forcibly overwrite the original file
1382 os.unlink(file)
1383 shutil.move(outFileName, file)
1384 except:
1385 # cleanup our temporary file
1386 os.unlink(outFileName)
1387 print "Failed to strip RCS keywords in %s" % file
1388 raise
1389
1390 print "Patched up RCS keywords in %s" % file
1391
3ea2cfd4
LD
1392 def p4UserForCommit(self,id):
1393 # Return the tuple (perforce user,git email) for a given git commit id
1394 self.getUserMapFromPerforceServer()
9bf28855
PW
1395 gitEmail = read_pipe(["git", "log", "--max-count=1",
1396 "--format=%ae", id])
3ea2cfd4
LD
1397 gitEmail = gitEmail.strip()
1398 if not self.emails.has_key(gitEmail):
1399 return (None,gitEmail)
1400 else:
1401 return (self.emails[gitEmail],gitEmail)
1402
1403 def checkValidP4Users(self,commits):
1404 # check if any git authors cannot be mapped to p4 users
1405 for id in commits:
1406 (user,email) = self.p4UserForCommit(id)
1407 if not user:
1408 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1409 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1410 print "%s" % msg
1411 else:
1412 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1413
1414 def lastP4Changelist(self):
1415 # Get back the last changelist number submitted in this client spec. This
1416 # then gets used to patch up the username in the change. If the same
1417 # client spec is being used by multiple processes then this might go
1418 # wrong.
1419 results = p4CmdList("client -o") # find the current client
1420 client = None
1421 for r in results:
1422 if r.has_key('Client'):
1423 client = r['Client']
1424 break
1425 if not client:
1426 die("could not get client spec")
6de040df 1427 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1428 for r in results:
1429 if r.has_key('change'):
1430 return r['change']
1431 die("Could not get changelist number for last submit - cannot patch up user details")
1432
1433 def modifyChangelistUser(self, changelist, newUser):
1434 # fixup the user field of a changelist after it has been submitted.
1435 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1436 if len(changes) != 1:
1437 die("Bad output from p4 change modifying %s to user %s" %
1438 (changelist, newUser))
1439
1440 c = changes[0]
1441 if c['User'] == newUser: return # nothing to do
1442 c['User'] = newUser
1443 input = marshal.dumps(c)
1444
3ea2cfd4
LD
1445 result = p4CmdList("change -f -i", stdin=input)
1446 for r in result:
1447 if r.has_key('code'):
1448 if r['code'] == 'error':
1449 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1450 if r.has_key('data'):
1451 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1452 return
1453 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1454
1455 def canChangeChangelists(self):
1456 # check to see if we have p4 admin or super-user permissions, either of
1457 # which are required to modify changelists.
52a4880b 1458 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1459 for r in results:
1460 if r.has_key('perm'):
1461 if r['perm'] == 'admin':
1462 return 1
1463 if r['perm'] == 'super':
1464 return 1
1465 return 0
1466
ea99c3ae 1467 def prepareSubmitTemplate(self):
f19cb0a0
PW
1468 """Run "p4 change -o" to grab a change specification template.
1469 This does not use "p4 -G", as it is nice to keep the submission
1470 template in original order, since a human might edit it.
1471
1472 Remove lines in the Files section that show changes to files
1473 outside the depot path we're committing into."""
1474
cbc69242
SH
1475 [upstream, settings] = findUpstreamBranchPoint()
1476
ea99c3ae
SH
1477 template = ""
1478 inFilesSection = False
6de040df 1479 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1480 if line.endswith("\r\n"):
1481 line = line[:-2] + "\n"
ea99c3ae
SH
1482 if inFilesSection:
1483 if line.startswith("\t"):
1484 # path starts and ends with a tab
1485 path = line[1:]
1486 lastTab = path.rfind("\t")
1487 if lastTab != -1:
1488 path = path[:lastTab]
cbc69242
SH
1489 if settings.has_key('depot-paths'):
1490 if not [p for p in settings['depot-paths']
1491 if p4PathStartsWith(path, p)]:
1492 continue
1493 else:
1494 if not p4PathStartsWith(path, self.depotPath):
1495 continue
ea99c3ae
SH
1496 else:
1497 inFilesSection = False
1498 else:
1499 if line.startswith("Files:"):
1500 inFilesSection = True
1501
1502 template += line
1503
1504 return template
1505
7c766e57
PW
1506 def edit_template(self, template_file):
1507 """Invoke the editor to let the user change the submission
1508 message. Return true if okay to continue with the submit."""
1509
1510 # if configured to skip the editing part, just submit
0d609032 1511 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1512 return True
1513
1514 # look at the modification time, to check later if the user saved
1515 # the file
1516 mtime = os.stat(template_file).st_mtime
1517
1518 # invoke the editor
f95ceaf0 1519 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1520 editor = os.environ.get("P4EDITOR")
1521 else:
1522 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1523 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1524
1525 # If the file was not saved, prompt to see if this patch should
1526 # be skipped. But skip this verification step if configured so.
0d609032 1527 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1528 return True
1529
d1652049
PW
1530 # modification time updated means user saved the file
1531 if os.stat(template_file).st_mtime > mtime:
1532 return True
1533
1534 while True:
1535 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1536 if response == 'y':
1537 return True
1538 if response == 'n':
1539 return False
7c766e57 1540
e2a892ee 1541 def get_diff_description(self, editedFiles, filesToAdd):
b4073bb3
MC
1542 # diff
1543 if os.environ.has_key("P4DIFF"):
1544 del(os.environ["P4DIFF"])
1545 diff = ""
1546 for editedFile in editedFiles:
1547 diff += p4_read_pipe(['diff', '-du',
1548 wildcard_encode(editedFile)])
1549
1550 # new file diff
1551 newdiff = ""
1552 for newFile in filesToAdd:
1553 newdiff += "==== new file ====\n"
1554 newdiff += "--- /dev/null\n"
1555 newdiff += "+++ %s\n" % newFile
1556 f = open(newFile, "r")
1557 for line in f.readlines():
1558 newdiff += "+" + line
1559 f.close()
1560
e2a892ee 1561 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1562
7cb5cbef 1563 def applyCommit(self, id):
67b0fe2e
PW
1564 """Apply one commit, return True if it succeeded."""
1565
1566 print "Applying", read_pipe(["git", "show", "-s",
1567 "--format=format:%h %s", id])
ae901090 1568
848de9c3 1569 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1570
84cb0003 1571 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1572 filesToAdd = set()
a02b8bc4 1573 filesToChangeType = set()
4f5cf76a 1574 filesToDelete = set()
d336c158 1575 editedFiles = set()
b6ad6dcc 1576 pureRenameCopy = set()
c65b670e 1577 filesToChangeExecBit = {}
60df071c 1578
4f5cf76a 1579 for line in diff:
b43b0a3c
CP
1580 diff = parseDiffTreeEntry(line)
1581 modifier = diff['status']
1582 path = diff['src']
4f5cf76a 1583 if modifier == "M":
6de040df 1584 p4_edit(path)
c65b670e
CP
1585 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1586 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1587 editedFiles.add(path)
4f5cf76a
SH
1588 elif modifier == "A":
1589 filesToAdd.add(path)
c65b670e 1590 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1591 if path in filesToDelete:
1592 filesToDelete.remove(path)
1593 elif modifier == "D":
1594 filesToDelete.add(path)
1595 if path in filesToAdd:
1596 filesToAdd.remove(path)
4fddb41b
VA
1597 elif modifier == "C":
1598 src, dest = diff['src'], diff['dst']
6de040df 1599 p4_integrate(src, dest)
b6ad6dcc 1600 pureRenameCopy.add(dest)
4fddb41b 1601 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1602 p4_edit(dest)
b6ad6dcc 1603 pureRenameCopy.discard(dest)
4fddb41b 1604 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1605 p4_edit(dest)
b6ad6dcc 1606 pureRenameCopy.discard(dest)
4fddb41b 1607 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1608 if self.isWindows:
1609 # turn off read-only attribute
1610 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1611 os.unlink(dest)
1612 editedFiles.add(dest)
d9a5f25b 1613 elif modifier == "R":
b43b0a3c 1614 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1615 if self.p4HasMoveCommand:
1616 p4_edit(src) # src must be open before move
1617 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1618 else:
8e9497c2
GG
1619 p4_integrate(src, dest)
1620 if diff['src_sha1'] != diff['dst_sha1']:
1621 p4_edit(dest)
1622 else:
1623 pureRenameCopy.add(dest)
c65b670e 1624 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1625 if not self.p4HasMoveCommand:
1626 p4_edit(dest) # with move: already open, writable
c65b670e 1627 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1628 if not self.p4HasMoveCommand:
d20f0f8e
PW
1629 if self.isWindows:
1630 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1631 os.unlink(dest)
1632 filesToDelete.add(src)
d9a5f25b 1633 editedFiles.add(dest)
a02b8bc4
RP
1634 elif modifier == "T":
1635 filesToChangeType.add(path)
4f5cf76a
SH
1636 else:
1637 die("unknown modifier %s for %s" % (modifier, path))
1638
749b668c 1639 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1640 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1641 tryPatchCmd = patchcmd + "--check -"
1642 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1643 patch_succeeded = True
51a2640a 1644
47a130b7 1645 if os.system(tryPatchCmd) != 0:
60df071c
LD
1646 fixed_rcs_keywords = False
1647 patch_succeeded = False
51a2640a 1648 print "Unfortunately applying the change failed!"
60df071c
LD
1649
1650 # Patch failed, maybe it's just RCS keyword woes. Look through
1651 # the patch to see if that's possible.
0d609032 1652 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1653 file = None
1654 pattern = None
1655 kwfiles = {}
1656 for file in editedFiles | filesToDelete:
1657 # did this file's delta contain RCS keywords?
1658 pattern = p4_keywords_regexp_for_file(file)
1659
1660 if pattern:
1661 # this file is a possibility...look for RCS keywords.
1662 regexp = re.compile(pattern, re.VERBOSE)
1663 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1664 if regexp.search(line):
1665 if verbose:
1666 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1667 kwfiles[file] = pattern
1668 break
1669
1670 for file in kwfiles:
1671 if verbose:
1672 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1673 # File is being deleted, so not open in p4. Must
1674 # disable the read-only bit on windows.
1675 if self.isWindows and file not in editedFiles:
1676 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1677 self.patchRCSKeywords(file, kwfiles[file])
1678 fixed_rcs_keywords = True
1679
1680 if fixed_rcs_keywords:
1681 print "Retrying the patch with RCS keywords cleaned up"
1682 if os.system(tryPatchCmd) == 0:
1683 patch_succeeded = True
1684
1685 if not patch_succeeded:
7e5dd9f2
PW
1686 for f in editedFiles:
1687 p4_revert(f)
7e5dd9f2 1688 return False
51a2640a 1689
55ac2ed6
PW
1690 #
1691 # Apply the patch for real, and do add/delete/+x handling.
1692 #
47a130b7 1693 system(applyPatchCmd)
4f5cf76a 1694
a02b8bc4
RP
1695 for f in filesToChangeType:
1696 p4_edit(f, "-t", "auto")
4f5cf76a 1697 for f in filesToAdd:
6de040df 1698 p4_add(f)
4f5cf76a 1699 for f in filesToDelete:
6de040df
LD
1700 p4_revert(f)
1701 p4_delete(f)
4f5cf76a 1702
c65b670e
CP
1703 # Set/clear executable bits
1704 for f in filesToChangeExecBit.keys():
1705 mode = filesToChangeExecBit[f]
1706 setP4ExecBit(f, mode)
1707
55ac2ed6
PW
1708 #
1709 # Build p4 change description, starting with the contents
1710 # of the git commit message.
1711 #
0e36f2d7 1712 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1713 logMessage = logMessage.strip()
f19cb0a0 1714 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1715
ea99c3ae 1716 template = self.prepareSubmitTemplate()
f19cb0a0 1717 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1718
c47178d4 1719 if self.preserveUser:
55ac2ed6 1720 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1721
55ac2ed6
PW
1722 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1723 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1724 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1725 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1726
55ac2ed6 1727 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1728 if not self.prepare_p4_only:
1729 submitTemplate += separatorLine
e2a892ee 1730 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
55ac2ed6 1731
c47178d4 1732 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1733 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1734 if self.isWindows:
1735 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1736 tmpFile.write(submitTemplate)
c47178d4
PW
1737 tmpFile.close()
1738
728b7ad8
PW
1739 if self.prepare_p4_only:
1740 #
1741 # Leave the p4 tree prepared, and the submit template around
1742 # and let the user decide what to do next
1743 #
1744 print
1745 print "P4 workspace prepared for submission."
1746 print "To submit or revert, go to client workspace"
1747 print " " + self.clientPath
1748 print
1749 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1750 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1751 " \"git p4\"." % fileName
1752 print "You can delete the file \"%s\" when finished." % fileName
1753
1754 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1755 print "To preserve change ownership by user %s, you must\n" \
1756 "do \"p4 change -f <change>\" after submitting and\n" \
1757 "edit the User field."
1758 if pureRenameCopy:
1759 print "After submitting, renamed files must be re-synced."
1760 print "Invoke \"p4 sync -f\" on each of these files:"
1761 for f in pureRenameCopy:
1762 print " " + f
1763
1764 print
1765 print "To revert the changes, use \"p4 revert ...\", and delete"
1766 print "the submit template file \"%s\"" % fileName
1767 if filesToAdd:
1768 print "Since the commit adds new files, they must be deleted:"
1769 for f in filesToAdd:
1770 print " " + f
1771 print
1772 return True
1773
55ac2ed6
PW
1774 #
1775 # Let the user edit the change description, then submit it.
1776 #
b7638fed 1777 submitted = False
cdc7e388 1778
b7638fed
GE
1779 try:
1780 if self.edit_template(fileName):
1781 # read the edited message and submit
1782 tmpFile = open(fileName, "rb")
1783 message = tmpFile.read()
1784 tmpFile.close()
1785 if self.isWindows:
1786 message = message.replace("\r\n", "\n")
1787 submitTemplate = message[:message.index(separatorLine)]
1788 p4_write_pipe(['submit', '-i'], submitTemplate)
1789
1790 if self.preserveUser:
1791 if p4User:
1792 # Get last changelist number. Cannot easily get it from
1793 # the submit command output as the output is
1794 # unmarshalled.
1795 changelist = self.lastP4Changelist()
1796 self.modifyChangelistUser(changelist, p4User)
1797
1798 # The rename/copy happened by applying a patch that created a
1799 # new file. This leaves it writable, which confuses p4.
1800 for f in pureRenameCopy:
1801 p4_sync(f, "-f")
1802 submitted = True
1803
1804 finally:
c47178d4 1805 # skip this patch
b7638fed
GE
1806 if not submitted:
1807 print "Submission cancelled, undoing p4 changes."
1808 for f in editedFiles:
1809 p4_revert(f)
1810 for f in filesToAdd:
1811 p4_revert(f)
1812 os.remove(f)
1813 for f in filesToDelete:
1814 p4_revert(f)
c47178d4
PW
1815
1816 os.remove(fileName)
b7638fed 1817 return submitted
4f5cf76a 1818
06804c76
LD
1819 # Export git tags as p4 labels. Create a p4 label and then tag
1820 # with that.
1821 def exportGitTags(self, gitTags):
c8942a22
LD
1822 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1823 if len(validLabelRegexp) == 0:
1824 validLabelRegexp = defaultLabelRegexp
1825 m = re.compile(validLabelRegexp)
06804c76
LD
1826
1827 for name in gitTags:
1828
1829 if not m.match(name):
1830 if verbose:
05a3cec5 1831 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1832 continue
1833
1834 # Get the p4 commit this corresponds to
c8942a22
LD
1835 logMessage = extractLogMessageFromGitCommit(name)
1836 values = extractSettingsGitLog(logMessage)
06804c76 1837
c8942a22 1838 if not values.has_key('change'):
06804c76
LD
1839 # a tag pointing to something not sent to p4; ignore
1840 if verbose:
1841 print "git tag %s does not give a p4 commit" % name
1842 continue
c8942a22
LD
1843 else:
1844 changelist = values['change']
06804c76
LD
1845
1846 # Get the tag details.
1847 inHeader = True
1848 isAnnotated = False
1849 body = []
1850 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1851 l = l.strip()
1852 if inHeader:
1853 if re.match(r'tag\s+', l):
1854 isAnnotated = True
1855 elif re.match(r'\s*$', l):
1856 inHeader = False
1857 continue
1858 else:
1859 body.append(l)
1860
1861 if not isAnnotated:
1862 body = ["lightweight tag imported by git p4\n"]
1863
1864 # Create the label - use the same view as the client spec we are using
1865 clientSpec = getClientSpec()
1866
1867 labelTemplate = "Label: %s\n" % name
1868 labelTemplate += "Description:\n"
1869 for b in body:
1870 labelTemplate += "\t" + b + "\n"
1871 labelTemplate += "View:\n"
9d57c4a6
KS
1872 for depot_side in clientSpec.mappings:
1873 labelTemplate += "\t%s\n" % depot_side
06804c76 1874
ef739f08
PW
1875 if self.dry_run:
1876 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1877 elif self.prepare_p4_only:
1878 print "Not creating p4 label %s for tag due to option" \
1879 " --prepare-p4-only" % name
ef739f08
PW
1880 else:
1881 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1882
ef739f08
PW
1883 # Use the label
1884 p4_system(["tag", "-l", name] +
9d57c4a6 1885 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1886
ef739f08
PW
1887 if verbose:
1888 print "created p4 label for tag %s" % name
06804c76 1889
4f5cf76a 1890 def run(self, args):
c9b50e63
SH
1891 if len(args) == 0:
1892 self.master = currentGitBranch()
c9b50e63
SH
1893 elif len(args) == 1:
1894 self.master = args[0]
28755dba
PW
1895 if not branchExists(self.master):
1896 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1897 else:
1898 return False
1899
00ad6e31
LD
1900 if self.master:
1901 allowSubmit = gitConfig("git-p4.allowSubmit")
1902 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1903 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 1904
27d2d811 1905 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1906 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1907 if len(self.origin) == 0:
1908 self.origin = upstream
a3fdd579 1909
3ea2cfd4
LD
1910 if self.preserveUser:
1911 if not self.canChangeChangelists():
1912 die("Cannot preserve user names without p4 super-user or admin permissions")
1913
6bbfd137
PW
1914 # if not set from the command line, try the config file
1915 if self.conflict_behavior is None:
1916 val = gitConfig("git-p4.conflict")
1917 if val:
1918 if val not in self.conflict_behavior_choices:
1919 die("Invalid value '%s' for config git-p4.conflict" % val)
1920 else:
1921 val = "ask"
1922 self.conflict_behavior = val
1923
a3fdd579
SH
1924 if self.verbose:
1925 print "Origin branch is " + self.origin
9512497b 1926
ea99c3ae 1927 if len(self.depotPath) == 0:
9512497b
SH
1928 print "Internal error: cannot locate perforce depot path from existing branches"
1929 sys.exit(128)
1930
543987bd 1931 self.useClientSpec = False
0d609032 1932 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1933 self.useClientSpec = True
1934 if self.useClientSpec:
1935 self.clientSpecDirs = getClientSpec()
9512497b 1936
2e3a16b2 1937 # Check for the existence of P4 branches
cd884106
VA
1938 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1939
1940 if self.useClientSpec and not branchesDetected:
543987bd
PW
1941 # all files are relative to the client spec
1942 self.clientPath = getClientRoot()
1943 else:
1944 self.clientPath = p4Where(self.depotPath)
9512497b 1945
543987bd
PW
1946 if self.clientPath == "":
1947 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1948
ea99c3ae 1949 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1950 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1951
0591cfa8 1952 # ensure the clientPath exists
8d7ec362 1953 new_client_dir = False
0591cfa8 1954 if not os.path.exists(self.clientPath):
8d7ec362 1955 new_client_dir = True
0591cfa8
GG
1956 os.makedirs(self.clientPath)
1957
bbd84863 1958 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1959 if self.dry_run:
1960 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1961 else:
ef739f08
PW
1962 print "Synchronizing p4 checkout..."
1963 if new_client_dir:
1964 # old one was destroyed, and maybe nobody told p4
1965 p4_sync("...", "-f")
1966 else:
1967 p4_sync("...")
4f5cf76a 1968 self.check()
4f5cf76a 1969
4c750c0d 1970 commits = []
00ad6e31
LD
1971 if self.master:
1972 commitish = self.master
1973 else:
1974 commitish = 'HEAD'
1975
1976 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
4c750c0d
SH
1977 commits.append(line.strip())
1978 commits.reverse()
4f5cf76a 1979
0d609032 1980 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1981 self.checkAuthorship = False
1982 else:
1983 self.checkAuthorship = True
1984
3ea2cfd4
LD
1985 if self.preserveUser:
1986 self.checkValidP4Users(commits)
1987
84cb0003
GG
1988 #
1989 # Build up a set of options to be passed to diff when
1990 # submitting each commit to p4.
1991 #
1992 if self.detectRenames:
1993 # command-line -M arg
1994 self.diffOpts = "-M"
1995 else:
1996 # If not explicitly set check the config variable
1997 detectRenames = gitConfig("git-p4.detectRenames")
1998
1999 if detectRenames.lower() == "false" or detectRenames == "":
2000 self.diffOpts = ""
2001 elif detectRenames.lower() == "true":
2002 self.diffOpts = "-M"
2003 else:
2004 self.diffOpts = "-M%s" % detectRenames
2005
2006 # no command-line arg for -C or --find-copies-harder, just
2007 # config variables
2008 detectCopies = gitConfig("git-p4.detectCopies")
2009 if detectCopies.lower() == "false" or detectCopies == "":
2010 pass
2011 elif detectCopies.lower() == "true":
2012 self.diffOpts += " -C"
2013 else:
2014 self.diffOpts += " -C%s" % detectCopies
2015
0d609032 2016 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2017 self.diffOpts += " --find-copies-harder"
2018
7e5dd9f2
PW
2019 #
2020 # Apply the commits, one at a time. On failure, ask if should
2021 # continue to try the rest of the patches, or quit.
2022 #
ef739f08
PW
2023 if self.dry_run:
2024 print "Would apply"
67b0fe2e 2025 applied = []
7e5dd9f2
PW
2026 last = len(commits) - 1
2027 for i, commit in enumerate(commits):
ef739f08
PW
2028 if self.dry_run:
2029 print " ", read_pipe(["git", "show", "-s",
2030 "--format=format:%h %s", commit])
2031 ok = True
2032 else:
2033 ok = self.applyCommit(commit)
67b0fe2e
PW
2034 if ok:
2035 applied.append(commit)
7e5dd9f2 2036 else:
728b7ad8
PW
2037 if self.prepare_p4_only and i < last:
2038 print "Processing only the first commit due to option" \
2039 " --prepare-p4-only"
2040 break
7e5dd9f2
PW
2041 if i < last:
2042 quit = False
2043 while True:
6bbfd137
PW
2044 # prompt for what to do, or use the option/variable
2045 if self.conflict_behavior == "ask":
2046 print "What do you want to do?"
2047 response = raw_input("[s]kip this commit but apply"
2048 " the rest, or [q]uit? ")
2049 if not response:
2050 continue
2051 elif self.conflict_behavior == "skip":
2052 response = "s"
2053 elif self.conflict_behavior == "quit":
2054 response = "q"
2055 else:
2056 die("Unknown conflict_behavior '%s'" %
2057 self.conflict_behavior)
2058
7e5dd9f2
PW
2059 if response[0] == "s":
2060 print "Skipping this commit, but applying the rest"
2061 break
2062 if response[0] == "q":
2063 print "Quitting"
2064 quit = True
2065 break
2066 if quit:
2067 break
4f5cf76a 2068
67b0fe2e 2069 chdir(self.oldWorkingDirectory)
4f5cf76a 2070
ef739f08
PW
2071 if self.dry_run:
2072 pass
728b7ad8
PW
2073 elif self.prepare_p4_only:
2074 pass
ef739f08 2075 elif len(commits) == len(applied):
67b0fe2e 2076 print "All commits applied!"
14594f4b 2077
4c750c0d 2078 sync = P4Sync()
44e8d26c
PW
2079 if self.branch:
2080 sync.branch = self.branch
4c750c0d 2081 sync.run([])
14594f4b 2082
4c750c0d
SH
2083 rebase = P4Rebase()
2084 rebase.rebase()
4f5cf76a 2085
67b0fe2e
PW
2086 else:
2087 if len(applied) == 0:
2088 print "No commits applied."
2089 else:
2090 print "Applied only the commits marked with '*':"
2091 for c in commits:
2092 if c in applied:
2093 star = "*"
2094 else:
2095 star = " "
2096 print star, read_pipe(["git", "show", "-s",
2097 "--format=format:%h %s", c])
2098 print "You will have to do 'git p4 sync' and rebase."
2099
0d609032 2100 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2101 self.exportLabels = True
06804c76
LD
2102
2103 if self.exportLabels:
2104 p4Labels = getP4Labels(self.depotPath)
2105 gitTags = getGitTags()
2106
2107 missingGitTags = gitTags - p4Labels
2108 self.exportGitTags(missingGitTags)
2109
98e023de 2110 # exit with error unless everything applied perfectly
67b0fe2e
PW
2111 if len(commits) != len(applied):
2112 sys.exit(1)
2113
b984733c
SH
2114 return True
2115
ecb7cf98
PW
2116class View(object):
2117 """Represent a p4 view ("p4 help views"), and map files in a
2118 repo according to the view."""
2119
9d57c4a6 2120 def __init__(self, client_name):
ecb7cf98 2121 self.mappings = []
9d57c4a6
KS
2122 self.client_prefix = "//%s/" % client_name
2123 # cache results of "p4 where" to lookup client file locations
2124 self.client_spec_path_cache = {}
ecb7cf98
PW
2125
2126 def append(self, view_line):
2127 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2128 sides. Append to self.mappings, preserving order. This
2129 is only needed for tag creation."""
ecb7cf98
PW
2130
2131 # Split the view line into exactly two words. P4 enforces
2132 # structure on these lines that simplifies this quite a bit.
2133 #
2134 # Either or both words may be double-quoted.
2135 # Single quotes do not matter.
2136 # Double-quote marks cannot occur inside the words.
2137 # A + or - prefix is also inside the quotes.
2138 # There are no quotes unless they contain a space.
2139 # The line is already white-space stripped.
2140 # The two words are separated by a single space.
2141 #
2142 if view_line[0] == '"':
2143 # First word is double quoted. Find its end.
2144 close_quote_index = view_line.find('"', 1)
2145 if close_quote_index <= 0:
2146 die("No first-word closing quote found: %s" % view_line)
2147 depot_side = view_line[1:close_quote_index]
2148 # skip closing quote and space
2149 rhs_index = close_quote_index + 1 + 1
2150 else:
2151 space_index = view_line.find(" ")
2152 if space_index <= 0:
2153 die("No word-splitting space found: %s" % view_line)
2154 depot_side = view_line[0:space_index]
2155 rhs_index = space_index + 1
2156
ecb7cf98 2157 # prefix + means overlay on previous mapping
ecb7cf98 2158 if depot_side.startswith("+"):
ecb7cf98
PW
2159 depot_side = depot_side[1:]
2160
9d57c4a6 2161 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2162 exclude = False
2163 if depot_side.startswith("-"):
2164 exclude = True
2165 depot_side = depot_side[1:]
2166
9d57c4a6
KS
2167 if not exclude:
2168 self.mappings.append(depot_side)
ecb7cf98 2169
9d57c4a6
KS
2170 def convert_client_path(self, clientFile):
2171 # chop off //client/ part to make it relative
2172 if not clientFile.startswith(self.client_prefix):
2173 die("No prefix '%s' on clientFile '%s'" %
2174 (self.client_prefix, clientFile))
2175 return clientFile[len(self.client_prefix):]
ecb7cf98 2176
9d57c4a6
KS
2177 def update_client_spec_path_cache(self, files):
2178 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2179
9d57c4a6
KS
2180 # List depot file paths exclude that already cached
2181 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2182
9d57c4a6
KS
2183 if len(fileArgs) == 0:
2184 return # All files in cache
ecb7cf98 2185
9d57c4a6
KS
2186 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2187 for res in where_result:
2188 if "code" in res and res["code"] == "error":
2189 # assume error is "... file(s) not in client view"
2190 continue
2191 if "clientFile" not in res:
20005443 2192 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2193 if "unmap" in res:
2194 # it will list all of them, but only one not unmap-ped
2195 continue
a0a50d87
LS
2196 if gitConfigBool("core.ignorecase"):
2197 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2198 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2199
9d57c4a6
KS
2200 # not found files or unmap files set to ""
2201 for depotFile in fileArgs:
a0a50d87
LS
2202 if gitConfigBool("core.ignorecase"):
2203 depotFile = depotFile.lower()
9d57c4a6
KS
2204 if depotFile not in self.client_spec_path_cache:
2205 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2206
9d57c4a6
KS
2207 def map_in_client(self, depot_path):
2208 """Return the relative location in the client where this
2209 depot file should live. Returns "" if the file should
2210 not be mapped in the client."""
ecb7cf98 2211
a0a50d87
LS
2212 if gitConfigBool("core.ignorecase"):
2213 depot_path = depot_path.lower()
2214
9d57c4a6
KS
2215 if depot_path in self.client_spec_path_cache:
2216 return self.client_spec_path_cache[depot_path]
2217
2218 die( "Error: %s is not found in client spec path" % depot_path )
2219 return ""
ecb7cf98 2220
3ea2cfd4 2221class P4Sync(Command, P4UserMap):
56c09345
PW
2222 delete_actions = ( "delete", "move/delete", "purge" )
2223
b984733c
SH
2224 def __init__(self):
2225 Command.__init__(self)
3ea2cfd4 2226 P4UserMap.__init__(self)
b984733c
SH
2227 self.options = [
2228 optparse.make_option("--branch", dest="branch"),
2229 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2230 optparse.make_option("--changesfile", dest="changesFile"),
2231 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2232 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2233 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2234 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2235 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2236 optparse.make_option("--max-changes", dest="maxChanges",
2237 help="Maximum number of changes to import"),
2238 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2239 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2240 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2241 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2242 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2243 help="Only sync files that are included in the Perforce Client Spec"),
2244 optparse.make_option("-/", dest="cloneExclude",
2245 action="append", type="string",
2246 help="exclude depot path"),
b984733c
SH
2247 ]
2248 self.description = """Imports from Perforce into a git repository.\n
2249 example:
2250 //depot/my/project/ -- to import the current head
2251 //depot/my/project/@all -- to import everything
2252 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2253
2254 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2255
2256 self.usage += " //depot/path[@revRange]"
b984733c 2257 self.silent = False
1d7367dc
RG
2258 self.createdBranches = set()
2259 self.committedChanges = set()
569d1bd4 2260 self.branch = ""
b984733c 2261 self.detectBranches = False
cb53e1f8 2262 self.detectLabels = False
06804c76 2263 self.importLabels = False
b984733c 2264 self.changesFile = ""
01265103 2265 self.syncWithOrigin = True
a028a98e 2266 self.importIntoRemotes = True
01a9c9c5 2267 self.maxChanges = ""
1051ef00 2268 self.changes_block_size = None
8b41a97f 2269 self.keepRepoPath = False
6326aa58 2270 self.depotPaths = None
3c699645 2271 self.p4BranchesInGit = []
354081d5 2272 self.cloneExclude = []
3a70cdfa 2273 self.useClientSpec = False
a93d33ee 2274 self.useClientSpec_from_options = False
ecb7cf98 2275 self.clientSpecDirs = None
fed23693 2276 self.tempBranches = []
d604176d 2277 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12
LS
2278 self.largeFileSystem = None
2279
2280 if gitConfig('git-p4.largeFileSystem'):
2281 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2282 self.largeFileSystem = largeFileSystemConstructor(
2283 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2284 )
b984733c 2285
01265103
SH
2286 if gitConfig("git-p4.syncFromOrigin") == "false":
2287 self.syncWithOrigin = False
2288
51334bb0
LD
2289 # This is required for the "append" cloneExclude action
2290 def ensure_value(self, attr, value):
2291 if not hasattr(self, attr) or getattr(self, attr) is None:
2292 setattr(self, attr, value)
2293 return getattr(self, attr)
2294
fed23693
VA
2295 # Force a checkpoint in fast-import and wait for it to finish
2296 def checkpoint(self):
2297 self.gitStream.write("checkpoint\n\n")
2298 self.gitStream.write("progress checkpoint\n\n")
2299 out = self.gitOutput.readline()
2300 if self.verbose:
2301 print "checkpoint finished: " + out
2302
b984733c 2303 def extractFilesFromCommit(self, commit):
354081d5
TT
2304 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2305 for path in self.cloneExclude]
b984733c
SH
2306 files = []
2307 fnum = 0
2308 while commit.has_key("depotFile%s" % fnum):
2309 path = commit["depotFile%s" % fnum]
6326aa58 2310
354081d5 2311 if [p for p in self.cloneExclude
d53de8b9 2312 if p4PathStartsWith(path, p)]:
354081d5
TT
2313 found = False
2314 else:
2315 found = [p for p in self.depotPaths
d53de8b9 2316 if p4PathStartsWith(path, p)]
6326aa58 2317 if not found:
b984733c
SH
2318 fnum = fnum + 1
2319 continue
2320
2321 file = {}
2322 file["path"] = path
2323 file["rev"] = commit["rev%s" % fnum]
2324 file["action"] = commit["action%s" % fnum]
2325 file["type"] = commit["type%s" % fnum]
2326 files.append(file)
2327 fnum = fnum + 1
2328 return files
2329
26e6a27d
JD
2330 def extractJobsFromCommit(self, commit):
2331 jobs = []
2332 jnum = 0
2333 while commit.has_key("job%s" % jnum):
2334 job = commit["job%s" % jnum]
2335 jobs.append(job)
2336 jnum = jnum + 1
2337 return jobs
2338
6326aa58 2339 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2340 """When streaming files, this is called to map a p4 depot path
2341 to where it should go in git. The prefixes are either
2342 self.depotPaths, or self.branchPrefixes in the case of
2343 branch detection."""
2344
3952710b 2345 if self.useClientSpec:
21ef5df4
PW
2346 # branch detection moves files up a level (the branch name)
2347 # from what client spec interpretation gives
0d1696ef 2348 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2349 if self.detectBranches:
2350 for b in self.knownBranches:
2351 if path.startswith(b + "/"):
2352 path = path[len(b)+1:]
2353
2354 elif self.keepRepoPath:
2355 # Preserve everything in relative path name except leading
2356 # //depot/; just look at first prefix as they all should
2357 # be in the same depot.
2358 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2359 if p4PathStartsWith(path, depot):
2360 path = path[len(depot):]
3952710b 2361
0d1696ef 2362 else:
0d1696ef
PW
2363 for p in prefixes:
2364 if p4PathStartsWith(path, p):
2365 path = path[len(p):]
21ef5df4 2366 break
8b41a97f 2367
0d1696ef 2368 path = wildcard_decode(path)
6326aa58 2369 return path
6754a299 2370
71b112d4 2371 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2372 """Look at each depotFile in the commit to figure out to what
2373 branch it belongs."""
2374
9d57c4a6
KS
2375 if self.clientSpecDirs:
2376 files = self.extractFilesFromCommit(commit)
2377 self.clientSpecDirs.update_client_spec_path_cache(files)
2378
d5904674 2379 branches = {}
71b112d4
SH
2380 fnum = 0
2381 while commit.has_key("depotFile%s" % fnum):
2382 path = commit["depotFile%s" % fnum]
6326aa58 2383 found = [p for p in self.depotPaths
d53de8b9 2384 if p4PathStartsWith(path, p)]
6326aa58 2385 if not found:
71b112d4
SH
2386 fnum = fnum + 1
2387 continue
2388
2389 file = {}
2390 file["path"] = path
2391 file["rev"] = commit["rev%s" % fnum]
2392 file["action"] = commit["action%s" % fnum]
2393 file["type"] = commit["type%s" % fnum]
2394 fnum = fnum + 1
2395
21ef5df4
PW
2396 # start with the full relative path where this file would
2397 # go in a p4 client
2398 if self.useClientSpec:
2399 relPath = self.clientSpecDirs.map_in_client(path)
2400 else:
2401 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2402
4b97ffb1 2403 for branch in self.knownBranches.keys():
21ef5df4
PW
2404 # add a trailing slash so that a commit into qt/4.2foo
2405 # doesn't end up in qt/4.2, e.g.
6754a299 2406 if relPath.startswith(branch + "/"):
d5904674
SH
2407 if branch not in branches:
2408 branches[branch] = []
71b112d4 2409 branches[branch].append(file)
6555b2cc 2410 break
b984733c
SH
2411
2412 return branches
2413
a5db4b12
LS
2414 def writeToGitStream(self, gitMode, relPath, contents):
2415 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2416 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2417 for d in contents:
2418 self.gitStream.write(d)
2419 self.gitStream.write('\n')
2420
b932705b
LD
2421 # output one file from the P4 stream
2422 # - helper for streamP4Files
2423
2424 def streamOneP4File(self, file, contents):
b932705b
LD
2425 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2426 if verbose:
d2176a50
LS
2427 size = int(self.stream_file['fileSize'])
2428 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2429 sys.stdout.flush()
b932705b 2430
9cffb8c8
PW
2431 (type_base, type_mods) = split_p4_type(file["type"])
2432
2433 git_mode = "100644"
2434 if "x" in type_mods:
2435 git_mode = "100755"
2436 if type_base == "symlink":
2437 git_mode = "120000"
1292df11
AJ
2438 # p4 print on a symlink sometimes contains "target\n";
2439 # if it does, remove the newline
b39c3612 2440 data = ''.join(contents)
40f846c3
PW
2441 if not data:
2442 # Some version of p4 allowed creating a symlink that pointed
2443 # to nothing. This causes p4 errors when checking out such
2444 # a change, and errors here too. Work around it by ignoring
2445 # the bad symlink; hopefully a future change fixes it.
2446 print "\nIgnoring empty symlink in %s" % file['depotFile']
2447 return
2448 elif data[-1] == '\n':
1292df11
AJ
2449 contents = [data[:-1]]
2450 else:
2451 contents = [data]
b932705b 2452
9cffb8c8 2453 if type_base == "utf16":
55aa5714
PW
2454 # p4 delivers different text in the python output to -G
2455 # than it does when using "print -o", or normal p4 client
2456 # operations. utf16 is converted to ascii or utf8, perhaps.
2457 # But ascii text saved as -t utf16 is completely mangled.
2458 # Invoke print -o to get the real contents.
7f0e5962
PW
2459 #
2460 # On windows, the newlines will always be mangled by print, so put
2461 # them back too. This is not needed to the cygwin windows version,
2462 # just the native "NT" type.
2463 #
1f5f3907
LS
2464 try:
2465 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2466 except Exception as e:
2467 if 'Translation of file content failed' in str(e):
2468 type_base = 'binary'
2469 else:
2470 raise e
2471 else:
2472 if p4_version_string().find('/NT') >= 0:
2473 text = text.replace('\r\n', '\n')
2474 contents = [ text ]
55aa5714 2475
9f7ef0ea
PW
2476 if type_base == "apple":
2477 # Apple filetype files will be streamed as a concatenation of
2478 # its appledouble header and the contents. This is useless
2479 # on both macs and non-macs. If using "print -q -o xx", it
2480 # will create "xx" with the data, and "%xx" with the header.
2481 # This is also not very useful.
2482 #
2483 # Ideally, someday, this script can learn how to generate
2484 # appledouble files directly and import those to git, but
2485 # non-mac machines can never find a use for apple filetype.
2486 print "\nIgnoring apple filetype file %s" % file['depotFile']
2487 return
2488
55aa5714
PW
2489 # Note that we do not try to de-mangle keywords on utf16 files,
2490 # even though in theory somebody may want that.
60df071c
LD
2491 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2492 if pattern:
2493 regexp = re.compile(pattern, re.VERBOSE)
2494 text = ''.join(contents)
2495 text = regexp.sub(r'$\1$', text)
2496 contents = [ text ]
b932705b 2497
00a9403a
LS
2498 try:
2499 relPath.decode('ascii')
2500 except:
2501 encoding = 'utf8'
2502 if gitConfig('git-p4.pathEncoding'):
2503 encoding = gitConfig('git-p4.pathEncoding')
4cb870d8 2504 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
00a9403a
LS
2505 if self.verbose:
2506 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
a9e38359 2507
a5db4b12
LS
2508 if self.largeFileSystem:
2509 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2510
a5db4b12 2511 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2512
2513 def streamOneP4Deletion(self, file):
2514 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2515 if verbose:
d2176a50
LS
2516 sys.stdout.write("delete %s\n" % relPath)
2517 sys.stdout.flush()
b932705b
LD
2518 self.gitStream.write("D %s\n" % relPath)
2519
a5db4b12
LS
2520 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2521 self.largeFileSystem.removeLargeFile(relPath)
2522
b932705b
LD
2523 # handle another chunk of streaming data
2524 def streamP4FilesCb(self, marshalled):
2525
78189bea
PW
2526 # catch p4 errors and complain
2527 err = None
2528 if "code" in marshalled:
2529 if marshalled["code"] == "error":
2530 if "data" in marshalled:
2531 err = marshalled["data"].rstrip()
4d25dc44
LS
2532
2533 if not err and 'fileSize' in self.stream_file:
2534 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2535 if required_bytes > 0:
2536 err = 'Not enough space left on %s! Free at least %i MB.' % (
2537 os.getcwd(), required_bytes/1024/1024
2538 )
2539
78189bea
PW
2540 if err:
2541 f = None
2542 if self.stream_have_file_info:
2543 if "depotFile" in self.stream_file:
2544 f = self.stream_file["depotFile"]
2545 # force a failure in fast-import, else an empty
2546 # commit will be made
2547 self.gitStream.write("\n")
2548 self.gitStream.write("die-now\n")
2549 self.gitStream.close()
2550 # ignore errors, but make sure it exits first
2551 self.importProcess.wait()
2552 if f:
2553 die("Error from p4 print for %s: %s" % (f, err))
2554 else:
2555 die("Error from p4 print: %s" % err)
2556
c3f6163b
AG
2557 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2558 # start of a new file - output the old one first
2559 self.streamOneP4File(self.stream_file, self.stream_contents)
2560 self.stream_file = {}
2561 self.stream_contents = []
2562 self.stream_have_file_info = False
b932705b 2563
c3f6163b
AG
2564 # pick up the new file information... for the
2565 # 'data' field we need to append to our array
2566 for k in marshalled.keys():
2567 if k == 'data':
d2176a50
LS
2568 if 'streamContentSize' not in self.stream_file:
2569 self.stream_file['streamContentSize'] = 0
2570 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2571 self.stream_contents.append(marshalled['data'])
2572 else:
2573 self.stream_file[k] = marshalled[k]
b932705b 2574
d2176a50
LS
2575 if (verbose and
2576 'streamContentSize' in self.stream_file and
2577 'fileSize' in self.stream_file and
2578 'depotFile' in self.stream_file):
2579 size = int(self.stream_file["fileSize"])
2580 if size > 0:
2581 progress = 100*self.stream_file['streamContentSize']/size
2582 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2583 sys.stdout.flush()
2584
c3f6163b 2585 self.stream_have_file_info = True
b932705b
LD
2586
2587 # Stream directly from "p4 files" into "git fast-import"
2588 def streamP4Files(self, files):
30b5940b
SH
2589 filesForCommit = []
2590 filesToRead = []
b932705b 2591 filesToDelete = []
30b5940b 2592
3a70cdfa 2593 for f in files:
ecb7cf98
PW
2594 filesForCommit.append(f)
2595 if f['action'] in self.delete_actions:
2596 filesToDelete.append(f)
2597 else:
2598 filesToRead.append(f)
6a49f8e2 2599
b932705b
LD
2600 # deleted files...
2601 for f in filesToDelete:
2602 self.streamOneP4Deletion(f)
1b9a4684 2603
b932705b
LD
2604 if len(filesToRead) > 0:
2605 self.stream_file = {}
2606 self.stream_contents = []
2607 self.stream_have_file_info = False
8ff45f2a 2608
c3f6163b
AG
2609 # curry self argument
2610 def streamP4FilesCbSelf(entry):
2611 self.streamP4FilesCb(entry)
6a49f8e2 2612
6de040df
LD
2613 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2614
2615 p4CmdList(["-x", "-", "print"],
2616 stdin=fileArgs,
2617 cb=streamP4FilesCbSelf)
30b5940b 2618
b932705b
LD
2619 # do the last chunk
2620 if self.stream_file.has_key('depotFile'):
2621 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2622
affb474f
LD
2623 def make_email(self, userid):
2624 if userid in self.users:
2625 return self.users[userid]
2626 else:
2627 return "%s <a@b>" % userid
2628
06804c76 2629 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2630 """ Stream a p4 tag.
2631 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2632 """
2633
06804c76
LD
2634 if verbose:
2635 print "writing tag %s for commit %s" % (labelName, commit)
2636 gitStream.write("tag %s\n" % labelName)
2637 gitStream.write("from %s\n" % commit)
2638
2639 if labelDetails.has_key('Owner'):
2640 owner = labelDetails["Owner"]
2641 else:
2642 owner = None
2643
2644 # Try to use the owner of the p4 label, or failing that,
2645 # the current p4 user id.
2646 if owner:
2647 email = self.make_email(owner)
2648 else:
2649 email = self.make_email(self.p4UserId())
2650 tagger = "%s %s %s" % (email, epoch, self.tz)
2651
2652 gitStream.write("tagger %s\n" % tagger)
2653
2654 print "labelDetails=",labelDetails
2655 if labelDetails.has_key('Description'):
2656 description = labelDetails['Description']
2657 else:
2658 description = 'Label from git p4'
2659
2660 gitStream.write("data %d\n" % len(description))
2661 gitStream.write(description)
2662 gitStream.write("\n")
2663
4ae048e6
LS
2664 def inClientSpec(self, path):
2665 if not self.clientSpecDirs:
2666 return True
2667 inClientSpec = self.clientSpecDirs.map_in_client(path)
2668 if not inClientSpec and self.verbose:
2669 print('Ignoring file outside of client spec: {0}'.format(path))
2670 return inClientSpec
2671
2672 def hasBranchPrefix(self, path):
2673 if not self.branchPrefixes:
2674 return True
2675 hasPrefix = [p for p in self.branchPrefixes
2676 if p4PathStartsWith(path, p)]
09667d01 2677 if not hasPrefix and self.verbose:
4ae048e6
LS
2678 print('Ignoring file outside of prefix: {0}'.format(path))
2679 return hasPrefix
2680
e63231e5 2681 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2682 epoch = details["time"]
2683 author = details["user"]
26e6a27d 2684 jobs = self.extractJobsFromCommit(details)
b984733c 2685
4b97ffb1 2686 if self.verbose:
4ae048e6 2687 print('commit into {0}'.format(branch))
96e07dd2 2688
9d57c4a6
KS
2689 if self.clientSpecDirs:
2690 self.clientSpecDirs.update_client_spec_path_cache(files)
2691
4ae048e6
LS
2692 files = [f for f in files
2693 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2694
2695 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2696 print('Ignoring revision {0} as it would produce an empty commit.'
2697 .format(details['change']))
2698 return
2699
b984733c 2700 self.gitStream.write("commit %s\n" % branch)
b43702ac 2701 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2702 self.committedChanges.add(int(details["change"]))
2703 committer = ""
b607e71e
SH
2704 if author not in self.users:
2705 self.getUserMapFromPerforceServer()
affb474f 2706 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2707
2708 self.gitStream.write("committer %s\n" % committer)
2709
2710 self.gitStream.write("data <<EOT\n")
2711 self.gitStream.write(details["desc"])
26e6a27d
JD
2712 if len(jobs) > 0:
2713 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
e63231e5
PW
2714 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2715 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2716 if len(details['options']) > 0:
2717 self.gitStream.write(": options = %s" % details['options'])
2718 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2719
2720 if len(parent) > 0:
4b97ffb1
SH
2721 if self.verbose:
2722 print "parent %s" % parent
b984733c
SH
2723 self.gitStream.write("from %s\n" % parent)
2724
4ae048e6 2725 self.streamP4Files(files)
b984733c
SH
2726 self.gitStream.write("\n")
2727
1f4ba1cb
SH
2728 change = int(details["change"])
2729
9bda3a85 2730 if self.labels.has_key(change):
1f4ba1cb
SH
2731 label = self.labels[change]
2732 labelDetails = label[0]
2733 labelRevisions = label[1]
71b112d4
SH
2734 if self.verbose:
2735 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2736
6de040df 2737 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2738 for p in self.branchPrefixes])
1f4ba1cb
SH
2739
2740 if len(files) == len(labelRevisions):
2741
2742 cleanedFiles = {}
2743 for info in files:
56c09345 2744 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2745 continue
2746 cleanedFiles[info["depotFile"]] = info["rev"]
2747
2748 if cleanedFiles == labelRevisions:
06804c76 2749 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2750
2751 else:
a46668fa 2752 if not self.silent:
cebdf5af
HWN
2753 print ("Tag %s does not match with change %s: files do not match."
2754 % (labelDetails["label"], change))
1f4ba1cb
SH
2755
2756 else:
a46668fa 2757 if not self.silent:
cebdf5af
HWN
2758 print ("Tag %s does not match with change %s: file count is different."
2759 % (labelDetails["label"], change))
b984733c 2760
06804c76 2761 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2762 def getLabels(self):
2763 self.labels = {}
2764
52a4880b 2765 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2766 if len(l) > 0 and not self.silent:
183f8436 2767 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2768
2769 for output in l:
1f4ba1cb
SH
2770 label = output["label"]
2771 revisions = {}
2772 newestChange = 0
71b112d4
SH
2773 if self.verbose:
2774 print "Querying files for label %s" % label
6de040df
LD
2775 for file in p4CmdList(["files"] +
2776 ["%s...@%s" % (p, label)
2777 for p in self.depotPaths]):
1f4ba1cb
SH
2778 revisions[file["depotFile"]] = file["rev"]
2779 change = int(file["change"])
2780 if change > newestChange:
2781 newestChange = change
2782
9bda3a85
SH
2783 self.labels[newestChange] = [output, revisions]
2784
2785 if self.verbose:
2786 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2787
06804c76
LD
2788 # Import p4 labels as git tags. A direct mapping does not
2789 # exist, so assume that if all the files are at the same revision
2790 # then we can use that, or it's something more complicated we should
2791 # just ignore.
2792 def importP4Labels(self, stream, p4Labels):
2793 if verbose:
2794 print "import p4 labels: " + ' '.join(p4Labels)
2795
2796 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2797 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2798 if len(validLabelRegexp) == 0:
2799 validLabelRegexp = defaultLabelRegexp
2800 m = re.compile(validLabelRegexp)
2801
2802 for name in p4Labels:
2803 commitFound = False
2804
2805 if not m.match(name):
2806 if verbose:
2807 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2808 continue
2809
2810 if name in ignoredP4Labels:
2811 continue
2812
2813 labelDetails = p4CmdList(['label', "-o", name])[0]
2814
2815 # get the most recent changelist for each file in this label
2816 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2817 for p in self.depotPaths])
2818
2819 if change.has_key('change'):
2820 # find the corresponding git commit; take the oldest commit
2821 changelist = int(change['change'])
b43702ac
LD
2822 if changelist in self.committedChanges:
2823 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2824 commitFound = True
b43702ac
LD
2825 else:
2826 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2827 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2828 if len(gitCommit) == 0:
2829 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2830 else:
2831 commitFound = True
2832 gitCommit = gitCommit.strip()
2833
2834 if commitFound:
06804c76
LD
2835 # Convert from p4 time format
2836 try:
2837 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2838 except ValueError:
a4e9054c 2839 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2840 tmwhen = 1
2841
2842 when = int(time.mktime(tmwhen))
2843 self.streamTag(stream, name, labelDetails, gitCommit, when)
2844 if verbose:
2845 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2846 else:
2847 if verbose:
2848 print "Label %s has no changelists - possibly deleted?" % name
2849
2850 if not commitFound:
2851 # We can't import this label; don't try again as it will get very
2852 # expensive repeatedly fetching all the files for labels that will
2853 # never be imported. If the label is moved in the future, the
2854 # ignore will need to be removed manually.
2855 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2856
86dff6b6
HWN
2857 def guessProjectName(self):
2858 for p in self.depotPaths:
6e5295c4
SH
2859 if p.endswith("/"):
2860 p = p[:-1]
2861 p = p[p.strip().rfind("/") + 1:]
2862 if not p.endswith("/"):
2863 p += "/"
2864 return p
86dff6b6 2865
4b97ffb1 2866 def getBranchMapping(self):
6555b2cc
SH
2867 lostAndFoundBranches = set()
2868
8ace74c0
VA
2869 user = gitConfig("git-p4.branchUser")
2870 if len(user) > 0:
2871 command = "branches -u %s" % user
2872 else:
2873 command = "branches"
2874
2875 for info in p4CmdList(command):
52a4880b 2876 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2877 viewIdx = 0
2878 while details.has_key("View%s" % viewIdx):
2879 paths = details["View%s" % viewIdx].split(" ")
2880 viewIdx = viewIdx + 1
2881 # require standard //depot/foo/... //depot/bar/... mapping
2882 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2883 continue
2884 source = paths[0]
2885 destination = paths[1]
6509e19c 2886 ## HACK
d53de8b9 2887 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2888 source = source[len(self.depotPaths[0]):-4]
2889 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2890
1a2edf4e
SH
2891 if destination in self.knownBranches:
2892 if not self.silent:
2893 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2894 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2895 continue
2896
6555b2cc
SH
2897 self.knownBranches[destination] = source
2898
2899 lostAndFoundBranches.discard(destination)
2900
29bdbac1 2901 if source not in self.knownBranches:
6555b2cc
SH
2902 lostAndFoundBranches.add(source)
2903
7199cf13
VA
2904 # Perforce does not strictly require branches to be defined, so we also
2905 # check git config for a branch list.
2906 #
2907 # Example of branch definition in git config file:
2908 # [git-p4]
2909 # branchList=main:branchA
2910 # branchList=main:branchB
2911 # branchList=branchA:branchC
2912 configBranches = gitConfigList("git-p4.branchList")
2913 for branch in configBranches:
2914 if branch:
2915 (source, destination) = branch.split(":")
2916 self.knownBranches[destination] = source
2917
2918 lostAndFoundBranches.discard(destination)
2919
2920 if source not in self.knownBranches:
2921 lostAndFoundBranches.add(source)
2922
6555b2cc
SH
2923
2924 for branch in lostAndFoundBranches:
2925 self.knownBranches[branch] = branch
29bdbac1 2926
38f9f5ec
SH
2927 def getBranchMappingFromGitBranches(self):
2928 branches = p4BranchesInGit(self.importIntoRemotes)
2929 for branch in branches.keys():
2930 if branch == "master":
2931 branch = "main"
2932 else:
2933 branch = branch[len(self.projectName):]
2934 self.knownBranches[branch] = branch
2935
bb6e09b2
HWN
2936 def updateOptionDict(self, d):
2937 option_keys = {}
2938 if self.keepRepoPath:
2939 option_keys['keepRepoPath'] = 1
2940
2941 d["options"] = ' '.join(sorted(option_keys.keys()))
2942
2943 def readOptions(self, d):
2944 self.keepRepoPath = (d.has_key('options')
2945 and ('keepRepoPath' in d['options']))
6326aa58 2946
8134f69c
SH
2947 def gitRefForBranch(self, branch):
2948 if branch == "main":
2949 return self.refPrefix + "master"
2950
2951 if len(branch) <= 0:
2952 return branch
2953
2954 return self.refPrefix + self.projectName + branch
2955
1ca3d710
SH
2956 def gitCommitByP4Change(self, ref, change):
2957 if self.verbose:
2958 print "looking in ref " + ref + " for change %s using bisect..." % change
2959
2960 earliestCommit = ""
2961 latestCommit = parseRevision(ref)
2962
2963 while True:
2964 if self.verbose:
2965 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2966 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2967 if len(next) == 0:
2968 if self.verbose:
2969 print "argh"
2970 return ""
2971 log = extractLogMessageFromGitCommit(next)
2972 settings = extractSettingsGitLog(log)
2973 currentChange = int(settings['change'])
2974 if self.verbose:
2975 print "current change %s" % currentChange
2976
2977 if currentChange == change:
2978 if self.verbose:
2979 print "found %s" % next
2980 return next
2981
2982 if currentChange < change:
2983 earliestCommit = "^%s" % next
2984 else:
2985 latestCommit = "%s" % next
2986
2987 return ""
2988
2989 def importNewBranch(self, branch, maxChange):
2990 # make fast-import flush all changes to disk and update the refs using the checkpoint
2991 # command so that we can try to find the branch parent in the git history
2992 self.gitStream.write("checkpoint\n\n");
2993 self.gitStream.flush();
2994 branchPrefix = self.depotPaths[0] + branch + "/"
2995 range = "@1,%s" % maxChange
2996 #print "prefix" + branchPrefix
96b2d54a 2997 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
2998 if len(changes) <= 0:
2999 return False
3000 firstChange = changes[0]
3001 #print "first change in branch: %s" % firstChange
3002 sourceBranch = self.knownBranches[branch]
3003 sourceDepotPath = self.depotPaths[0] + sourceBranch
3004 sourceRef = self.gitRefForBranch(sourceBranch)
3005 #print "source " + sourceBranch
3006
52a4880b 3007 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3008 #print "branch parent: %s" % branchParentChange
3009 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3010 if len(gitParent) > 0:
3011 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3012 #print "parent git commit: %s" % gitParent
3013
3014 self.importChanges(changes)
3015 return True
3016
fed23693
VA
3017 def searchParent(self, parent, branch, target):
3018 parentFound = False
c7d34884
PW
3019 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3020 "--no-merges", parent]):
fed23693
VA
3021 blob = blob.strip()
3022 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3023 parentFound = True
3024 if self.verbose:
3025 print "Found parent of %s in commit %s" % (branch, blob)
3026 break
3027 if parentFound:
3028 return blob
3029 else:
3030 return None
3031
e87f37ae
SH
3032 def importChanges(self, changes):
3033 cnt = 1
3034 for change in changes:
18fa13d0 3035 description = p4_describe(change)
e87f37ae
SH
3036 self.updateOptionDict(description)
3037
3038 if not self.silent:
3039 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3040 sys.stdout.flush()
3041 cnt = cnt + 1
3042
3043 try:
3044 if self.detectBranches:
3045 branches = self.splitFilesIntoBranches(description)
3046 for branch in branches.keys():
3047 ## HACK --hwn
3048 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3049 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3050
3051 parent = ""
3052
3053 filesForCommit = branches[branch]
3054
3055 if self.verbose:
3056 print "branch is %s" % branch
3057
3058 self.updatedBranches.add(branch)
3059
3060 if branch not in self.createdBranches:
3061 self.createdBranches.add(branch)
3062 parent = self.knownBranches[branch]
3063 if parent == branch:
3064 parent = ""
1ca3d710
SH
3065 else:
3066 fullBranch = self.projectName + branch
3067 if fullBranch not in self.p4BranchesInGit:
3068 if not self.silent:
3069 print("\n Importing new branch %s" % fullBranch);
3070 if self.importNewBranch(branch, change - 1):
3071 parent = ""
3072 self.p4BranchesInGit.append(fullBranch)
3073 if not self.silent:
3074 print("\n Resuming with change %s" % change);
3075
3076 if self.verbose:
3077 print "parent determined through known branches: %s" % parent
e87f37ae 3078
8134f69c
SH
3079 branch = self.gitRefForBranch(branch)
3080 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3081
3082 if self.verbose:
3083 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3084
3085 if len(parent) == 0 and branch in self.initialParents:
3086 parent = self.initialParents[branch]
3087 del self.initialParents[branch]
3088
fed23693
VA
3089 blob = None
3090 if len(parent) > 0:
4f9273d2 3091 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3092 if self.verbose:
3093 print "Creating temporary branch: " + tempBranch
e63231e5 3094 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3095 self.tempBranches.append(tempBranch)
3096 self.checkpoint()
3097 blob = self.searchParent(parent, branch, tempBranch)
3098 if blob:
e63231e5 3099 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3100 else:
3101 if self.verbose:
3102 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3103 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3104 else:
3105 files = self.extractFilesFromCommit(description)
e63231e5 3106 self.commit(description, files, self.branch,
e87f37ae 3107 self.initialParent)
47497844 3108 # only needed once, to connect to the previous commit
e87f37ae
SH
3109 self.initialParent = ""
3110 except IOError:
3111 print self.gitError.read()
3112 sys.exit(1)
3113
c208a243
SH
3114 def importHeadRevision(self, revision):
3115 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3116
4e2e6ce4
PW
3117 details = {}
3118 details["user"] = "git perforce import user"
1494fcbb 3119 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3120 % (' '.join(self.depotPaths), revision))
3121 details["change"] = revision
3122 newestRevision = 0
3123
3124 fileCnt = 0
6de040df
LD
3125 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3126
3127 for info in p4CmdList(["files"] + fileArgs):
c208a243 3128
68b28593 3129 if 'code' in info and info['code'] == 'error':
c208a243
SH
3130 sys.stderr.write("p4 returned an error: %s\n"
3131 % info['data'])
d88e707f
PW
3132 if info['data'].find("must refer to client") >= 0:
3133 sys.stderr.write("This particular p4 error is misleading.\n")
3134 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3135 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3136 sys.exit(1)
68b28593
PW
3137 if 'p4ExitCode' in info:
3138 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3139 sys.exit(1)
3140
3141
3142 change = int(info["change"])
3143 if change > newestRevision:
3144 newestRevision = change
3145
56c09345 3146 if info["action"] in self.delete_actions:
c208a243
SH
3147 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3148 #fileCnt = fileCnt + 1
3149 continue
3150
3151 for prop in ["depotFile", "rev", "action", "type" ]:
3152 details["%s%s" % (prop, fileCnt)] = info[prop]
3153
3154 fileCnt = fileCnt + 1
3155
3156 details["change"] = newestRevision
4e2e6ce4 3157
9dcb9f24 3158 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3159 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3160 res = p4_describe(newestRevision)
3161 details["time"] = res["time"]
4e2e6ce4 3162
c208a243
SH
3163 self.updateOptionDict(details)
3164 try:
e63231e5 3165 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3166 except IOError:
3167 print "IO error with git fast-import. Is your git version recent enough?"
3168 print self.gitError.read()
3169
3170
b984733c 3171 def run(self, args):
6326aa58 3172 self.depotPaths = []
179caebf 3173 self.changeRange = ""
6326aa58 3174 self.previousDepotPaths = []
991a2de4 3175 self.hasOrigin = False
ce6f33c8 3176
29bdbac1
SH
3177 # map from branch depot path to parent branch
3178 self.knownBranches = {}
3179 self.initialParents = {}
3180
a028a98e
SH
3181 if self.importIntoRemotes:
3182 self.refPrefix = "refs/remotes/p4/"
3183 else:
db775559 3184 self.refPrefix = "refs/heads/p4/"
a028a98e 3185
991a2de4
PW
3186 if self.syncWithOrigin:
3187 self.hasOrigin = originP4BranchesExist()
3188 if self.hasOrigin:
3189 if not self.silent:
3190 print 'Syncing with origin first, using "git fetch origin"'
3191 system("git fetch origin")
10f880f8 3192
5a8e84cd 3193 branch_arg_given = bool(self.branch)
569d1bd4 3194 if len(self.branch) == 0:
db775559 3195 self.branch = self.refPrefix + "master"
a028a98e 3196 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3197 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3198 system("git branch -D p4")
967f72e2 3199
a93d33ee
PW
3200 # accept either the command-line option, or the configuration variable
3201 if self.useClientSpec:
3202 # will use this after clone to set the variable
3203 self.useClientSpec_from_options = True
3204 else:
0d609032 3205 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3206 self.useClientSpec = True
3207 if self.useClientSpec:
543987bd 3208 self.clientSpecDirs = getClientSpec()
3a70cdfa 3209
6a49f8e2
HWN
3210 # TODO: should always look at previous commits,
3211 # merge with previous imports, if possible.
3212 if args == []:
d414c74a 3213 if self.hasOrigin:
5ca44617 3214 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3215
3216 # branches holds mapping from branch name to sha1
3217 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3218
3219 # restrict to just this one, disabling detect-branches
3220 if branch_arg_given:
3221 short = self.branch.split("/")[-1]
3222 if short in branches:
3223 self.p4BranchesInGit = [ short ]
3224 else:
3225 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3226
3227 if len(self.p4BranchesInGit) > 1:
3228 if not self.silent:
3229 print "Importing from/into multiple branches"
3230 self.detectBranches = True
8c9e8b6e
PW
3231 for branch in branches.keys():
3232 self.initialParents[self.refPrefix + branch] = \
3233 branches[branch]
967f72e2 3234
29bdbac1
SH
3235 if self.verbose:
3236 print "branches: %s" % self.p4BranchesInGit
3237
3238 p4Change = 0
3239 for branch in self.p4BranchesInGit:
cebdf5af 3240 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3241
3242 settings = extractSettingsGitLog(logMsg)
29bdbac1 3243
bb6e09b2
HWN
3244 self.readOptions(settings)
3245 if (settings.has_key('depot-paths')
3246 and settings.has_key ('change')):
3247 change = int(settings['change']) + 1
29bdbac1
SH
3248 p4Change = max(p4Change, change)
3249
bb6e09b2
HWN
3250 depotPaths = sorted(settings['depot-paths'])
3251 if self.previousDepotPaths == []:
6326aa58 3252 self.previousDepotPaths = depotPaths
29bdbac1 3253 else:
6326aa58
HWN
3254 paths = []
3255 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3256 prev_list = prev.split("/")
3257 cur_list = cur.split("/")
3258 for i in range(0, min(len(cur_list), len(prev_list))):
3259 if cur_list[i] <> prev_list[i]:
583e1707 3260 i = i - 1
6326aa58
HWN
3261 break
3262
04d277b3 3263 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3264
3265 self.previousDepotPaths = paths
29bdbac1
SH
3266
3267 if p4Change > 0:
bb6e09b2 3268 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3269 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3270 if not self.silent and not self.detectBranches:
967f72e2 3271 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3272
40d69ac3
PW
3273 # accept multiple ref name abbreviations:
3274 # refs/foo/bar/branch -> use it exactly
3275 # p4/branch -> prepend refs/remotes/ or refs/heads/
3276 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3277 if not self.branch.startswith("refs/"):
40d69ac3
PW
3278 if self.importIntoRemotes:
3279 prepend = "refs/remotes/"
3280 else:
3281 prepend = "refs/heads/"
3282 if not self.branch.startswith("p4/"):
3283 prepend += "p4/"
3284 self.branch = prepend + self.branch
179caebf 3285
6326aa58 3286 if len(args) == 0 and self.depotPaths:
b984733c 3287 if not self.silent:
6326aa58 3288 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3289 else:
6326aa58 3290 if self.depotPaths and self.depotPaths != args:
cebdf5af 3291 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3292 "This doesn't work!" % (' '.join (self.depotPaths),
3293 ' '.join (args)))
b984733c 3294 sys.exit(1)
6326aa58 3295
bb6e09b2 3296 self.depotPaths = sorted(args)
b984733c 3297
1c49fc19 3298 revision = ""
b984733c 3299 self.users = {}
b984733c 3300
58c8bc7c
PW
3301 # Make sure no revision specifiers are used when --changesfile
3302 # is specified.
3303 bad_changesfile = False
3304 if len(self.changesFile) > 0:
3305 for p in self.depotPaths:
3306 if p.find("@") >= 0 or p.find("#") >= 0:
3307 bad_changesfile = True
3308 break
3309 if bad_changesfile:
3310 die("Option --changesfile is incompatible with revision specifiers")
3311
6326aa58
HWN
3312 newPaths = []
3313 for p in self.depotPaths:
3314 if p.find("@") != -1:
3315 atIdx = p.index("@")
3316 self.changeRange = p[atIdx:]
3317 if self.changeRange == "@all":
3318 self.changeRange = ""
6a49f8e2 3319 elif ',' not in self.changeRange:
1c49fc19 3320 revision = self.changeRange
6326aa58 3321 self.changeRange = ""
7fcff9de 3322 p = p[:atIdx]
6326aa58
HWN
3323 elif p.find("#") != -1:
3324 hashIdx = p.index("#")
1c49fc19 3325 revision = p[hashIdx:]
7fcff9de 3326 p = p[:hashIdx]
6326aa58 3327 elif self.previousDepotPaths == []:
58c8bc7c
PW
3328 # pay attention to changesfile, if given, else import
3329 # the entire p4 tree at the head revision
3330 if len(self.changesFile) == 0:
3331 revision = "#head"
6326aa58
HWN
3332
3333 p = re.sub ("\.\.\.$", "", p)
3334 if not p.endswith("/"):
3335 p += "/"
3336
3337 newPaths.append(p)
3338
3339 self.depotPaths = newPaths
3340
e63231e5
PW
3341 # --detect-branches may change this for each branch
3342 self.branchPrefixes = self.depotPaths
3343
b607e71e 3344 self.loadUserMapFromCache()
cb53e1f8
SH
3345 self.labels = {}
3346 if self.detectLabels:
3347 self.getLabels();
b984733c 3348
4b97ffb1 3349 if self.detectBranches:
df450923
SH
3350 ## FIXME - what's a P4 projectName ?
3351 self.projectName = self.guessProjectName()
3352
38f9f5ec
SH
3353 if self.hasOrigin:
3354 self.getBranchMappingFromGitBranches()
3355 else:
3356 self.getBranchMapping()
29bdbac1
SH
3357 if self.verbose:
3358 print "p4-git branches: %s" % self.p4BranchesInGit
3359 print "initial parents: %s" % self.initialParents
3360 for b in self.p4BranchesInGit:
3361 if b != "master":
6326aa58
HWN
3362
3363 ## FIXME
29bdbac1
SH
3364 b = b[len(self.projectName):]
3365 self.createdBranches.add(b)
4b97ffb1 3366
f291b4e3 3367 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3368
78189bea
PW
3369 self.importProcess = subprocess.Popen(["git", "fast-import"],
3370 stdin=subprocess.PIPE,
3371 stdout=subprocess.PIPE,
3372 stderr=subprocess.PIPE);
3373 self.gitOutput = self.importProcess.stdout
3374 self.gitStream = self.importProcess.stdin
3375 self.gitError = self.importProcess.stderr
b984733c 3376
1c49fc19 3377 if revision:
c208a243 3378 self.importHeadRevision(revision)
b984733c
SH
3379 else:
3380 changes = []
3381
0828ab14 3382 if len(self.changesFile) > 0:
b984733c 3383 output = open(self.changesFile).readlines()
1d7367dc 3384 changeSet = set()
b984733c
SH
3385 for line in output:
3386 changeSet.add(int(line))
3387
3388 for change in changeSet:
3389 changes.append(change)
3390
3391 changes.sort()
3392 else:
9dcb9f24
PW
3393 # catch "git p4 sync" with no new branches, in a repo that
3394 # does not have any existing p4 branches
5a8e84cd
PW
3395 if len(args) == 0:
3396 if not self.p4BranchesInGit:
3397 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3398
3399 # The default branch is master, unless --branch is used to
3400 # specify something else. Make sure it exists, or complain
3401 # nicely about how to use --branch.
3402 if not self.detectBranches:
3403 if not branch_exists(self.branch):
3404 if branch_arg_given:
3405 die("Error: branch %s does not exist." % self.branch)
3406 else:
3407 die("Error: no branch %s; perhaps specify one with --branch." %
3408 self.branch)
3409
29bdbac1 3410 if self.verbose:
86dff6b6 3411 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3412 self.changeRange)
96b2d54a 3413 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
b984733c 3414
01a9c9c5 3415 if len(self.maxChanges) > 0:
7fcff9de 3416 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3417
b984733c 3418 if len(changes) == 0:
0828ab14 3419 if not self.silent:
341dc1c1 3420 print "No changes to import!"
06804c76
LD
3421 else:
3422 if not self.silent and not self.detectBranches:
3423 print "Import destination: %s" % self.branch
3424
3425 self.updatedBranches = set()
b984733c 3426
47497844
PW
3427 if not self.detectBranches:
3428 if args:
3429 # start a new branch
3430 self.initialParent = ""
3431 else:
3432 # build on a previous revision
3433 self.initialParent = parseRevision(self.branch)
3434
06804c76 3435 self.importChanges(changes)
a9d1a27a 3436
06804c76
LD
3437 if not self.silent:
3438 print ""
3439 if len(self.updatedBranches) > 0:
3440 sys.stdout.write("Updated branches: ")
3441 for b in self.updatedBranches:
3442 sys.stdout.write("%s " % b)
3443 sys.stdout.write("\n")
341dc1c1 3444
0d609032 3445 if gitConfigBool("git-p4.importLabels"):
06dcd152 3446 self.importLabels = True
b984733c 3447
06804c76
LD
3448 if self.importLabels:
3449 p4Labels = getP4Labels(