Merge branch 'jk/war-on-sprintf' into maint-2.7
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
8b41a97f 28
a235e85c
BC
29try:
30 from subprocess import CalledProcessError
31except ImportError:
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self, returncode, cmd):
39 self.returncode = returncode
40 self.cmd = cmd
41 def __str__(self):
42 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
4addad22 44verbose = False
86949eef 45
06804c76 46# Only labels/tags matching this will be imported/exported
c8942a22 47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 48
1051ef00
LD
49# Grab changes in blocks of this many revisions, unless otherwise requested
50defaultBlockSize = 512
51
21a50753
AK
52def p4_build_cmd(cmd):
53 """Build a suitable p4 command line.
54
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
58 """
6de040df 59 real_cmd = ["p4"]
abcaf073
AK
60
61 user = gitConfig("git-p4.user")
62 if len(user) > 0:
6de040df 63 real_cmd += ["-u",user]
abcaf073
AK
64
65 password = gitConfig("git-p4.password")
66 if len(password) > 0:
6de040df 67 real_cmd += ["-P", password]
abcaf073
AK
68
69 port = gitConfig("git-p4.port")
70 if len(port) > 0:
6de040df 71 real_cmd += ["-p", port]
abcaf073
AK
72
73 host = gitConfig("git-p4.host")
74 if len(host) > 0:
41799aa2 75 real_cmd += ["-H", host]
abcaf073
AK
76
77 client = gitConfig("git-p4.client")
78 if len(client) > 0:
6de040df 79 real_cmd += ["-c", client]
abcaf073 80
6de040df
LD
81
82 if isinstance(cmd,basestring):
83 real_cmd = ' '.join(real_cmd) + ' ' + cmd
84 else:
85 real_cmd += cmd
21a50753
AK
86 return real_cmd
87
bbd84863
MF
88def chdir(path, is_client_path=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
93
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
98
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
102 the client path.
103 """
104
105 os.chdir(path)
106 if not is_client_path:
107 path = os.getcwd()
108 os.environ['PWD'] = path
053fd0c1 109
4d25dc44
LS
110def calcDiskFree():
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform.system() == 'Windows':
113 free_bytes = ctypes.c_ulonglong(0)
114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115 return free_bytes.value
116 else:
117 st = os.statvfs(os.getcwd())
118 return st.f_bavail * st.f_frsize
119
86dff6b6
HWN
120def die(msg):
121 if verbose:
122 raise Exception(msg)
123 else:
124 sys.stderr.write(msg + "\n")
125 sys.exit(1)
126
6de040df 127def write_pipe(c, stdin):
4addad22 128 if verbose:
6de040df 129 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 130
6de040df
LD
131 expand = isinstance(c,basestring)
132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133 pipe = p.stdin
134 val = pipe.write(stdin)
135 pipe.close()
136 if p.wait():
137 die('Command failed: %s' % str(c))
b016d397
HWN
138
139 return val
140
6de040df 141def p4_write_pipe(c, stdin):
d9429194 142 real_cmd = p4_build_cmd(c)
6de040df 143 return write_pipe(real_cmd, stdin)
d9429194 144
4addad22
HWN
145def read_pipe(c, ignore_error=False):
146 if verbose:
6de040df 147 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 148
6de040df 149 expand = isinstance(c,basestring)
1f5f3907
LS
150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151 (out, err) = p.communicate()
152 if p.returncode != 0 and not ignore_error:
153 die('Command failed: %s\nError: %s' % (str(c), err))
154 return out
b016d397 155
d9429194
AK
156def p4_read_pipe(c, ignore_error=False):
157 real_cmd = p4_build_cmd(c)
158 return read_pipe(real_cmd, ignore_error)
b016d397 159
bce4c5fc 160def read_pipe_lines(c):
4addad22 161 if verbose:
6de040df
LD
162 sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164 expand = isinstance(c, basestring)
165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166 pipe = p.stdout
b016d397 167 val = pipe.readlines()
6de040df
LD
168 if pipe.close() or p.wait():
169 die('Command failed: %s' % str(c))
b016d397
HWN
170
171 return val
caace111 172
2318121b
AK
173def p4_read_pipe_lines(c):
174 """Specifically invoke p4 on the command supplied. """
155af834 175 real_cmd = p4_build_cmd(c)
2318121b
AK
176 return read_pipe_lines(real_cmd)
177
8e9497c2
GG
178def p4_has_command(cmd):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd = p4_build_cmd(["help", cmd])
182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183 stderr=subprocess.PIPE)
184 p.communicate()
185 return p.returncode == 0
186
249da4c0
PW
187def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
192
193 if not p4_has_command("move"):
194 return False
195 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197 (out, err) = p.communicate()
198 # return code will be 1 in either case
199 if err.find("Invalid option") >= 0:
200 return False
201 if err.find("disabled") >= 0:
202 return False
203 # assume it failed because @... was invalid changelist
204 return True
205
cbff4b25 206def system(cmd, ignore_error=False):
6de040df 207 expand = isinstance(cmd,basestring)
4addad22 208 if verbose:
6de040df 209 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 210 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 211 if retcode and not ignore_error:
a235e85c 212 raise CalledProcessError(retcode, cmd)
6754a299 213
cbff4b25
LD
214 return retcode
215
bf9320f1
AK
216def p4_system(cmd):
217 """Specifically invoke p4 as the system command. """
155af834 218 real_cmd = p4_build_cmd(cmd)
6de040df 219 expand = isinstance(real_cmd, basestring)
a235e85c
BC
220 retcode = subprocess.call(real_cmd, shell=expand)
221 if retcode:
222 raise CalledProcessError(retcode, real_cmd)
6de040df 223
7f0e5962
PW
224_p4_version_string = None
225def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
228
229 $ p4 -V
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233 """
234 global _p4_version_string
235 if not _p4_version_string:
236 a = p4_read_pipe_lines(["-V"])
237 _p4_version_string = a[-1].rstrip()
238 return _p4_version_string
239
6de040df 240def p4_integrate(src, dest):
9d7d446a 241 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 242
8d7ec362 243def p4_sync(f, *options):
9d7d446a 244 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
245
246def p4_add(f):
9d7d446a
PW
247 # forcibly add file names with wildcards
248 if wildcard_present(f):
249 p4_system(["add", "-f", f])
250 else:
251 p4_system(["add", f])
6de040df
LD
252
253def p4_delete(f):
9d7d446a 254 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
255
256def p4_edit(f):
9d7d446a 257 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
258
259def p4_revert(f):
9d7d446a 260 p4_system(["revert", wildcard_encode(f)])
6de040df 261
9d7d446a
PW
262def p4_reopen(type, f):
263 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 264
8e9497c2
GG
265def p4_move(src, dest):
266 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
267
1051ef00
LD
268def p4_last_change():
269 results = p4CmdList(["changes", "-m", "1"])
270 return int(results[0]['change'])
271
18fa13d0
PW
272def p4_describe(change):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
275 results."""
276
277 ds = p4CmdList(["describe", "-s", str(change)])
278 if len(ds) != 1:
279 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280
281 d = ds[0]
282
283 if "p4ExitCode" in d:
284 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
285 str(d)))
286 if "code" in d:
287 if d["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
289
290 if "time" not in d:
291 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
292
293 return d
294
9cffb8c8
PW
295#
296# Canonicalize the p4 type and return a tuple of the
297# base type, plus any modifiers. See "p4 help filetypes"
298# for a list and explanation.
299#
300def split_p4_type(p4type):
301
302 p4_filetypes_historical = {
303 "ctempobj": "binary+Sw",
304 "ctext": "text+C",
305 "cxtext": "text+Cx",
306 "ktext": "text+k",
307 "kxtext": "text+kx",
308 "ltext": "text+F",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
314 "xltext": "text+Fx",
315 "xtempobj": "binary+Swx",
316 "xtext": "text+x",
317 "xunicode": "unicode+x",
318 "xutf16": "utf16+x",
319 }
320 if p4type in p4_filetypes_historical:
321 p4type = p4_filetypes_historical[p4type]
322 mods = ""
323 s = p4type.split("+")
324 base = s[0]
325 mods = ""
326 if len(s) > 1:
327 mods = s[1]
328 return (base, mods)
b9fc6ea9 329
60df071c
LD
330#
331# return the raw p4 type of a file (text, text+ko, etc)
332#
79467e61
PW
333def p4_type(f):
334 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
335 return results[0]['headType']
336
337#
338# Given a type base and modifier, return a regexp matching
339# the keywords that can be expanded in the file
340#
341def p4_keywords_regexp_for_type(base, type_mods):
342 if base in ("text", "unicode", "binary"):
343 kwords = None
344 if "ko" in type_mods:
345 kwords = 'Id|Header'
346 elif "k" in type_mods:
347 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348 else:
349 return None
350 pattern = r"""
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
6b2bf41e 353 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
354 \$ # another dollar
355 """ % kwords
356 return pattern
357 else:
358 return None
359
360#
361# Given a file, return a regexp matching the possible
362# RCS keywords that will be expanded, or None for files
363# with kw expansion turned off.
364#
365def p4_keywords_regexp_for_file(file):
366 if not os.path.exists(file):
367 return None
368 else:
369 (type_base, type_mods) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 371
c65b670e
CP
372def setP4ExecBit(file, mode):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
375
376 p4Type = "+x"
377
378 if not isModeExec(mode):
379 p4Type = getP4OpenedType(file)
380 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382 if p4Type[-1] == "+":
383 p4Type = p4Type[0:-1]
384
6de040df 385 p4_reopen(p4Type, file)
c65b670e
CP
386
387def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
389
9d7d446a 390 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 391 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
392 if match:
393 return match.group(1)
394 else:
f3e5ae4f 395 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 396
06804c76
LD
397# Return the set of all p4 labels
398def getP4Labels(depotPaths):
399 labels = set()
400 if isinstance(depotPaths,basestring):
401 depotPaths = [depotPaths]
402
403 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
404 label = l['label']
405 labels.add(label)
406
407 return labels
408
409# Return the set of all git tags
410def getGitTags():
411 gitTags = set()
412 for line in read_pipe_lines(["git", "tag"]):
413 tag = line.strip()
414 gitTags.add(tag)
415 return gitTags
416
b43b0a3c
CP
417def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421 while True:
422 yield pattern
423
424def parseDiffTreeEntry(entry):
425 """Parses a single diff tree entry into its component elements.
426
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
429
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
440
441 If the pattern is not matched, None is returned."""
442
443 match = diffTreePattern().next().match(entry)
444 if match:
445 return {
446 'src_mode': match.group(1),
447 'dst_mode': match.group(2),
448 'src_sha1': match.group(3),
449 'dst_sha1': match.group(4),
450 'status': match.group(5),
451 'status_score': match.group(6),
452 'src': match.group(7),
453 'dst': match.group(10)
454 }
455 return None
456
c65b670e
CP
457def isModeExec(mode):
458 # Returns True if the given git mode represents an executable file,
459 # otherwise False.
460 return mode[-3:] == "755"
461
462def isModeExecChanged(src_mode, dst_mode):
463 return isModeExec(src_mode) != isModeExec(dst_mode)
464
b932705b 465def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
466
467 if isinstance(cmd,basestring):
468 cmd = "-G " + cmd
469 expand = True
470 else:
471 cmd = ["-G"] + cmd
472 expand = False
473
474 cmd = p4_build_cmd(cmd)
6a49f8e2 475 if verbose:
6de040df 476 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
477
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
481 stdin_file = None
482 if stdin is not None:
483 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
484 if isinstance(stdin,basestring):
485 stdin_file.write(stdin)
486 else:
487 for i in stdin:
488 stdin_file.write(i + '\n')
9f90c733
SL
489 stdin_file.flush()
490 stdin_file.seek(0)
491
6de040df
LD
492 p4 = subprocess.Popen(cmd,
493 shell=expand,
9f90c733
SL
494 stdin=stdin_file,
495 stdout=subprocess.PIPE)
86949eef
SH
496
497 result = []
498 try:
499 while True:
9f90c733 500 entry = marshal.load(p4.stdout)
c3f6163b
AG
501 if cb is not None:
502 cb(entry)
503 else:
504 result.append(entry)
86949eef
SH
505 except EOFError:
506 pass
9f90c733
SL
507 exitCode = p4.wait()
508 if exitCode != 0:
ac3e0d79
SH
509 entry = {}
510 entry["p4ExitCode"] = exitCode
511 result.append(entry)
86949eef
SH
512
513 return result
514
515def p4Cmd(cmd):
516 list = p4CmdList(cmd)
517 result = {}
518 for entry in list:
519 result.update(entry)
520 return result;
521
cb2c9db5
SH
522def p4Where(depotPath):
523 if not depotPath.endswith("/"):
524 depotPath += "/"
cd884106
VA
525 depotPathLong = depotPath + "..."
526 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
527 output = None
528 for entry in outputList:
75bc9573 529 if "depotFile" in entry:
cd884106
VA
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
533 output = entry
534 break
535 elif "data" in entry:
536 data = entry.get("data")
537 space = data.find(" ")
538 if data[:space] == depotPath:
539 output = entry
540 break
7f705dc3
TAL
541 if output == None:
542 return ""
dc524036
SH
543 if output["code"] == "error":
544 return ""
cb2c9db5
SH
545 clientPath = ""
546 if "path" in output:
547 clientPath = output.get("path")
548 elif "data" in output:
549 data = output.get("data")
550 lastSpace = data.rfind(" ")
551 clientPath = data[lastSpace + 1:]
552
553 if clientPath.endswith("..."):
554 clientPath = clientPath[:-3]
555 return clientPath
556
86949eef 557def currentGitBranch():
00ad6e31
LD
558 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
559 if retcode != 0:
560 # on a detached head
561 return None
562 else:
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
86949eef 564
4f5cf76a 565def isValidGitDir(path):
bb6e09b2
HWN
566 if (os.path.exists(path + "/HEAD")
567 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
568 return True;
569 return False
570
463e8af6 571def parseRevision(ref):
b25b2065 572 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 573
28755dba
PW
574def branchExists(ref):
575 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
576 ignore_error=True)
577 return len(rev) > 0
578
6ae8de88
SH
579def extractLogMessageFromGitCommit(commit):
580 logMessage = ""
b016d397
HWN
581
582 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 583 foundTitle = False
b016d397 584 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
585 if not foundTitle:
586 if len(log) == 1:
1c094184 587 foundTitle = True
6ae8de88
SH
588 continue
589
590 logMessage += log
591 return logMessage
592
bb6e09b2 593def extractSettingsGitLog(log):
6ae8de88
SH
594 values = {}
595 for line in log.split("\n"):
596 line = line.strip()
6326aa58
HWN
597 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
598 if not m:
599 continue
600
601 assignments = m.group(1).split (':')
602 for a in assignments:
603 vals = a.split ('=')
604 key = vals[0].strip()
605 val = ('='.join (vals[1:])).strip()
606 if val.endswith ('\"') and val.startswith('"'):
607 val = val[1:-1]
608
609 values[key] = val
610
845b42cb
SH
611 paths = values.get("depot-paths")
612 if not paths:
613 paths = values.get("depot-path")
a3fdd579
SH
614 if paths:
615 values['depot-paths'] = paths.split(',')
bb6e09b2 616 return values
6ae8de88 617
8136a639 618def gitBranchExists(branch):
bb6e09b2
HWN
619 proc = subprocess.Popen(["git", "rev-parse", branch],
620 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 621 return proc.wait() == 0;
8136a639 622
36bd8446 623_gitConfig = {}
b345d6c3 624
692e1796 625def gitConfig(key, typeSpecifier=None):
36bd8446 626 if not _gitConfig.has_key(key):
692e1796
LS
627 cmd = [ "git", "config" ]
628 if typeSpecifier:
629 cmd += [ typeSpecifier ]
630 cmd += [ key ]
b345d6c3
PW
631 s = read_pipe(cmd, ignore_error=True)
632 _gitConfig[key] = s.strip()
36bd8446 633 return _gitConfig[key]
01265103 634
0d609032
PW
635def gitConfigBool(key):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
638 in the config."""
639
36bd8446 640 if not _gitConfig.has_key(key):
692e1796 641 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 642 return _gitConfig[key]
01265103 643
cb1dafdf
LS
644def gitConfigInt(key):
645 if not _gitConfig.has_key(key):
646 cmd = [ "git", "config", "--int", key ]
0d609032
PW
647 s = read_pipe(cmd, ignore_error=True)
648 v = s.strip()
cb1dafdf
LS
649 try:
650 _gitConfig[key] = int(gitConfig(key, '--int'))
651 except ValueError:
652 _gitConfig[key] = None
36bd8446 653 return _gitConfig[key]
01265103 654
7199cf13
VA
655def gitConfigList(key):
656 if not _gitConfig.has_key(key):
2abba301
PW
657 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658 _gitConfig[key] = s.strip().split(os.linesep)
7960e707
LS
659 if _gitConfig[key] == ['']:
660 _gitConfig[key] = []
7199cf13
VA
661 return _gitConfig[key]
662
2c8037ed
PW
663def p4BranchesInGit(branchesAreInRemotes=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
668 "p4/" prefix."""
669
062410bb
SH
670 branches = {}
671
672 cmdline = "git rev-parse --symbolic "
673 if branchesAreInRemotes:
2c8037ed 674 cmdline += "--remotes"
062410bb 675 else:
2c8037ed 676 cmdline += "--branches"
062410bb
SH
677
678 for line in read_pipe_lines(cmdline):
679 line = line.strip()
680
2c8037ed
PW
681 # only import to p4/
682 if not line.startswith('p4/'):
683 continue
684 # special symbolic ref to p4/master
685 if line == "p4/HEAD":
062410bb 686 continue
062410bb 687
2c8037ed
PW
688 # strip off p4/ prefix
689 branch = line[len("p4/"):]
062410bb
SH
690
691 branches[branch] = parseRevision(line)
2c8037ed 692
062410bb
SH
693 return branches
694
5a8e84cd
PW
695def branch_exists(branch):
696 """Make sure that the given ref name really exists."""
697
698 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700 out, _ = p.communicate()
701 if p.returncode:
702 return False
703 # expect exactly one line of output: the branch name
704 return out.rstrip() == branch
705
9ceab363 706def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
707 branches = p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath = {}
710 for branch in branches.keys():
711 tip = branches[branch]
712 log = extractLogMessageFromGitCommit(tip)
713 settings = extractSettingsGitLog(log)
714 if settings.has_key("depot-paths"):
715 paths = ",".join(settings["depot-paths"])
716 branchByDepotPath[paths] = "remotes/p4/" + branch
717
27d2d811 718 settings = None
27d2d811
SH
719 parent = 0
720 while parent < 65535:
9ceab363 721 commit = head + "~%s" % parent
27d2d811
SH
722 log = extractLogMessageFromGitCommit(commit)
723 settings = extractSettingsGitLog(log)
86506fe5
SH
724 if settings.has_key("depot-paths"):
725 paths = ",".join(settings["depot-paths"])
726 if branchByDepotPath.has_key(paths):
727 return [branchByDepotPath[paths], settings]
27d2d811 728
86506fe5 729 parent = parent + 1
27d2d811 730
86506fe5 731 return ["", settings]
27d2d811 732
5ca44617
SH
733def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
734 if not silent:
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
736 % localRefPrefix)
737
738 originPrefix = "origin/p4/"
739
740 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
741 line = line.strip()
742 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
743 continue
744
745 headName = line[len(originPrefix):]
746 remoteHead = localRefPrefix + headName
747 originHead = line
748
749 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750 if (not original.has_key('depot-paths')
751 or not original.has_key('change')):
752 continue
753
754 update = False
755 if not gitBranchExists(remoteHead):
756 if verbose:
757 print "creating %s" % remoteHead
758 update = True
759 else:
760 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761 if settings.has_key('change') > 0:
762 if settings['depot-paths'] == original['depot-paths']:
763 originP4Change = int(original['change'])
764 p4Change = int(settings['change'])
765 if originP4Change > p4Change:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead, originP4Change,
769 remoteHead, p4Change))
770 update = True
771 else:
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead, ','.join(original['depot-paths']),
775 remoteHead, ','.join(settings['depot-paths'])))
776
777 if update:
778 system("git update-ref %s %s" % (remoteHead, originHead))
779
780def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
782
1051ef00
LD
783
784def p4ParseNumericChangeRange(parts):
785 changeStart = int(parts[0][1:])
786 if parts[1] == '#head':
787 changeEnd = p4_last_change()
788 else:
789 changeEnd = int(parts[1])
790
791 return (changeStart, changeEnd)
792
793def chooseBlockSize(blockSize):
794 if blockSize:
795 return blockSize
796 else:
797 return defaultBlockSize
798
799def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 800 assert depotPaths
96b2d54a 801
1051ef00
LD
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
807
96b2d54a 808 if changeRange is None or changeRange == '':
1051ef00
LD
809 changeStart = 1
810 changeEnd = p4_last_change()
811 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
812 else:
813 parts = changeRange.split(',')
814 assert len(parts) == 2
1051ef00
LD
815 try:
816 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817 block_size = chooseBlockSize(requestedBlockSize)
818 except:
819 changeStart = parts[0][1:]
820 changeEnd = parts[1]
821 if requestedBlockSize:
822 die("cannot use --changes-block-size with non-numeric revisions")
823 block_size = None
4f6432d8 824
1f90a648 825 changes = []
96b2d54a 826
1f90a648
SH
827 # Retrieve changes a block at a time, to prevent running
828 # into a MaxResults/MaxScanRows error from the server.
1051ef00 829
1f90a648
SH
830 while True:
831 cmd = ['changes']
1051ef00 832
1f90a648
SH
833 if block_size:
834 end = min(changeEnd, changeStart + block_size)
835 revisionRange = "%d,%d" % (changeStart, end)
836 else:
837 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 838
1f90a648 839 for p in depotPaths:
1051ef00
LD
840 cmd += ["%s...@%s" % (p, revisionRange)]
841
1f90a648
SH
842 # Insert changes in chronological order
843 for line in reversed(p4_read_pipe_lines(cmd)):
844 changes.append(int(line.split(" ")[1]))
1051ef00 845
1f90a648
SH
846 if not block_size:
847 break
1051ef00 848
1f90a648
SH
849 if end >= changeEnd:
850 break
1051ef00 851
1f90a648 852 changeStart = end + 1
4f6432d8 853
1f90a648
SH
854 changes = sorted(changes)
855 return changes
4f6432d8 856
d53de8b9
TAL
857def p4PathStartsWith(path, prefix):
858 # This method tries to remedy a potential mixed-case issue:
859 #
860 # If UserA adds //depot/DirA/file1
861 # and UserB adds //depot/dira/file2
862 #
863 # we may or may not have a problem. If you have core.ignorecase=true,
864 # we treat DirA and dira as the same directory
0d609032 865 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
866 return path.lower().startswith(prefix.lower())
867 return path.startswith(prefix)
868
543987bd
PW
869def getClientSpec():
870 """Look at the p4 client spec, create a View() object that contains
871 all the mappings, and return it."""
872
873 specList = p4CmdList("client -o")
874 if len(specList) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' %
876 len(specList))
877
878 # dictionary of all client parameters
879 entry = specList[0]
880
9d57c4a6
KS
881 # the //client/ name
882 client_name = entry["Client"]
883
543987bd
PW
884 # just the keys that start with "View"
885 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
886
887 # hold this new View
9d57c4a6 888 view = View(client_name)
543987bd
PW
889
890 # append the lines, in order, to the view
891 for view_num in range(len(view_keys)):
892 k = "View%d" % view_num
893 if k not in view_keys:
894 die("Expected view key %s missing" % k)
895 view.append(entry[k])
896
897 return view
898
899def getClientRoot():
900 """Grab the client directory."""
901
902 output = p4CmdList("client -o")
903 if len(output) != 1:
904 die('Output from "client -o" is %d lines, expecting 1' % len(output))
905
906 entry = output[0]
907 if "Root" not in entry:
908 die('Client has no "Root"')
909
910 return entry["Root"]
911
9d7d446a
PW
912#
913# P4 wildcards are not allowed in filenames. P4 complains
914# if you simply add them, but you can force it with "-f", in
915# which case it translates them into %xx encoding internally.
916#
917def wildcard_decode(path):
918 # Search for and fix just these four characters. Do % last so
919 # that fixing it does not inadvertently create new %-escapes.
920 # Cannot have * in a filename in windows; untested as to
921 # what p4 would do in such a case.
922 if not platform.system() == "Windows":
923 path = path.replace("%2A", "*")
924 path = path.replace("%23", "#") \
925 .replace("%40", "@") \
926 .replace("%25", "%")
927 return path
928
929def wildcard_encode(path):
930 # do % first to avoid double-encoding the %s introduced here
931 path = path.replace("%", "%25") \
932 .replace("*", "%2A") \
933 .replace("#", "%23") \
934 .replace("@", "%40")
935 return path
936
937def wildcard_present(path):
598354c0
BC
938 m = re.search("[*#@%]", path)
939 return m is not None
9d7d446a 940
a5db4b12
LS
941class LargeFileSystem(object):
942 """Base class for large file system support."""
943
944 def __init__(self, writeToGitStream):
945 self.largeFiles = set()
946 self.writeToGitStream = writeToGitStream
947
948 def generatePointer(self, cloneDestination, contentFile):
949 """Return the content of a pointer file that is stored in Git instead of
950 the actual content."""
951 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
952
953 def pushFile(self, localLargeFile):
954 """Push the actual content which is not stored in the Git repository to
955 a server."""
956 assert False, "Method 'pushFile' required in " + self.__class__.__name__
957
958 def hasLargeFileExtension(self, relPath):
959 return reduce(
960 lambda a, b: a or b,
961 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
962 False
963 )
964
965 def generateTempFile(self, contents):
966 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
967 for d in contents:
968 contentFile.write(d)
969 contentFile.close()
970 return contentFile.name
971
972 def exceedsLargeFileThreshold(self, relPath, contents):
973 if gitConfigInt('git-p4.largeFileThreshold'):
974 contentsSize = sum(len(d) for d in contents)
975 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
976 return True
977 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978 contentsSize = sum(len(d) for d in contents)
979 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
980 return False
981 contentTempFile = self.generateTempFile(contents)
982 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
983 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
984 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
985 zf.close()
986 compressedContentsSize = zf.infolist()[0].compress_size
987 os.remove(contentTempFile)
988 os.remove(compressedContentFile.name)
989 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
990 return True
991 return False
992
993 def addLargeFile(self, relPath):
994 self.largeFiles.add(relPath)
995
996 def removeLargeFile(self, relPath):
997 self.largeFiles.remove(relPath)
998
999 def isLargeFile(self, relPath):
1000 return relPath in self.largeFiles
1001
1002 def processContent(self, git_mode, relPath, contents):
1003 """Processes the content of git fast import. This method decides if a
1004 file is stored in the large file system and handles all necessary
1005 steps."""
1006 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1007 contentTempFile = self.generateTempFile(contents)
1008 (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1009
1010 # Move temp file to final location in large file system
1011 largeFileDir = os.path.dirname(localLargeFile)
1012 if not os.path.isdir(largeFileDir):
1013 os.makedirs(largeFileDir)
1014 shutil.move(contentTempFile, localLargeFile)
1015 self.addLargeFile(relPath)
1016 if gitConfigBool('git-p4.largeFilePush'):
1017 self.pushFile(localLargeFile)
1018 if verbose:
1019 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1020 return (git_mode, contents)
1021
1022class MockLFS(LargeFileSystem):
1023 """Mock large file system for testing."""
1024
1025 def generatePointer(self, contentFile):
1026 """The pointer content is the original content prefixed with "pointer-".
1027 The local filename of the large file storage is derived from the file content.
1028 """
1029 with open(contentFile, 'r') as f:
1030 content = next(f)
1031 gitMode = '100644'
1032 pointerContents = 'pointer-' + content
1033 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1034 return (gitMode, pointerContents, localLargeFile)
1035
1036 def pushFile(self, localLargeFile):
1037 """The remote filename of the large file storage is the same as the local
1038 one but in a different directory.
1039 """
1040 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1041 if not os.path.exists(remotePath):
1042 os.makedirs(remotePath)
1043 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1044
b47d807d
LS
1045class GitLFS(LargeFileSystem):
1046 """Git LFS as backend for the git-p4 large file system.
1047 See https://git-lfs.github.com/ for details."""
1048
1049 def __init__(self, *args):
1050 LargeFileSystem.__init__(self, *args)
1051 self.baseGitAttributes = []
1052
1053 def generatePointer(self, contentFile):
1054 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1055 mode and content which is stored in the Git repository instead of
1056 the actual content. Return also the new location of the actual
1057 content.
1058 """
1059 pointerProcess = subprocess.Popen(
1060 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1061 stdout=subprocess.PIPE
1062 )
1063 pointerFile = pointerProcess.stdout.read()
1064 if pointerProcess.wait():
1065 os.remove(contentFile)
1066 die('git-lfs pointer command failed. Did you install the extension?')
1067 pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
1068 oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
1069 localLargeFile = os.path.join(
1070 os.getcwd(),
1071 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1072 oid,
1073 )
1074 # LFS Spec states that pointer files should not have the executable bit set.
1075 gitMode = '100644'
1076 return (gitMode, pointerContents, localLargeFile)
1077
1078 def pushFile(self, localLargeFile):
1079 uploadProcess = subprocess.Popen(
1080 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1081 )
1082 if uploadProcess.wait():
1083 die('git-lfs push command failed. Did you define a remote?')
1084
1085 def generateGitAttributes(self):
1086 return (
1087 self.baseGitAttributes +
1088 [
1089 '\n',
1090 '#\n',
1091 '# Git LFS (see https://git-lfs.github.com/)\n',
1092 '#\n',
1093 ] +
1094 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1095 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1096 ] +
1097 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1098 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1099 ]
1100 )
1101
1102 def addLargeFile(self, relPath):
1103 LargeFileSystem.addLargeFile(self, relPath)
1104 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1105
1106 def removeLargeFile(self, relPath):
1107 LargeFileSystem.removeLargeFile(self, relPath)
1108 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1109
1110 def processContent(self, git_mode, relPath, contents):
1111 if relPath == '.gitattributes':
1112 self.baseGitAttributes = contents
1113 return (git_mode, self.generateGitAttributes())
1114 else:
1115 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1116
b984733c
SH
1117class Command:
1118 def __init__(self):
1119 self.usage = "usage: %prog [options]"
8910ac0e 1120 self.needsGit = True
6a10b6aa 1121 self.verbose = False
b984733c 1122
3ea2cfd4
LD
1123class P4UserMap:
1124 def __init__(self):
1125 self.userMapFromPerforceServer = False
affb474f
LD
1126 self.myP4UserId = None
1127
1128 def p4UserId(self):
1129 if self.myP4UserId:
1130 return self.myP4UserId
1131
1132 results = p4CmdList("user -o")
1133 for r in results:
1134 if r.has_key('User'):
1135 self.myP4UserId = r['User']
1136 return r['User']
1137 die("Could not find your p4 user id")
1138
1139 def p4UserIsMe(self, p4User):
1140 # return True if the given p4 user is actually me
1141 me = self.p4UserId()
1142 if not p4User or p4User != me:
1143 return False
1144 else:
1145 return True
3ea2cfd4
LD
1146
1147 def getUserCacheFilename(self):
1148 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1149 return home + "/.gitp4-usercache.txt"
1150
1151 def getUserMapFromPerforceServer(self):
1152 if self.userMapFromPerforceServer:
1153 return
1154 self.users = {}
1155 self.emails = {}
1156
1157 for output in p4CmdList("users"):
1158 if not output.has_key("User"):
1159 continue
1160 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1161 self.emails[output["Email"]] = output["User"]
1162
1163
1164 s = ''
1165 for (key, val) in self.users.items():
1166 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1167
1168 open(self.getUserCacheFilename(), "wb").write(s)
1169 self.userMapFromPerforceServer = True
1170
1171 def loadUserMapFromCache(self):
1172 self.users = {}
1173 self.userMapFromPerforceServer = False
1174 try:
1175 cache = open(self.getUserCacheFilename(), "rb")
1176 lines = cache.readlines()
1177 cache.close()
1178 for line in lines:
1179 entry = line.strip().split("\t")
1180 self.users[entry[0]] = entry[1]
1181 except IOError:
1182 self.getUserMapFromPerforceServer()
1183
b984733c 1184class P4Debug(Command):
86949eef 1185 def __init__(self):
6ae8de88 1186 Command.__init__(self)
6a10b6aa 1187 self.options = []
c8c39116 1188 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1189 self.needsGit = False
86949eef
SH
1190
1191 def run(self, args):
b1ce9447 1192 j = 0
6de040df 1193 for output in p4CmdList(args):
b1ce9447
HWN
1194 print 'Element: %d' % j
1195 j += 1
86949eef 1196 print output
b984733c 1197 return True
86949eef 1198
5834684d
SH
1199class P4RollBack(Command):
1200 def __init__(self):
1201 Command.__init__(self)
1202 self.options = [
0c66a783 1203 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1204 ]
1205 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1206 self.rollbackLocalBranches = False
5834684d
SH
1207
1208 def run(self, args):
1209 if len(args) != 1:
1210 return False
1211 maxChange = int(args[0])
0c66a783 1212
ad192f28 1213 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1214 die("Problems executing p4");
1215
0c66a783
SH
1216 if self.rollbackLocalBranches:
1217 refPrefix = "refs/heads/"
b016d397 1218 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1219 else:
1220 refPrefix = "refs/remotes/"
b016d397 1221 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1222
1223 for line in lines:
1224 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1225 line = line.strip()
1226 ref = refPrefix + line
5834684d 1227 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1228 settings = extractSettingsGitLog(log)
1229
1230 depotPaths = settings['depot-paths']
1231 change = settings['change']
1232
5834684d 1233 changed = False
52102d47 1234
6326aa58
HWN
1235 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1236 for p in depotPaths]))) == 0:
52102d47
SH
1237 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1238 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1239 continue
1240
bb6e09b2 1241 while change and int(change) > maxChange:
5834684d 1242 changed = True
52102d47
SH
1243 if self.verbose:
1244 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1245 system("git update-ref %s \"%s^\"" % (ref, ref))
1246 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1247 settings = extractSettingsGitLog(log)
1248
1249
1250 depotPaths = settings['depot-paths']
1251 change = settings['change']
5834684d
SH
1252
1253 if changed:
52102d47 1254 print "%s rewound to %s" % (ref, change)
5834684d
SH
1255
1256 return True
1257
3ea2cfd4 1258class P4Submit(Command, P4UserMap):
6bbfd137
PW
1259
1260 conflict_behavior_choices = ("ask", "skip", "quit")
1261
4f5cf76a 1262 def __init__(self):
b984733c 1263 Command.__init__(self)
3ea2cfd4 1264 P4UserMap.__init__(self)
4f5cf76a 1265 self.options = [
4f5cf76a 1266 optparse.make_option("--origin", dest="origin"),
ae901090 1267 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1268 # preserve the user, requires relevant p4 permissions
1269 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1270 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1271 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1272 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1273 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1274 choices=self.conflict_behavior_choices),
1275 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
1276 ]
1277 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1278 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1279 self.origin = ""
ae901090 1280 self.detectRenames = False
0d609032 1281 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1282 self.dry_run = False
728b7ad8 1283 self.prepare_p4_only = False
6bbfd137 1284 self.conflict_behavior = None
f7baba8b 1285 self.isWindows = (platform.system() == "Windows")
06804c76 1286 self.exportLabels = False
249da4c0 1287 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1288 self.branch = None
4f5cf76a 1289
a5db4b12
LS
1290 if gitConfig('git-p4.largeFileSystem'):
1291 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1292
4f5cf76a
SH
1293 def check(self):
1294 if len(p4CmdList("opened ...")) > 0:
1295 die("You have files opened with perforce! Close them before starting the sync.")
1296
f19cb0a0
PW
1297 def separate_jobs_from_description(self, message):
1298 """Extract and return a possible Jobs field in the commit
1299 message. It goes into a separate section in the p4 change
1300 specification.
1301
1302 A jobs line starts with "Jobs:" and looks like a new field
1303 in a form. Values are white-space separated on the same
1304 line or on following lines that start with a tab.
1305
1306 This does not parse and extract the full git commit message
1307 like a p4 form. It just sees the Jobs: line as a marker
1308 to pass everything from then on directly into the p4 form,
1309 but outside the description section.
1310
1311 Return a tuple (stripped log message, jobs string)."""
1312
1313 m = re.search(r'^Jobs:', message, re.MULTILINE)
1314 if m is None:
1315 return (message, None)
1316
1317 jobtext = message[m.start():]
1318 stripped_message = message[:m.start()].rstrip()
1319 return (stripped_message, jobtext)
1320
1321 def prepareLogMessage(self, template, message, jobs):
1322 """Edits the template returned from "p4 change -o" to insert
1323 the message in the Description field, and the jobs text in
1324 the Jobs field."""
4f5cf76a
SH
1325 result = ""
1326
edae1e2f
SH
1327 inDescriptionSection = False
1328
4f5cf76a
SH
1329 for line in template.split("\n"):
1330 if line.startswith("#"):
1331 result += line + "\n"
1332 continue
1333
edae1e2f 1334 if inDescriptionSection:
c9dbab04 1335 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1336 inDescriptionSection = False
f19cb0a0
PW
1337 # insert Jobs section
1338 if jobs:
1339 result += jobs + "\n"
edae1e2f
SH
1340 else:
1341 continue
1342 else:
1343 if line.startswith("Description:"):
1344 inDescriptionSection = True
1345 line += "\n"
1346 for messageLine in message.split("\n"):
1347 line += "\t" + messageLine + "\n"
1348
1349 result += line + "\n"
4f5cf76a
SH
1350
1351 return result
1352
60df071c
LD
1353 def patchRCSKeywords(self, file, pattern):
1354 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1355 (handle, outFileName) = tempfile.mkstemp(dir='.')
1356 try:
1357 outFile = os.fdopen(handle, "w+")
1358 inFile = open(file, "r")
1359 regexp = re.compile(pattern, re.VERBOSE)
1360 for line in inFile.readlines():
1361 line = regexp.sub(r'$\1$', line)
1362 outFile.write(line)
1363 inFile.close()
1364 outFile.close()
1365 # Forcibly overwrite the original file
1366 os.unlink(file)
1367 shutil.move(outFileName, file)
1368 except:
1369 # cleanup our temporary file
1370 os.unlink(outFileName)
1371 print "Failed to strip RCS keywords in %s" % file
1372 raise
1373
1374 print "Patched up RCS keywords in %s" % file
1375
3ea2cfd4
LD
1376 def p4UserForCommit(self,id):
1377 # Return the tuple (perforce user,git email) for a given git commit id
1378 self.getUserMapFromPerforceServer()
9bf28855
PW
1379 gitEmail = read_pipe(["git", "log", "--max-count=1",
1380 "--format=%ae", id])
3ea2cfd4
LD
1381 gitEmail = gitEmail.strip()
1382 if not self.emails.has_key(gitEmail):
1383 return (None,gitEmail)
1384 else:
1385 return (self.emails[gitEmail],gitEmail)
1386
1387 def checkValidP4Users(self,commits):
1388 # check if any git authors cannot be mapped to p4 users
1389 for id in commits:
1390 (user,email) = self.p4UserForCommit(id)
1391 if not user:
1392 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1393 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1394 print "%s" % msg
1395 else:
1396 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1397
1398 def lastP4Changelist(self):
1399 # Get back the last changelist number submitted in this client spec. This
1400 # then gets used to patch up the username in the change. If the same
1401 # client spec is being used by multiple processes then this might go
1402 # wrong.
1403 results = p4CmdList("client -o") # find the current client
1404 client = None
1405 for r in results:
1406 if r.has_key('Client'):
1407 client = r['Client']
1408 break
1409 if not client:
1410 die("could not get client spec")
6de040df 1411 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1412 for r in results:
1413 if r.has_key('change'):
1414 return r['change']
1415 die("Could not get changelist number for last submit - cannot patch up user details")
1416
1417 def modifyChangelistUser(self, changelist, newUser):
1418 # fixup the user field of a changelist after it has been submitted.
1419 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1420 if len(changes) != 1:
1421 die("Bad output from p4 change modifying %s to user %s" %
1422 (changelist, newUser))
1423
1424 c = changes[0]
1425 if c['User'] == newUser: return # nothing to do
1426 c['User'] = newUser
1427 input = marshal.dumps(c)
1428
3ea2cfd4
LD
1429 result = p4CmdList("change -f -i", stdin=input)
1430 for r in result:
1431 if r.has_key('code'):
1432 if r['code'] == 'error':
1433 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1434 if r.has_key('data'):
1435 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1436 return
1437 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1438
1439 def canChangeChangelists(self):
1440 # check to see if we have p4 admin or super-user permissions, either of
1441 # which are required to modify changelists.
52a4880b 1442 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1443 for r in results:
1444 if r.has_key('perm'):
1445 if r['perm'] == 'admin':
1446 return 1
1447 if r['perm'] == 'super':
1448 return 1
1449 return 0
1450
ea99c3ae 1451 def prepareSubmitTemplate(self):
f19cb0a0
PW
1452 """Run "p4 change -o" to grab a change specification template.
1453 This does not use "p4 -G", as it is nice to keep the submission
1454 template in original order, since a human might edit it.
1455
1456 Remove lines in the Files section that show changes to files
1457 outside the depot path we're committing into."""
1458
cbc69242
SH
1459 [upstream, settings] = findUpstreamBranchPoint()
1460
ea99c3ae
SH
1461 template = ""
1462 inFilesSection = False
6de040df 1463 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1464 if line.endswith("\r\n"):
1465 line = line[:-2] + "\n"
ea99c3ae
SH
1466 if inFilesSection:
1467 if line.startswith("\t"):
1468 # path starts and ends with a tab
1469 path = line[1:]
1470 lastTab = path.rfind("\t")
1471 if lastTab != -1:
1472 path = path[:lastTab]
cbc69242
SH
1473 if settings.has_key('depot-paths'):
1474 if not [p for p in settings['depot-paths']
1475 if p4PathStartsWith(path, p)]:
1476 continue
1477 else:
1478 if not p4PathStartsWith(path, self.depotPath):
1479 continue
ea99c3ae
SH
1480 else:
1481 inFilesSection = False
1482 else:
1483 if line.startswith("Files:"):
1484 inFilesSection = True
1485
1486 template += line
1487
1488 return template
1489
7c766e57
PW
1490 def edit_template(self, template_file):
1491 """Invoke the editor to let the user change the submission
1492 message. Return true if okay to continue with the submit."""
1493
1494 # if configured to skip the editing part, just submit
0d609032 1495 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1496 return True
1497
1498 # look at the modification time, to check later if the user saved
1499 # the file
1500 mtime = os.stat(template_file).st_mtime
1501
1502 # invoke the editor
f95ceaf0 1503 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1504 editor = os.environ.get("P4EDITOR")
1505 else:
1506 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1507 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1508
1509 # If the file was not saved, prompt to see if this patch should
1510 # be skipped. But skip this verification step if configured so.
0d609032 1511 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1512 return True
1513
d1652049
PW
1514 # modification time updated means user saved the file
1515 if os.stat(template_file).st_mtime > mtime:
1516 return True
1517
1518 while True:
1519 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1520 if response == 'y':
1521 return True
1522 if response == 'n':
1523 return False
7c766e57 1524
e2a892ee 1525 def get_diff_description(self, editedFiles, filesToAdd):
b4073bb3
MC
1526 # diff
1527 if os.environ.has_key("P4DIFF"):
1528 del(os.environ["P4DIFF"])
1529 diff = ""
1530 for editedFile in editedFiles:
1531 diff += p4_read_pipe(['diff', '-du',
1532 wildcard_encode(editedFile)])
1533
1534 # new file diff
1535 newdiff = ""
1536 for newFile in filesToAdd:
1537 newdiff += "==== new file ====\n"
1538 newdiff += "--- /dev/null\n"
1539 newdiff += "+++ %s\n" % newFile
1540 f = open(newFile, "r")
1541 for line in f.readlines():
1542 newdiff += "+" + line
1543 f.close()
1544
e2a892ee 1545 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1546
7cb5cbef 1547 def applyCommit(self, id):
67b0fe2e
PW
1548 """Apply one commit, return True if it succeeded."""
1549
1550 print "Applying", read_pipe(["git", "show", "-s",
1551 "--format=format:%h %s", id])
ae901090 1552
848de9c3 1553 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1554
84cb0003 1555 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1556 filesToAdd = set()
1557 filesToDelete = set()
d336c158 1558 editedFiles = set()
b6ad6dcc 1559 pureRenameCopy = set()
c65b670e 1560 filesToChangeExecBit = {}
60df071c 1561
4f5cf76a 1562 for line in diff:
b43b0a3c
CP
1563 diff = parseDiffTreeEntry(line)
1564 modifier = diff['status']
1565 path = diff['src']
4f5cf76a 1566 if modifier == "M":
6de040df 1567 p4_edit(path)
c65b670e
CP
1568 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1569 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1570 editedFiles.add(path)
4f5cf76a
SH
1571 elif modifier == "A":
1572 filesToAdd.add(path)
c65b670e 1573 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1574 if path in filesToDelete:
1575 filesToDelete.remove(path)
1576 elif modifier == "D":
1577 filesToDelete.add(path)
1578 if path in filesToAdd:
1579 filesToAdd.remove(path)
4fddb41b
VA
1580 elif modifier == "C":
1581 src, dest = diff['src'], diff['dst']
6de040df 1582 p4_integrate(src, dest)
b6ad6dcc 1583 pureRenameCopy.add(dest)
4fddb41b 1584 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1585 p4_edit(dest)
b6ad6dcc 1586 pureRenameCopy.discard(dest)
4fddb41b 1587 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1588 p4_edit(dest)
b6ad6dcc 1589 pureRenameCopy.discard(dest)
4fddb41b 1590 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1591 if self.isWindows:
1592 # turn off read-only attribute
1593 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1594 os.unlink(dest)
1595 editedFiles.add(dest)
d9a5f25b 1596 elif modifier == "R":
b43b0a3c 1597 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1598 if self.p4HasMoveCommand:
1599 p4_edit(src) # src must be open before move
1600 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1601 else:
8e9497c2
GG
1602 p4_integrate(src, dest)
1603 if diff['src_sha1'] != diff['dst_sha1']:
1604 p4_edit(dest)
1605 else:
1606 pureRenameCopy.add(dest)
c65b670e 1607 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1608 if not self.p4HasMoveCommand:
1609 p4_edit(dest) # with move: already open, writable
c65b670e 1610 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1611 if not self.p4HasMoveCommand:
d20f0f8e
PW
1612 if self.isWindows:
1613 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1614 os.unlink(dest)
1615 filesToDelete.add(src)
d9a5f25b 1616 editedFiles.add(dest)
4f5cf76a
SH
1617 else:
1618 die("unknown modifier %s for %s" % (modifier, path))
1619
749b668c 1620 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1621 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1622 tryPatchCmd = patchcmd + "--check -"
1623 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1624 patch_succeeded = True
51a2640a 1625
47a130b7 1626 if os.system(tryPatchCmd) != 0:
60df071c
LD
1627 fixed_rcs_keywords = False
1628 patch_succeeded = False
51a2640a 1629 print "Unfortunately applying the change failed!"
60df071c
LD
1630
1631 # Patch failed, maybe it's just RCS keyword woes. Look through
1632 # the patch to see if that's possible.
0d609032 1633 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1634 file = None
1635 pattern = None
1636 kwfiles = {}
1637 for file in editedFiles | filesToDelete:
1638 # did this file's delta contain RCS keywords?
1639 pattern = p4_keywords_regexp_for_file(file)
1640
1641 if pattern:
1642 # this file is a possibility...look for RCS keywords.
1643 regexp = re.compile(pattern, re.VERBOSE)
1644 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1645 if regexp.search(line):
1646 if verbose:
1647 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1648 kwfiles[file] = pattern
1649 break
1650
1651 for file in kwfiles:
1652 if verbose:
1653 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1654 # File is being deleted, so not open in p4. Must
1655 # disable the read-only bit on windows.
1656 if self.isWindows and file not in editedFiles:
1657 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1658 self.patchRCSKeywords(file, kwfiles[file])
1659 fixed_rcs_keywords = True
1660
1661 if fixed_rcs_keywords:
1662 print "Retrying the patch with RCS keywords cleaned up"
1663 if os.system(tryPatchCmd) == 0:
1664 patch_succeeded = True
1665
1666 if not patch_succeeded:
7e5dd9f2
PW
1667 for f in editedFiles:
1668 p4_revert(f)
7e5dd9f2 1669 return False
51a2640a 1670
55ac2ed6
PW
1671 #
1672 # Apply the patch for real, and do add/delete/+x handling.
1673 #
47a130b7 1674 system(applyPatchCmd)
4f5cf76a
SH
1675
1676 for f in filesToAdd:
6de040df 1677 p4_add(f)
4f5cf76a 1678 for f in filesToDelete:
6de040df
LD
1679 p4_revert(f)
1680 p4_delete(f)
4f5cf76a 1681
c65b670e
CP
1682 # Set/clear executable bits
1683 for f in filesToChangeExecBit.keys():
1684 mode = filesToChangeExecBit[f]
1685 setP4ExecBit(f, mode)
1686
55ac2ed6
PW
1687 #
1688 # Build p4 change description, starting with the contents
1689 # of the git commit message.
1690 #
0e36f2d7 1691 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1692 logMessage = logMessage.strip()
f19cb0a0 1693 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1694
ea99c3ae 1695 template = self.prepareSubmitTemplate()
f19cb0a0 1696 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1697
c47178d4 1698 if self.preserveUser:
55ac2ed6 1699 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1700
55ac2ed6
PW
1701 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1702 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1703 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1704 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1705
55ac2ed6 1706 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1707 if not self.prepare_p4_only:
1708 submitTemplate += separatorLine
e2a892ee 1709 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
55ac2ed6 1710
c47178d4 1711 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1712 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1713 if self.isWindows:
1714 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1715 tmpFile.write(submitTemplate)
c47178d4
PW
1716 tmpFile.close()
1717
728b7ad8
PW
1718 if self.prepare_p4_only:
1719 #
1720 # Leave the p4 tree prepared, and the submit template around
1721 # and let the user decide what to do next
1722 #
1723 print
1724 print "P4 workspace prepared for submission."
1725 print "To submit or revert, go to client workspace"
1726 print " " + self.clientPath
1727 print
1728 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1729 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1730 " \"git p4\"." % fileName
1731 print "You can delete the file \"%s\" when finished." % fileName
1732
1733 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1734 print "To preserve change ownership by user %s, you must\n" \
1735 "do \"p4 change -f <change>\" after submitting and\n" \
1736 "edit the User field."
1737 if pureRenameCopy:
1738 print "After submitting, renamed files must be re-synced."
1739 print "Invoke \"p4 sync -f\" on each of these files:"
1740 for f in pureRenameCopy:
1741 print " " + f
1742
1743 print
1744 print "To revert the changes, use \"p4 revert ...\", and delete"
1745 print "the submit template file \"%s\"" % fileName
1746 if filesToAdd:
1747 print "Since the commit adds new files, they must be deleted:"
1748 for f in filesToAdd:
1749 print " " + f
1750 print
1751 return True
1752
55ac2ed6
PW
1753 #
1754 # Let the user edit the change description, then submit it.
1755 #
b7638fed 1756 submitted = False
cdc7e388 1757
b7638fed
GE
1758 try:
1759 if self.edit_template(fileName):
1760 # read the edited message and submit
1761 tmpFile = open(fileName, "rb")
1762 message = tmpFile.read()
1763 tmpFile.close()
1764 if self.isWindows:
1765 message = message.replace("\r\n", "\n")
1766 submitTemplate = message[:message.index(separatorLine)]
1767 p4_write_pipe(['submit', '-i'], submitTemplate)
1768
1769 if self.preserveUser:
1770 if p4User:
1771 # Get last changelist number. Cannot easily get it from
1772 # the submit command output as the output is
1773 # unmarshalled.
1774 changelist = self.lastP4Changelist()
1775 self.modifyChangelistUser(changelist, p4User)
1776
1777 # The rename/copy happened by applying a patch that created a
1778 # new file. This leaves it writable, which confuses p4.
1779 for f in pureRenameCopy:
1780 p4_sync(f, "-f")
1781 submitted = True
1782
1783 finally:
c47178d4 1784 # skip this patch
b7638fed
GE
1785 if not submitted:
1786 print "Submission cancelled, undoing p4 changes."
1787 for f in editedFiles:
1788 p4_revert(f)
1789 for f in filesToAdd:
1790 p4_revert(f)
1791 os.remove(f)
1792 for f in filesToDelete:
1793 p4_revert(f)
c47178d4
PW
1794
1795 os.remove(fileName)
b7638fed 1796 return submitted
4f5cf76a 1797
06804c76
LD
1798 # Export git tags as p4 labels. Create a p4 label and then tag
1799 # with that.
1800 def exportGitTags(self, gitTags):
c8942a22
LD
1801 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1802 if len(validLabelRegexp) == 0:
1803 validLabelRegexp = defaultLabelRegexp
1804 m = re.compile(validLabelRegexp)
06804c76
LD
1805
1806 for name in gitTags:
1807
1808 if not m.match(name):
1809 if verbose:
05a3cec5 1810 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1811 continue
1812
1813 # Get the p4 commit this corresponds to
c8942a22
LD
1814 logMessage = extractLogMessageFromGitCommit(name)
1815 values = extractSettingsGitLog(logMessage)
06804c76 1816
c8942a22 1817 if not values.has_key('change'):
06804c76
LD
1818 # a tag pointing to something not sent to p4; ignore
1819 if verbose:
1820 print "git tag %s does not give a p4 commit" % name
1821 continue
c8942a22
LD
1822 else:
1823 changelist = values['change']
06804c76
LD
1824
1825 # Get the tag details.
1826 inHeader = True
1827 isAnnotated = False
1828 body = []
1829 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1830 l = l.strip()
1831 if inHeader:
1832 if re.match(r'tag\s+', l):
1833 isAnnotated = True
1834 elif re.match(r'\s*$', l):
1835 inHeader = False
1836 continue
1837 else:
1838 body.append(l)
1839
1840 if not isAnnotated:
1841 body = ["lightweight tag imported by git p4\n"]
1842
1843 # Create the label - use the same view as the client spec we are using
1844 clientSpec = getClientSpec()
1845
1846 labelTemplate = "Label: %s\n" % name
1847 labelTemplate += "Description:\n"
1848 for b in body:
1849 labelTemplate += "\t" + b + "\n"
1850 labelTemplate += "View:\n"
9d57c4a6
KS
1851 for depot_side in clientSpec.mappings:
1852 labelTemplate += "\t%s\n" % depot_side
06804c76 1853
ef739f08
PW
1854 if self.dry_run:
1855 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1856 elif self.prepare_p4_only:
1857 print "Not creating p4 label %s for tag due to option" \
1858 " --prepare-p4-only" % name
ef739f08
PW
1859 else:
1860 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1861
ef739f08
PW
1862 # Use the label
1863 p4_system(["tag", "-l", name] +
9d57c4a6 1864 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1865
ef739f08
PW
1866 if verbose:
1867 print "created p4 label for tag %s" % name
06804c76 1868
4f5cf76a 1869 def run(self, args):
c9b50e63
SH
1870 if len(args) == 0:
1871 self.master = currentGitBranch()
c9b50e63
SH
1872 elif len(args) == 1:
1873 self.master = args[0]
28755dba
PW
1874 if not branchExists(self.master):
1875 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1876 else:
1877 return False
1878
00ad6e31
LD
1879 if self.master:
1880 allowSubmit = gitConfig("git-p4.allowSubmit")
1881 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1882 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 1883
27d2d811 1884 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1885 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1886 if len(self.origin) == 0:
1887 self.origin = upstream
a3fdd579 1888
3ea2cfd4
LD
1889 if self.preserveUser:
1890 if not self.canChangeChangelists():
1891 die("Cannot preserve user names without p4 super-user or admin permissions")
1892
6bbfd137
PW
1893 # if not set from the command line, try the config file
1894 if self.conflict_behavior is None:
1895 val = gitConfig("git-p4.conflict")
1896 if val:
1897 if val not in self.conflict_behavior_choices:
1898 die("Invalid value '%s' for config git-p4.conflict" % val)
1899 else:
1900 val = "ask"
1901 self.conflict_behavior = val
1902
a3fdd579
SH
1903 if self.verbose:
1904 print "Origin branch is " + self.origin
9512497b 1905
ea99c3ae 1906 if len(self.depotPath) == 0:
9512497b
SH
1907 print "Internal error: cannot locate perforce depot path from existing branches"
1908 sys.exit(128)
1909
543987bd 1910 self.useClientSpec = False
0d609032 1911 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1912 self.useClientSpec = True
1913 if self.useClientSpec:
1914 self.clientSpecDirs = getClientSpec()
9512497b 1915
cd884106
VA
1916 # Check for the existance of P4 branches
1917 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1918
1919 if self.useClientSpec and not branchesDetected:
543987bd
PW
1920 # all files are relative to the client spec
1921 self.clientPath = getClientRoot()
1922 else:
1923 self.clientPath = p4Where(self.depotPath)
9512497b 1924
543987bd
PW
1925 if self.clientPath == "":
1926 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1927
ea99c3ae 1928 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1929 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1930
0591cfa8 1931 # ensure the clientPath exists
8d7ec362 1932 new_client_dir = False
0591cfa8 1933 if not os.path.exists(self.clientPath):
8d7ec362 1934 new_client_dir = True
0591cfa8
GG
1935 os.makedirs(self.clientPath)
1936
bbd84863 1937 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1938 if self.dry_run:
1939 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1940 else:
ef739f08
PW
1941 print "Synchronizing p4 checkout..."
1942 if new_client_dir:
1943 # old one was destroyed, and maybe nobody told p4
1944 p4_sync("...", "-f")
1945 else:
1946 p4_sync("...")
4f5cf76a 1947 self.check()
4f5cf76a 1948
4c750c0d 1949 commits = []
00ad6e31
LD
1950 if self.master:
1951 commitish = self.master
1952 else:
1953 commitish = 'HEAD'
1954
1955 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
4c750c0d
SH
1956 commits.append(line.strip())
1957 commits.reverse()
4f5cf76a 1958
0d609032 1959 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1960 self.checkAuthorship = False
1961 else:
1962 self.checkAuthorship = True
1963
3ea2cfd4
LD
1964 if self.preserveUser:
1965 self.checkValidP4Users(commits)
1966
84cb0003
GG
1967 #
1968 # Build up a set of options to be passed to diff when
1969 # submitting each commit to p4.
1970 #
1971 if self.detectRenames:
1972 # command-line -M arg
1973 self.diffOpts = "-M"
1974 else:
1975 # If not explicitly set check the config variable
1976 detectRenames = gitConfig("git-p4.detectRenames")
1977
1978 if detectRenames.lower() == "false" or detectRenames == "":
1979 self.diffOpts = ""
1980 elif detectRenames.lower() == "true":
1981 self.diffOpts = "-M"
1982 else:
1983 self.diffOpts = "-M%s" % detectRenames
1984
1985 # no command-line arg for -C or --find-copies-harder, just
1986 # config variables
1987 detectCopies = gitConfig("git-p4.detectCopies")
1988 if detectCopies.lower() == "false" or detectCopies == "":
1989 pass
1990 elif detectCopies.lower() == "true":
1991 self.diffOpts += " -C"
1992 else:
1993 self.diffOpts += " -C%s" % detectCopies
1994
0d609032 1995 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1996 self.diffOpts += " --find-copies-harder"
1997
7e5dd9f2
PW
1998 #
1999 # Apply the commits, one at a time. On failure, ask if should
2000 # continue to try the rest of the patches, or quit.
2001 #
ef739f08
PW
2002 if self.dry_run:
2003 print "Would apply"
67b0fe2e 2004 applied = []
7e5dd9f2
PW
2005 last = len(commits) - 1
2006 for i, commit in enumerate(commits):
ef739f08
PW
2007 if self.dry_run:
2008 print " ", read_pipe(["git", "show", "-s",
2009 "--format=format:%h %s", commit])
2010 ok = True
2011 else:
2012 ok = self.applyCommit(commit)
67b0fe2e
PW
2013 if ok:
2014 applied.append(commit)
7e5dd9f2 2015 else:
728b7ad8
PW
2016 if self.prepare_p4_only and i < last:
2017 print "Processing only the first commit due to option" \
2018 " --prepare-p4-only"
2019 break
7e5dd9f2
PW
2020 if i < last:
2021 quit = False
2022 while True:
6bbfd137
PW
2023 # prompt for what to do, or use the option/variable
2024 if self.conflict_behavior == "ask":
2025 print "What do you want to do?"
2026 response = raw_input("[s]kip this commit but apply"
2027 " the rest, or [q]uit? ")
2028 if not response:
2029 continue
2030 elif self.conflict_behavior == "skip":
2031 response = "s"
2032 elif self.conflict_behavior == "quit":
2033 response = "q"
2034 else:
2035 die("Unknown conflict_behavior '%s'" %
2036 self.conflict_behavior)
2037
7e5dd9f2
PW
2038 if response[0] == "s":
2039 print "Skipping this commit, but applying the rest"
2040 break
2041 if response[0] == "q":
2042 print "Quitting"
2043 quit = True
2044 break
2045 if quit:
2046 break
4f5cf76a 2047
67b0fe2e 2048 chdir(self.oldWorkingDirectory)
4f5cf76a 2049
ef739f08
PW
2050 if self.dry_run:
2051 pass
728b7ad8
PW
2052 elif self.prepare_p4_only:
2053 pass
ef739f08 2054 elif len(commits) == len(applied):
67b0fe2e 2055 print "All commits applied!"
14594f4b 2056
4c750c0d 2057 sync = P4Sync()
44e8d26c
PW
2058 if self.branch:
2059 sync.branch = self.branch
4c750c0d 2060 sync.run([])
14594f4b 2061
4c750c0d
SH
2062 rebase = P4Rebase()
2063 rebase.rebase()
4f5cf76a 2064
67b0fe2e
PW
2065 else:
2066 if len(applied) == 0:
2067 print "No commits applied."
2068 else:
2069 print "Applied only the commits marked with '*':"
2070 for c in commits:
2071 if c in applied:
2072 star = "*"
2073 else:
2074 star = " "
2075 print star, read_pipe(["git", "show", "-s",
2076 "--format=format:%h %s", c])
2077 print "You will have to do 'git p4 sync' and rebase."
2078
0d609032 2079 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2080 self.exportLabels = True
06804c76
LD
2081
2082 if self.exportLabels:
2083 p4Labels = getP4Labels(self.depotPath)
2084 gitTags = getGitTags()
2085
2086 missingGitTags = gitTags - p4Labels
2087 self.exportGitTags(missingGitTags)
2088
98e023de 2089 # exit with error unless everything applied perfectly
67b0fe2e
PW
2090 if len(commits) != len(applied):
2091 sys.exit(1)
2092
b984733c
SH
2093 return True
2094
ecb7cf98
PW
2095class View(object):
2096 """Represent a p4 view ("p4 help views"), and map files in a
2097 repo according to the view."""
2098
9d57c4a6 2099 def __init__(self, client_name):
ecb7cf98 2100 self.mappings = []
9d57c4a6
KS
2101 self.client_prefix = "//%s/" % client_name
2102 # cache results of "p4 where" to lookup client file locations
2103 self.client_spec_path_cache = {}
ecb7cf98
PW
2104
2105 def append(self, view_line):
2106 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2107 sides. Append to self.mappings, preserving order. This
2108 is only needed for tag creation."""
ecb7cf98
PW
2109
2110 # Split the view line into exactly two words. P4 enforces
2111 # structure on these lines that simplifies this quite a bit.
2112 #
2113 # Either or both words may be double-quoted.
2114 # Single quotes do not matter.
2115 # Double-quote marks cannot occur inside the words.
2116 # A + or - prefix is also inside the quotes.
2117 # There are no quotes unless they contain a space.
2118 # The line is already white-space stripped.
2119 # The two words are separated by a single space.
2120 #
2121 if view_line[0] == '"':
2122 # First word is double quoted. Find its end.
2123 close_quote_index = view_line.find('"', 1)
2124 if close_quote_index <= 0:
2125 die("No first-word closing quote found: %s" % view_line)
2126 depot_side = view_line[1:close_quote_index]
2127 # skip closing quote and space
2128 rhs_index = close_quote_index + 1 + 1
2129 else:
2130 space_index = view_line.find(" ")
2131 if space_index <= 0:
2132 die("No word-splitting space found: %s" % view_line)
2133 depot_side = view_line[0:space_index]
2134 rhs_index = space_index + 1
2135
ecb7cf98 2136 # prefix + means overlay on previous mapping
ecb7cf98 2137 if depot_side.startswith("+"):
ecb7cf98
PW
2138 depot_side = depot_side[1:]
2139
9d57c4a6 2140 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2141 exclude = False
2142 if depot_side.startswith("-"):
2143 exclude = True
2144 depot_side = depot_side[1:]
2145
9d57c4a6
KS
2146 if not exclude:
2147 self.mappings.append(depot_side)
ecb7cf98 2148
9d57c4a6
KS
2149 def convert_client_path(self, clientFile):
2150 # chop off //client/ part to make it relative
2151 if not clientFile.startswith(self.client_prefix):
2152 die("No prefix '%s' on clientFile '%s'" %
2153 (self.client_prefix, clientFile))
2154 return clientFile[len(self.client_prefix):]
ecb7cf98 2155
9d57c4a6
KS
2156 def update_client_spec_path_cache(self, files):
2157 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2158
9d57c4a6
KS
2159 # List depot file paths exclude that already cached
2160 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2161
9d57c4a6
KS
2162 if len(fileArgs) == 0:
2163 return # All files in cache
ecb7cf98 2164
9d57c4a6
KS
2165 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2166 for res in where_result:
2167 if "code" in res and res["code"] == "error":
2168 # assume error is "... file(s) not in client view"
2169 continue
2170 if "clientFile" not in res:
20005443 2171 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2172 if "unmap" in res:
2173 # it will list all of them, but only one not unmap-ped
2174 continue
a0a50d87
LS
2175 if gitConfigBool("core.ignorecase"):
2176 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2177 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2178
9d57c4a6
KS
2179 # not found files or unmap files set to ""
2180 for depotFile in fileArgs:
a0a50d87
LS
2181 if gitConfigBool("core.ignorecase"):
2182 depotFile = depotFile.lower()
9d57c4a6
KS
2183 if depotFile not in self.client_spec_path_cache:
2184 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2185
9d57c4a6
KS
2186 def map_in_client(self, depot_path):
2187 """Return the relative location in the client where this
2188 depot file should live. Returns "" if the file should
2189 not be mapped in the client."""
ecb7cf98 2190
a0a50d87
LS
2191 if gitConfigBool("core.ignorecase"):
2192 depot_path = depot_path.lower()
2193
9d57c4a6
KS
2194 if depot_path in self.client_spec_path_cache:
2195 return self.client_spec_path_cache[depot_path]
2196
2197 die( "Error: %s is not found in client spec path" % depot_path )
2198 return ""
ecb7cf98 2199
3ea2cfd4 2200class P4Sync(Command, P4UserMap):
56c09345
PW
2201 delete_actions = ( "delete", "move/delete", "purge" )
2202
b984733c
SH
2203 def __init__(self):
2204 Command.__init__(self)
3ea2cfd4 2205 P4UserMap.__init__(self)
b984733c
SH
2206 self.options = [
2207 optparse.make_option("--branch", dest="branch"),
2208 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2209 optparse.make_option("--changesfile", dest="changesFile"),
2210 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2211 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2212 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2213 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2214 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2215 optparse.make_option("--max-changes", dest="maxChanges",
2216 help="Maximum number of changes to import"),
2217 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2218 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2219 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2220 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2221 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2222 help="Only sync files that are included in the Perforce Client Spec"),
2223 optparse.make_option("-/", dest="cloneExclude",
2224 action="append", type="string",
2225 help="exclude depot path"),
b984733c
SH
2226 ]
2227 self.description = """Imports from Perforce into a git repository.\n
2228 example:
2229 //depot/my/project/ -- to import the current head
2230 //depot/my/project/@all -- to import everything
2231 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2232
2233 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2234
2235 self.usage += " //depot/path[@revRange]"
b984733c 2236 self.silent = False
1d7367dc
RG
2237 self.createdBranches = set()
2238 self.committedChanges = set()
569d1bd4 2239 self.branch = ""
b984733c 2240 self.detectBranches = False
cb53e1f8 2241 self.detectLabels = False
06804c76 2242 self.importLabels = False
b984733c 2243 self.changesFile = ""
01265103 2244 self.syncWithOrigin = True
a028a98e 2245 self.importIntoRemotes = True
01a9c9c5 2246 self.maxChanges = ""
1051ef00 2247 self.changes_block_size = None
8b41a97f 2248 self.keepRepoPath = False
6326aa58 2249 self.depotPaths = None
3c699645 2250 self.p4BranchesInGit = []
354081d5 2251 self.cloneExclude = []
3a70cdfa 2252 self.useClientSpec = False
a93d33ee 2253 self.useClientSpec_from_options = False
ecb7cf98 2254 self.clientSpecDirs = None
fed23693
VA
2255 self.tempBranches = []
2256 self.tempBranchLocation = "git-p4-tmp"
a5db4b12
LS
2257 self.largeFileSystem = None
2258
2259 if gitConfig('git-p4.largeFileSystem'):
2260 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2261 self.largeFileSystem = largeFileSystemConstructor(
2262 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2263 )
b984733c 2264
01265103
SH
2265 if gitConfig("git-p4.syncFromOrigin") == "false":
2266 self.syncWithOrigin = False
2267
51334bb0
LD
2268 # This is required for the "append" cloneExclude action
2269 def ensure_value(self, attr, value):
2270 if not hasattr(self, attr) or getattr(self, attr) is None:
2271 setattr(self, attr, value)
2272 return getattr(self, attr)
2273
fed23693
VA
2274 # Force a checkpoint in fast-import and wait for it to finish
2275 def checkpoint(self):
2276 self.gitStream.write("checkpoint\n\n")
2277 self.gitStream.write("progress checkpoint\n\n")
2278 out = self.gitOutput.readline()
2279 if self.verbose:
2280 print "checkpoint finished: " + out
2281
b984733c 2282 def extractFilesFromCommit(self, commit):
354081d5
TT
2283 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2284 for path in self.cloneExclude]
b984733c
SH
2285 files = []
2286 fnum = 0
2287 while commit.has_key("depotFile%s" % fnum):
2288 path = commit["depotFile%s" % fnum]
6326aa58 2289
354081d5 2290 if [p for p in self.cloneExclude
d53de8b9 2291 if p4PathStartsWith(path, p)]:
354081d5
TT
2292 found = False
2293 else:
2294 found = [p for p in self.depotPaths
d53de8b9 2295 if p4PathStartsWith(path, p)]
6326aa58 2296 if not found:
b984733c
SH
2297 fnum = fnum + 1
2298 continue
2299
2300 file = {}
2301 file["path"] = path
2302 file["rev"] = commit["rev%s" % fnum]
2303 file["action"] = commit["action%s" % fnum]
2304 file["type"] = commit["type%s" % fnum]
2305 files.append(file)
2306 fnum = fnum + 1
2307 return files
2308
6326aa58 2309 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2310 """When streaming files, this is called to map a p4 depot path
2311 to where it should go in git. The prefixes are either
2312 self.depotPaths, or self.branchPrefixes in the case of
2313 branch detection."""
2314
3952710b 2315 if self.useClientSpec:
21ef5df4
PW
2316 # branch detection moves files up a level (the branch name)
2317 # from what client spec interpretation gives
0d1696ef 2318 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2319 if self.detectBranches:
2320 for b in self.knownBranches:
2321 if path.startswith(b + "/"):
2322 path = path[len(b)+1:]
2323
2324 elif self.keepRepoPath:
2325 # Preserve everything in relative path name except leading
2326 # //depot/; just look at first prefix as they all should
2327 # be in the same depot.
2328 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2329 if p4PathStartsWith(path, depot):
2330 path = path[len(depot):]
3952710b 2331
0d1696ef 2332 else:
0d1696ef
PW
2333 for p in prefixes:
2334 if p4PathStartsWith(path, p):
2335 path = path[len(p):]
21ef5df4 2336 break
8b41a97f 2337
0d1696ef 2338 path = wildcard_decode(path)
6326aa58 2339 return path
6754a299 2340
71b112d4 2341 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2342 """Look at each depotFile in the commit to figure out to what
2343 branch it belongs."""
2344
9d57c4a6
KS
2345 if self.clientSpecDirs:
2346 files = self.extractFilesFromCommit(commit)
2347 self.clientSpecDirs.update_client_spec_path_cache(files)
2348
d5904674 2349 branches = {}
71b112d4
SH
2350 fnum = 0
2351 while commit.has_key("depotFile%s" % fnum):
2352 path = commit["depotFile%s" % fnum]
6326aa58 2353 found = [p for p in self.depotPaths
d53de8b9 2354 if p4PathStartsWith(path, p)]
6326aa58 2355 if not found:
71b112d4
SH
2356 fnum = fnum + 1
2357 continue
2358
2359 file = {}
2360 file["path"] = path
2361 file["rev"] = commit["rev%s" % fnum]
2362 file["action"] = commit["action%s" % fnum]
2363 file["type"] = commit["type%s" % fnum]
2364 fnum = fnum + 1
2365
21ef5df4
PW
2366 # start with the full relative path where this file would
2367 # go in a p4 client
2368 if self.useClientSpec:
2369 relPath = self.clientSpecDirs.map_in_client(path)
2370 else:
2371 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2372
4b97ffb1 2373 for branch in self.knownBranches.keys():
21ef5df4
PW
2374 # add a trailing slash so that a commit into qt/4.2foo
2375 # doesn't end up in qt/4.2, e.g.
6754a299 2376 if relPath.startswith(branch + "/"):
d5904674
SH
2377 if branch not in branches:
2378 branches[branch] = []
71b112d4 2379 branches[branch].append(file)
6555b2cc 2380 break
b984733c
SH
2381
2382 return branches
2383
a5db4b12
LS
2384 def writeToGitStream(self, gitMode, relPath, contents):
2385 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2386 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2387 for d in contents:
2388 self.gitStream.write(d)
2389 self.gitStream.write('\n')
2390
b932705b
LD
2391 # output one file from the P4 stream
2392 # - helper for streamP4Files
2393
2394 def streamOneP4File(self, file, contents):
b932705b
LD
2395 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2396 if verbose:
d2176a50
LS
2397 size = int(self.stream_file['fileSize'])
2398 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2399 sys.stdout.flush()
b932705b 2400
9cffb8c8
PW
2401 (type_base, type_mods) = split_p4_type(file["type"])
2402
2403 git_mode = "100644"
2404 if "x" in type_mods:
2405 git_mode = "100755"
2406 if type_base == "symlink":
2407 git_mode = "120000"
1292df11
AJ
2408 # p4 print on a symlink sometimes contains "target\n";
2409 # if it does, remove the newline
b39c3612 2410 data = ''.join(contents)
40f846c3
PW
2411 if not data:
2412 # Some version of p4 allowed creating a symlink that pointed
2413 # to nothing. This causes p4 errors when checking out such
2414 # a change, and errors here too. Work around it by ignoring
2415 # the bad symlink; hopefully a future change fixes it.
2416 print "\nIgnoring empty symlink in %s" % file['depotFile']
2417 return
2418 elif data[-1] == '\n':
1292df11
AJ
2419 contents = [data[:-1]]
2420 else:
2421 contents = [data]
b932705b 2422
9cffb8c8 2423 if type_base == "utf16":
55aa5714
PW
2424 # p4 delivers different text in the python output to -G
2425 # than it does when using "print -o", or normal p4 client
2426 # operations. utf16 is converted to ascii or utf8, perhaps.
2427 # But ascii text saved as -t utf16 is completely mangled.
2428 # Invoke print -o to get the real contents.
7f0e5962
PW
2429 #
2430 # On windows, the newlines will always be mangled by print, so put
2431 # them back too. This is not needed to the cygwin windows version,
2432 # just the native "NT" type.
2433 #
1f5f3907
LS
2434 try:
2435 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2436 except Exception as e:
2437 if 'Translation of file content failed' in str(e):
2438 type_base = 'binary'
2439 else:
2440 raise e
2441 else:
2442 if p4_version_string().find('/NT') >= 0:
2443 text = text.replace('\r\n', '\n')
2444 contents = [ text ]
55aa5714 2445
9f7ef0ea
PW
2446 if type_base == "apple":
2447 # Apple filetype files will be streamed as a concatenation of
2448 # its appledouble header and the contents. This is useless
2449 # on both macs and non-macs. If using "print -q -o xx", it
2450 # will create "xx" with the data, and "%xx" with the header.
2451 # This is also not very useful.
2452 #
2453 # Ideally, someday, this script can learn how to generate
2454 # appledouble files directly and import those to git, but
2455 # non-mac machines can never find a use for apple filetype.
2456 print "\nIgnoring apple filetype file %s" % file['depotFile']
2457 return
2458
55aa5714
PW
2459 # Note that we do not try to de-mangle keywords on utf16 files,
2460 # even though in theory somebody may want that.
60df071c
LD
2461 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2462 if pattern:
2463 regexp = re.compile(pattern, re.VERBOSE)
2464 text = ''.join(contents)
2465 text = regexp.sub(r'$\1$', text)
2466 contents = [ text ]
b932705b 2467
00a9403a
LS
2468 try:
2469 relPath.decode('ascii')
2470 except:
2471 encoding = 'utf8'
2472 if gitConfig('git-p4.pathEncoding'):
2473 encoding = gitConfig('git-p4.pathEncoding')
4cb870d8 2474 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
00a9403a
LS
2475 if self.verbose:
2476 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
a9e38359 2477
a5db4b12
LS
2478 if self.largeFileSystem:
2479 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2480
a5db4b12 2481 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2482
2483 def streamOneP4Deletion(self, file):
2484 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2485 if verbose:
d2176a50
LS
2486 sys.stdout.write("delete %s\n" % relPath)
2487 sys.stdout.flush()
b932705b
LD
2488 self.gitStream.write("D %s\n" % relPath)
2489
a5db4b12
LS
2490 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2491 self.largeFileSystem.removeLargeFile(relPath)
2492
b932705b
LD
2493 # handle another chunk of streaming data
2494 def streamP4FilesCb(self, marshalled):
2495
78189bea
PW
2496 # catch p4 errors and complain
2497 err = None
2498 if "code" in marshalled:
2499 if marshalled["code"] == "error":
2500 if "data" in marshalled:
2501 err = marshalled["data"].rstrip()
4d25dc44
LS
2502
2503 if not err and 'fileSize' in self.stream_file:
2504 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2505 if required_bytes > 0:
2506 err = 'Not enough space left on %s! Free at least %i MB.' % (
2507 os.getcwd(), required_bytes/1024/1024
2508 )
2509
78189bea
PW
2510 if err:
2511 f = None
2512 if self.stream_have_file_info:
2513 if "depotFile" in self.stream_file:
2514 f = self.stream_file["depotFile"]
2515 # force a failure in fast-import, else an empty
2516 # commit will be made
2517 self.gitStream.write("\n")
2518 self.gitStream.write("die-now\n")
2519 self.gitStream.close()
2520 # ignore errors, but make sure it exits first
2521 self.importProcess.wait()
2522 if f:
2523 die("Error from p4 print for %s: %s" % (f, err))
2524 else:
2525 die("Error from p4 print: %s" % err)
2526
c3f6163b
AG
2527 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2528 # start of a new file - output the old one first
2529 self.streamOneP4File(self.stream_file, self.stream_contents)
2530 self.stream_file = {}
2531 self.stream_contents = []
2532 self.stream_have_file_info = False
b932705b 2533
c3f6163b
AG
2534 # pick up the new file information... for the
2535 # 'data' field we need to append to our array
2536 for k in marshalled.keys():
2537 if k == 'data':
d2176a50
LS
2538 if 'streamContentSize' not in self.stream_file:
2539 self.stream_file['streamContentSize'] = 0
2540 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2541 self.stream_contents.append(marshalled['data'])
2542 else:
2543 self.stream_file[k] = marshalled[k]
b932705b 2544
d2176a50
LS
2545 if (verbose and
2546 'streamContentSize' in self.stream_file and
2547 'fileSize' in self.stream_file and
2548 'depotFile' in self.stream_file):
2549 size = int(self.stream_file["fileSize"])
2550 if size > 0:
2551 progress = 100*self.stream_file['streamContentSize']/size
2552 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2553 sys.stdout.flush()
2554
c3f6163b 2555 self.stream_have_file_info = True
b932705b
LD
2556
2557 # Stream directly from "p4 files" into "git fast-import"
2558 def streamP4Files(self, files):
30b5940b
SH
2559 filesForCommit = []
2560 filesToRead = []
b932705b 2561 filesToDelete = []
30b5940b 2562
3a70cdfa 2563 for f in files:
ecb7cf98
PW
2564 filesForCommit.append(f)
2565 if f['action'] in self.delete_actions:
2566 filesToDelete.append(f)
2567 else:
2568 filesToRead.append(f)
6a49f8e2 2569
b932705b
LD
2570 # deleted files...
2571 for f in filesToDelete:
2572 self.streamOneP4Deletion(f)
1b9a4684 2573
b932705b
LD
2574 if len(filesToRead) > 0:
2575 self.stream_file = {}
2576 self.stream_contents = []
2577 self.stream_have_file_info = False
8ff45f2a 2578
c3f6163b
AG
2579 # curry self argument
2580 def streamP4FilesCbSelf(entry):
2581 self.streamP4FilesCb(entry)
6a49f8e2 2582
6de040df
LD
2583 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2584
2585 p4CmdList(["-x", "-", "print"],
2586 stdin=fileArgs,
2587 cb=streamP4FilesCbSelf)
30b5940b 2588
b932705b
LD
2589 # do the last chunk
2590 if self.stream_file.has_key('depotFile'):
2591 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2592
affb474f
LD
2593 def make_email(self, userid):
2594 if userid in self.users:
2595 return self.users[userid]
2596 else:
2597 return "%s <a@b>" % userid
2598
06804c76 2599 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2600 """ Stream a p4 tag.
2601 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2602 """
2603
06804c76
LD
2604 if verbose:
2605 print "writing tag %s for commit %s" % (labelName, commit)
2606 gitStream.write("tag %s\n" % labelName)
2607 gitStream.write("from %s\n" % commit)
2608
2609 if labelDetails.has_key('Owner'):
2610 owner = labelDetails["Owner"]
2611 else:
2612 owner = None
2613
2614 # Try to use the owner of the p4 label, or failing that,
2615 # the current p4 user id.
2616 if owner:
2617 email = self.make_email(owner)
2618 else:
2619 email = self.make_email(self.p4UserId())
2620 tagger = "%s %s %s" % (email, epoch, self.tz)
2621
2622 gitStream.write("tagger %s\n" % tagger)
2623
2624 print "labelDetails=",labelDetails
2625 if labelDetails.has_key('Description'):
2626 description = labelDetails['Description']
2627 else:
2628 description = 'Label from git p4'
2629
2630 gitStream.write("data %d\n" % len(description))
2631 gitStream.write(description)
2632 gitStream.write("\n")
2633
4ae048e6
LS
2634 def inClientSpec(self, path):
2635 if not self.clientSpecDirs:
2636 return True
2637 inClientSpec = self.clientSpecDirs.map_in_client(path)
2638 if not inClientSpec and self.verbose:
2639 print('Ignoring file outside of client spec: {0}'.format(path))
2640 return inClientSpec
2641
2642 def hasBranchPrefix(self, path):
2643 if not self.branchPrefixes:
2644 return True
2645 hasPrefix = [p for p in self.branchPrefixes
2646 if p4PathStartsWith(path, p)]
2647 if hasPrefix and self.verbose:
2648 print('Ignoring file outside of prefix: {0}'.format(path))
2649 return hasPrefix
2650
e63231e5 2651 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2652 epoch = details["time"]
2653 author = details["user"]
2654
4b97ffb1 2655 if self.verbose:
4ae048e6 2656 print('commit into {0}'.format(branch))
96e07dd2 2657
9d57c4a6
KS
2658 if self.clientSpecDirs:
2659 self.clientSpecDirs.update_client_spec_path_cache(files)
2660
4ae048e6
LS
2661 files = [f for f in files
2662 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2663
2664 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2665 print('Ignoring revision {0} as it would produce an empty commit.'
2666 .format(details['change']))
2667 return
2668
b984733c 2669 self.gitStream.write("commit %s\n" % branch)
b43702ac 2670 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2671 self.committedChanges.add(int(details["change"]))
2672 committer = ""
b607e71e
SH
2673 if author not in self.users:
2674 self.getUserMapFromPerforceServer()
affb474f 2675 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2676
2677 self.gitStream.write("committer %s\n" % committer)
2678
2679 self.gitStream.write("data <<EOT\n")
2680 self.gitStream.write(details["desc"])
e63231e5
PW
2681 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2682 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2683 if len(details['options']) > 0:
2684 self.gitStream.write(": options = %s" % details['options'])
2685 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2686
2687 if len(parent) > 0:
4b97ffb1
SH
2688 if self.verbose:
2689 print "parent %s" % parent
b984733c
SH
2690 self.gitStream.write("from %s\n" % parent)
2691
4ae048e6 2692 self.streamP4Files(files)
b984733c
SH
2693 self.gitStream.write("\n")
2694
1f4ba1cb
SH
2695 change = int(details["change"])
2696
9bda3a85 2697 if self.labels.has_key(change):
1f4ba1cb
SH
2698 label = self.labels[change]
2699 labelDetails = label[0]
2700 labelRevisions = label[1]
71b112d4
SH
2701 if self.verbose:
2702 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2703
6de040df 2704 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2705 for p in self.branchPrefixes])
1f4ba1cb
SH
2706
2707 if len(files) == len(labelRevisions):
2708
2709 cleanedFiles = {}
2710 for info in files:
56c09345 2711 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2712 continue
2713 cleanedFiles[info["depotFile"]] = info["rev"]
2714
2715 if cleanedFiles == labelRevisions:
06804c76 2716 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2717
2718 else:
a46668fa 2719 if not self.silent:
cebdf5af
HWN
2720 print ("Tag %s does not match with change %s: files do not match."
2721 % (labelDetails["label"], change))
1f4ba1cb
SH
2722
2723 else:
a46668fa 2724 if not self.silent:
cebdf5af
HWN
2725 print ("Tag %s does not match with change %s: file count is different."
2726 % (labelDetails["label"], change))
b984733c 2727
06804c76 2728 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2729 def getLabels(self):
2730 self.labels = {}
2731
52a4880b 2732 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2733 if len(l) > 0 and not self.silent:
183f8436 2734 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2735
2736 for output in l:
1f4ba1cb
SH
2737 label = output["label"]
2738 revisions = {}
2739 newestChange = 0
71b112d4
SH
2740 if self.verbose:
2741 print "Querying files for label %s" % label
6de040df
LD
2742 for file in p4CmdList(["files"] +
2743 ["%s...@%s" % (p, label)
2744 for p in self.depotPaths]):
1f4ba1cb
SH
2745 revisions[file["depotFile"]] = file["rev"]
2746 change = int(file["change"])
2747 if change > newestChange:
2748 newestChange = change
2749
9bda3a85
SH
2750 self.labels[newestChange] = [output, revisions]
2751
2752 if self.verbose:
2753 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2754
06804c76
LD
2755 # Import p4 labels as git tags. A direct mapping does not
2756 # exist, so assume that if all the files are at the same revision
2757 # then we can use that, or it's something more complicated we should
2758 # just ignore.
2759 def importP4Labels(self, stream, p4Labels):
2760 if verbose:
2761 print "import p4 labels: " + ' '.join(p4Labels)
2762
2763 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2764 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2765 if len(validLabelRegexp) == 0:
2766 validLabelRegexp = defaultLabelRegexp
2767 m = re.compile(validLabelRegexp)
2768
2769 for name in p4Labels:
2770 commitFound = False
2771
2772 if not m.match(name):
2773 if verbose:
2774 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2775 continue
2776
2777 if name in ignoredP4Labels:
2778 continue
2779
2780 labelDetails = p4CmdList(['label', "-o", name])[0]
2781
2782 # get the most recent changelist for each file in this label
2783 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2784 for p in self.depotPaths])
2785
2786 if change.has_key('change'):
2787 # find the corresponding git commit; take the oldest commit
2788 changelist = int(change['change'])
b43702ac
LD
2789 if changelist in self.committedChanges:
2790 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2791 commitFound = True
b43702ac
LD
2792 else:
2793 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2794 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2795 if len(gitCommit) == 0:
2796 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2797 else:
2798 commitFound = True
2799 gitCommit = gitCommit.strip()
2800
2801 if commitFound:
06804c76
LD
2802 # Convert from p4 time format
2803 try:
2804 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2805 except ValueError:
a4e9054c 2806 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2807 tmwhen = 1
2808
2809 when = int(time.mktime(tmwhen))
2810 self.streamTag(stream, name, labelDetails, gitCommit, when)
2811 if verbose:
2812 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2813 else:
2814 if verbose:
2815 print "Label %s has no changelists - possibly deleted?" % name
2816
2817 if not commitFound:
2818 # We can't import this label; don't try again as it will get very
2819 # expensive repeatedly fetching all the files for labels that will
2820 # never be imported. If the label is moved in the future, the
2821 # ignore will need to be removed manually.
2822 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2823
86dff6b6
HWN
2824 def guessProjectName(self):
2825 for p in self.depotPaths:
6e5295c4
SH
2826 if p.endswith("/"):
2827 p = p[:-1]
2828 p = p[p.strip().rfind("/") + 1:]
2829 if not p.endswith("/"):
2830 p += "/"
2831 return p
86dff6b6 2832
4b97ffb1 2833 def getBranchMapping(self):
6555b2cc
SH
2834 lostAndFoundBranches = set()
2835
8ace74c0
VA
2836 user = gitConfig("git-p4.branchUser")
2837 if len(user) > 0:
2838 command = "branches -u %s" % user
2839 else:
2840 command = "branches"
2841
2842 for info in p4CmdList(command):
52a4880b 2843 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2844 viewIdx = 0
2845 while details.has_key("View%s" % viewIdx):
2846 paths = details["View%s" % viewIdx].split(" ")
2847 viewIdx = viewIdx + 1
2848 # require standard //depot/foo/... //depot/bar/... mapping
2849 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2850 continue
2851 source = paths[0]
2852 destination = paths[1]
6509e19c 2853 ## HACK
d53de8b9 2854 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2855 source = source[len(self.depotPaths[0]):-4]
2856 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2857
1a2edf4e
SH
2858 if destination in self.knownBranches:
2859 if not self.silent:
2860 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2861 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2862 continue
2863
6555b2cc
SH
2864 self.knownBranches[destination] = source
2865
2866 lostAndFoundBranches.discard(destination)
2867
29bdbac1 2868 if source not in self.knownBranches:
6555b2cc
SH
2869 lostAndFoundBranches.add(source)
2870
7199cf13
VA
2871 # Perforce does not strictly require branches to be defined, so we also
2872 # check git config for a branch list.
2873 #
2874 # Example of branch definition in git config file:
2875 # [git-p4]
2876 # branchList=main:branchA
2877 # branchList=main:branchB
2878 # branchList=branchA:branchC
2879 configBranches = gitConfigList("git-p4.branchList")
2880 for branch in configBranches:
2881 if branch:
2882 (source, destination) = branch.split(":")
2883 self.knownBranches[destination] = source
2884
2885 lostAndFoundBranches.discard(destination)
2886
2887 if source not in self.knownBranches:
2888 lostAndFoundBranches.add(source)
2889
6555b2cc
SH
2890
2891 for branch in lostAndFoundBranches:
2892 self.knownBranches[branch] = branch
29bdbac1 2893
38f9f5ec
SH
2894 def getBranchMappingFromGitBranches(self):
2895 branches = p4BranchesInGit(self.importIntoRemotes)
2896 for branch in branches.keys():
2897 if branch == "master":
2898 branch = "main"
2899 else:
2900 branch = branch[len(self.projectName):]
2901 self.knownBranches[branch] = branch
2902
bb6e09b2
HWN
2903 def updateOptionDict(self, d):
2904 option_keys = {}
2905 if self.keepRepoPath:
2906 option_keys['keepRepoPath'] = 1
2907
2908 d["options"] = ' '.join(sorted(option_keys.keys()))
2909
2910 def readOptions(self, d):
2911 self.keepRepoPath = (d.has_key('options')
2912 and ('keepRepoPath' in d['options']))
6326aa58 2913
8134f69c
SH
2914 def gitRefForBranch(self, branch):
2915 if branch == "main":
2916 return self.refPrefix + "master"
2917
2918 if len(branch) <= 0:
2919 return branch
2920
2921 return self.refPrefix + self.projectName + branch
2922
1ca3d710
SH
2923 def gitCommitByP4Change(self, ref, change):
2924 if self.verbose:
2925 print "looking in ref " + ref + " for change %s using bisect..." % change
2926
2927 earliestCommit = ""
2928 latestCommit = parseRevision(ref)
2929
2930 while True:
2931 if self.verbose:
2932 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2933 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2934 if len(next) == 0:
2935 if self.verbose:
2936 print "argh"
2937 return ""
2938 log = extractLogMessageFromGitCommit(next)
2939 settings = extractSettingsGitLog(log)
2940 currentChange = int(settings['change'])
2941 if self.verbose:
2942 print "current change %s" % currentChange
2943
2944 if currentChange == change:
2945 if self.verbose:
2946 print "found %s" % next
2947 return next
2948
2949 if currentChange < change:
2950 earliestCommit = "^%s" % next
2951 else:
2952 latestCommit = "%s" % next
2953
2954 return ""
2955
2956 def importNewBranch(self, branch, maxChange):
2957 # make fast-import flush all changes to disk and update the refs using the checkpoint
2958 # command so that we can try to find the branch parent in the git history
2959 self.gitStream.write("checkpoint\n\n");
2960 self.gitStream.flush();
2961 branchPrefix = self.depotPaths[0] + branch + "/"
2962 range = "@1,%s" % maxChange
2963 #print "prefix" + branchPrefix
96b2d54a 2964 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
2965 if len(changes) <= 0:
2966 return False
2967 firstChange = changes[0]
2968 #print "first change in branch: %s" % firstChange
2969 sourceBranch = self.knownBranches[branch]
2970 sourceDepotPath = self.depotPaths[0] + sourceBranch
2971 sourceRef = self.gitRefForBranch(sourceBranch)
2972 #print "source " + sourceBranch
2973
52a4880b 2974 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2975 #print "branch parent: %s" % branchParentChange
2976 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2977 if len(gitParent) > 0:
2978 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2979 #print "parent git commit: %s" % gitParent
2980
2981 self.importChanges(changes)
2982 return True
2983
fed23693
VA
2984 def searchParent(self, parent, branch, target):
2985 parentFound = False
c7d34884
PW
2986 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2987 "--no-merges", parent]):
fed23693
VA
2988 blob = blob.strip()
2989 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2990 parentFound = True
2991 if self.verbose:
2992 print "Found parent of %s in commit %s" % (branch, blob)
2993 break
2994 if parentFound:
2995 return blob
2996 else:
2997 return None
2998
e87f37ae
SH
2999 def importChanges(self, changes):
3000 cnt = 1
3001 for change in changes:
18fa13d0 3002 description = p4_describe(change)
e87f37ae
SH
3003 self.updateOptionDict(description)
3004
3005 if not self.silent:
3006 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3007 sys.stdout.flush()
3008 cnt = cnt + 1
3009
3010 try:
3011 if self.detectBranches:
3012 branches = self.splitFilesIntoBranches(description)
3013 for branch in branches.keys():
3014 ## HACK --hwn
3015 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3016 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3017
3018 parent = ""
3019
3020 filesForCommit = branches[branch]
3021
3022 if self.verbose:
3023 print "branch is %s" % branch
3024
3025 self.updatedBranches.add(branch)
3026
3027 if branch not in self.createdBranches:
3028 self.createdBranches.add(branch)
3029 parent = self.knownBranches[branch]
3030 if parent == branch:
3031 parent = ""
1ca3d710
SH
3032 else:
3033 fullBranch = self.projectName + branch
3034 if fullBranch not in self.p4BranchesInGit:
3035 if not self.silent:
3036 print("\n Importing new branch %s" % fullBranch);
3037 if self.importNewBranch(branch, change - 1):
3038 parent = ""
3039 self.p4BranchesInGit.append(fullBranch)
3040 if not self.silent:
3041 print("\n Resuming with change %s" % change);
3042
3043 if self.verbose:
3044 print "parent determined through known branches: %s" % parent
e87f37ae 3045
8134f69c
SH
3046 branch = self.gitRefForBranch(branch)
3047 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3048
3049 if self.verbose:
3050 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3051
3052 if len(parent) == 0 and branch in self.initialParents:
3053 parent = self.initialParents[branch]
3054 del self.initialParents[branch]
3055
fed23693
VA
3056 blob = None
3057 if len(parent) > 0:
4f9273d2 3058 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3059 if self.verbose:
3060 print "Creating temporary branch: " + tempBranch
e63231e5 3061 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3062 self.tempBranches.append(tempBranch)
3063 self.checkpoint()
3064 blob = self.searchParent(parent, branch, tempBranch)
3065 if blob:
e63231e5 3066 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3067 else:
3068 if self.verbose:
3069 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3070 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3071 else:
3072 files = self.extractFilesFromCommit(description)
e63231e5 3073 self.commit(description, files, self.branch,
e87f37ae 3074 self.initialParent)
47497844 3075 # only needed once, to connect to the previous commit
e87f37ae
SH
3076 self.initialParent = ""
3077 except IOError:
3078 print self.gitError.read()
3079 sys.exit(1)
3080
c208a243
SH
3081 def importHeadRevision(self, revision):
3082 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3083
4e2e6ce4
PW
3084 details = {}
3085 details["user"] = "git perforce import user"
1494fcbb 3086 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3087 % (' '.join(self.depotPaths), revision))
3088 details["change"] = revision
3089 newestRevision = 0
3090
3091 fileCnt = 0
6de040df
LD
3092 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3093
3094 for info in p4CmdList(["files"] + fileArgs):
c208a243 3095
68b28593 3096 if 'code' in info and info['code'] == 'error':
c208a243
SH
3097 sys.stderr.write("p4 returned an error: %s\n"
3098 % info['data'])
d88e707f
PW
3099 if info['data'].find("must refer to client") >= 0:
3100 sys.stderr.write("This particular p4 error is misleading.\n")
3101 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3102 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3103 sys.exit(1)
68b28593
PW
3104 if 'p4ExitCode' in info:
3105 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3106 sys.exit(1)
3107
3108
3109 change = int(info["change"])
3110 if change > newestRevision:
3111 newestRevision = change
3112
56c09345 3113 if info["action"] in self.delete_actions:
c208a243
SH
3114 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3115 #fileCnt = fileCnt + 1
3116 continue
3117
3118 for prop in ["depotFile", "rev", "action", "type" ]:
3119 details["%s%s" % (prop, fileCnt)] = info[prop]
3120
3121 fileCnt = fileCnt + 1
3122
3123 details["change"] = newestRevision
4e2e6ce4 3124
9dcb9f24 3125 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3126 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3127 res = p4_describe(newestRevision)
3128 details["time"] = res["time"]
4e2e6ce4 3129
c208a243
SH
3130 self.updateOptionDict(details)
3131 try:
e63231e5 3132 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3133 except IOError:
3134 print "IO error with git fast-import. Is your git version recent enough?"
3135 print self.gitError.read()
3136
3137
b984733c 3138 def run(self, args):
6326aa58 3139 self.depotPaths = []
179caebf 3140 self.changeRange = ""
6326aa58 3141 self.previousDepotPaths = []
991a2de4 3142 self.hasOrigin = False
ce6f33c8 3143
29bdbac1
SH
3144 # map from branch depot path to parent branch
3145 self.knownBranches = {}
3146 self.initialParents = {}
3147
a028a98e
SH
3148 if self.importIntoRemotes:
3149 self.refPrefix = "refs/remotes/p4/"
3150 else:
db775559 3151 self.refPrefix = "refs/heads/p4/"
a028a98e 3152
991a2de4
PW
3153 if self.syncWithOrigin:
3154 self.hasOrigin = originP4BranchesExist()
3155 if self.hasOrigin:
3156 if not self.silent:
3157 print 'Syncing with origin first, using "git fetch origin"'
3158 system("git fetch origin")
10f880f8 3159
5a8e84cd 3160 branch_arg_given = bool(self.branch)
569d1bd4 3161 if len(self.branch) == 0:
db775559 3162 self.branch = self.refPrefix + "master"
a028a98e 3163 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3164 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3165 system("git branch -D p4")
967f72e2 3166
a93d33ee
PW
3167 # accept either the command-line option, or the configuration variable
3168 if self.useClientSpec:
3169 # will use this after clone to set the variable
3170 self.useClientSpec_from_options = True
3171 else:
0d609032 3172 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3173 self.useClientSpec = True
3174 if self.useClientSpec:
543987bd 3175 self.clientSpecDirs = getClientSpec()
3a70cdfa 3176
6a49f8e2
HWN
3177 # TODO: should always look at previous commits,
3178 # merge with previous imports, if possible.
3179 if args == []:
d414c74a 3180 if self.hasOrigin:
5ca44617 3181 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3182
3183 # branches holds mapping from branch name to sha1
3184 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3185
3186 # restrict to just this one, disabling detect-branches
3187 if branch_arg_given:
3188 short = self.branch.split("/")[-1]
3189 if short in branches:
3190 self.p4BranchesInGit = [ short ]
3191 else:
3192 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3193
3194 if len(self.p4BranchesInGit) > 1:
3195 if not self.silent:
3196 print "Importing from/into multiple branches"
3197 self.detectBranches = True
8c9e8b6e
PW
3198 for branch in branches.keys():
3199 self.initialParents[self.refPrefix + branch] = \
3200 branches[branch]
967f72e2 3201
29bdbac1
SH
3202 if self.verbose:
3203 print "branches: %s" % self.p4BranchesInGit
3204
3205 p4Change = 0
3206 for branch in self.p4BranchesInGit:
cebdf5af 3207 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3208
3209 settings = extractSettingsGitLog(logMsg)
29bdbac1 3210
bb6e09b2
HWN
3211 self.readOptions(settings)
3212 if (settings.has_key('depot-paths')
3213 and settings.has_key ('change')):
3214 change = int(settings['change']) + 1
29bdbac1
SH
3215 p4Change = max(p4Change, change)
3216
bb6e09b2
HWN
3217 depotPaths = sorted(settings['depot-paths'])
3218 if self.previousDepotPaths == []:
6326aa58 3219 self.previousDepotPaths = depotPaths
29bdbac1 3220 else:
6326aa58
HWN
3221 paths = []
3222 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3223 prev_list = prev.split("/")
3224 cur_list = cur.split("/")
3225 for i in range(0, min(len(cur_list), len(prev_list))):
3226 if cur_list[i] <> prev_list[i]:
583e1707 3227 i = i - 1
6326aa58
HWN
3228 break
3229
04d277b3 3230 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3231
3232 self.previousDepotPaths = paths
29bdbac1
SH
3233
3234 if p4Change > 0:
bb6e09b2 3235 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3236 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3237 if not self.silent and not self.detectBranches:
967f72e2 3238 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3239
40d69ac3
PW
3240 # accept multiple ref name abbreviations:
3241 # refs/foo/bar/branch -> use it exactly
3242 # p4/branch -> prepend refs/remotes/ or refs/heads/
3243 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3244 if not self.branch.startswith("refs/"):
40d69ac3
PW
3245 if self.importIntoRemotes:
3246 prepend = "refs/remotes/"
3247 else:
3248 prepend = "refs/heads/"
3249 if not self.branch.startswith("p4/"):
3250 prepend += "p4/"
3251 self.branch = prepend + self.branch
179caebf 3252
6326aa58 3253 if len(args) == 0 and self.depotPaths:
b984733c 3254 if not self.silent:
6326aa58 3255 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3256 else:
6326aa58 3257 if self.depotPaths and self.depotPaths != args:
cebdf5af 3258 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3259 "This doesn't work!" % (' '.join (self.depotPaths),
3260 ' '.join (args)))
b984733c 3261 sys.exit(1)
6326aa58 3262
bb6e09b2 3263 self.depotPaths = sorted(args)
b984733c 3264
1c49fc19 3265 revision = ""
b984733c 3266 self.users = {}
b984733c 3267
58c8bc7c
PW
3268 # Make sure no revision specifiers are used when --changesfile
3269 # is specified.
3270 bad_changesfile = False
3271 if len(self.changesFile) > 0:
3272 for p in self.depotPaths:
3273 if p.find("@") >= 0 or p.find("#") >= 0:
3274 bad_changesfile = True
3275 break
3276 if bad_changesfile:
3277 die("Option --changesfile is incompatible with revision specifiers")
3278
6326aa58
HWN
3279 newPaths = []
3280 for p in self.depotPaths:
3281 if p.find("@") != -1:
3282 atIdx = p.index("@")
3283 self.changeRange = p[atIdx:]
3284 if self.changeRange == "@all":
3285 self.changeRange = ""
6a49f8e2 3286 elif ',' not in self.changeRange:
1c49fc19 3287 revision = self.changeRange
6326aa58 3288 self.changeRange = ""
7fcff9de 3289 p = p[:atIdx]
6326aa58
HWN
3290 elif p.find("#") != -1:
3291 hashIdx = p.index("#")
1c49fc19 3292 revision = p[hashIdx:]
7fcff9de 3293 p = p[:hashIdx]
6326aa58 3294 elif self.previousDepotPaths == []:
58c8bc7c
PW
3295 # pay attention to changesfile, if given, else import
3296 # the entire p4 tree at the head revision
3297 if len(self.changesFile) == 0:
3298 revision = "#head"
6326aa58
HWN
3299
3300 p = re.sub ("\.\.\.$", "", p)
3301 if not p.endswith("/"):
3302 p += "/"
3303
3304 newPaths.append(p)
3305
3306 self.depotPaths = newPaths
3307
e63231e5
PW
3308 # --detect-branches may change this for each branch
3309 self.branchPrefixes = self.depotPaths
3310
b607e71e 3311 self.loadUserMapFromCache()
cb53e1f8
SH
3312 self.labels = {}
3313 if self.detectLabels:
3314 self.getLabels();
b984733c 3315
4b97ffb1 3316 if self.detectBranches:
df450923
SH
3317 ## FIXME - what's a P4 projectName ?
3318 self.projectName = self.guessProjectName()
3319
38f9f5ec
SH
3320 if self.hasOrigin:
3321 self.getBranchMappingFromGitBranches()
3322 else:
3323 self.getBranchMapping()
29bdbac1
SH
3324 if self.verbose:
3325 print "p4-git branches: %s" % self.p4BranchesInGit
3326 print "initial parents: %s" % self.initialParents
3327 for b in self.p4BranchesInGit:
3328 if b != "master":
6326aa58
HWN
3329
3330 ## FIXME
29bdbac1
SH
3331 b = b[len(self.projectName):]
3332 self.createdBranches.add(b)
4b97ffb1 3333
f291b4e3 3334 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3335
78189bea
PW
3336 self.importProcess = subprocess.Popen(["git", "fast-import"],
3337 stdin=subprocess.PIPE,
3338 stdout=subprocess.PIPE,
3339 stderr=subprocess.PIPE);
3340 self.gitOutput = self.importProcess.stdout
3341 self.gitStream = self.importProcess.stdin
3342 self.gitError = self.importProcess.stderr
b984733c 3343
1c49fc19 3344 if revision:
c208a243 3345 self.importHeadRevision(revision)
b984733c
SH
3346 else:
3347 changes = []
3348
0828ab14 3349 if len(self.changesFile) > 0:
b984733c 3350 output = open(self.changesFile).readlines()
1d7367dc 3351 changeSet = set()
b984733c
SH
3352 for line in output:
3353 changeSet.add(int(line))
3354
3355 for change in changeSet:
3356 changes.append(change)
3357
3358 changes.sort()
3359 else:
9dcb9f24
PW
3360 # catch "git p4 sync" with no new branches, in a repo that
3361 # does not have any existing p4 branches
5a8e84cd
PW
3362 if len(args) == 0:
3363 if not self.p4BranchesInGit:
3364 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3365
3366 # The default branch is master, unless --branch is used to
3367 # specify something else. Make sure it exists, or complain
3368 # nicely about how to use --branch.
3369 if not self.detectBranches:
3370 if not branch_exists(self.branch):
3371 if branch_arg_given:
3372 die("Error: branch %s does not exist." % self.branch)
3373 else:
3374 die("Error: no branch %s; perhaps specify one with --branch." %
3375 self.branch)
3376
29bdbac1 3377 if self.verbose:
86dff6b6 3378 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3379 self.changeRange)
96b2d54a 3380 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
b984733c 3381
01a9c9c5 3382 if len(self.maxChanges) > 0:
7fcff9de 3383 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3384
b984733c 3385 if len(changes) == 0:
0828ab14 3386 if not self.silent:
341dc1c1 3387 print "No changes to import!"
06804c76
LD
3388 else:
3389 if not self.silent and not self.detectBranches:
3390 print "Import destination: %s" % self.branch
3391
3392 self.updatedBranches = set()
b984733c 3393
47497844
PW
3394 if not self.detectBranches:
3395 if args:
3396 # start a new branch
3397 self.initialParent = ""
3398 else:
3399 # build on a previous revision
3400 self.initialParent = parseRevision(self.branch)
3401
06804c76 3402 self.importChanges(changes)
a9d1a27a 3403
06804c76
LD
3404 if not self.silent:
3405 print ""
3406 if len(self.updatedBranches) > 0:
3407 sys.stdout.write("Updated branches: ")
3408 for b in self.updatedBranches:
3409 sys.stdout.write("%s " % b)
3410 sys.stdout.write("\n")
341dc1c1 3411
0d609032 3412 if gitConfigBool("git-p4.importLabels"):
06dcd152 3413 self.importLabels = True
b984733c 3414
06804c76
LD
3415 if self.importLabels:
3416 p4Labels = getP4Labels(self.depotPaths)
3417 gitTags = getGitTags()
3418
3419 missingP4Labels = p4Labels - gitTags
3420 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3421
b984733c 3422 self.gitStream.close()
78189bea 3423 if self.importProcess.wait() != 0:
29bdbac1 3424 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3425 self.gitOutput.close()
3426 self.gitError.close()
3427
fed23693
VA
3428 # Cleanup temporary branches created during import
3429 if self.tempBranches != []:
3430 for branch in self.tempBranches:
3431 read_pipe("git update-ref -d %s" % branch)
3432 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3433
55d12437
PW
3434 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3435 # a convenient shortcut refname "p4".
3436 if self.importIntoRemotes:
3437 head_ref = self.refPrefix + "HEAD"
3438 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3439 system(["git", "symbolic-ref", head_ref, self.branch])
3440
b984733c
SH
3441 return True
3442
01ce1fe9
SH
3443class P4Rebase(Command):
3444 def __init__(self):
3445 Command.__init__(self)
06804c76
LD
3446 self.options = [
3447 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3448 ]
06804c76 3449 self.importLabels = False
cebdf5af
HWN
3450 self.description = ("Fetches the latest revision from perforce and "
3451 + "rebases the current work (branch) against it")
01ce1fe9
SH