Merge branch 'mh/notes-duplicate-entries'
[git/git.git] / git-p4.py
<
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
efdcc992
LD
30# support basestring in python3
31try:
32 unicode = unicode
33except NameError:
34 # 'unicode' is undefined, must be Python 3
35 str = str
36 unicode = str
37 bytes = bytes
38 basestring = (str,bytes)
39else:
40 # 'unicode' exists, must be Python 2
41 str = str
42 unicode = unicode
43 bytes = str
44 basestring = basestring
45
a235e85c
BC
46try:
47 from subprocess import CalledProcessError
48except ImportError:
49 # from python2.7:subprocess.py
50 # Exception classes used by this module.
51 class CalledProcessError(Exception):
52 """This exception is raised when a process run by check_call() returns
53 a non-zero exit status. The exit status will be stored in the
54 returncode attribute."""
55 def __init__(self, returncode, cmd):
56 self.returncode = returncode
57 self.cmd = cmd
58 def __str__(self):
59 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
4addad22 61verbose = False
86949eef 62
06804c76 63# Only labels/tags matching this will be imported/exported
c8942a22 64defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 65
3deed5e0
LD
66# The block size is reduced automatically if required
67defaultBlockSize = 1<<20
1051ef00 68
0ef67acd 69p4_access_checked = False
1051ef00 70
21a50753
AK
71def p4_build_cmd(cmd):
72 """Build a suitable p4 command line.
73
74 This consolidates building and returning a p4 command line into one
75 location. It means that hooking into the environment, or other configuration
76 can be done more easily.
77 """
6de040df 78 real_cmd = ["p4"]
abcaf073
AK
79
80 user = gitConfig("git-p4.user")
81 if len(user) > 0:
6de040df 82 real_cmd += ["-u",user]
abcaf073
AK
83
84 password = gitConfig("git-p4.password")
85 if len(password) > 0:
6de040df 86 real_cmd += ["-P", password]
abcaf073
AK
87
88 port = gitConfig("git-p4.port")
89 if len(port) > 0:
6de040df 90 real_cmd += ["-p", port]
abcaf073
AK
91
92 host = gitConfig("git-p4.host")
93 if len(host) > 0:
41799aa2 94 real_cmd += ["-H", host]
abcaf073
AK
95
96 client = gitConfig("git-p4.client")
97 if len(client) > 0:
6de040df 98 real_cmd += ["-c", client]
abcaf073 99
89a6ecc5
LS
100 retries = gitConfigInt("git-p4.retries")
101 if retries is None:
102 # Perform 3 retries by default
103 retries = 3
bc233524
IK
104 if retries > 0:
105 # Provide a way to not pass this option by setting git-p4.retries to 0
106 real_cmd += ["-r", str(retries)]
6de040df
LD
107
108 if isinstance(cmd,basestring):
109 real_cmd = ' '.join(real_cmd) + ' ' + cmd
110 else:
111 real_cmd += cmd
0ef67acd
LD
112
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked:
116 p4_access_checked = True # suppress access checks in p4_check_access itself
117 p4_check_access()
118
21a50753
AK
119 return real_cmd
120
378f7be1
LD
121def git_dir(path):
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
124 """
125 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126 if not d or len(d) == 0:
127 return None
128 else:
129 return d
130
bbd84863
MF
131def chdir(path, is_client_path=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
136
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
141
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
145 the client path.
146 """
147
148 os.chdir(path)
149 if not is_client_path:
150 path = os.getcwd()
151 os.environ['PWD'] = path
053fd0c1 152
4d25dc44
LS
153def calcDiskFree():
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform.system() == 'Windows':
156 free_bytes = ctypes.c_ulonglong(0)
157 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158 return free_bytes.value
159 else:
160 st = os.statvfs(os.getcwd())
161 return st.f_bavail * st.f_frsize
162
86dff6b6
HWN
163def die(msg):
164 if verbose:
165 raise Exception(msg)
166 else:
167 sys.stderr.write(msg + "\n")
168 sys.exit(1)
169
6de040df 170def write_pipe(c, stdin):
4addad22 171 if verbose:
6de040df 172 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 173
6de040df
LD
174 expand = isinstance(c,basestring)
175 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176 pipe = p.stdin
177 val = pipe.write(stdin)
178 pipe.close()
179 if p.wait():
180 die('Command failed: %s' % str(c))
b016d397
HWN
181
182 return val
183
6de040df 184def p4_write_pipe(c, stdin):
d9429194 185 real_cmd = p4_build_cmd(c)
6de040df 186 return write_pipe(real_cmd, stdin)
d9429194 187
78871bf4
LD
188def read_pipe_full(c):
189 """ Read output from command. Returns a tuple
190 of the return status, stdout text and stderr
191 text.
192 """
4addad22 193 if verbose:
6de040df 194 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 195
6de040df 196 expand = isinstance(c,basestring)
1f5f3907
LS
197 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198 (out, err) = p.communicate()
78871bf4
LD
199 return (p.returncode, out, err)
200
201def read_pipe(c, ignore_error=False):
202 """ Read output from command. Returns the output text on
203 success. On failure, terminates execution, unless
204 ignore_error is True, when it returns an empty string.
205 """
206 (retcode, out, err) = read_pipe_full(c)
207 if retcode != 0:
208 if ignore_error:
209 out = ""
210 else:
211 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 212 return out
b016d397 213
78871bf4
LD
214def read_pipe_text(c):
215 """ Read output from a command with trailing whitespace stripped.
216 On error, returns None.
217 """
218 (retcode, out, err) = read_pipe_full(c)
219 if retcode != 0:
220 return None
221 else:
222 return out.rstrip()
223
d9429194
AK
224def p4_read_pipe(c, ignore_error=False):
225 real_cmd = p4_build_cmd(c)
226 return read_pipe(real_cmd, ignore_error)
b016d397 227
bce4c5fc 228def read_pipe_lines(c):
4addad22 229 if verbose:
6de040df
LD
230 sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232 expand = isinstance(c, basestring)
233 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234 pipe = p.stdout
b016d397 235 val = pipe.readlines()
6de040df
LD
236 if pipe.close() or p.wait():
237 die('Command failed: %s' % str(c))
b016d397
HWN
238
239 return val
caace111 240
2318121b
AK
241def p4_read_pipe_lines(c):
242 """Specifically invoke p4 on the command supplied. """
155af834 243 real_cmd = p4_build_cmd(c)
2318121b
AK
244 return read_pipe_lines(real_cmd)
245
8e9497c2
GG
246def p4_has_command(cmd):
247 """Ask p4 for help on this command. If it returns an error, the
248 command does not exist in this version of p4."""
249 real_cmd = p4_build_cmd(["help", cmd])
250 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251 stderr=subprocess.PIPE)
252 p.communicate()
253 return p.returncode == 0
254
249da4c0
PW
255def p4_has_move_command():
256 """See if the move command exists, that it supports -k, and that
257 it has not been administratively disabled. The arguments
258 must be correct, but the filenames do not have to exist. Use
259 ones with wildcards so even if they exist, it will fail."""
260
261 if not p4_has_command("move"):
262 return False
263 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265 (out, err) = p.communicate()
266 # return code will be 1 in either case
267 if err.find("Invalid option") >= 0:
268 return False
269 if err.find("disabled") >= 0:
270 return False
271 # assume it failed because @... was invalid changelist
272 return True
273
cbff4b25 274def system(cmd, ignore_error=False):
6de040df 275 expand = isinstance(cmd,basestring)
4addad22 276 if verbose:
6de040df 277 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 278 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 279 if retcode and not ignore_error:
a235e85c 280 raise CalledProcessError(retcode, cmd)
6754a299 281
cbff4b25
LD
282 return retcode
283
bf9320f1
AK
284def p4_system(cmd):
285 """Specifically invoke p4 as the system command. """
155af834 286 real_cmd = p4_build_cmd(cmd)
6de040df 287 expand = isinstance(real_cmd, basestring)
a235e85c
BC
288 retcode = subprocess.call(real_cmd, shell=expand)
289 if retcode:
290 raise CalledProcessError(retcode, real_cmd)
6de040df 291
0ef67acd
LD
292def die_bad_access(s):
293 die("failure accessing depot: {0}".format(s.rstrip()))
294
295def p4_check_access(min_expiration=1):
296 """ Check if we can access Perforce - account still logged in
297 """
298 results = p4CmdList(["login", "-s"])
299
300 if len(results) == 0:
301 # should never get here: always get either some results, or a p4ExitCode
302 assert("could not parse response from perforce")
303
304 result = results[0]
305
306 if 'p4ExitCode' in result:
307 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308 die_bad_access("could not run p4")
309
310 code = result.get("code")
311 if not code:
312 # we get here if we couldn't connect and there was nothing to unmarshal
313 die_bad_access("could not connect")
314
315 elif code == "stat":
316 expiry = result.get("TicketExpiration")
317 if expiry:
318 expiry = int(expiry)
319 if expiry > min_expiration:
320 # ok to carry on
321 return
322 else:
323 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325 else:
326 # account without a timeout - all ok
327 return
328
329 elif code == "error":
330 data = result.get("data")
331 if data:
332 die_bad_access("p4 error: {0}".format(data))
333 else:
334 die_bad_access("unknown error")
d4990d56
PO
335 elif code == "info":
336 return
0ef67acd
LD
337 else:
338 die_bad_access("unknown error code {0}".format(code))
339
7f0e5962
PW
340_p4_version_string = None
341def p4_version_string():
342 """Read the version string, showing just the last line, which
343 hopefully is the interesting version bit.
344
345 $ p4 -V
346 Perforce - The Fast Software Configuration Management System.
347 Copyright 1995-2011 Perforce Software. All rights reserved.
348 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
349 """
350 global _p4_version_string
351 if not _p4_version_string:
352 a = p4_read_pipe_lines(["-V"])
353 _p4_version_string = a[-1].rstrip()
354 return _p4_version_string
355
6de040df 356def p4_integrate(src, dest):
9d7d446a 357 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 358
8d7ec362 359def p4_sync(f, *options):
9d7d446a 360 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
361
362def p4_add(f):
9d7d446a
PW
363 # forcibly add file names with wildcards
364 if wildcard_present(f):
365 p4_system(["add", "-f", f])
366 else:
367 p4_system(["add", f])
6de040df
LD
368
369def p4_delete(f):
9d7d446a 370 p4_system(["delete", wildcard_encode(f)])
6de040df 371
a02b8bc4
RP
372def p4_edit(f, *options):
373 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
374
375def p4_revert(f):
9d7d446a 376 p4_system(["revert", wildcard_encode(f)])
6de040df 377
9d7d446a
PW
378def p4_reopen(type, f):
379 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 380
46c609e9
LD
381def p4_reopen_in_change(changelist, files):
382 cmd = ["reopen", "-c", str(changelist)] + files
383 p4_system(cmd)
384
8e9497c2
GG
385def p4_move(src, dest):
386 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
387
1051ef00 388def p4_last_change():
1997e91f 389 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
390 return int(results[0]['change'])
391
123f6317 392def p4_describe(change, shelved=False):
18fa13d0
PW
393 """Make sure it returns a valid result by checking for
394 the presence of field "time". Return a dict of the
395 results."""
396
123f6317
LD
397 cmd = ["describe", "-s"]
398 if shelved:
399 cmd += ["-S"]
400 cmd += [str(change)]
401
402 ds = p4CmdList(cmd, skip_info=True)
18fa13d0
PW
403 if len(ds) != 1:
404 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
405
406 d = ds[0]
407
408 if "p4ExitCode" in d:
409 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
410 str(d)))
411 if "code" in d:
412 if d["code"] == "error":
413 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
414
415 if "time" not in d:
416 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
417
418 return d
419
9cffb8c8
PW
420#
421# Canonicalize the p4 type and return a tuple of the
422# base type, plus any modifiers. See "p4 help filetypes"
423# for a list and explanation.
424#
425def split_p4_type(p4type):
426
427 p4_filetypes_historical = {
428 "ctempobj": "binary+Sw",
429 "ctext": "text+C",
430 "cxtext": "text+Cx",
431 "ktext": "text+k",
432 "kxtext": "text+kx",
433 "ltext": "text+F",
434 "tempobj": "binary+FSw",
435 "ubinary": "binary+F",
436 "uresource": "resource+F",
437 "uxbinary": "binary+Fx",
438 "xbinary": "binary+x",
439 "xltext": "text+Fx",
440 "xtempobj": "binary+Swx",
441 "xtext": "text+x",
442 "xunicode": "unicode+x",
443 "xutf16": "utf16+x",
444 }
445 if p4type in p4_filetypes_historical:
446 p4type = p4_filetypes_historical[p4type]
447 mods = ""
448 s = p4type.split("+")
449 base = s[0]
450 mods = ""
451 if len(s) > 1:
452 mods = s[1]
453 return (base, mods)
b9fc6ea9 454
60df071c
LD
455#
456# return the raw p4 type of a file (text, text+ko, etc)
457#
79467e61
PW
458def p4_type(f):
459 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
460 return results[0]['headType']
461
462#
463# Given a type base and modifier, return a regexp matching
464# the keywords that can be expanded in the file
465#
466def p4_keywords_regexp_for_type(base, type_mods):
467 if base in ("text", "unicode", "binary"):
468 kwords = None
469 if "ko" in type_mods:
470 kwords = 'Id|Header'
471 elif "k" in type_mods:
472 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
473 else:
474 return None
475 pattern = r"""
476 \$ # Starts with a dollar, followed by...
477 (%s) # one of the keywords, followed by...
6b2bf41e 478 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
479 \$ # another dollar
480 """ % kwords
481 return pattern
482 else:
483 return None
484
485#
486# Given a file, return a regexp matching the possible
487# RCS keywords that will be expanded, or None for files
488# with kw expansion turned off.
489#
490def p4_keywords_regexp_for_file(file):
491 if not os.path.exists(file):
492 return None
493 else:
494 (type_base, type_mods) = split_p4_type(p4_type(file))
495 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 496
c65b670e
CP
497def setP4ExecBit(file, mode):
498 # Reopens an already open file and changes the execute bit to match
499 # the execute bit setting in the passed in mode.
500
501 p4Type = "+x"
502
503 if not isModeExec(mode):
504 p4Type = getP4OpenedType(file)
505 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
506 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
507 if p4Type[-1] == "+":
508 p4Type = p4Type[0:-1]
509
6de040df 510 p4_reopen(p4Type, file)
c65b670e
CP
511
512def getP4OpenedType(file):
513 # Returns the perforce file type for the given file.
514
9d7d446a 515 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 516 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
517 if match:
518 return match.group(1)
519 else:
f3e5ae4f 520 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 521
06804c76
LD
522# Return the set of all p4 labels
523def getP4Labels(depotPaths):
524 labels = set()
525 if isinstance(depotPaths,basestring):
526 depotPaths = [depotPaths]
527
528 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
529 label = l['label']
530 labels.add(label)
531
532 return labels
533
534# Return the set of all git tags
535def getGitTags():
536 gitTags = set()
537 for line in read_pipe_lines(["git", "tag"]):
538 tag = line.strip()
539 gitTags.add(tag)
540 return gitTags
541
b43b0a3c
CP
542def diffTreePattern():
543 # This is a simple generator for the diff tree regex pattern. This could be
544 # a class variable if this and parseDiffTreeEntry were a part of a class.
545 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
546 while True:
547 yield pattern
548
549def parseDiffTreeEntry(entry):
550 """Parses a single diff tree entry into its component elements.
551
552 See git-diff-tree(1) manpage for details about the format of the diff
553 output. This method returns a dictionary with the following elements:
554
555 src_mode - The mode of the source file
556 dst_mode - The mode of the destination file
557 src_sha1 - The sha1 for the source file
558 dst_sha1 - The sha1 fr the destination file
559 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
560 status_score - The score for the status (applicable for 'C' and 'R'
561 statuses). This is None if there is no score.
562 src - The path for the source file.
563 dst - The path for the destination file. This is only present for
564 copy or renames. If it is not present, this is None.
565
566 If the pattern is not matched, None is returned."""
567
568 match = diffTreePattern().next().match(entry)
569 if match:
570 return {
571 'src_mode': match.group(1),
572 'dst_mode': match.group(2),
573 'src_sha1': match.group(3),
574 'dst_sha1': match.group(4),
575 'status': match.group(5),
576 'status_score': match.group(6),
577 'src': match.group(7),
578 'dst': match.group(10)
579 }
580 return None
581
c65b670e
CP
582def isModeExec(mode):
583 # Returns True if the given git mode represents an executable file,
584 # otherwise False.
585 return mode[-3:] == "755"
586
55bb3e36
LD
587class P4Exception(Exception):
588 """ Base class for exceptions from the p4 client """
589 def __init__(self, exit_code):
590 self.p4ExitCode = exit_code
591
592class P4ServerException(P4Exception):
593 """ Base class for exceptions where we get some kind of marshalled up result from the server """
594 def __init__(self, exit_code, p4_result):
595 super(P4ServerException, self).__init__(exit_code)
596 self.p4_result = p4_result
597 self.code = p4_result[0]['code']
598 self.data = p4_result[0]['data']
599
600class P4RequestSizeException(P4ServerException):
601 """ One of the maxresults or maxscanrows errors """
602 def __init__(self, exit_code, p4_result, limit):
603 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
604 self.limit = limit
605
c65b670e
CP
606def isModeExecChanged(src_mode, dst_mode):
607 return isModeExec(src_mode) != isModeExec(dst_mode)
608
55bb3e36
LD
609def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
610 errors_as_exceptions=False):
6de040df
LD
611
612 if isinstance(cmd,basestring):
613 cmd = "-G " + cmd
614 expand = True
615 else:
616 cmd = ["-G"] + cmd
617 expand = False
618
619 cmd = p4_build_cmd(cmd)
6a49f8e2 620 if verbose:
6de040df 621 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
622
623 # Use a temporary file to avoid deadlocks without
624 # subprocess.communicate(), which would put another copy
625 # of stdout into memory.
626 stdin_file = None
627 if stdin is not None:
628 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
629 if isinstance(stdin,basestring):
630 stdin_file.write(stdin)
631 else:
632 for i in stdin:
633 stdin_file.write(i + '\n')
9f90c733
SL
634 stdin_file.flush()
635 stdin_file.seek(0)
636
6de040df
LD
637 p4 = subprocess.Popen(cmd,
638 shell=expand,
9f90c733
SL
639 stdin=stdin_file,
640 stdout=subprocess.PIPE)
86949eef
SH
641
642 result = []
643 try:
644 while True:
9f90c733 645 entry = marshal.load(p4.stdout)
1997e91f
MT
646 if skip_info:
647 if 'code' in entry and entry['code'] == 'info':
648 continue
c3f6163b
AG
649 if cb is not None:
650 cb(entry)
651 else:
652 result.append(entry)
86949eef
SH
653 except EOFError:
654 pass
9f90c733
SL
655 exitCode = p4.wait()
656 if exitCode != 0:
55bb3e36
LD
657 if errors_as_exceptions:
658 if len(result) > 0:
659 data = result[0].get('data')
660 if data:
661 m = re.search('Too many rows scanned \(over (\d+)\)', data)
662 if not m:
663 m = re.search('Request too large \(over (\d+)\)', data)
664
665 if m:
666 limit = int(m.group(1))
667 raise P4RequestSizeException(exitCode, result, limit)
668
669 raise P4ServerException(exitCode, result)
670 else:
671 raise P4Exception(exitCode)
672 else:
673 entry = {}
674 entry["p4ExitCode"] = exitCode
675 result.append(entry)
86949eef
SH
676
677 return result
678
679def p4Cmd(cmd):
680 list = p4CmdList(cmd)
681 result = {}
682 for entry in list:
683 result.update(entry)
684 return result;
685
cb2c9db5
SH
686def p4Where(depotPath):
687 if not depotPath.endswith("/"):
688 depotPath += "/"
cd884106
VA
689 depotPathLong = depotPath + "..."
690 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
691 output = None
692 for entry in outputList:
75bc9573 693 if "depotFile" in entry:
cd884106
VA
694 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
695 # The base path always ends with "/...".
696 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
697 output = entry
698 break
699 elif "data" in entry:
700 data = entry.get("data")
701 space = data.find(" ")
702 if data[:space] == depotPath:
703 output = entry
704 break
7f705dc3
TAL
705 if output == None:
706 return ""
dc524036
SH
707 if output["code"] == "error":
708 return ""
cb2c9db5
SH
709 clientPath = ""
710 if "path" in output:
711 clientPath = output.get("path")
712 elif "data" in output:
713 data = output.get("data")
714 lastSpace = data.rfind(" ")
715 clientPath = data[lastSpace + 1:]
716
717 if clientPath.endswith("..."):
718 clientPath = clientPath[:-3]
719 return clientPath
720
86949eef 721def currentGitBranch():
eff45110 722 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 723
4f5cf76a 724def isValidGitDir(path):
378f7be1 725 return git_dir(path) != None
4f5cf76a 726
463e8af6 727def parseRevision(ref):
b25b2065 728 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 729
28755dba
PW
730def branchExists(ref):
731 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
732 ignore_error=True)
733 return len(rev) > 0
734
6ae8de88
SH
735def extractLogMessageFromGitCommit(commit):
736 logMessage = ""
b016d397
HWN
737
738 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 739 foundTitle = False
c3f2358d 740 for log in read_pipe_lines(["git", "cat-file", "commit", commit]):
6ae8de88
SH
741 if not foundTitle:
742 if len(log) == 1:
1c094184 743 foundTitle = True
6ae8de88
SH
744 continue
745
746 logMessage += log
747 return logMessage
748
bb6e09b2 749def extractSettingsGitLog(log):
6ae8de88
SH
750 values = {}
751 for line in log.split("\n"):
752 line = line.strip()
6326aa58
HWN
753 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
754 if not m:
755 continue
756
757 assignments = m.group(1).split (':')
758 for a in assignments:
759 vals = a.split ('=')
760 key = vals[0].strip()
761 val = ('='.join (vals[1:])).strip()
762 if val.endswith ('\"') and val.startswith('"'):
763 val = val[1:-1]
764
765 values[key] = val
766
845b42cb
SH
767 paths = values.get("depot-paths")
768 if not paths:
769 paths = values.get("depot-path")
a3fdd579
SH
770 if paths:
771 values['depot-paths'] = paths.split(',')
bb6e09b2 772 return values
6ae8de88 773
8136a639 774def gitBranchExists(branch):
bb6e09b2
HWN
775 proc = subprocess.Popen(["git", "rev-parse", branch],
776 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 777 return proc.wait() == 0;
8136a639 778
123f6317
LD
779def gitUpdateRef(ref, newvalue):
780 subprocess.check_call(["git", "update-ref", ref, newvalue])
781
782def gitDeleteRef(ref):
783 subprocess.check_call(["git", "update-ref", "-d", ref])
784
36bd8446 785_gitConfig = {}
b345d6c3 786
692e1796 787def gitConfig(key, typeSpecifier=None):
dba1c9d9 788 if key not in _gitConfig:
692e1796
LS
789 cmd = [ "git", "config" ]
790 if typeSpecifier:
791 cmd += [ typeSpecifier ]
792 cmd += [ key ]
b345d6c3
PW
793 s = read_pipe(cmd, ignore_error=True)
794 _gitConfig[key] = s.strip()
36bd8446 795 return _gitConfig[key]
01265103 796
0d609032
PW
797def gitConfigBool(key):
798 """Return a bool, using git config --bool. It is True only if the
799 variable is set to true, and False if set to false or not present
800 in the config."""
801
dba1c9d9 802 if key not in _gitConfig:
692e1796 803 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 804 return _gitConfig[key]
01265103 805
cb1dafdf 806def gitConfigInt(key):
dba1c9d9 807 if key not in _gitConfig:
cb1dafdf 808 cmd = [ "git", "config", "--int", key ]
0d609032
PW
809 s = read_pipe(cmd, ignore_error=True)
810 v = s.strip()
cb1dafdf
LS
811 try:
812 _gitConfig[key] = int(gitConfig(key, '--int'))
813 except ValueError:
814 _gitConfig[key] = None
36bd8446 815 return _gitConfig[key]
01265103 816
7199cf13 817def gitConfigList(key):
dba1c9d9 818 if key not in _gitConfig:
2abba301 819 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 820 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
821 if _gitConfig[key] == ['']:
822 _gitConfig[key] = []
7199cf13
VA
823 return _gitConfig[key]
824
2c8037ed
PW
825def p4BranchesInGit(branchesAreInRemotes=True):
826 """Find all the branches whose names start with "p4/", looking
827 in remotes or heads as specified by the argument. Return
828 a dictionary of { branch: revision } for each one found.
829 The branch names are the short names, without any
830 "p4/" prefix."""
831
062410bb
SH
832 branches = {}
833
834 cmdline = "git rev-parse --symbolic "
835 if branchesAreInRemotes:
2c8037ed 836 cmdline += "--remotes"
062410bb 837 else:
2c8037ed 838 cmdline += "--branches"
062410bb
SH
839
840 for line in read_pipe_lines(cmdline):
841 line = line.strip()
842
2c8037ed
PW
843 # only import to p4/
844 if not line.startswith('p4/'):
845 continue
846 # special symbolic ref to p4/master
847 if line == "p4/HEAD":
062410bb 848 continue
062410bb 849
2c8037ed
PW
850 # strip off p4/ prefix
851 branch = line[len("p4/"):]
062410bb
SH
852
853 branches[branch] = parseRevision(line)
2c8037ed 854
062410bb
SH
855 return branches
856
5a8e84cd
PW
857def branch_exists(branch):
858 """Make sure that the given ref name really exists."""
859
860 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
861 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
862 out, _ = p.communicate()
863 if p.returncode:
864 return False
865 # expect exactly one line of output: the branch name
866 return out.rstrip() == branch
867
9ceab363 868def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
869 branches = p4BranchesInGit()
870 # map from depot-path to branch name
871 branchByDepotPath = {}
872 for branch in branches.keys():
873 tip = branches[branch]
874 log = extractLogMessageFromGitCommit(tip)
875 settings = extractSettingsGitLog(log)
dba1c9d9 876 if "depot-paths" in settings:
86506fe5
SH
877 paths = ",".join(settings["depot-paths"])
878 branchByDepotPath[paths] = "remotes/p4/" + branch
879
27d2d811 880 settings = None
27d2d811
SH
881 parent = 0
882 while parent < 65535:
9ceab363 883 commit = head + "~%s" % parent
27d2d811
SH
884 log = extractLogMessageFromGitCommit(commit)
885 settings = extractSettingsGitLog(log)
dba1c9d9 886 if "depot-paths" in settings:
86506fe5 887 paths = ",".join(settings["depot-paths"])
dba1c9d9 888 if paths in branchByDepotPath:
86506fe5 889 return [branchByDepotPath[paths], settings]
27d2d811 890
86506fe5 891 parent = parent + 1
27d2d811 892
86506fe5 893 return ["", settings]
27d2d811 894
5ca44617
SH
895def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
896 if not silent:
f2606b17 897 print("Creating/updating branch(es) in %s based on origin branch(es)"
5ca44617
SH
898 % localRefPrefix)
899
900 originPrefix = "origin/p4/"
901
902 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
903 line = line.strip()
904 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
905 continue
906
907 headName = line[len(originPrefix):]
908 remoteHead = localRefPrefix + headName
909 originHead = line
910
911 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
dba1c9d9
LD
912 if ('depot-paths' not in original
913 or 'change' not in original):
5ca44617
SH
914 continue
915
916 update = False
917 if not gitBranchExists(remoteHead):
918 if verbose:
f2606b17 919 print("creating %s" % remoteHead)
5ca44617
SH
920 update = True
921 else:
922 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
dba1c9d9 923 if 'change' in settings:
5ca44617
SH
924 if settings['depot-paths'] == original['depot-paths']:
925 originP4Change = int(original['change'])
926 p4Change = int(settings['change'])
927 if originP4Change > p4Change:
f2606b17 928 print("%s (%s) is newer than %s (%s). "
5ca44617
SH
929 "Updating p4 branch from origin."
930 % (originHead, originP4Change,
931 remoteHead, p4Change))
932 update = True
933 else:
f2606b17 934 print("Ignoring: %s was imported from %s while "
5ca44617
SH
935 "%s was imported from %s"
936 % (originHead, ','.join(original['depot-paths']),
937 remoteHead, ','.join(settings['depot-paths'])))
938
939 if update:
940 system("git update-ref %s %s" % (remoteHead, originHead))
941
942def originP4BranchesExist():
943 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
944
1051ef00
LD
945
946def p4ParseNumericChangeRange(parts):
947 changeStart = int(parts[0][1:])
948 if parts[1] == '#head':
949 changeEnd = p4_last_change()
950 else:
951 changeEnd = int(parts[1])
952
953 return (changeStart, changeEnd)
954
955def chooseBlockSize(blockSize):
956 if blockSize:
957 return blockSize
958 else:
959 return defaultBlockSize
960
961def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 962 assert depotPaths
96b2d54a 963
1051ef00
LD
964 # Parse the change range into start and end. Try to find integer
965 # revision ranges as these can be broken up into blocks to avoid
966 # hitting server-side limits (maxrows, maxscanresults). But if
967 # that doesn't work, fall back to using the raw revision specifier
968 # strings, without using block mode.
969
96b2d54a 970 if changeRange is None or changeRange == '':
1051ef00
LD
971 changeStart = 1
972 changeEnd = p4_last_change()
973 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
974 else:
975 parts = changeRange.split(',')
976 assert len(parts) == 2
1051ef00
LD
977 try:
978 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
979 block_size = chooseBlockSize(requestedBlockSize)
8fa0abf8 980 except ValueError:
1051ef00
LD
981 changeStart = parts[0][1:]
982 changeEnd = parts[1]
983 if requestedBlockSize:
984 die("cannot use --changes-block-size with non-numeric revisions")
985 block_size = None
4f6432d8 986
9943e5b9 987 changes = set()
96b2d54a 988
1f90a648 989 # Retrieve changes a block at a time, to prevent running
3deed5e0
LD
990 # into a MaxResults/MaxScanRows error from the server. If
991 # we _do_ hit one of those errors, turn down the block size
1051ef00 992
1f90a648
SH
993 while True:
994 cmd = ['changes']
1051ef00 995
1f90a648
SH
996 if block_size:
997 end = min(changeEnd, changeStart + block_size)
998 revisionRange = "%d,%d" % (changeStart, end)
999 else:
1000 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 1001
1f90a648 1002 for p in depotPaths:
1051ef00
LD
1003 cmd += ["%s...@%s" % (p, revisionRange)]
1004
3deed5e0
LD
1005 # fetch the changes
1006 try:
1007 result = p4CmdList(cmd, errors_as_exceptions=True)
1008 except P4RequestSizeException as e:
1009 if not block_size:
1010 block_size = e.limit
1011 elif block_size > e.limit:
1012 block_size = e.limit
1013 else:
1014 block_size = max(2, block_size // 2)
1015
1016 if verbose: print("block size error, retrying with block size {0}".format(block_size))
1017 continue
1018 except P4Exception as e:
1019 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1020
1f90a648 1021 # Insert changes in chronological order
3deed5e0 1022 for entry in reversed(result):
dba1c9d9 1023 if 'change' not in entry:
b596b3b9
MT
1024 continue
1025 changes.add(int(entry['change']))
1051ef00 1026
1f90a648
SH
1027 if not block_size:
1028 break
1051ef00 1029
1f90a648
SH
1030 if end >= changeEnd:
1031 break
1051ef00 1032
1f90a648 1033 changeStart = end + 1
4f6432d8 1034
1f90a648
SH
1035 changes = sorted(changes)
1036 return changes
4f6432d8 1037
d53de8b9
TAL
1038def p4PathStartsWith(path, prefix):
1039 # This method tries to remedy a potential mixed-case issue:
1040 #
1041 # If UserA adds //depot/DirA/file1
1042 # and UserB adds //depot/dira/file2
1043 #
1044 # we may or may not have a problem. If you have core.ignorecase=true,
1045 # we treat DirA and dira as the same directory
0d609032 1046 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
1047 return path.lower().startswith(prefix.lower())
1048 return path.startswith(prefix)
1049
543987bd
PW
1050def getClientSpec():
1051 """Look at the p4 client spec, create a View() object that contains
1052 all the mappings, and return it."""
1053
1054 specList = p4CmdList("client -o")
1055 if len(specList) != 1:
1056 die('Output from "client -o" is %d lines, expecting 1' %
1057 len(specList))
1058
1059 # dictionary of all client parameters
1060 entry = specList[0]
1061
9d57c4a6
KS
1062 # the //client/ name
1063 client_name = entry["Client"]
1064
543987bd
PW
1065 # just the keys that start with "View"
1066 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1067
1068 # hold this new View
9d57c4a6 1069 view = View(client_name)
543987bd
PW
1070
1071 # append the lines, in order, to the view
1072 for view_num in range(len(view_keys)):
1073 k = "View%d" % view_num
1074 if k not in view_keys:
1075 die("Expected view key %s missing" % k)
1076 view.append(entry[k])
1077
1078 return view
1079
1080def getClientRoot():
1081 """Grab the client directory."""
1082
1083 output = p4CmdList("client -o")
1084 if len(output) != 1:
1085 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1086
1087 entry = output[0]
1088 if "Root" not in entry:
1089 die('Client has no "Root"')
1090
1091 return entry["Root"]
1092
9d7d446a
PW
1093#
1094# P4 wildcards are not allowed in filenames. P4 complains
1095# if you simply add them, but you can force it with "-f", in
1096# which case it translates them into %xx encoding internally.
1097#
1098def wildcard_decode(path):
1099 # Search for and fix just these four characters. Do % last so
1100 # that fixing it does not inadvertently create new %-escapes.
1101 # Cannot have * in a filename in windows; untested as to
1102 # what p4 would do in such a case.
1103 if not platform.system() == "Windows":
1104 path = path.replace("%2A", "*")
1105 path = path.replace("%23", "#") \
1106 .replace("%40", "@") \
1107 .replace("%25", "%")
1108 return path
1109
1110def wildcard_encode(path):
1111 # do % first to avoid double-encoding the %s introduced here
1112 path = path.replace("%", "%25") \
1113 .replace("*", "%2A") \
1114 .replace("#", "%23") \
1115 .replace("@", "%40")
1116 return path
1117
1118def wildcard_present(path):
598354c0
BC
1119 m = re.search("[*#@%]", path)
1120 return m is not None
9d7d446a 1121
a5db4b12
LS
1122class LargeFileSystem(object):
1123 """Base class for large file system support."""
1124
1125 def __init__(self, writeToGitStream):
1126 self.largeFiles = set()
1127 self.writeToGitStream = writeToGitStream
1128
1129 def generatePointer(self, cloneDestination, contentFile):
1130 """Return the content of a pointer file that is stored in Git instead of
1131 the actual content."""
1132 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1133
1134 def pushFile(self, localLargeFile):
1135 """Push the actual content which is not stored in the Git repository to
1136 a server."""
1137 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1138
1139 def hasLargeFileExtension(self, relPath):
1140 return reduce(
1141 lambda a, b: a or b,
1142 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1143 False
1144 )
1145
1146 def generateTempFile(self, contents):
1147 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1148 for d in contents:
1149 contentFile.write(d)
1150 contentFile.close()
1151 return contentFile.name
1152
1153 def exceedsLargeFileThreshold(self, relPath, contents):
1154 if gitConfigInt('git-p4.largeFileThreshold'):
1155 contentsSize = sum(len(d) for d in contents)
1156 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1157 return True
1158 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159 contentsSize = sum(len(d) for d in contents)
1160 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1161 return False
1162 contentTempFile = self.generateTempFile(contents)
1163 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1164 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1165 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1166 zf.close()
1167 compressedContentsSize = zf.infolist()[0].compress_size
1168 os.remove(contentTempFile)
1169 os.remove(compressedContentFile.name)
1170 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1171 return True
1172 return False
1173
1174 def addLargeFile(self, relPath):
1175 self.largeFiles.add(relPath)
1176
1177 def removeLargeFile(self, relPath):
1178 self.largeFiles.remove(relPath)
1179
1180 def isLargeFile(self, relPath):
1181 return relPath in self.largeFiles
1182
1183 def processContent(self, git_mode, relPath, contents):
1184 """Processes the content of git fast import. This method decides if a
1185 file is stored in the large file system and handles all necessary
1186 steps."""
1187 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1188 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1189 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1190 if pointer_git_mode:
1191 git_mode = pointer_git_mode
1192 if localLargeFile:
1193 # Move temp file to final location in large file system
1194 largeFileDir = os.path.dirname(localLargeFile)
1195 if not os.path.isdir(largeFileDir):
1196 os.makedirs(largeFileDir)
1197 shutil.move(contentTempFile, localLargeFile)
1198 self.addLargeFile(relPath)
1199 if gitConfigBool('git-p4.largeFilePush'):
1200 self.pushFile(localLargeFile)
1201 if verbose:
1202 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1203 return (git_mode, contents)
1204
1205class MockLFS(LargeFileSystem):
1206 """Mock large file system for testing."""
1207
1208 def generatePointer(self, contentFile):
1209 """The pointer content is the original content prefixed with "pointer-".
1210 The local filename of the large file storage is derived from the file content.
1211 """
1212 with open(contentFile, 'r') as f:
1213 content = next(f)
1214 gitMode = '100644'
1215 pointerContents = 'pointer-' + content
1216 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1217 return (gitMode, pointerContents, localLargeFile)
1218
1219 def pushFile(self, localLargeFile):
1220 """The remote filename of the large file storage is the same as the local
1221 one but in a different directory.
1222 """
1223 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1224 if not os.path.exists(remotePath):
1225 os.makedirs(remotePath)
1226 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1227
b47d807d
LS
1228class GitLFS(LargeFileSystem):
1229 """Git LFS as backend for the git-p4 large file system.
1230 See https://git-lfs.github.com/ for details."""
1231
1232 def __init__(self, *args):
1233 LargeFileSystem.__init__(self, *args)
1234 self.baseGitAttributes = []
1235
1236 def generatePointer(self, contentFile):
1237 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1238 mode and content which is stored in the Git repository instead of
1239 the actual content. Return also the new location of the actual
1240 content.
1241 """
d5eb3cf5
LS
1242 if os.path.getsize(contentFile) == 0:
1243 return (None, '', None)
1244
b47d807d
LS
1245 pointerProcess = subprocess.Popen(
1246 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1247 stdout=subprocess.PIPE
1248 )
1249 pointerFile = pointerProcess.stdout.read()
1250 if pointerProcess.wait():
1251 os.remove(contentFile)
1252 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1253
1254 # Git LFS removed the preamble in the output of the 'pointer' command
1255 # starting from version 1.2.0. Check for the preamble here to support
1256 # earlier versions.
1257 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1258 if pointerFile.startswith('Git LFS pointer for'):
1259 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1260
1261 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1262 localLargeFile = os.path.join(
1263 os.getcwd(),
1264 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1265 oid,
1266 )
1267 # LFS Spec states that pointer files should not have the executable bit set.
1268 gitMode = '100644'
82f2567e 1269 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1270
1271 def pushFile(self, localLargeFile):
1272 uploadProcess = subprocess.Popen(
1273 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1274 )
1275 if uploadProcess.wait():
1276 die('git-lfs push command failed. Did you define a remote?')
1277
1278 def generateGitAttributes(self):
1279 return (
1280 self.baseGitAttributes +
1281 [
1282 '\n',
1283 '#\n',
1284 '# Git LFS (see https://git-lfs.github.com/)\n',
1285 '#\n',
1286 ] +
862f9312 1287 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1288 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1289 ] +
862f9312 1290 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1291 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1292 ]
1293 )
1294
1295 def addLargeFile(self, relPath):
1296 LargeFileSystem.addLargeFile(self, relPath)
1297 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1298
1299 def removeLargeFile(self, relPath):
1300 LargeFileSystem.removeLargeFile(self, relPath)
1301 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1302
1303 def processContent(self, git_mode, relPath, contents):
1304 if relPath == '.gitattributes':
1305 self.baseGitAttributes = contents
1306 return (git_mode, self.generateGitAttributes())
1307 else:
1308 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1309
b984733c 1310class Command:
89143ac2 1311 delete_actions = ( "delete", "move/delete", "purge" )
0108f47e 1312 add_actions = ( "add", "branch", "move/add" )
89143ac2 1313
b984733c
SH
1314 def __init__(self):
1315 self.usage = "usage: %prog [options]"
8910ac0e 1316 self.needsGit = True
6a10b6aa 1317 self.verbose = False
b984733c 1318
ff8c50ed 1319 # This is required for the "append" update_shelve action
8cf422db
LD
1320 def ensure_value(self, attr, value):
1321 if not hasattr(self, attr) or getattr(self, attr) is None:
1322 setattr(self, attr, value)
1323 return getattr(self, attr)
1324
3ea2cfd4
LD
1325class P4UserMap:
1326 def __init__(self):
1327 self.userMapFromPerforceServer = False
affb474f
LD
1328 self.myP4UserId = None
1329
1330 def p4UserId(self):
1331 if self.myP4UserId:
1332 return self.myP4UserId
1333
1334 results = p4CmdList("user -o")
1335 for r in results:
dba1c9d9 1336 if 'User' in r:
affb474f
LD
1337 self.myP4UserId = r['User']
1338 return r['User']
1339 die("Could not find your p4 user id")
1340
1341 def p4UserIsMe(self, p4User):
1342 # return True if the given p4 user is actually me
1343 me = self.p4UserId()
1344 if not p4User or p4User != me:
1345 return False
1346 else:
1347 return True
3ea2cfd4
LD
1348
1349 def getUserCacheFilename(self):
1350 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1351 return home + "/.gitp4-usercache.txt"
1352
1353 def getUserMapFromPerforceServer(self):
1354 if self.userMapFromPerforceServer:
1355 return
1356 self.users = {}
1357 self.emails = {}
1358
1359 for output in p4CmdList("users"):
dba1c9d9 1360 if "User" not in output:
3ea2cfd4
LD
1361 continue
1362 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1363 self.emails[output["Email"]] = output["User"]
1364
10d08a14
LS
1365 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1366 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1367 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1368 if mapUser and len(mapUser[0]) == 3:
1369 user = mapUser[0][0]
1370 fullname = mapUser[0][1]
1371 email = mapUser[0][2]
1372 self.users[user] = fullname + " <" + email + ">"
1373 self.emails[email] = user
3ea2cfd4
LD
1374
1375 s = ''
1376 for (key, val) in self.users.items():
1377 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1378
1379 open(self.getUserCacheFilename(), "wb").write(s)
1380 self.userMapFromPerforceServer = True
1381
1382 def loadUserMapFromCache(self):
1383 self.users = {}
1384 self.userMapFromPerforceServer = False
1385 try:
1386 cache = open(self.getUserCacheFilename(), "rb")
1387 lines = cache.readlines()
1388 cache.close()
1389 for line in lines:
1390 entry = line.strip().split("\t")
1391 self.users[entry[0]] = entry[1]
1392 except IOError:
1393 self.getUserMapFromPerforceServer()
1394
b984733c 1395class P4Debug(Command):
86949eef 1396 def __init__(self):
6ae8de88 1397 Command.__init__(self)
6a10b6aa 1398 self.options = []
c8c39116 1399 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1400 self.needsGit = False
86949eef
SH
1401
1402 def run(self, args):
b1ce9447 1403 j = 0
6de040df 1404 for output in p4CmdList(args):
f2606b17 1405 print('Element: %d' % j)
b1ce9447 1406 j += 1
f2606b17 1407 print(output)
b984733c 1408 return True
86949eef 1409
5834684d
SH
1410class P4RollBack(Command):
1411 def __init__(self):
1412 Command.__init__(self)
1413 self.options = [
0c66a783 1414 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1415 ]
1416 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1417 self.rollbackLocalBranches = False
5834684d
SH
1418
1419 def run(self, args):
1420 if len(args) != 1:
1421 return False
1422 maxChange = int(args[0])
0c66a783 1423
ad192f28 1424 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1425 die("Problems executing p4");
1426
0c66a783
SH
1427 if self.rollbackLocalBranches:
1428 refPrefix = "refs/heads/"
b016d397 1429 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1430 else:
1431 refPrefix = "refs/remotes/"
b016d397 1432 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1433
1434 for line in lines:
1435 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1436 line = line.strip()
1437 ref = refPrefix + line
5834684d 1438 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1439 settings = extractSettingsGitLog(log)
1440
1441 depotPaths = settings['depot-paths']
1442 change = settings['change']
1443
5834684d 1444 changed = False
52102d47 1445
6326aa58
HWN
1446 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1447 for p in depotPaths]))) == 0:
f2606b17 1448 print("Branch %s did not exist at change %s, deleting." % (ref, maxChange))
52102d47
SH
1449 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1450 continue
1451
bb6e09b2 1452 while change and int(change) > maxChange:
5834684d 1453 changed = True
52102d47 1454 if self.verbose:
f2606b17 1455 print("%s is at %s ; rewinding towards %s" % (ref, change, maxChange))
5834684d
SH
1456 system("git update-ref %s \"%s^\"" % (ref, ref))
1457 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1458 settings = extractSettingsGitLog(log)
1459
1460
1461 depotPaths = settings['depot-paths']
1462 change = settings['change']
5834684d
SH
1463
1464 if changed:
f2606b17 1465 print("%s rewound to %s" % (ref, change))
5834684d
SH
1466
1467 return True
1468
3ea2cfd4 1469class P4Submit(Command, P4UserMap):
6bbfd137
PW
1470
1471 conflict_behavior_choices = ("ask", "skip", "quit")
1472
4f5cf76a 1473 def __init__(self):
b984733c 1474 Command.__init__(self)
3ea2cfd4 1475 P4UserMap.__init__(self)
4f5cf76a 1476 self.options = [
4f5cf76a 1477 optparse.make_option("--origin", dest="origin"),
ae901090 1478 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1479 # preserve the user, requires relevant p4 permissions
1480 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1481 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1482 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1483 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1484 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1485 choices=self.conflict_behavior_choices),
1486 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1487 optparse.make_option("--shelve", dest="shelve", action="store_true",
1488 help="Shelve instead of submit. Shelved files are reverted, "
1489 "restoring the workspace to the state before the shelve"),
8cf422db 1490 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1491 metavar="CHANGELIST",
8cf422db 1492 help="update an existing shelved changelist, implies --shelve, "
f55b87c1
RM
1493 "repeat in-order for multiple shelved changelists"),
1494 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1495 help="submit only the specified commit(s), one commit or xxx..xxx"),
1496 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1497 help="Disable rebase after submit is completed. Can be useful if you "
b9d34db9
LD
1498 "work from a local git branch that is not master"),
1499 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1500 help="Skip Perforce sync of p4/master after submit or shelve"),
4f5cf76a 1501 ]
251c8c50
CB
1502 self.description = """Submit changes from git to the perforce depot.\n
1503 The `p4-pre-submit` hook is executed if it exists and is executable.
1504 The hook takes no parameters and nothing from standard input. Exiting with
1505 non-zero status from this script prevents `git-p4 submit` from launching.
1506
1507 One usage scenario is to run unit tests in the hook."""
1508
c9b50e63 1509 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1510 self.origin = ""
ae901090 1511 self.detectRenames = False
0d609032 1512 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1513 self.dry_run = False
b34fa577 1514 self.shelve = False
8cf422db 1515 self.update_shelve = list()
f55b87c1 1516 self.commit = ""
3b3477ea 1517 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
b9d34db9 1518 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
728b7ad8 1519 self.prepare_p4_only = False
6bbfd137 1520 self.conflict_behavior = None
f7baba8b 1521 self.isWindows = (platform.system() == "Windows")
06804c76 1522 self.exportLabels = False
249da4c0 1523 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1524 self.branch = None
4f5cf76a 1525
a5db4b12
LS
1526 if gitConfig('git-p4.largeFileSystem'):
1527 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1528
4f5cf76a
SH
1529 def check(self):
1530 if len(p4CmdList("opened ...")) > 0:
1531 die("You have files opened with perforce! Close them before starting the sync.")
1532
f19cb0a0
PW
1533 def separate_jobs_from_description(self, message):
1534 """Extract and return a possible Jobs field in the commit
1535 message. It goes into a separate section in the p4 change
1536 specification.
1537
1538 A jobs line starts with "Jobs:" and looks like a new field
1539 in a form. Values are white-space separated on the same
1540 line or on following lines that start with a tab.
1541
1542 This does not parse and extract the full git commit message
1543 like a p4 form. It just sees the Jobs: line as a marker
1544 to pass everything from then on directly into the p4 form,
1545 but outside the description section.
1546
1547 Return a tuple (stripped log message, jobs string)."""
1548
1549 m = re.search(r'^Jobs:', message, re.MULTILINE)
1550 if m is None:
1551 return (message, None)
1552
1553 jobtext = message[m.start():]
1554 stripped_message = message[:m.start()].rstrip()
1555 return (stripped_message, jobtext)
1556
1557 def prepareLogMessage(self, template, message, jobs):
1558 """Edits the template returned from "p4 change -o" to insert
1559 the message in the Description field, and the jobs text in
1560 the Jobs field."""
4f5cf76a
SH
1561 result = ""
1562
edae1e2f
SH
1563 inDescriptionSection = False
1564
4f5cf76a
SH
1565 for line in template.split("\n"):
1566 if line.startswith("#"):
1567 result += line + "\n"
1568 continue
1569
edae1e2f 1570 if inDescriptionSection:
c9dbab04 1571 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1572 inDescriptionSection = False
f19cb0a0
PW
1573 # insert Jobs section
1574 if jobs:
1575 result += jobs + "\n"
edae1e2f
SH
1576 else:
1577 continue
1578 else:
1579 if line.startswith("Description:"):
1580 inDescriptionSection = True
1581 line += "\n"
1582 for messageLine in message.split("\n"):
1583 line += "\t" + messageLine + "\n"
1584
1585 result += line + "\n"
4f5cf76a
SH
1586
1587 return result
1588
60df071c
LD
1589 def patchRCSKeywords(self, file, pattern):
1590 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1591 (handle, outFileName) = tempfile.mkstemp(dir='.')
1592 try:
1593 outFile = os.fdopen(handle, "w+")
1594 inFile = open(file, "r")
1595 regexp = re.compile(pattern, re.VERBOSE)
1596 for line in inFile.readlines():
1597 line = regexp.sub(r'$\1$', line)
1598 outFile.write(line)
1599 inFile.close()
1600 outFile.close()
1601 # Forcibly overwrite the original file
1602 os.unlink(file)
1603 shutil.move(outFileName, file)
1604 except:
1605 # cleanup our temporary file
1606 os.unlink(outFileName)
f2606b17 1607 print("Failed to strip RCS keywords in %s" % file)
60df071c
LD
1608 raise
1609
f2606b17 1610 print("Patched up RCS keywords in %s" % file)
60df071c 1611
3ea2cfd4
LD
1612 def p4UserForCommit(self,id):
1613 # Return the tuple (perforce user,git email) for a given git commit id
1614 self.getUserMapFromPerforceServer()
9bf28855
PW
1615 gitEmail = read_pipe(["git", "log", "--max-count=1",
1616 "--format=%ae", id])
3ea2cfd4 1617 gitEmail = gitEmail.strip()
dba1c9d9 1618 if gitEmail not in self.emails:
3ea2cfd4
LD
1619 return (None,gitEmail)
1620 else:
1621 return (self.emails[gitEmail],gitEmail)
1622
1623 def checkValidP4Users(self,commits):
1624 # check if any git authors cannot be mapped to p4 users
1625 for id in commits:
1626 (user,email) = self.p4UserForCommit(id)
1627 if not user:
1628 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1629 if gitConfigBool("git-p4.allowMissingP4Users"):
f2606b17 1630 print("%s" % msg)
3ea2cfd4
LD
1631 else:
1632 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1633
1634 def lastP4Changelist(self):
1635 # Get back the last changelist number submitted in this client spec. This
1636 # then gets used to patch up the username in the change. If the same
1637 # client spec is being used by multiple processes then this might go
1638 # wrong.
1639 results = p4CmdList("client -o") # find the current client
1640 client = None
1641 for r in results:
dba1c9d9 1642 if 'Client' in r:
3ea2cfd4
LD
1643 client = r['Client']
1644 break
1645 if not client:
1646 die("could not get client spec")
6de040df 1647 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4 1648 for r in results:
dba1c9d9 1649 if 'change' in r:
3ea2cfd4
LD
1650 return r['change']
1651 die("Could not get changelist number for last submit - cannot patch up user details")
1652
1653 def modifyChangelistUser(self, changelist, newUser):
1654 # fixup the user field of a changelist after it has been submitted.
1655 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1656 if len(changes) != 1:
1657 die("Bad output from p4 change modifying %s to user %s" %
1658 (changelist, newUser))
1659
1660 c = changes[0]
1661 if c['User'] == newUser: return # nothing to do
1662 c['User'] = newUser
1663 input = marshal.dumps(c)
1664
3ea2cfd4
LD
1665 result = p4CmdList("change -f -i", stdin=input)
1666 for r in result:
dba1c9d9 1667 if 'code' in r:
3ea2cfd4
LD
1668 if r['code'] == 'error':
1669 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
dba1c9d9 1670 if 'data' in r:
3ea2cfd4
LD
1671 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1672 return
1673 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1674
1675 def canChangeChangelists(self):
1676 # check to see if we have p4 admin or super-user permissions, either of
1677 # which are required to modify changelists.
52a4880b 1678 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4 1679 for r in results:
dba1c9d9 1680 if 'perm' in r:
3ea2cfd4
LD
1681 if r['perm'] == 'admin':
1682 return 1
1683 if r['perm'] == 'super':
1684 return 1
1685 return 0
1686
46c609e9 1687 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1688 """Run "p4 change -o" to grab a change specification template.
1689 This does not use "p4 -G", as it is nice to keep the submission
1690 template in original order, since a human might edit it.
1691
1692 Remove lines in the Files section that show changes to files
1693 outside the depot path we're committing into."""
1694
cbc69242
SH
1695 [upstream, settings] = findUpstreamBranchPoint()
1696
b596b3b9
MT
1697 template = """\
1698# A Perforce Change Specification.
1699#
1700# Change: The change number. 'new' on a new changelist.
1701# Date: The date this specification was last modified.
1702# Client: The client on which the changelist was created. Read-only.
1703# User: The user who created the changelist.
1704# Status: Either 'pending' or 'submitted'. Read-only.
1705# Type: Either 'public' or 'restricted'. Default is 'public'.
1706# Description: Comments about the changelist. Required.
1707# Jobs: What opened jobs are to be closed by this changelist.
1708# You may delete jobs from this list. (New changelists only.)
1709# Files: What opened files from the default changelist are to be added
1710# to this changelist. You may delete files from this list.
1711# (New changelists only.)
1712"""
1713 files_list = []
ea99c3ae 1714 inFilesSection = False
b596b3b9 1715 change_entry = None
46c609e9
LD
1716 args = ['change', '-o']
1717 if changelist:
1718 args.append(str(changelist))
b596b3b9 1719 for entry in p4CmdList(args):
dba1c9d9 1720 if 'code' not in entry:
b596b3b9
MT
1721 continue
1722 if entry['code'] == 'stat':
1723 change_entry = entry
1724 break
1725 if not change_entry:
1726 die('Failed to decode output of p4 change -o')
1727 for key, value in change_entry.iteritems():
1728 if key.startswith('File'):
dba1c9d9 1729 if 'depot-paths' in settings:
b596b3b9
MT
1730 if not [p for p in settings['depot-paths']
1731 if p4PathStartsWith(value, p)]:
1732 continue
ea99c3ae 1733 else:
b596b3b9
MT
1734 if not p4PathStartsWith(value, self.depotPath):
1735 continue
1736 files_list.append(value)
1737 continue
1738 # Output in the order expected by prepareLogMessage
1739 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
dba1c9d9 1740 if key not in change_entry:
b596b3b9
MT
1741 continue
1742 template += '\n'
1743 template += key + ':'
1744 if key == 'Description':
1745 template += '\n'
1746 for field_line in change_entry[key].splitlines():
1747 template += '\t'+field_line+'\n'
1748 if len(files_list) > 0:
1749 template += '\n'
1750 template += 'Files:\n'
1751 for path in files_list:
1752 template += '\t'+path+'\n'
ea99c3ae
SH
1753 return template
1754
7c766e57
PW
1755 def edit_template(self, template_file):
1756 """Invoke the editor to let the user change the submission
1757 message. Return true if okay to continue with the submit."""
1758
1759 # if configured to skip the editing part, just submit
0d609032 1760 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1761 return True
1762
1763 # look at the modification time, to check later if the user saved
1764 # the file
1765 mtime = os.stat(template_file).st_mtime
1766
1767 # invoke the editor
dba1c9d9 1768 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1769 editor = os.environ.get("P4EDITOR")
1770 else:
1771 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1772 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1773
1774 # If the file was not saved, prompt to see if this patch should
1775 # be skipped. But skip this verification step if configured so.
0d609032 1776 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1777 return True
1778
d1652049
PW
1779 # modification time updated means user saved the file
1780 if os.stat(template_file).st_mtime > mtime:
1781 return True
1782
1783 while True:
1784 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1785 if response == 'y':
1786 return True
1787 if response == 'n':
1788 return False
7c766e57 1789
df8a9e86 1790 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3 1791 # diff
dba1c9d9 1792 if "P4DIFF" in os.environ:
b4073bb3
MC
1793 del(os.environ["P4DIFF"])
1794 diff = ""
1795 for editedFile in editedFiles:
1796 diff += p4_read_pipe(['diff', '-du',
1797 wildcard_encode(editedFile)])
1798
1799 # new file diff
1800 newdiff = ""
1801 for newFile in filesToAdd:
1802 newdiff += "==== new file ====\n"
1803 newdiff += "--- /dev/null\n"
1804 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1805
1806 is_link = os.path.islink(newFile)
1807 expect_link = newFile in symlinks
1808
1809 if is_link and expect_link:
1810 newdiff += "+%s\n" % os.readlink(newFile)
1811 else:
1812 f = open(newFile, "r")
1813 for line in f.readlines():
1814 newdiff += "+" + line
1815 f.close()
b4073bb3 1816
e2a892ee 1817 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1818
7cb5cbef 1819 def applyCommit(self, id):
67b0fe2e
PW
1820 """Apply one commit, return True if it succeeded."""
1821
f2606b17
LD
1822 print("Applying", read_pipe(["git", "show", "-s",
1823 "--format=format:%h %s", id]))
ae901090 1824
848de9c3 1825 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1826
84cb0003 1827 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1828 filesToAdd = set()
a02b8bc4 1829 filesToChangeType = set()
4f5cf76a 1830 filesToDelete = set()
d336c158 1831 editedFiles = set()
b6ad6dcc 1832 pureRenameCopy = set()
df8a9e86 1833 symlinks = set()
c65b670e 1834 filesToChangeExecBit = {}
46c609e9 1835 all_files = list()
60df071c 1836
4f5cf76a 1837 for line in diff:
b43b0a3c
CP
1838 diff = parseDiffTreeEntry(line)
1839 modifier = diff['status']
1840 path = diff['src']
46c609e9
LD
1841 all_files.append(path)
1842
4f5cf76a 1843 if modifier == "M":
6de040df 1844 p4_edit(path)
c65b670e
CP
1845 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1846 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1847 editedFiles.add(path)
4f5cf76a
SH
1848 elif modifier == "A":
1849 filesToAdd.add(path)
c65b670e 1850 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1851 if path in filesToDelete:
1852 filesToDelete.remove(path)
df8a9e86
LD
1853
1854 dst_mode = int(diff['dst_mode'], 8)
db2d997e 1855 if dst_mode == 0o120000:
df8a9e86
LD
1856 symlinks.add(path)
1857
4f5cf76a
SH
1858 elif modifier == "D":
1859 filesToDelete.add(path)
1860 if path in filesToAdd:
1861 filesToAdd.remove(path)
4fddb41b
VA
1862 elif modifier == "C":
1863 src, dest = diff['src'], diff['dst']
7a10946a 1864 all_files.append(dest)
6de040df 1865 p4_integrate(src, dest)
b6ad6dcc 1866 pureRenameCopy.add(dest)
4fddb41b 1867 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1868 p4_edit(dest)
b6ad6dcc 1869 pureRenameCopy.discard(dest)
4fddb41b 1870 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1871 p4_edit(dest)
b6ad6dcc 1872 pureRenameCopy.discard(dest)
4fddb41b 1873 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1874 if self.isWindows:
1875 # turn off read-only attribute
1876 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1877 os.unlink(dest)
1878 editedFiles.add(dest)
d9a5f25b 1879 elif modifier == "R":
b43b0a3c 1880 src, dest = diff['src'], diff['dst']
7a10946a 1881 all_files.append(dest)
8e9497c2
GG
1882 if self.p4HasMoveCommand:
1883 p4_edit(src) # src must be open before move
1884 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1885 else:
8e9497c2
GG
1886 p4_integrate(src, dest)
1887 if diff['src_sha1'] != diff['dst_sha1']:
1888 p4_edit(dest)
1889 else:
1890 pureRenameCopy.add(dest)
c65b670e 1891 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1892 if not self.p4HasMoveCommand:
1893 p4_edit(dest) # with move: already open, writable
c65b670e 1894 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1895 if not self.p4HasMoveCommand:
d20f0f8e
PW
1896 if self.isWindows:
1897 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1898 os.unlink(dest)
1899 filesToDelete.add(src)
d9a5f25b 1900 editedFiles.add(dest)
a02b8bc4
RP
1901 elif modifier == "T":
1902 filesToChangeType.add(path)
4f5cf76a
SH
1903 else:
1904 die("unknown modifier %s for %s" % (modifier, path))
1905
749b668c 1906 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1907 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1908 tryPatchCmd = patchcmd + "--check -"
1909 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1910 patch_succeeded = True
51a2640a 1911
47a130b7 1912 if os.system(tryPatchCmd) != 0:
60df071c
LD
1913 fixed_rcs_keywords = False
1914 patch_succeeded = False
f2606b17 1915 print("Unfortunately applying the change failed!")
60df071c
LD
1916
1917 # Patch failed, maybe it's just RCS keyword woes. Look through
1918 # the patch to see if that's possible.
0d609032 1919 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1920 file = None
1921 pattern = None
1922 kwfiles = {}
1923 for file in editedFiles | filesToDelete:
1924 # did this file's delta contain RCS keywords?
1925 pattern = p4_keywords_regexp_for_file(file)
1926
1927 if pattern:
1928 # this file is a possibility...look for RCS keywords.
1929 regexp = re.compile(pattern, re.VERBOSE)
1930 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1931 if regexp.search(line):
1932 if verbose:
f2606b17 1933 print("got keyword match on %s in %s in %s" % (pattern, line, file))
60df071c
LD
1934 kwfiles[file] = pattern
1935 break
1936
1937 for file in kwfiles:
1938 if verbose:
f2606b17 1939 print("zapping %s with %s" % (line,pattern))
d20f0f8e
PW
1940 # File is being deleted, so not open in p4. Must
1941 # disable the read-only bit on windows.
1942 if self.isWindows and file not in editedFiles:
1943 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1944 self.patchRCSKeywords(file, kwfiles[file])
1945 fixed_rcs_keywords = True
1946
1947 if fixed_rcs_keywords:
f2606b17 1948 print("Retrying the patch with RCS keywords cleaned up")
60df071c
LD
1949 if os.system(tryPatchCmd) == 0:
1950 patch_succeeded = True
1951
1952 if not patch_succeeded:
7e5dd9f2
PW
1953 for f in editedFiles:
1954 p4_revert(f)
7e5dd9f2 1955 return False
51a2640a 1956
55ac2ed6
PW
1957 #
1958 # Apply the patch for real, and do add/delete/+x handling.
1959 #
47a130b7 1960 system(applyPatchCmd)
4f5cf76a 1961
a02b8bc4
RP
1962 for f in filesToChangeType:
1963 p4_edit(f, "-t", "auto")
4f5cf76a 1964 for f in filesToAdd:
6de040df 1965 p4_add(f)
4f5cf76a 1966 for f in filesToDelete:
6de040df
LD
1967 p4_revert(f)
1968 p4_delete(f)
4f5cf76a 1969
c65b670e
CP
1970 # Set/clear executable bits
1971 for f in filesToChangeExecBit.keys():
1972 mode = filesToChangeExecBit[f]
1973 setP4ExecBit(f, mode)
1974
8cf422db
LD
1975 update_shelve = 0
1976 if len(self.update_shelve) > 0:
1977 update_shelve = self.update_shelve.pop(0)
1978 p4_reopen_in_change(update_shelve, all_files)
46c609e9 1979
55ac2ed6
PW
1980 #
1981 # Build p4 change description, starting with the contents
1982 # of the git commit message.
1983 #
0e36f2d7 1984 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1985 logMessage = logMessage.strip()
f19cb0a0 1986 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1987
8cf422db 1988 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 1989 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1990
c47178d4 1991 if self.preserveUser:
55ac2ed6 1992 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1993
55ac2ed6
PW
1994 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1995 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1996 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1997 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1998
55ac2ed6 1999 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
2000 if not self.prepare_p4_only:
2001 submitTemplate += separatorLine
df8a9e86 2002 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 2003
c47178d4 2004 (handle, fileName) = tempfile.mkstemp()
e2a892ee 2005 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
2006 if self.isWindows:
2007 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 2008 tmpFile.write(submitTemplate)
c47178d4
PW
2009 tmpFile.close()
2010
728b7ad8
PW
2011 if self.prepare_p4_only:
2012 #
2013 # Leave the p4 tree prepared, and the submit template around
2014 # and let the user decide what to do next
2015 #
f2606b17
LD
2016 print()
2017 print("P4 workspace prepared for submission.")
2018 print("To submit or revert, go to client workspace")
2019 print(" " + self.clientPath)
2020 print()
2021 print("To submit, use \"p4 submit\" to write a new description,")
2022 print("or \"p4 submit -i <%s\" to use the one prepared by" \
2023 " \"git p4\"." % fileName)
2024 print("You can delete the file \"%s\" when finished." % fileName)
728b7ad8
PW
2025
2026 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
f2606b17 2027 print("To preserve change ownership by user %s, you must\n" \
728b7ad8 2028 "do \"p4 change -f <change>\" after submitting and\n" \
f2606b17 2029 "edit the User field.")
728b7ad8 2030 if pureRenameCopy:
f2606b17
LD
2031 print("After submitting, renamed files must be re-synced.")
2032 print("Invoke \"p4 sync -f\" on each of these files:")
728b7ad8 2033 for f in pureRenameCopy:
f2606b17 2034 print(" " + f)
728b7ad8 2035
f2606b17
LD
2036 print()
2037 print("To revert the changes, use \"p4 revert ...\", and delete")
2038 print("the submit template file \"%s\"" % fileName)
728b7ad8 2039 if filesToAdd:
f2606b17 2040 print("Since the commit adds new files, they must be deleted:")
728b7ad8 2041 for f in filesToAdd:
f2606b17
LD
2042 print(" " + f)
2043 print()
728b7ad8
PW
2044 return True
2045
55ac2ed6
PW
2046 #
2047 # Let the user edit the change description, then submit it.
2048 #
b7638fed 2049 submitted = False
cdc7e388 2050
b7638fed
GE
2051 try:
2052 if self.edit_template(fileName):
2053 # read the edited message and submit
2054 tmpFile = open(fileName, "rb")
2055 message = tmpFile.read()
2056 tmpFile.close()
2057 if self.isWindows:
2058 message = message.replace("\r\n", "\n")
2059 submitTemplate = message[:message.index(separatorLine)]
46c609e9 2060
8cf422db 2061 if update_shelve:
46c609e9
LD
2062 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2063 elif self.shelve:
b34fa577
VK
2064 p4_write_pipe(['shelve', '-i'], submitTemplate)
2065 else:
2066 p4_write_pipe(['submit', '-i'], submitTemplate)
2067 # The rename/copy happened by applying a patch that created a
2068 # new file. This leaves it writable, which confuses p4.
2069 for f in pureRenameCopy:
2070 p4_sync(f, "-f")
b7638fed
GE
2071
2072 if self.preserveUser:
2073 if p4User:
2074 # Get last changelist number. Cannot easily get it from
2075 # the submit command output as the output is
2076 # unmarshalled.
2077 changelist = self.lastP4Changelist()
2078 self.modifyChangelistUser(changelist, p4User)
2079
b7638fed
GE
2080 submitted = True
2081
2082 finally:
c47178d4 2083 # skip this patch
b34fa577
VK
2084 if not submitted or self.shelve:
2085 if self.shelve:
2086 print ("Reverting shelved files.")
2087 else:
2088 print ("Submission cancelled, undoing p4 changes.")
2089 for f in editedFiles | filesToDelete:
b7638fed
GE
2090 p4_revert(f)
2091 for f in filesToAdd:
2092 p4_revert(f)
2093 os.remove(f)
c47178d4
PW
2094
2095 os.remove(fileName)
b7638fed 2096 return submitted
4f5cf76a 2097
06804c76
LD
2098 # Export git tags as p4 labels. Create a p4 label and then tag
2099 # with that.
2100 def exportGitTags(self, gitTags):
c8942a22
LD
2101 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2102 if len(validLabelRegexp) == 0:
2103 validLabelRegexp = defaultLabelRegexp
2104 m = re.compile(validLabelRegexp)
06804c76
LD
2105
2106 for name in gitTags:
2107
2108 if not m.match(name):
2109 if verbose:
f2606b17 2110 print("tag %s does not match regexp %s" % (name, validLabelRegexp))
06804c76
LD
2111 continue
2112
2113 # Get the p4 commit this corresponds to
c8942a22
LD
2114 logMessage = extractLogMessageFromGitCommit(name)
2115 values = extractSettingsGitLog(logMessage)
06804c76 2116
dba1c9d9 2117 if 'change' not in values:
06804c76
LD
2118 # a tag pointing to something not sent to p4; ignore
2119 if verbose:
f2606b17 2120 print("git tag %s does not give a p4 commit" % name)
06804c76 2121 continue
c8942a22
LD
2122 else:
2123 changelist = values['change']
06804c76
LD
2124
2125 # Get the tag details.
2126 inHeader = True
2127 isAnnotated = False
2128 body = []
2129 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2130 l = l.strip()
2131 if inHeader:
2132 if re.match(r'tag\s+', l):
2133 isAnnotated = True
2134 elif re.match(r'\s*$', l):
2135 inHeader = False
2136 continue
2137 else:
2138 body.append(l)
2139
2140 if not isAnnotated:
2141 body = ["lightweight tag imported by git p4\n"]
2142
2143 # Create the label - use the same view as the client spec we are using
2144 clientSpec = getClientSpec()
2145
2146 labelTemplate = "Label: %s\n" % name
2147 labelTemplate += "Description:\n"
2148 for b in body:
2149 labelTemplate += "\t" + b + "\n"
2150 labelTemplate += "View:\n"
9d57c4a6
KS
2151 for depot_side in clientSpec.mappings:
2152 labelTemplate += "\t%s\n" % depot_side
06804c76 2153
ef739f08 2154 if self.dry_run:
f2606b17 2155 print("Would create p4 label %s for tag" % name)
728b7ad8 2156 elif self.prepare_p4_only:
f2606b17
LD
2157 print("Not creating p4 label %s for tag due to option" \
2158 " --prepare-p4-only" % name)
ef739f08
PW
2159 else:
2160 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2161
ef739f08
PW
2162 # Use the label
2163 p4_system(["tag", "-l", name] +
9d57c4a6 2164 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2165
ef739f08 2166 if verbose:
f2606b17 2167 print("created p4 label for tag %s" % name)
06804c76 2168
4f5cf76a 2169 def run(self, args):
c9b50e63
SH
2170 if len(args) == 0:
2171 self.master = currentGitBranch()
c9b50e63
SH
2172 elif len(args) == 1:
2173 self.master = args[0]
28755dba
PW
2174 if not branchExists(self.master):
2175 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2176 else:
2177 return False
2178
8cf422db
LD
2179 for i in self.update_shelve:
2180 if i <= 0:
2181 sys.exit("invalid changelist %d" % i)
2182
00ad6e31
LD
2183 if self.master:
2184 allowSubmit = gitConfig("git-p4.allowSubmit")
2185 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2186 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2187
27d2d811 2188 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 2189 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2190 if len(self.origin) == 0:
2191 self.origin = upstream
a3fdd579 2192
8cf422db 2193 if len(self.update_shelve) > 0:
46c609e9
LD
2194 self.shelve = True
2195
3ea2cfd4
LD
2196 if self.preserveUser:
2197 if not self.canChangeChangelists():
2198 die("Cannot preserve user names without p4 super-user or admin permissions")
2199
6bbfd137
PW
2200 # if not set from the command line, try the config file
2201 if self.conflict_behavior is None:
2202 val = gitConfig("git-p4.conflict")
2203 if val:
2204 if val not in self.conflict_behavior_choices:
2205 die("Invalid value '%s' for config git-p4.conflict" % val)
2206 else:
2207 val = "ask"
2208 self.conflict_behavior = val
2209
a3fdd579 2210 if self.verbose:
f2606b17 2211 print("Origin branch is " + self.origin)
9512497b 2212
ea99c3ae 2213 if len(self.depotPath) == 0:
f2606b17 2214 print("Internal error: cannot locate perforce depot path from existing branches")
9512497b
SH
2215 sys.exit(128)
2216
543987bd 2217 self.useClientSpec = False
0d609032 2218 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2219 self.useClientSpec = True
2220 if self.useClientSpec:
2221 self.clientSpecDirs = getClientSpec()
9512497b 2222
2e3a16b2 2223 # Check for the existence of P4 branches
cd884106
VA
2224 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2225
2226 if self.useClientSpec and not branchesDetected:
543987bd
PW
2227 # all files are relative to the client spec
2228 self.clientPath = getClientRoot()
2229 else:
2230 self.clientPath = p4Where(self.depotPath)
9512497b 2231
543987bd
PW
2232 if self.clientPath == "":
2233 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2234
f2606b17 2235 print("Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath))
7944f142 2236 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2237
0591cfa8 2238 # ensure the clientPath exists
8d7ec362 2239 new_client_dir = False
0591cfa8 2240 if not os.path.exists(self.clientPath):
8d7ec362 2241 new_client_dir = True
0591cfa8
GG
2242 os.makedirs(self.clientPath)
2243
bbd84863 2244 chdir(self.clientPath, is_client_path=True)
ef739f08 2245 if self.dry_run:
f2606b17 2246 print("Would synchronize p4 checkout in %s" % self.clientPath)
8d7ec362 2247 else:
f2606b17 2248 print("Synchronizing p4 checkout...")
ef739f08
PW
2249 if new_client_dir:
2250 # old one was destroyed, and maybe nobody told p4
2251 p4_sync("...", "-f")
2252 else:
2253 p4_sync("...")
4f5cf76a 2254 self.check()
4f5cf76a 2255
4c750c0d 2256 commits = []
00ad6e31 2257 if self.master:
89f32a92 2258 committish = self.master
00ad6e31 2259 else:
89f32a92 2260 committish = 'HEAD'
00ad6e31 2261
f55b87c1
RM
2262 if self.commit != "":
2263 if self.commit.find("..") != -1:
2264 limits_ish = self.commit.split("..")
2265 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2266 commits.append(line.strip())
2267 commits.reverse()
2268 else:
2269 commits.append(self.commit)
2270 else:
e6388994 2271 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
f55b87c1
RM
2272 commits.append(line.strip())
2273 commits.reverse()
4f5cf76a 2274
0d609032 2275 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2276 self.checkAuthorship = False
2277 else:
2278 self.checkAuthorship = True
2279
3ea2cfd4
LD
2280 if self.preserveUser:
2281 self.checkValidP4Users(commits)
2282
84cb0003
GG
2283 #
2284 # Build up a set of options to be passed to diff when
2285 # submitting each commit to p4.
2286 #
2287 if self.detectRenames:
2288 # command-line -M arg
2289 self.diffOpts = "-M"
2290 else:
2291 # If not explicitly set check the config variable
2292 detectRenames = gitConfig("git-p4.detectRenames")
2293
2294 if detectRenames.lower() == "false" or detectRenames == "":
2295 self.diffOpts = ""
2296 elif detectRenames.lower() == "true":
2297 self.diffOpts = "-M"
2298 else:
2299 self.diffOpts = "-M%s" % detectRenames
2300
2301 # no command-line arg for -C or --find-copies-harder, just
2302 # config variables
2303 detectCopies = gitConfig("git-p4.detectCopies")
2304 if detectCopies.lower() == "false" or detectCopies == "":
2305 pass
2306 elif detectCopies.lower() == "true":
2307 self.diffOpts += " -C"
2308 else:
2309 self.diffOpts += " -C%s" % detectCopies
2310
0d609032 2311 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2312 self.diffOpts += " --find-copies-harder"
2313
8cf422db
LD
2314 num_shelves = len(self.update_shelve)
2315 if num_shelves > 0 and num_shelves != len(commits):
2316 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2317 (len(commits), num_shelves))
2318
251c8c50
CB
2319 hooks_path = gitConfig("core.hooksPath")
2320 if len(hooks_path) <= 0:
2321 hooks_path = os.path.join(os.environ.get("GIT_DIR", ".git"), "hooks")
2322
2323 hook_file = os.path.join(hooks_path, "p4-pre-submit")
2324 if os.path.isfile(hook_file) and os.access(hook_file, os.X_OK) and subprocess.call([hook_file]) != 0:
2325 sys.exit(1)
2326
7e5dd9f2
PW
2327 #
2328 # Apply the commits, one at a time. On failure, ask if should
2329 # continue to try the rest of the patches, or quit.
2330 #
ef739f08 2331 if self.dry_run:
f2606b17 2332 print("Would apply")
67b0fe2e 2333 applied = []
7e5dd9f2
PW
2334 last = len(commits) - 1
2335 for i, commit in enumerate(commits):
ef739f08 2336 if self.dry_run:
f2606b17
LD
2337 print(" ", read_pipe(["git", "show", "-s",
2338 "--format=format:%h %s", commit]))
ef739f08
PW
2339 ok = True
2340 else:
2341 ok = self.applyCommit(commit)
67b0fe2e
PW
2342 if ok:
2343 applied.append(commit)
7e5dd9f2 2344 else:
728b7ad8 2345 if self.prepare_p4_only and i < last:
f2606b17
LD
2346 print("Processing only the first commit due to option" \
2347 " --prepare-p4-only")
728b7ad8 2348 break
7e5dd9f2
PW
2349 if i < last:
2350 quit = False
2351 while True:
6bbfd137
PW
2352 # prompt for what to do, or use the option/variable
2353 if self.conflict_behavior == "ask":
f2606b17 2354 print("What do you want to do?")
6bbfd137
PW
2355 response = raw_input("[s]kip this commit but apply"
2356 " the rest, or [q]uit? ")
2357 if not response:
2358 continue
2359 elif self.conflict_behavior == "skip":
2360 response = "s"
2361 elif self.conflict_behavior == "quit":
2362 response = "q"
2363 else:
2364 die("Unknown conflict_behavior '%s'" %
2365 self.conflict_behavior)
2366
7e5dd9f2 2367 if response[0] == "s":
f2606b17 2368 print("Skipping this commit, but applying the rest")
7e5dd9f2
PW
2369 break
2370 if response[0] == "q":
f2606b17 2371 print("Quitting")
7e5dd9f2
PW
2372 quit = True
2373 break
2374 if quit:
2375 break
4f5cf76a 2376
67b0fe2e 2377 chdir(self.oldWorkingDirectory)
b34fa577 2378 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2379 if self.dry_run:
2380 pass
728b7ad8
PW
2381 elif self.prepare_p4_only:
2382 pass
ef739f08 2383 elif len(commits) == len(applied):
f2606b17 2384 print("All commits {0}!".format(shelved_applied))
14594f4b 2385
4c750c0d 2386 sync = P4Sync()
44e8d26c
PW
2387 if self.branch:
2388 sync.branch = self.branch
b9d34db9
LD
2389 if self.disable_p4sync:
2390 sync.sync_origin_only()
2391 else:
2392 sync.run([])
14594f4b 2393
b9d34db9
LD
2394 if not self.disable_rebase:
2395 rebase = P4Rebase()
2396 rebase.rebase()
4f5cf76a 2397
67b0fe2e
PW
2398 else:
2399 if len(applied) == 0:
f2606b17 2400 print("No commits {0}.".format(shelved_applied))
67b0fe2e 2401 else:
f2606b17 2402 print("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2403 for c in commits:
2404 if c in applied:
2405 star = "*"
2406 else:
2407 star = " "
f2606b17
LD
2408 print(star, read_pipe(["git", "show", "-s",
2409 "--format=format:%h %s", c]))
2410 print("You will have to do 'git p4 sync' and rebase.")
67b0fe2e 2411
0d609032 2412 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2413 self.exportLabels = True
06804c76
LD
2414
2415 if self.exportLabels:
2416 p4Labels = getP4Labels(self.depotPath)
2417 gitTags = getGitTags()
2418
2419 missingGitTags = gitTags - p4Labels
2420 self.exportGitTags(missingGitTags)
2421
98e023de 2422 # exit with error unless everything applied perfectly
67b0fe2e
PW
2423 if len(commits) != len(applied):
2424 sys.exit(1)
2425
b984733c
SH
2426 return True
2427
ecb7cf98
PW
2428class View(object):
2429 """Represent a p4 view ("p4 help views"), and map files in a
2430 repo according to the view."""
2431
9d57c4a6 2432 def __init__(self, client_name):
ecb7cf98 2433 self.mappings = []
9d57c4a6
KS
2434 self.client_prefix = "//%s/" % client_name
2435 # cache results of "p4 where" to lookup client file locations
2436 self.client_spec_path_cache = {}
ecb7cf98
PW
2437
2438 def append(self, view_line):
2439 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2440 sides. Append to self.mappings, preserving order. This
2441 is only needed for tag creation."""
ecb7cf98
PW
2442
2443 # Split the view line into exactly two words. P4 enforces
2444 # structure on these lines that simplifies this quite a bit.
2445 #
2446 # Either or both words may be double-quoted.
2447 # Single quotes do not matter.
2448 # Double-quote marks cannot occur inside the words.
2449 # A + or - prefix is also inside the quotes.
2450 # There are no quotes unless they contain a space.
2451 # The line is already white-space stripped.
2452 # The two words are separated by a single space.
2453 #
2454 if view_line[0] == '"':
2455 # First word is double quoted. Find its end.
2456 close_quote_index = view_line.find('"', 1)
2457 if close_quote_index <= 0:
2458 die("No first-word closing quote found: %s" % view_line)
2459 depot_side = view_line[1:close_quote_index]
2460 # skip closing quote and space
2461 rhs_index = close_quote_index + 1 + 1
2462 else:
2463 space_index = view_line.find(" ")
2464 if space_index <= 0:
2465 die("No word-splitting space found: %s" % view_line)
2466 depot_side = view_line[0:space_index]
2467 rhs_index = space_index + 1
2468
ecb7cf98 2469 # prefix + means overlay on previous mapping
ecb7cf98 2470 if depot_side.startswith("+"):
ecb7cf98
PW
2471 depot_side = depot_side[1:]
2472
9d57c4a6 2473 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2474 exclude = False
2475 if depot_side.startswith("-"):
2476 exclude = True
2477 depot_side = depot_side[1:]
2478
9d57c4a6
KS
2479 if not exclude:
2480 self.mappings.append(depot_side)
ecb7cf98 2481
9d57c4a6
KS
2482 def convert_client_path(self, clientFile):
2483 # chop off //client/ part to make it relative
2484 if not clientFile.startswith(self.client_prefix):
2485 die("No prefix '%s' on clientFile '%s'" %
2486 (self.client_prefix, clientFile))
2487 return clientFile[len(self.client_prefix):]
ecb7cf98 2488
9d57c4a6
KS
2489 def update_client_spec_path_cache(self, files):
2490 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2491
9d57c4a6
KS
2492 # List depot file paths exclude that already cached
2493 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2494
9d57c4a6
KS
2495 if len(fileArgs) == 0:
2496 return # All files in cache
ecb7cf98 2497
9d57c4a6
KS
2498 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2499 for res in where_result:
2500 if "code" in res and res["code"] == "error":
2501 # assume error is "... file(s) not in client view"
2502 continue
2503 if "clientFile" not in res:
20005443 2504 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2505 if "unmap" in res:
2506 # it will list all of them, but only one not unmap-ped
2507 continue
a0a50d87
LS
2508 if gitConfigBool("core.ignorecase"):
2509 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2510 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2511
9d57c4a6
KS
2512 # not found files or unmap files set to ""
2513 for depotFile in fileArgs:
a0a50d87
LS
2514 if gitConfigBool("core.ignorecase"):
2515 depotFile = depotFile.lower()
9d57c4a6
KS
2516 if depotFile not in self.client_spec_path_cache:
2517 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2518
9d57c4a6
KS
2519 def map_in_client(self, depot_path):
2520 """Return the relative location in the client where this
2521 depot file should live. Returns "" if the file should
2522 not be mapped in the client."""
ecb7cf98 2523
a0a50d87
LS
2524 if gitConfigBool("core.ignorecase"):
2525 depot_path = depot_path.lower()
2526
9d57c4a6
KS
2527 if depot_path in self.client_spec_path_cache:
2528 return self.client_spec_path_cache[depot_path]
2529
2530 die( "Error: %s is not found in client spec path" % depot_path )
2531 return ""
ecb7cf98 2532
ff8c50ed
AM
2533def cloneExcludeCallback(option, opt_str, value, parser):
2534 # prepend "/" because the first "/" was consumed as part of the option itself.
2535 # ("-//depot/A/..." becomes "/depot/A/..." after option parsing)
2536 parser.values.cloneExclude += ["/" + re.sub(r"\.\.\.$", "", value)]
2537
3ea2cfd4 2538class P4Sync(Command, P4UserMap):
56c09345 2539
b984733c
SH
2540 def __init__(self):
2541 Command.__init__(self)
3ea2cfd4 2542 P4UserMap.__init__(self)
b984733c
SH
2543 self.options = [
2544 optparse.make_option("--branch", dest="branch"),
2545 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2546 optparse.make_option("--changesfile", dest="changesFile"),
2547 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2548 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2549 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2550 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2551 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2552 optparse.make_option("--max-changes", dest="maxChanges",
2553 help="Maximum number of changes to import"),
2554 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2555 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2556 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2557 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2558 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2559 help="Only sync files that are included in the Perforce Client Spec"),
2560 optparse.make_option("-/", dest="cloneExclude",
ff8c50ed 2561 action="callback", callback=cloneExcludeCallback, type="string",
51334bb0 2562 help="exclude depot path"),
b984733c
SH
2563 ]
2564 self.description = """Imports from Perforce into a git repository.\n
2565 example:
2566 //depot/my/project/ -- to import the current head
2567 //depot/my/project/@all -- to import everything
2568 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2569
2570 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2571
2572 self.usage += " //depot/path[@revRange]"
b984733c 2573 self.silent = False
1d7367dc
RG
2574 self.createdBranches = set()
2575 self.committedChanges = set()
569d1bd4 2576 self.branch = ""
b984733c 2577 self.detectBranches = False
cb53e1f8 2578 self.detectLabels = False
06804c76 2579 self.importLabels = False
b984733c 2580 self.changesFile = ""
01265103 2581 self.syncWithOrigin = True
a028a98e 2582 self.importIntoRemotes = True
01a9c9c5 2583 self.maxChanges = ""
1051ef00 2584 self.changes_block_size = None
8b41a97f 2585 self.keepRepoPath = False
6326aa58 2586 self.depotPaths = None
3c699645 2587 self.p4BranchesInGit = []
354081d5 2588 self.cloneExclude = []
3a70cdfa 2589 self.useClientSpec = False
a93d33ee 2590 self.useClientSpec_from_options = False
ecb7cf98 2591 self.clientSpecDirs = None
fed23693 2592 self.tempBranches = []
d604176d 2593 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12 2594 self.largeFileSystem = None
123f6317 2595 self.suppress_meta_comment = False
a5db4b12
LS
2596
2597 if gitConfig('git-p4.largeFileSystem'):
2598 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2599 self.largeFileSystem = largeFileSystemConstructor(
2600 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2601 )
b984733c 2602
01265103
SH
2603 if gitConfig("git-p4.syncFromOrigin") == "false":
2604 self.syncWithOrigin = False
2605
123f6317
LD
2606 self.depotPaths = []
2607 self.changeRange = ""
2608 self.previousDepotPaths = []
2609 self.hasOrigin = False
2610
2611 # map from branch depot path to parent branch
2612 self.knownBranches = {}
2613 self.initialParents = {}
2614
2615 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2616 self.labels = {}
2617
fed23693
VA
2618 # Force a checkpoint in fast-import and wait for it to finish
2619 def checkpoint(self):
2620 self.gitStream.write("checkpoint\n\n")
2621 self.gitStream.write("progress checkpoint\n\n")
2622 out = self.gitOutput.readline()
2623 if self.verbose:
f2606b17 2624 print("checkpoint finished: " + out)
fed23693 2625
a2bee10a
AM
2626 def isPathWanted(self, path):
2627 for p in self.cloneExclude:
2628 if p.endswith("/"):
2629 if p4PathStartsWith(path, p):
2630 return False
2631 # "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"
2632 elif path.lower() == p.lower():
2633 return False
2634 for p in self.depotPaths:
2635 if p4PathStartsWith(path, p):
2636 return True
2637 return False
2638
89143ac2 2639 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0):
b984733c
SH
2640 files = []
2641 fnum = 0
dba1c9d9 2642 while "depotFile%s" % fnum in commit:
b984733c 2643 path = commit["depotFile%s" % fnum]
a2bee10a 2644 found = self.isPathWanted(path)
6326aa58 2645 if not found:
b984733c
SH
2646 fnum = fnum + 1
2647 continue
2648
2649 file = {}
2650 file["path"] = path
2651 file["rev"] = commit["rev%s" % fnum]
2652 file["action"] = commit["action%s" % fnum]
2653 file["type"] = commit["type%s" % fnum]
123f6317
LD
2654 if shelved:
2655 file["shelved_cl"] = int(shelved_cl)
b984733c
SH
2656 files.append(file)
2657 fnum = fnum + 1
2658 return files
2659
26e6a27d
JD
2660 def extractJobsFromCommit(self, commit):
2661 jobs = []
2662 jnum = 0
dba1c9d9 2663 while "job%s" % jnum in commit:
26e6a27d
JD
2664 job = commit["job%s" % jnum]
2665 jobs.append(job)
2666 jnum = jnum + 1
2667 return jobs
2668
6326aa58 2669 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2670 """When streaming files, this is called to map a p4 depot path
2671 to where it should go in git. The prefixes are either
2672 self.depotPaths, or self.branchPrefixes in the case of
2673 branch detection."""
2674
3952710b 2675 if self.useClientSpec:
21ef5df4
PW
2676 # branch detection moves files up a level (the branch name)
2677 # from what client spec interpretation gives
0d1696ef 2678 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2679 if self.detectBranches:
2680 for b in self.knownBranches:
f2768cb3 2681 if p4PathStartsWith(path, b + "/"):
21ef5df4
PW
2682 path = path[len(b)+1:]
2683
2684 elif self.keepRepoPath:
2685 # Preserve everything in relative path name except leading
2686 # //depot/; just look at first prefix as they all should
2687 # be in the same depot.
2688 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2689 if p4PathStartsWith(path, depot):
2690 path = path[len(depot):]
3952710b 2691
0d1696ef 2692 else:
0d1696ef
PW
2693 for p in prefixes:
2694 if p4PathStartsWith(path, p):
2695 path = path[len(p):]
21ef5df4 2696 break
8b41a97f 2697
0d1696ef 2698 path = wildcard_decode(path)
6326aa58 2699 return path
6754a299 2700
71b112d4 2701 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2702 """Look at each depotFile in the commit to figure out to what
2703 branch it belongs."""
2704
9d57c4a6
KS
2705 if self.clientSpecDirs:
2706 files = self.extractFilesFromCommit(commit)
2707 self.clientSpecDirs.update_client_spec_path_cache(files)
2708
d5904674 2709 branches = {}
71b112d4 2710 fnum = 0
dba1c9d9 2711 while "depotFile%s" % fnum in commit:
71b112d4 2712 path = commit["depotFile%s" % fnum]
d15068a6 2713 found = self.isPathWanted(path)
6326aa58 2714 if not found:
71b112d4
SH
2715 fnum = fnum + 1
2716 continue
2717
2718 file = {}
2719 file["path"] = path
2720 file["rev"] = commit["rev%s" % fnum]
2721 file["action"] = commit["action%s" % fnum]
2722 file["type"] = commit["type%s" % fnum]
2723 fnum = fnum + 1
2724
21ef5df4
PW
2725 # start with the full relative path where this file would
2726 # go in a p4 client
2727 if self.useClientSpec:
2728 relPath = self.clientSpecDirs.map_in_client(path)
2729 else:
2730 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2731
4b97ffb1 2732 for branch in self.knownBranches.keys():
21ef5df4
PW
2733 # add a trailing slash so that a commit into qt/4.2foo
2734 # doesn't end up in qt/4.2, e.g.
f2768cb3 2735 if p4PathStartsWith(relPath, branch + "/"):
d5904674
SH
2736 if branch not in branches:
2737 branches[branch] = []
71b112d4 2738 branches[branch].append(file)
6555b2cc 2739 break
b984733c
SH
2740
2741 return branches
2742
a5db4b12
LS
2743 def writeToGitStream(self, gitMode, relPath, contents):
2744 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2745 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2746 for d in contents:
2747 self.gitStream.write(d)
2748 self.gitStream.write('\n')
2749
a8b05162
LS
2750 def encodeWithUTF8(self, path):
2751 try:
2752 path.decode('ascii')
2753 except:
2754 encoding = 'utf8'
2755 if gitConfig('git-p4.pathEncoding'):
2756 encoding = gitConfig('git-p4.pathEncoding')
2757 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2758 if self.verbose:
f2606b17 2759 print('Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path))
a8b05162
LS
2760 return path
2761
b932705b
LD
2762 # output one file from the P4 stream
2763 # - helper for streamP4Files
2764
2765 def streamOneP4File(self, file, contents):
b932705b 2766 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2767 relPath = self.encodeWithUTF8(relPath)
b932705b 2768 if verbose:
0742b7c8
LD
2769 if 'fileSize' in self.stream_file:
2770 size = int(self.stream_file['fileSize'])
2771 else:
2772 size = 0 # deleted files don't get a fileSize apparently
d2176a50
LS
2773 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2774 sys.stdout.flush()
b932705b 2775
9cffb8c8
PW
2776 (type_base, type_mods) = split_p4_type(file["type"])
2777
2778 git_mode = "100644"
2779 if "x" in type_mods:
2780 git_mode = "100755"
2781 if type_base == "symlink":
2782 git_mode = "120000"
1292df11
AJ
2783 # p4 print on a symlink sometimes contains "target\n";
2784 # if it does, remove the newline
b39c3612 2785 data = ''.join(contents)
40f846c3
PW
2786 if not data:
2787 # Some version of p4 allowed creating a symlink that pointed
2788 # to nothing. This causes p4 errors when checking out such
2789 # a change, and errors here too. Work around it by ignoring
2790 # the bad symlink; hopefully a future change fixes it.
f2606b17 2791 print("\nIgnoring empty symlink in %s" % file['depotFile'])
40f846c3
PW
2792 return
2793 elif data[-1] == '\n':
1292df11
AJ
2794 contents = [data[:-1]]
2795 else:
2796 contents = [data]
b932705b 2797
9cffb8c8 2798 if type_base == "utf16":
55aa5714
PW
2799 # p4 delivers different text in the python output to -G
2800 # than it does when using "print -o", or normal p4 client
2801 # operations. utf16 is converted to ascii or utf8, perhaps.
2802 # But ascii text saved as -t utf16 is completely mangled.
2803 # Invoke print -o to get the real contents.
7f0e5962
PW
2804 #
2805 # On windows, the newlines will always be mangled by print, so put
2806 # them back too. This is not needed to the cygwin windows version,
2807 # just the native "NT" type.
2808 #
1f5f3907
LS
2809 try:
2810 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2811 except Exception as e:
2812 if 'Translation of file content failed' in str(e):
2813 type_base = 'binary'
2814 else:
2815 raise e
2816 else:
2817 if p4_version_string().find('/NT') >= 0:
2818 text = text.replace('\r\n', '\n')
2819 contents = [ text ]
55aa5714 2820
9f7ef0ea
PW
2821 if type_base == "apple":
2822 # Apple filetype files will be streamed as a concatenation of
2823 # its appledouble header and the contents. This is useless
2824 # on both macs and non-macs. If using "print -q -o xx", it
2825 # will create "xx" with the data, and "%xx" with the header.
2826 # This is also not very useful.
2827 #
2828 # Ideally, someday, this script can learn how to generate
2829 # appledouble files directly and import those to git, but
2830 # non-mac machines can never find a use for apple filetype.
f2606b17 2831 print("\nIgnoring apple filetype file %s" % file['depotFile'])
9f7ef0ea
PW
2832 return
2833
55aa5714
PW
2834 # Note that we do not try to de-mangle keywords on utf16 files,
2835 # even though in theory somebody may want that.
60df071c
LD
2836 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2837 if pattern:
2838 regexp = re.compile(pattern, re.VERBOSE)
2839 text = ''.join(contents)
2840 text = regexp.sub(r'$\1$', text)
2841 contents = [ text ]
b932705b 2842
a5db4b12
LS
2843 if self.largeFileSystem:
2844 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2845
a5db4b12 2846 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2847
2848 def streamOneP4Deletion(self, file):
2849 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2850 relPath = self.encodeWithUTF8(relPath)
b932705b 2851 if verbose:
d2176a50
LS
2852 sys.stdout.write("delete %s\n" % relPath)
2853 sys.stdout.flush()
b932705b
LD
2854 self.gitStream.write("D %s\n" % relPath)
2855
a5db4b12
LS
2856 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2857 self.largeFileSystem.removeLargeFile(relPath)
2858
b932705b
LD
2859 # handle another chunk of streaming data
2860 def streamP4FilesCb(self, marshalled):
2861
78189bea
PW
2862 # catch p4 errors and complain
2863 err = None
2864 if "code" in marshalled:
2865 if marshalled["code"] == "error":
2866 if "data" in marshalled:
2867 err = marshalled["data"].rstrip()
4d25dc44
LS
2868
2869 if not err and 'fileSize' in self.stream_file:
2870 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2871 if required_bytes > 0:
2872 err = 'Not enough space left on %s! Free at least %i MB.' % (
2873 os.getcwd(), required_bytes/1024/1024
2874 )
2875
78189bea
PW
2876 if err:
2877 f = None
2878 if self.stream_have_file_info:
2879 if "depotFile" in self.stream_file:
2880 f = self.stream_file["depotFile"]
2881 # force a failure in fast-import, else an empty
2882 # commit will be made
2883 self.gitStream.write("\n")
2884 self.gitStream.write("die-now\n")
2885 self.gitStream.close()
2886 # ignore errors, but make sure it exits first
2887 self.importProcess.wait()
2888 if f:
2889 die("Error from p4 print for %s: %s" % (f, err))
2890 else:
2891 die("Error from p4 print: %s" % err)
2892
dba1c9d9 2893 if 'depotFile' in marshalled and self.stream_have_file_info:
c3f6163b
AG
2894 # start of a new file - output the old one first
2895 self.streamOneP4File(self.stream_file, self.stream_contents)
2896 self.stream_file = {}
2897 self.stream_contents = []
2898 self.stream_have_file_info = False
b932705b 2899
c3f6163b
AG
2900 # pick up the new file information... for the
2901 # 'data' field we need to append to our array
2902 for k in marshalled.keys():
2903 if k == 'data':
d2176a50
LS
2904 if 'streamContentSize' not in self.stream_file:
2905 self.stream_file['streamContentSize'] = 0
2906 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2907 self.stream_contents.append(marshalled['data'])
2908 else:
2909 self.stream_file[k] = marshalled[k]
b932705b 2910
d2176a50
LS
2911 if (verbose and
2912 'streamContentSize' in self.stream_file and
2913 'fileSize' in self.stream_file and
2914 'depotFile' in self.stream_file):
2915 size = int(self.stream_file["fileSize"])
2916 if size > 0:
2917 progress = 100*self.stream_file['streamContentSize']/size
2918 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2919 sys.stdout.flush()
2920
c3f6163b 2921 self.stream_have_file_info = True
b932705b
LD
2922
2923 # Stream directly from "p4 files" into "git fast-import"
2924 def streamP4Files(self, files):
30b5940b
SH
2925 filesForCommit = []
2926 filesToRead = []
b932705b 2927 filesToDelete = []
30b5940b 2928
3a70cdfa 2929 for f in files:
ecb7cf98
PW
2930 filesForCommit.append(f)
2931 if f['action'] in self.delete_actions:
2932 filesToDelete.append(f)
2933 else:
2934 filesToRead.append(f)
6a49f8e2 2935
b932705b
LD
2936 # deleted files...
2937 for f in filesToDelete:
2938 self.streamOneP4Deletion(f)
1b9a4684 2939
b932705b
LD
2940 if len(filesToRead) > 0:
2941 self.stream_file = {}
2942 self.stream_contents = []
2943 self.stream_have_file_info = False
8ff45f2a 2944
c3f6163b
AG
2945 # curry self argument
2946 def streamP4FilesCbSelf(entry):
2947 self.streamP4FilesCb(entry)
6a49f8e2 2948
123f6317
LD
2949 fileArgs = []
2950 for f in filesToRead:
2951 if 'shelved_cl' in f:
2952 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2953 # the contents
2954 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2955 else:
2956 fileArg = '%s#%s' % (f['path'], f['rev'])
2957
2958 fileArgs.append(fileArg)
6de040df
LD
2959
2960 p4CmdList(["-x", "-", "print"],
2961 stdin=fileArgs,
2962 cb=streamP4FilesCbSelf)
30b5940b 2963
b932705b 2964 # do the last chunk
dba1c9d9 2965 if 'depotFile' in self.stream_file:
b932705b 2966 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2967
affb474f
LD
2968 def make_email(self, userid):
2969 if userid in self.users:
2970 return self.users[userid]
2971 else:
2972 return "%s <a@b>" % userid
2973
06804c76 2974 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2975 """ Stream a p4 tag.
2976 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2977 """
2978
06804c76 2979 if verbose:
f2606b17 2980 print("writing tag %s for commit %s" % (labelName, commit))
06804c76
LD
2981 gitStream.write("tag %s\n" % labelName)
2982 gitStream.write("from %s\n" % commit)
2983
dba1c9d9 2984 if 'Owner' in labelDetails:
06804c76
LD
2985 owner = labelDetails["Owner"]
2986 else:
2987 owner = None
2988
2989 # Try to use the owner of the p4 label, or failing that,
2990 # the current p4 user id.
2991 if owner:
2992 email = self.make_email(owner)
2993 else:
2994 email = self.make_email(self.p4UserId())
2995 tagger = "%s %s %s" % (email, epoch, self.tz)
2996
2997 gitStream.write("tagger %s\n" % tagger)
2998
f2606b17 2999 print("labelDetails=",labelDetails)
dba1c9d9 3000 if 'Description' in labelDetails:
06804c76
LD
3001 description = labelDetails['Description']
3002 else:
3003 description = 'Label from git p4'
3004
3005 gitStream.write("data %d\n" % len(description))
3006 gitStream.write(description)
3007 gitStream.write("\n")
3008
4ae048e6
LS
3009 def inClientSpec(self, path):
3010 if not self.clientSpecDirs:
3011 return True
3012 inClientSpec = self.clientSpecDirs.map_in_client(path)
3013 if not inClientSpec and self.verbose:
3014 print('Ignoring file outside of client spec: {0}'.format(path))
3015 return inClientSpec
3016
3017 def hasBranchPrefix(self, path):
3018 if not self.branchPrefixes:
3019 return True
3020 hasPrefix = [p for p in self.branchPrefixes
3021 if p4PathStartsWith(path, p)]
09667d01 3022 if not hasPrefix and self.verbose:
4ae048e6
LS
3023 print('Ignoring file outside of prefix: {0}'.format(path))
3024 return hasPrefix
3025
89143ac2 3026 def commit(self, details, files, branch, parent = "", allow_empty=False):
b984733c
SH
3027 epoch = details["time"]
3028 author = details["user"]
26e6a27d 3029 jobs = self.extractJobsFromCommit(details)
b984733c 3030
4b97ffb1 3031 if self.verbose:
4ae048e6 3032 print('commit into {0}'.format(branch))
96e07dd2 3033
9d57c4a6
KS
3034 if self.clientSpecDirs:
3035 self.clientSpecDirs.update_client_spec_path_cache(files)
3036
4ae048e6
LS
3037 files = [f for f in files
3038 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3039
89143ac2
LD
3040 if gitConfigBool('git-p4.keepEmptyCommits'):
3041 allow_empty = True
3042
3043 if not files and not allow_empty:
4ae048e6
LS
3044 print('Ignoring revision {0} as it would produce an empty commit.'
3045 .format(details['change']))
3046 return
3047
b984733c 3048 self.gitStream.write("commit %s\n" % branch)
b43702ac 3049 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
3050 self.committedChanges.add(int(details["change"]))
3051 committer = ""
b607e71e
SH
3052 if author not in self.users:
3053 self.getUserMapFromPerforceServer()
affb474f 3054 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
3055
3056 self.gitStream.write("committer %s\n" % committer)
3057
3058 self.gitStream.write("data <<EOT\n")
3059 self.gitStream.write(details["desc"])
26e6a27d
JD
3060 if len(jobs) > 0:
3061 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
123f6317
LD
3062
3063 if not self.suppress_meta_comment:
3064 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3065 (','.join(self.branchPrefixes), details["change"]))
3066 if len(details['options']) > 0:
3067 self.gitStream.write(": options = %s" % details['options'])
3068 self.gitStream.write("]\n")
3069
3070 self.gitStream.write("EOT\n\n")
b984733c
SH
3071
3072 if len(parent) > 0:
4b97ffb1 3073 if self.verbose:
f2606b17 3074 print("parent %s" % parent)
b984733c
SH
3075 self.gitStream.write("from %s\n" % parent)
3076
4ae048e6 3077 self.streamP4Files(files)
b984733c
SH
3078 self.gitStream.write("\n")
3079
1f4ba1cb
SH
3080 change = int(details["change"])
3081
dba1c9d9 3082 if change in self.labels:
1f4ba1cb
SH
3083 label = self.labels[change]
3084 labelDetails = label[0]
3085 labelRevisions = label[1]
71b112d4 3086 if self.verbose:
f2606b17 3087 print("Change %s is labelled %s" % (change, labelDetails))
1f4ba1cb 3088
6de040df 3089 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 3090 for p in self.branchPrefixes])
1f4ba1cb
SH
3091
3092 if len(files) == len(labelRevisions):
3093
3094 cleanedFiles = {}
3095 for info in files:
56c09345 3096 if info["action"] in self.delete_actions:
1f4ba1cb
SH
3097 continue
3098 cleanedFiles[info["depotFile"]] = info["rev"]
3099
3100 if cleanedFiles == labelRevisions:
06804c76 3101 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
3102
3103 else:
a46668fa 3104 if not self.silent:
f2606b17 3105 print("Tag %s does not match with change %s: files do not match."
cebdf5af 3106 % (labelDetails["label"], change))
1f4ba1cb
SH
3107
3108 else:
a46668fa 3109 if not self.silent:
f2606b17 3110 print("Tag %s does not match with change %s: file count is different."
cebdf5af 3111 % (labelDetails["label"], change))
b984733c 3112
06804c76 3113 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
3114 def getLabels(self):
3115 self.labels = {}
3116
52a4880b 3117 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 3118 if len(l) > 0 and not self.silent:
4d88519f 3119 print("Finding files belonging to labels in %s" % self.depotPaths)
01ce1fe9
SH
3120
3121 for output in l:
1f4ba1cb
SH
3122 label = output["label"]
3123 revisions = {}
3124 newestChange = 0
71b112d4 3125 if self.verbose:
f2606b17 3126 print("Querying files for label %s" % label)
6de040df
LD
3127 for file in p4CmdList(["files"] +
3128 ["%s...@%s" % (p, label)
3129 for p in self.depotPaths]):
1f4ba1cb
SH
3130 revisions[file["depotFile"]] = file["rev"]
3131 change = int(file["change"])
3132 if change > newestChange:
3133 newestChange = change
3134
9bda3a85
SH
3135 self.labels[newestChange] = [output, revisions]
3136
3137 if self.verbose:
f2606b17 3138 print("Label changes: %s" % self.labels.keys())
1f4ba1cb 3139
06804c76
LD
3140 # Import p4 labels as git tags. A direct mapping does not
3141 # exist, so assume that if all the files are at the same revision
3142 # then we can use that, or it's something more complicated we should
3143 # just ignore.
3144 def importP4Labels(self, stream, p4Labels):
3145 if verbose:
f2606b17 3146 print("import p4 labels: " + ' '.join(p4Labels))
06804c76
LD
3147
3148 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 3149 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
3150 if len(validLabelRegexp) == 0:
3151 validLabelRegexp = defaultLabelRegexp
3152 m = re.compile(validLabelRegexp)
3153
3154 for name in p4Labels:
3155 commitFound = False
3156
3157 if not m.match(name):
3158 if verbose:
f2606b17 3159 print("label %s does not match regexp %s" % (name,validLabelRegexp))
06804c76
LD
3160 continue
3161
3162 if name in ignoredP4Labels:
3163 continue
3164
3165 labelDetails = p4CmdList(['label', "-o", name])[0]
3166
3167 # get the most recent changelist for each file in this label
3168 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3169 for p in self.depotPaths])
3170
dba1c9d9 3171 if 'change' in change:
06804c76
LD
3172 # find the corresponding git commit; take the oldest commit
3173 changelist = int(change['change'])
b43702ac
LD
3174 if changelist in self.committedChanges:
3175 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 3176 commitFound = True
b43702ac
LD
3177 else:
3178 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3179 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3180 if len(gitCommit) == 0:
f2606b17 3181 print("importing label %s: could not find git commit for changelist %d" % (name, changelist))
b43702ac
LD
3182 else:
3183 commitFound = True
3184 gitCommit = gitCommit.strip()
3185
3186 if commitFound:
06804c76
LD
3187 # Convert from p4 time format
3188 try:
3189 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3190 except ValueError:
f2606b17 3191 print("Could not convert label time %s" % labelDetails['Update'])
06804c76
LD
3192 tmwhen = 1
3193
3194 when = int(time.mktime(tmwhen))
3195 self.streamTag(stream, name, labelDetails, gitCommit, when)
3196 if verbose:
f2606b17 3197 print("p4 label %s mapped to git commit %s" % (name, gitCommit))
06804c76
LD
3198 else:
3199 if verbose:
f2606b17 3200 print("Label %s has no changelists - possibly deleted?" % name)
06804c76
LD
3201
3202 if not commitFound:
3203 # We can't import this label; don't try again as it will get very
3204 # expensive repeatedly fetching all the files for labels that will
3205 # never be imported. If the label is moved in the future, the
3206 # ignore will need to be removed manually.
3207 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3208
86dff6b6
HWN
3209 def guessProjectName(self):
3210 for p in self.depotPaths:
6e5295c4
SH
3211 if p.endswith("/"):
3212 p = p[:-1]
3213 p = p[p.strip().rfind("/") + 1:]
3214 if not p.endswith("/"):
3215 p += "/"
3216 return p
86dff6b6 3217
4b97ffb1 3218 def getBranchMapping(self):
6555b2cc
SH
3219 lostAndFoundBranches = set()
3220
8ace74c0
VA
3221 user = gitConfig("git-p4.branchUser")
3222 if len(user) > 0:
3223 command = "branches -u %s" % user
3224 else:
3225 command = "branches"
3226
3227 for info in p4CmdList(command):
52a4880b 3228 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1 3229 viewIdx = 0
dba1c9d9 3230 while "View%s" % viewIdx in details:
4b97ffb1
SH
3231 paths = details["View%s" % viewIdx].split(" ")
3232 viewIdx = viewIdx + 1
3233 # require standard //depot/foo/... //depot/bar/... mapping
3234 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3235 continue
3236 source = paths[0]
3237 destination = paths[1]
6509e19c 3238 ## HACK
d53de8b9 3239 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3240 source = source[len(self.depotPaths[0]):-4]
3241 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3242
1a2edf4e
SH
3243 if destination in self.knownBranches:
3244 if not self.silent:
f2606b17
LD
3245 print("p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination))
3246 print("but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination))
1a2edf4e
SH
3247 continue
3248
6555b2cc
SH
3249 self.knownBranches[destination] = source
3250
3251 lostAndFoundBranches.discard(destination)
3252
29bdbac1 3253 if source not in self.knownBranches:
6555b2cc
SH
3254 lostAndFoundBranches.add(source)
3255
7199cf13
VA
3256 # Perforce does not strictly require branches to be defined, so we also
3257 # check git config for a branch list.
3258 #
3259 # Example of branch definition in git config file:
3260 # [git-p4]
3261 # branchList=main:branchA
3262 # branchList=main:branchB
3263 # branchList=branchA:branchC
3264 configBranches = gitConfigList("git-p4.branchList")
3265 for branch in configBranches:
3266 if branch:
3267 (source, destination) = branch.split(":")
3268 self.knownBranches[destination] = source
3269
3270 lostAndFoundBranches.discard(destination)
3271
3272 if source not in self.knownBranches:
3273 lostAndFoundBranches.add(source)
3274
6555b2cc
SH
3275
3276 for branch in lostAndFoundBranches:
3277 self.knownBranches[branch] = branch
29bdbac1 3278
38f9f5ec
SH
3279 def getBranchMappingFromGitBranches(self):
3280 branches = p4BranchesInGit(self.importIntoRemotes)
3281 for branch in branches.keys():
3282 if branch == "master":
3283 branch = "main"
3284 else:
3285 branch = branch[len(self.projectName):]
3286 self.knownBranches[branch] = branch
3287
bb6e09b2
HWN
3288 def updateOptionDict(self, d):
3289 option_keys = {}
3290 if self.keepRepoPath:
3291 option_keys['keepRepoPath'] = 1
3292
3293 d["options"] = ' '.join(sorted(option_keys.keys()))
3294
3295 def readOptions(self, d):
dba1c9d9 3296 self.keepRepoPath = ('options' in d
bb6e09b2 3297 and ('keepRepoPath' in d['options']))
6326aa58 3298
8134f69c
SH
3299 def gitRefForBranch(self, branch):
3300 if branch == "main":
3301 return self.refPrefix + "master"
3302
3303 if len(branch) <= 0:
3304 return branch
3305
3306 return self.refPrefix + self.projectName + branch
3307
1ca3d710
SH
3308 def gitCommitByP4Change(self, ref, change):
3309 if self.verbose:
f2606b17 3310 print("looking in ref " + ref + " for change %s using bisect..." % change)
1ca3d710
SH
3311
3312 earliestCommit = ""
3313 latestCommit = parseRevision(ref)
3314
3315 while True:
3316 if self.verbose:
f2606b17 3317 print("trying: earliest %s latest %s" % (earliestCommit, latestCommit))
1ca3d710
SH
3318 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3319 if len(next) == 0:
3320 if self.verbose:
f2606b17 3321 print("argh")
1ca3d710
SH
3322 return ""
3323 log = extractLogMessageFromGitCommit(next)
3324 settings = extractSettingsGitLog(log)
3325 currentChange = int(settings['change'])
3326 if self.verbose:
f2606b17 3327 print("current change %s" % currentChange)
1ca3d710
SH
3328
3329 if currentChange == change:
3330 if self.verbose:
f2606b17 3331 print("found %s" % next)
1ca3d710
SH
3332 return next
3333
3334 if currentChange < change:
3335 earliestCommit = "^%s" % next
3336 else:
2dda7412
AM
3337 if next == latestCommit:
3338 die("Infinite loop while looking in ref %s for change %s. Check your branch mappings" % (ref, change))
3339 latestCommit = "%s^@" % next
1ca3d710
SH
3340
3341 return ""
3342
3343 def importNewBranch(self, branch, maxChange):
3344 # make fast-import flush all changes to disk and update the refs using the checkpoint
3345 # command so that we can try to find the branch parent in the git history
3346 self.gitStream.write("checkpoint\n\n");
3347 self.gitStream.flush();
3348 branchPrefix = self.depotPaths[0] + branch + "/"
3349 range = "@1,%s" % maxChange
3350 #print "prefix" + branchPrefix
96b2d54a 3351 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3352 if len(changes) <= 0:
3353 return False
3354 firstChange = changes[0]
3355 #print "first change in branch: %s" % firstChange
3356 sourceBranch = self.knownBranches[branch]
3357 sourceDepotPath = self.depotPaths[0] + sourceBranch
3358 sourceRef = self.gitRefForBranch(sourceBranch)
3359 #print "source " + sourceBranch
3360
52a4880b 3361 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3362 #print "branch parent: %s" % branchParentChange
3363 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3364 if len(gitParent) > 0:
3365 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3366 #print "parent git commit: %s" % gitParent
3367
3368 self.importChanges(changes)
3369 return True
3370
fed23693
VA
3371 def searchParent(self, parent, branch, target):
3372 parentFound = False
c7d34884
PW
3373 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3374 "--no-merges", parent]):
fed23693
VA
3375 blob = blob.strip()
3376 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3377 parentFound = True
3378 if self.verbose:
f2606b17 3379 print("Found parent of %s in commit %s" % (branch, blob))
fed23693
VA
3380 break
3381 if parentFound:
3382 return blob
3383 else:
3384 return None
3385
89143ac2 3386 def importChanges(self, changes, origin_revision=0):
e87f37ae
SH
3387 cnt = 1
3388 for change in changes:
89143ac2 3389 description = p4_describe(change)
e87f37ae
SH
3390 self.updateOptionDict(description)
3391
3392 if not self.silent:
3393 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3394 sys.stdout.flush()
3395 cnt = cnt + 1
3396
3397 try:
3398 if self.detectBranches:
3399 branches = self.splitFilesIntoBranches(description)
3400 for branch in branches.keys():
3401 ## HACK --hwn
3402 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3403 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3404
3405 parent = ""
3406
3407 filesForCommit = branches[branch]
3408
3409 if self.verbose:
f2606b17 3410 print("branch is %s" % branch)
e87f37ae
SH
3411
3412 self.updatedBranches.add(branch)