git-p4: python3: basestring workaround
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
efdcc992
LD
30# support basestring in python3
31try:
32 unicode = unicode
33except NameError:
34 # 'unicode' is undefined, must be Python 3
35 str = str
36 unicode = str
37 bytes = bytes
38 basestring = (str,bytes)
39else:
40 # 'unicode' exists, must be Python 2
41 str = str
42 unicode = unicode
43 bytes = str
44 basestring = basestring
45
a235e85c
BC
46try:
47 from subprocess import CalledProcessError
48except ImportError:
49 # from python2.7:subprocess.py
50 # Exception classes used by this module.
51 class CalledProcessError(Exception):
52 """This exception is raised when a process run by check_call() returns
53 a non-zero exit status. The exit status will be stored in the
54 returncode attribute."""
55 def __init__(self, returncode, cmd):
56 self.returncode = returncode
57 self.cmd = cmd
58 def __str__(self):
59 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
60
4addad22 61verbose = False
86949eef 62
06804c76 63# Only labels/tags matching this will be imported/exported
c8942a22 64defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 65
3deed5e0
LD
66# The block size is reduced automatically if required
67defaultBlockSize = 1<<20
1051ef00 68
0ef67acd 69p4_access_checked = False
1051ef00 70
21a50753
AK
71def p4_build_cmd(cmd):
72 """Build a suitable p4 command line.
73
74 This consolidates building and returning a p4 command line into one
75 location. It means that hooking into the environment, or other configuration
76 can be done more easily.
77 """
6de040df 78 real_cmd = ["p4"]
abcaf073
AK
79
80 user = gitConfig("git-p4.user")
81 if len(user) > 0:
6de040df 82 real_cmd += ["-u",user]
abcaf073
AK
83
84 password = gitConfig("git-p4.password")
85 if len(password) > 0:
6de040df 86 real_cmd += ["-P", password]
abcaf073
AK
87
88 port = gitConfig("git-p4.port")
89 if len(port) > 0:
6de040df 90 real_cmd += ["-p", port]
abcaf073
AK
91
92 host = gitConfig("git-p4.host")
93 if len(host) > 0:
41799aa2 94 real_cmd += ["-H", host]
abcaf073
AK
95
96 client = gitConfig("git-p4.client")
97 if len(client) > 0:
6de040df 98 real_cmd += ["-c", client]
abcaf073 99
89a6ecc5
LS
100 retries = gitConfigInt("git-p4.retries")
101 if retries is None:
102 # Perform 3 retries by default
103 retries = 3
bc233524
IK
104 if retries > 0:
105 # Provide a way to not pass this option by setting git-p4.retries to 0
106 real_cmd += ["-r", str(retries)]
6de040df
LD
107
108 if isinstance(cmd,basestring):
109 real_cmd = ' '.join(real_cmd) + ' ' + cmd
110 else:
111 real_cmd += cmd
0ef67acd
LD
112
113 # now check that we can actually talk to the server
114 global p4_access_checked
115 if not p4_access_checked:
116 p4_access_checked = True # suppress access checks in p4_check_access itself
117 p4_check_access()
118
21a50753
AK
119 return real_cmd
120
378f7be1
LD
121def git_dir(path):
122 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
123 This won't automatically add ".git" to a directory.
124 """
125 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
126 if not d or len(d) == 0:
127 return None
128 else:
129 return d
130
bbd84863
MF
131def chdir(path, is_client_path=False):
132 """Do chdir to the given path, and set the PWD environment
133 variable for use by P4. It does not look at getcwd() output.
134 Since we're not using the shell, it is necessary to set the
135 PWD environment variable explicitly.
136
137 Normally, expand the path to force it to be absolute. This
138 addresses the use of relative path names inside P4 settings,
139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
140 as given; it looks for .p4config using PWD.
141
142 If is_client_path, the path was handed to us directly by p4,
143 and may be a symbolic link. Do not call os.getcwd() in this
144 case, because it will cause p4 to think that PWD is not inside
145 the client path.
146 """
147
148 os.chdir(path)
149 if not is_client_path:
150 path = os.getcwd()
151 os.environ['PWD'] = path
053fd0c1 152
4d25dc44
LS
153def calcDiskFree():
154 """Return free space in bytes on the disk of the given dirname."""
155 if platform.system() == 'Windows':
156 free_bytes = ctypes.c_ulonglong(0)
157 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
158 return free_bytes.value
159 else:
160 st = os.statvfs(os.getcwd())
161 return st.f_bavail * st.f_frsize
162
86dff6b6
HWN
163def die(msg):
164 if verbose:
165 raise Exception(msg)
166 else:
167 sys.stderr.write(msg + "\n")
168 sys.exit(1)
169
6de040df 170def write_pipe(c, stdin):
4addad22 171 if verbose:
6de040df 172 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 173
6de040df
LD
174 expand = isinstance(c,basestring)
175 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
176 pipe = p.stdin
177 val = pipe.write(stdin)
178 pipe.close()
179 if p.wait():
180 die('Command failed: %s' % str(c))
b016d397
HWN
181
182 return val
183
6de040df 184def p4_write_pipe(c, stdin):
d9429194 185 real_cmd = p4_build_cmd(c)
6de040df 186 return write_pipe(real_cmd, stdin)
d9429194 187
78871bf4
LD
188def read_pipe_full(c):
189 """ Read output from command. Returns a tuple
190 of the return status, stdout text and stderr
191 text.
192 """
4addad22 193 if verbose:
6de040df 194 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 195
6de040df 196 expand = isinstance(c,basestring)
1f5f3907
LS
197 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
198 (out, err) = p.communicate()
78871bf4
LD
199 return (p.returncode, out, err)
200
201def read_pipe(c, ignore_error=False):
202 """ Read output from command. Returns the output text on
203 success. On failure, terminates execution, unless
204 ignore_error is True, when it returns an empty string.
205 """
206 (retcode, out, err) = read_pipe_full(c)
207 if retcode != 0:
208 if ignore_error:
209 out = ""
210 else:
211 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 212 return out
b016d397 213
78871bf4
LD
214def read_pipe_text(c):
215 """ Read output from a command with trailing whitespace stripped.
216 On error, returns None.
217 """
218 (retcode, out, err) = read_pipe_full(c)
219 if retcode != 0:
220 return None
221 else:
222 return out.rstrip()
223
d9429194
AK
224def p4_read_pipe(c, ignore_error=False):
225 real_cmd = p4_build_cmd(c)
226 return read_pipe(real_cmd, ignore_error)
b016d397 227
bce4c5fc 228def read_pipe_lines(c):
4addad22 229 if verbose:
6de040df
LD
230 sys.stderr.write('Reading pipe: %s\n' % str(c))
231
232 expand = isinstance(c, basestring)
233 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
234 pipe = p.stdout
b016d397 235 val = pipe.readlines()
6de040df
LD
236 if pipe.close() or p.wait():
237 die('Command failed: %s' % str(c))
b016d397
HWN
238
239 return val
caace111 240
2318121b
AK
241def p4_read_pipe_lines(c):
242 """Specifically invoke p4 on the command supplied. """
155af834 243 real_cmd = p4_build_cmd(c)
2318121b
AK
244 return read_pipe_lines(real_cmd)
245
8e9497c2
GG
246def p4_has_command(cmd):
247 """Ask p4 for help on this command. If it returns an error, the
248 command does not exist in this version of p4."""
249 real_cmd = p4_build_cmd(["help", cmd])
250 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
251 stderr=subprocess.PIPE)
252 p.communicate()
253 return p.returncode == 0
254
249da4c0
PW
255def p4_has_move_command():
256 """See if the move command exists, that it supports -k, and that
257 it has not been administratively disabled. The arguments
258 must be correct, but the filenames do not have to exist. Use
259 ones with wildcards so even if they exist, it will fail."""
260
261 if not p4_has_command("move"):
262 return False
263 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
264 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
265 (out, err) = p.communicate()
266 # return code will be 1 in either case
267 if err.find("Invalid option") >= 0:
268 return False
269 if err.find("disabled") >= 0:
270 return False
271 # assume it failed because @... was invalid changelist
272 return True
273
cbff4b25 274def system(cmd, ignore_error=False):
6de040df 275 expand = isinstance(cmd,basestring)
4addad22 276 if verbose:
6de040df 277 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 278 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 279 if retcode and not ignore_error:
a235e85c 280 raise CalledProcessError(retcode, cmd)
6754a299 281
cbff4b25
LD
282 return retcode
283
bf9320f1
AK
284def p4_system(cmd):
285 """Specifically invoke p4 as the system command. """
155af834 286 real_cmd = p4_build_cmd(cmd)
6de040df 287 expand = isinstance(real_cmd, basestring)
a235e85c
BC
288 retcode = subprocess.call(real_cmd, shell=expand)
289 if retcode:
290 raise CalledProcessError(retcode, real_cmd)
6de040df 291
0ef67acd
LD
292def die_bad_access(s):
293 die("failure accessing depot: {0}".format(s.rstrip()))
294
295def p4_check_access(min_expiration=1):
296 """ Check if we can access Perforce - account still logged in
297 """
298 results = p4CmdList(["login", "-s"])
299
300 if len(results) == 0:
301 # should never get here: always get either some results, or a p4ExitCode
302 assert("could not parse response from perforce")
303
304 result = results[0]
305
306 if 'p4ExitCode' in result:
307 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
308 die_bad_access("could not run p4")
309
310 code = result.get("code")
311 if not code:
312 # we get here if we couldn't connect and there was nothing to unmarshal
313 die_bad_access("could not connect")
314
315 elif code == "stat":
316 expiry = result.get("TicketExpiration")
317 if expiry:
318 expiry = int(expiry)
319 if expiry > min_expiration:
320 # ok to carry on
321 return
322 else:
323 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
324
325 else:
326 # account without a timeout - all ok
327 return
328
329 elif code == "error":
330 data = result.get("data")
331 if data:
332 die_bad_access("p4 error: {0}".format(data))
333 else:
334 die_bad_access("unknown error")
335 else:
336 die_bad_access("unknown error code {0}".format(code))
337
7f0e5962
PW
338_p4_version_string = None
339def p4_version_string():
340 """Read the version string, showing just the last line, which
341 hopefully is the interesting version bit.
342
343 $ p4 -V
344 Perforce - The Fast Software Configuration Management System.
345 Copyright 1995-2011 Perforce Software. All rights reserved.
346 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
347 """
348 global _p4_version_string
349 if not _p4_version_string:
350 a = p4_read_pipe_lines(["-V"])
351 _p4_version_string = a[-1].rstrip()
352 return _p4_version_string
353
6de040df 354def p4_integrate(src, dest):
9d7d446a 355 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 356
8d7ec362 357def p4_sync(f, *options):
9d7d446a 358 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
359
360def p4_add(f):
9d7d446a
PW
361 # forcibly add file names with wildcards
362 if wildcard_present(f):
363 p4_system(["add", "-f", f])
364 else:
365 p4_system(["add", f])
6de040df
LD
366
367def p4_delete(f):
9d7d446a 368 p4_system(["delete", wildcard_encode(f)])
6de040df 369
a02b8bc4
RP
370def p4_edit(f, *options):
371 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
372
373def p4_revert(f):
9d7d446a 374 p4_system(["revert", wildcard_encode(f)])
6de040df 375
9d7d446a
PW
376def p4_reopen(type, f):
377 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 378
46c609e9
LD
379def p4_reopen_in_change(changelist, files):
380 cmd = ["reopen", "-c", str(changelist)] + files
381 p4_system(cmd)
382
8e9497c2
GG
383def p4_move(src, dest):
384 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
385
1051ef00 386def p4_last_change():
1997e91f 387 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
388 return int(results[0]['change'])
389
123f6317 390def p4_describe(change, shelved=False):
18fa13d0
PW
391 """Make sure it returns a valid result by checking for
392 the presence of field "time". Return a dict of the
393 results."""
394
123f6317
LD
395 cmd = ["describe", "-s"]
396 if shelved:
397 cmd += ["-S"]
398 cmd += [str(change)]
399
400 ds = p4CmdList(cmd, skip_info=True)
18fa13d0
PW
401 if len(ds) != 1:
402 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
403
404 d = ds[0]
405
406 if "p4ExitCode" in d:
407 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
408 str(d)))
409 if "code" in d:
410 if d["code"] == "error":
411 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
412
413 if "time" not in d:
414 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
415
416 return d
417
9cffb8c8
PW
418#
419# Canonicalize the p4 type and return a tuple of the
420# base type, plus any modifiers. See "p4 help filetypes"
421# for a list and explanation.
422#
423def split_p4_type(p4type):
424
425 p4_filetypes_historical = {
426 "ctempobj": "binary+Sw",
427 "ctext": "text+C",
428 "cxtext": "text+Cx",
429 "ktext": "text+k",
430 "kxtext": "text+kx",
431 "ltext": "text+F",
432 "tempobj": "binary+FSw",
433 "ubinary": "binary+F",
434 "uresource": "resource+F",
435 "uxbinary": "binary+Fx",
436 "xbinary": "binary+x",
437 "xltext": "text+Fx",
438 "xtempobj": "binary+Swx",
439 "xtext": "text+x",
440 "xunicode": "unicode+x",
441 "xutf16": "utf16+x",
442 }
443 if p4type in p4_filetypes_historical:
444 p4type = p4_filetypes_historical[p4type]
445 mods = ""
446 s = p4type.split("+")
447 base = s[0]
448 mods = ""
449 if len(s) > 1:
450 mods = s[1]
451 return (base, mods)
b9fc6ea9 452
60df071c
LD
453#
454# return the raw p4 type of a file (text, text+ko, etc)
455#
79467e61
PW
456def p4_type(f):
457 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
458 return results[0]['headType']
459
460#
461# Given a type base and modifier, return a regexp matching
462# the keywords that can be expanded in the file
463#
464def p4_keywords_regexp_for_type(base, type_mods):
465 if base in ("text", "unicode", "binary"):
466 kwords = None
467 if "ko" in type_mods:
468 kwords = 'Id|Header'
469 elif "k" in type_mods:
470 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
471 else:
472 return None
473 pattern = r"""
474 \$ # Starts with a dollar, followed by...
475 (%s) # one of the keywords, followed by...
6b2bf41e 476 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
477 \$ # another dollar
478 """ % kwords
479 return pattern
480 else:
481 return None
482
483#
484# Given a file, return a regexp matching the possible
485# RCS keywords that will be expanded, or None for files
486# with kw expansion turned off.
487#
488def p4_keywords_regexp_for_file(file):
489 if not os.path.exists(file):
490 return None
491 else:
492 (type_base, type_mods) = split_p4_type(p4_type(file))
493 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 494
c65b670e
CP
495def setP4ExecBit(file, mode):
496 # Reopens an already open file and changes the execute bit to match
497 # the execute bit setting in the passed in mode.
498
499 p4Type = "+x"
500
501 if not isModeExec(mode):
502 p4Type = getP4OpenedType(file)
503 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
504 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
505 if p4Type[-1] == "+":
506 p4Type = p4Type[0:-1]
507
6de040df 508 p4_reopen(p4Type, file)
c65b670e
CP
509
510def getP4OpenedType(file):
511 # Returns the perforce file type for the given file.
512
9d7d446a 513 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 514 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
515 if match:
516 return match.group(1)
517 else:
f3e5ae4f 518 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 519
06804c76
LD
520# Return the set of all p4 labels
521def getP4Labels(depotPaths):
522 labels = set()
523 if isinstance(depotPaths,basestring):
524 depotPaths = [depotPaths]
525
526 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
527 label = l['label']
528 labels.add(label)
529
530 return labels
531
532# Return the set of all git tags
533def getGitTags():
534 gitTags = set()
535 for line in read_pipe_lines(["git", "tag"]):
536 tag = line.strip()
537 gitTags.add(tag)
538 return gitTags
539
b43b0a3c
CP
540def diffTreePattern():
541 # This is a simple generator for the diff tree regex pattern. This could be
542 # a class variable if this and parseDiffTreeEntry were a part of a class.
543 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
544 while True:
545 yield pattern
546
547def parseDiffTreeEntry(entry):
548 """Parses a single diff tree entry into its component elements.
549
550 See git-diff-tree(1) manpage for details about the format of the diff
551 output. This method returns a dictionary with the following elements:
552
553 src_mode - The mode of the source file
554 dst_mode - The mode of the destination file
555 src_sha1 - The sha1 for the source file
556 dst_sha1 - The sha1 fr the destination file
557 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
558 status_score - The score for the status (applicable for 'C' and 'R'
559 statuses). This is None if there is no score.
560 src - The path for the source file.
561 dst - The path for the destination file. This is only present for
562 copy or renames. If it is not present, this is None.
563
564 If the pattern is not matched, None is returned."""
565
566 match = diffTreePattern().next().match(entry)
567 if match:
568 return {
569 'src_mode': match.group(1),
570 'dst_mode': match.group(2),
571 'src_sha1': match.group(3),
572 'dst_sha1': match.group(4),
573 'status': match.group(5),
574 'status_score': match.group(6),
575 'src': match.group(7),
576 'dst': match.group(10)
577 }
578 return None
579
c65b670e
CP
580def isModeExec(mode):
581 # Returns True if the given git mode represents an executable file,
582 # otherwise False.
583 return mode[-3:] == "755"
584
55bb3e36
LD
585class P4Exception(Exception):
586 """ Base class for exceptions from the p4 client """
587 def __init__(self, exit_code):
588 self.p4ExitCode = exit_code
589
590class P4ServerException(P4Exception):
591 """ Base class for exceptions where we get some kind of marshalled up result from the server """
592 def __init__(self, exit_code, p4_result):
593 super(P4ServerException, self).__init__(exit_code)
594 self.p4_result = p4_result
595 self.code = p4_result[0]['code']
596 self.data = p4_result[0]['data']
597
598class P4RequestSizeException(P4ServerException):
599 """ One of the maxresults or maxscanrows errors """
600 def __init__(self, exit_code, p4_result, limit):
601 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
602 self.limit = limit
603
c65b670e
CP
604def isModeExecChanged(src_mode, dst_mode):
605 return isModeExec(src_mode) != isModeExec(dst_mode)
606
55bb3e36
LD
607def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
608 errors_as_exceptions=False):
6de040df
LD
609
610 if isinstance(cmd,basestring):
611 cmd = "-G " + cmd
612 expand = True
613 else:
614 cmd = ["-G"] + cmd
615 expand = False
616
617 cmd = p4_build_cmd(cmd)
6a49f8e2 618 if verbose:
6de040df 619 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
620
621 # Use a temporary file to avoid deadlocks without
622 # subprocess.communicate(), which would put another copy
623 # of stdout into memory.
624 stdin_file = None
625 if stdin is not None:
626 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
627 if isinstance(stdin,basestring):
628 stdin_file.write(stdin)
629 else:
630 for i in stdin:
631 stdin_file.write(i + '\n')
9f90c733
SL
632 stdin_file.flush()
633 stdin_file.seek(0)
634
6de040df
LD
635 p4 = subprocess.Popen(cmd,
636 shell=expand,
9f90c733
SL
637 stdin=stdin_file,
638 stdout=subprocess.PIPE)
86949eef
SH
639
640 result = []
641 try:
642 while True:
9f90c733 643 entry = marshal.load(p4.stdout)
1997e91f
MT
644 if skip_info:
645 if 'code' in entry and entry['code'] == 'info':
646 continue
c3f6163b
AG
647 if cb is not None:
648 cb(entry)
649 else:
650 result.append(entry)
86949eef
SH
651 except EOFError:
652 pass
9f90c733
SL
653 exitCode = p4.wait()
654 if exitCode != 0:
55bb3e36
LD
655 if errors_as_exceptions:
656 if len(result) > 0:
657 data = result[0].get('data')
658 if data:
659 m = re.search('Too many rows scanned \(over (\d+)\)', data)
660 if not m:
661 m = re.search('Request too large \(over (\d+)\)', data)
662
663 if m:
664 limit = int(m.group(1))
665 raise P4RequestSizeException(exitCode, result, limit)
666
667 raise P4ServerException(exitCode, result)
668 else:
669 raise P4Exception(exitCode)
670 else:
671 entry = {}
672 entry["p4ExitCode"] = exitCode
673 result.append(entry)
86949eef
SH
674
675 return result
676
677def p4Cmd(cmd):
678 list = p4CmdList(cmd)
679 result = {}
680 for entry in list:
681 result.update(entry)
682 return result;
683
cb2c9db5
SH
684def p4Where(depotPath):
685 if not depotPath.endswith("/"):
686 depotPath += "/"
cd884106
VA
687 depotPathLong = depotPath + "..."
688 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
689 output = None
690 for entry in outputList:
75bc9573 691 if "depotFile" in entry:
cd884106
VA
692 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
693 # The base path always ends with "/...".
694 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
695 output = entry
696 break
697 elif "data" in entry:
698 data = entry.get("data")
699 space = data.find(" ")
700 if data[:space] == depotPath:
701 output = entry
702 break
7f705dc3
TAL
703 if output == None:
704 return ""
dc524036
SH
705 if output["code"] == "error":
706 return ""
cb2c9db5
SH
707 clientPath = ""
708 if "path" in output:
709 clientPath = output.get("path")
710 elif "data" in output:
711 data = output.get("data")
712 lastSpace = data.rfind(" ")
713 clientPath = data[lastSpace + 1:]
714
715 if clientPath.endswith("..."):
716 clientPath = clientPath[:-3]
717 return clientPath
718
86949eef 719def currentGitBranch():
eff45110 720 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 721
4f5cf76a 722def isValidGitDir(path):
378f7be1 723 return git_dir(path) != None
4f5cf76a 724
463e8af6 725def parseRevision(ref):
b25b2065 726 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 727
28755dba
PW
728def branchExists(ref):
729 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
730 ignore_error=True)
731 return len(rev) > 0
732
6ae8de88
SH
733def extractLogMessageFromGitCommit(commit):
734 logMessage = ""
b016d397
HWN
735
736 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 737 foundTitle = False
b016d397 738 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
739 if not foundTitle:
740 if len(log) == 1:
1c094184 741 foundTitle = True
6ae8de88
SH
742 continue
743
744 logMessage += log
745 return logMessage
746
bb6e09b2 747def extractSettingsGitLog(log):
6ae8de88
SH
748 values = {}
749 for line in log.split("\n"):
750 line = line.strip()
6326aa58
HWN
751 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
752 if not m:
753 continue
754
755 assignments = m.group(1).split (':')
756 for a in assignments:
757 vals = a.split ('=')
758 key = vals[0].strip()
759 val = ('='.join (vals[1:])).strip()
760 if val.endswith ('\"') and val.startswith('"'):
761 val = val[1:-1]
762
763 values[key] = val
764
845b42cb
SH
765 paths = values.get("depot-paths")
766 if not paths:
767 paths = values.get("depot-path")
a3fdd579
SH
768 if paths:
769 values['depot-paths'] = paths.split(',')
bb6e09b2 770 return values
6ae8de88 771
8136a639 772def gitBranchExists(branch):
bb6e09b2
HWN
773 proc = subprocess.Popen(["git", "rev-parse", branch],
774 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 775 return proc.wait() == 0;
8136a639 776
123f6317
LD
777def gitUpdateRef(ref, newvalue):
778 subprocess.check_call(["git", "update-ref", ref, newvalue])
779
780def gitDeleteRef(ref):
781 subprocess.check_call(["git", "update-ref", "-d", ref])
782
36bd8446 783_gitConfig = {}
b345d6c3 784
692e1796 785def gitConfig(key, typeSpecifier=None):
dba1c9d9 786 if key not in _gitConfig:
692e1796
LS
787 cmd = [ "git", "config" ]
788 if typeSpecifier:
789 cmd += [ typeSpecifier ]
790 cmd += [ key ]
b345d6c3
PW
791 s = read_pipe(cmd, ignore_error=True)
792 _gitConfig[key] = s.strip()
36bd8446 793 return _gitConfig[key]
01265103 794
0d609032
PW
795def gitConfigBool(key):
796 """Return a bool, using git config --bool. It is True only if the
797 variable is set to true, and False if set to false or not present
798 in the config."""
799
dba1c9d9 800 if key not in _gitConfig:
692e1796 801 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 802 return _gitConfig[key]
01265103 803
cb1dafdf 804def gitConfigInt(key):
dba1c9d9 805 if key not in _gitConfig:
cb1dafdf 806 cmd = [ "git", "config", "--int", key ]
0d609032
PW
807 s = read_pipe(cmd, ignore_error=True)
808 v = s.strip()
cb1dafdf
LS
809 try:
810 _gitConfig[key] = int(gitConfig(key, '--int'))
811 except ValueError:
812 _gitConfig[key] = None
36bd8446 813 return _gitConfig[key]
01265103 814
7199cf13 815def gitConfigList(key):
dba1c9d9 816 if key not in _gitConfig:
2abba301 817 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 818 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
819 if _gitConfig[key] == ['']:
820 _gitConfig[key] = []
7199cf13
VA
821 return _gitConfig[key]
822
2c8037ed
PW
823def p4BranchesInGit(branchesAreInRemotes=True):
824 """Find all the branches whose names start with "p4/", looking
825 in remotes or heads as specified by the argument. Return
826 a dictionary of { branch: revision } for each one found.
827 The branch names are the short names, without any
828 "p4/" prefix."""
829
062410bb
SH
830 branches = {}
831
832 cmdline = "git rev-parse --symbolic "
833 if branchesAreInRemotes:
2c8037ed 834 cmdline += "--remotes"
062410bb 835 else:
2c8037ed 836 cmdline += "--branches"
062410bb
SH
837
838 for line in read_pipe_lines(cmdline):
839 line = line.strip()
840
2c8037ed
PW
841 # only import to p4/
842 if not line.startswith('p4/'):
843 continue
844 # special symbolic ref to p4/master
845 if line == "p4/HEAD":
062410bb 846 continue
062410bb 847
2c8037ed
PW
848 # strip off p4/ prefix
849 branch = line[len("p4/"):]
062410bb
SH
850
851 branches[branch] = parseRevision(line)
2c8037ed 852
062410bb
SH
853 return branches
854
5a8e84cd
PW
855def branch_exists(branch):
856 """Make sure that the given ref name really exists."""
857
858 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
859 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
860 out, _ = p.communicate()
861 if p.returncode:
862 return False
863 # expect exactly one line of output: the branch name
864 return out.rstrip() == branch
865
9ceab363 866def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
867 branches = p4BranchesInGit()
868 # map from depot-path to branch name
869 branchByDepotPath = {}
870 for branch in branches.keys():
871 tip = branches[branch]
872 log = extractLogMessageFromGitCommit(tip)
873 settings = extractSettingsGitLog(log)
dba1c9d9 874 if "depot-paths" in settings:
86506fe5
SH
875 paths = ",".join(settings["depot-paths"])
876 branchByDepotPath[paths] = "remotes/p4/" + branch
877
27d2d811 878 settings = None
27d2d811
SH
879 parent = 0
880 while parent < 65535:
9ceab363 881 commit = head + "~%s" % parent
27d2d811
SH
882 log = extractLogMessageFromGitCommit(commit)
883 settings = extractSettingsGitLog(log)
dba1c9d9 884 if "depot-paths" in settings:
86506fe5 885 paths = ",".join(settings["depot-paths"])
dba1c9d9 886 if paths in branchByDepotPath:
86506fe5 887 return [branchByDepotPath[paths], settings]
27d2d811 888
86506fe5 889 parent = parent + 1
27d2d811 890
86506fe5 891 return ["", settings]
27d2d811 892
5ca44617
SH
893def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
894 if not silent:
895 print ("Creating/updating branch(es) in %s based on origin branch(es)"
896 % localRefPrefix)
897
898 originPrefix = "origin/p4/"
899
900 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
901 line = line.strip()
902 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
903 continue
904
905 headName = line[len(originPrefix):]
906 remoteHead = localRefPrefix + headName
907 originHead = line
908
909 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
dba1c9d9
LD
910 if ('depot-paths' not in original
911 or 'change' not in original):
5ca44617
SH
912 continue
913
914 update = False
915 if not gitBranchExists(remoteHead):
916 if verbose:
917 print "creating %s" % remoteHead
918 update = True
919 else:
920 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
dba1c9d9 921 if 'change' in settings:
5ca44617
SH
922 if settings['depot-paths'] == original['depot-paths']:
923 originP4Change = int(original['change'])
924 p4Change = int(settings['change'])
925 if originP4Change > p4Change:
926 print ("%s (%s) is newer than %s (%s). "
927 "Updating p4 branch from origin."
928 % (originHead, originP4Change,
929 remoteHead, p4Change))
930 update = True
931 else:
932 print ("Ignoring: %s was imported from %s while "
933 "%s was imported from %s"
934 % (originHead, ','.join(original['depot-paths']),
935 remoteHead, ','.join(settings['depot-paths'])))
936
937 if update:
938 system("git update-ref %s %s" % (remoteHead, originHead))
939
940def originP4BranchesExist():
941 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
942
1051ef00
LD
943
944def p4ParseNumericChangeRange(parts):
945 changeStart = int(parts[0][1:])
946 if parts[1] == '#head':
947 changeEnd = p4_last_change()
948 else:
949 changeEnd = int(parts[1])
950
951 return (changeStart, changeEnd)
952
953def chooseBlockSize(blockSize):
954 if blockSize:
955 return blockSize
956 else:
957 return defaultBlockSize
958
959def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 960 assert depotPaths
96b2d54a 961
1051ef00
LD
962 # Parse the change range into start and end. Try to find integer
963 # revision ranges as these can be broken up into blocks to avoid
964 # hitting server-side limits (maxrows, maxscanresults). But if
965 # that doesn't work, fall back to using the raw revision specifier
966 # strings, without using block mode.
967
96b2d54a 968 if changeRange is None or changeRange == '':
1051ef00
LD
969 changeStart = 1
970 changeEnd = p4_last_change()
971 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
972 else:
973 parts = changeRange.split(',')
974 assert len(parts) == 2
1051ef00
LD
975 try:
976 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
977 block_size = chooseBlockSize(requestedBlockSize)
8fa0abf8 978 except ValueError:
1051ef00
LD
979 changeStart = parts[0][1:]
980 changeEnd = parts[1]
981 if requestedBlockSize:
982 die("cannot use --changes-block-size with non-numeric revisions")
983 block_size = None
4f6432d8 984
9943e5b9 985 changes = set()
96b2d54a 986
1f90a648 987 # Retrieve changes a block at a time, to prevent running
3deed5e0
LD
988 # into a MaxResults/MaxScanRows error from the server. If
989 # we _do_ hit one of those errors, turn down the block size
1051ef00 990
1f90a648
SH
991 while True:
992 cmd = ['changes']
1051ef00 993
1f90a648
SH
994 if block_size:
995 end = min(changeEnd, changeStart + block_size)
996 revisionRange = "%d,%d" % (changeStart, end)
997 else:
998 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 999
1f90a648 1000 for p in depotPaths:
1051ef00
LD
1001 cmd += ["%s...@%s" % (p, revisionRange)]
1002
3deed5e0
LD
1003 # fetch the changes
1004 try:
1005 result = p4CmdList(cmd, errors_as_exceptions=True)
1006 except P4RequestSizeException as e:
1007 if not block_size:
1008 block_size = e.limit
1009 elif block_size > e.limit:
1010 block_size = e.limit
1011 else:
1012 block_size = max(2, block_size // 2)
1013
1014 if verbose: print("block size error, retrying with block size {0}".format(block_size))
1015 continue
1016 except P4Exception as e:
1017 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1018
1f90a648 1019 # Insert changes in chronological order
3deed5e0 1020 for entry in reversed(result):
dba1c9d9 1021 if 'change' not in entry:
b596b3b9
MT
1022 continue
1023 changes.add(int(entry['change']))
1051ef00 1024
1f90a648
SH
1025 if not block_size:
1026 break
1051ef00 1027
1f90a648
SH
1028 if end >= changeEnd:
1029 break
1051ef00 1030
1f90a648 1031 changeStart = end + 1
4f6432d8 1032
1f90a648
SH
1033 changes = sorted(changes)
1034 return changes
4f6432d8 1035
d53de8b9
TAL
1036def p4PathStartsWith(path, prefix):
1037 # This method tries to remedy a potential mixed-case issue:
1038 #
1039 # If UserA adds //depot/DirA/file1
1040 # and UserB adds //depot/dira/file2
1041 #
1042 # we may or may not have a problem. If you have core.ignorecase=true,
1043 # we treat DirA and dira as the same directory
0d609032 1044 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
1045 return path.lower().startswith(prefix.lower())
1046 return path.startswith(prefix)
1047
543987bd
PW
1048def getClientSpec():
1049 """Look at the p4 client spec, create a View() object that contains
1050 all the mappings, and return it."""
1051
1052 specList = p4CmdList("client -o")
1053 if len(specList) != 1:
1054 die('Output from "client -o" is %d lines, expecting 1' %
1055 len(specList))
1056
1057 # dictionary of all client parameters
1058 entry = specList[0]
1059
9d57c4a6
KS
1060 # the //client/ name
1061 client_name = entry["Client"]
1062
543987bd
PW
1063 # just the keys that start with "View"
1064 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1065
1066 # hold this new View
9d57c4a6 1067 view = View(client_name)
543987bd
PW
1068
1069 # append the lines, in order, to the view
1070 for view_num in range(len(view_keys)):
1071 k = "View%d" % view_num
1072 if k not in view_keys:
1073 die("Expected view key %s missing" % k)
1074 view.append(entry[k])
1075
1076 return view
1077
1078def getClientRoot():
1079 """Grab the client directory."""
1080
1081 output = p4CmdList("client -o")
1082 if len(output) != 1:
1083 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1084
1085 entry = output[0]
1086 if "Root" not in entry:
1087 die('Client has no "Root"')
1088
1089 return entry["Root"]
1090
9d7d446a
PW
1091#
1092# P4 wildcards are not allowed in filenames. P4 complains
1093# if you simply add them, but you can force it with "-f", in
1094# which case it translates them into %xx encoding internally.
1095#
1096def wildcard_decode(path):
1097 # Search for and fix just these four characters. Do % last so
1098 # that fixing it does not inadvertently create new %-escapes.
1099 # Cannot have * in a filename in windows; untested as to
1100 # what p4 would do in such a case.
1101 if not platform.system() == "Windows":
1102 path = path.replace("%2A", "*")
1103 path = path.replace("%23", "#") \
1104 .replace("%40", "@") \
1105 .replace("%25", "%")
1106 return path
1107
1108def wildcard_encode(path):
1109 # do % first to avoid double-encoding the %s introduced here
1110 path = path.replace("%", "%25") \
1111 .replace("*", "%2A") \
1112 .replace("#", "%23") \
1113 .replace("@", "%40")
1114 return path
1115
1116def wildcard_present(path):
598354c0
BC
1117 m = re.search("[*#@%]", path)
1118 return m is not None
9d7d446a 1119
a5db4b12
LS
1120class LargeFileSystem(object):
1121 """Base class for large file system support."""
1122
1123 def __init__(self, writeToGitStream):
1124 self.largeFiles = set()
1125 self.writeToGitStream = writeToGitStream
1126
1127 def generatePointer(self, cloneDestination, contentFile):
1128 """Return the content of a pointer file that is stored in Git instead of
1129 the actual content."""
1130 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1131
1132 def pushFile(self, localLargeFile):
1133 """Push the actual content which is not stored in the Git repository to
1134 a server."""
1135 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1136
1137 def hasLargeFileExtension(self, relPath):
1138 return reduce(
1139 lambda a, b: a or b,
1140 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1141 False
1142 )
1143
1144 def generateTempFile(self, contents):
1145 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146 for d in contents:
1147 contentFile.write(d)
1148 contentFile.close()
1149 return contentFile.name
1150
1151 def exceedsLargeFileThreshold(self, relPath, contents):
1152 if gitConfigInt('git-p4.largeFileThreshold'):
1153 contentsSize = sum(len(d) for d in contents)
1154 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1155 return True
1156 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1157 contentsSize = sum(len(d) for d in contents)
1158 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1159 return False
1160 contentTempFile = self.generateTempFile(contents)
1161 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1162 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1163 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1164 zf.close()
1165 compressedContentsSize = zf.infolist()[0].compress_size
1166 os.remove(contentTempFile)
1167 os.remove(compressedContentFile.name)
1168 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1169 return True
1170 return False
1171
1172 def addLargeFile(self, relPath):
1173 self.largeFiles.add(relPath)
1174
1175 def removeLargeFile(self, relPath):
1176 self.largeFiles.remove(relPath)
1177
1178 def isLargeFile(self, relPath):
1179 return relPath in self.largeFiles
1180
1181 def processContent(self, git_mode, relPath, contents):
1182 """Processes the content of git fast import. This method decides if a
1183 file is stored in the large file system and handles all necessary
1184 steps."""
1185 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1186 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1187 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1188 if pointer_git_mode:
1189 git_mode = pointer_git_mode
1190 if localLargeFile:
1191 # Move temp file to final location in large file system
1192 largeFileDir = os.path.dirname(localLargeFile)
1193 if not os.path.isdir(largeFileDir):
1194 os.makedirs(largeFileDir)
1195 shutil.move(contentTempFile, localLargeFile)
1196 self.addLargeFile(relPath)
1197 if gitConfigBool('git-p4.largeFilePush'):
1198 self.pushFile(localLargeFile)
1199 if verbose:
1200 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1201 return (git_mode, contents)
1202
1203class MockLFS(LargeFileSystem):
1204 """Mock large file system for testing."""
1205
1206 def generatePointer(self, contentFile):
1207 """The pointer content is the original content prefixed with "pointer-".
1208 The local filename of the large file storage is derived from the file content.
1209 """
1210 with open(contentFile, 'r') as f:
1211 content = next(f)
1212 gitMode = '100644'
1213 pointerContents = 'pointer-' + content
1214 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1215 return (gitMode, pointerContents, localLargeFile)
1216
1217 def pushFile(self, localLargeFile):
1218 """The remote filename of the large file storage is the same as the local
1219 one but in a different directory.
1220 """
1221 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1222 if not os.path.exists(remotePath):
1223 os.makedirs(remotePath)
1224 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1225
b47d807d
LS
1226class GitLFS(LargeFileSystem):
1227 """Git LFS as backend for the git-p4 large file system.
1228 See https://git-lfs.github.com/ for details."""
1229
1230 def __init__(self, *args):
1231 LargeFileSystem.__init__(self, *args)
1232 self.baseGitAttributes = []
1233
1234 def generatePointer(self, contentFile):
1235 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1236 mode and content which is stored in the Git repository instead of
1237 the actual content. Return also the new location of the actual
1238 content.
1239 """
d5eb3cf5
LS
1240 if os.path.getsize(contentFile) == 0:
1241 return (None, '', None)
1242
b47d807d
LS
1243 pointerProcess = subprocess.Popen(
1244 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1245 stdout=subprocess.PIPE
1246 )
1247 pointerFile = pointerProcess.stdout.read()
1248 if pointerProcess.wait():
1249 os.remove(contentFile)
1250 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1251
1252 # Git LFS removed the preamble in the output of the 'pointer' command
1253 # starting from version 1.2.0. Check for the preamble here to support
1254 # earlier versions.
1255 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1256 if pointerFile.startswith('Git LFS pointer for'):
1257 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1258
1259 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1260 localLargeFile = os.path.join(
1261 os.getcwd(),
1262 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1263 oid,
1264 )
1265 # LFS Spec states that pointer files should not have the executable bit set.
1266 gitMode = '100644'
82f2567e 1267 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1268
1269 def pushFile(self, localLargeFile):
1270 uploadProcess = subprocess.Popen(
1271 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1272 )
1273 if uploadProcess.wait():
1274 die('git-lfs push command failed. Did you define a remote?')
1275
1276 def generateGitAttributes(self):
1277 return (
1278 self.baseGitAttributes +
1279 [
1280 '\n',
1281 '#\n',
1282 '# Git LFS (see https://git-lfs.github.com/)\n',
1283 '#\n',
1284 ] +
862f9312 1285 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1286 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1287 ] +
862f9312 1288 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1289 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1290 ]
1291 )
1292
1293 def addLargeFile(self, relPath):
1294 LargeFileSystem.addLargeFile(self, relPath)
1295 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1296
1297 def removeLargeFile(self, relPath):
1298 LargeFileSystem.removeLargeFile(self, relPath)
1299 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1300
1301 def processContent(self, git_mode, relPath, contents):
1302 if relPath == '.gitattributes':
1303 self.baseGitAttributes = contents
1304 return (git_mode, self.generateGitAttributes())
1305 else:
1306 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1307
b984733c
SH
1308class Command:
1309 def __init__(self):
1310 self.usage = "usage: %prog [options]"
8910ac0e 1311 self.needsGit = True
6a10b6aa 1312 self.verbose = False
b984733c 1313
8cf422db
LD
1314 # This is required for the "append" cloneExclude action
1315 def ensure_value(self, attr, value):
1316 if not hasattr(self, attr) or getattr(self, attr) is None:
1317 setattr(self, attr, value)
1318 return getattr(self, attr)
1319
3ea2cfd4
LD
1320class P4UserMap:
1321 def __init__(self):
1322 self.userMapFromPerforceServer = False
affb474f
LD
1323 self.myP4UserId = None
1324
1325 def p4UserId(self):
1326 if self.myP4UserId:
1327 return self.myP4UserId
1328
1329 results = p4CmdList("user -o")
1330 for r in results:
dba1c9d9 1331 if 'User' in r:
affb474f
LD
1332 self.myP4UserId = r['User']
1333 return r['User']
1334 die("Could not find your p4 user id")
1335
1336 def p4UserIsMe(self, p4User):
1337 # return True if the given p4 user is actually me
1338 me = self.p4UserId()
1339 if not p4User or p4User != me:
1340 return False
1341 else:
1342 return True
3ea2cfd4
LD
1343
1344 def getUserCacheFilename(self):
1345 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1346 return home + "/.gitp4-usercache.txt"
1347
1348 def getUserMapFromPerforceServer(self):
1349 if self.userMapFromPerforceServer:
1350 return
1351 self.users = {}
1352 self.emails = {}
1353
1354 for output in p4CmdList("users"):
dba1c9d9 1355 if "User" not in output:
3ea2cfd4
LD
1356 continue
1357 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1358 self.emails[output["Email"]] = output["User"]
1359
10d08a14
LS
1360 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1361 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1362 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1363 if mapUser and len(mapUser[0]) == 3:
1364 user = mapUser[0][0]
1365 fullname = mapUser[0][1]
1366 email = mapUser[0][2]
1367 self.users[user] = fullname + " <" + email + ">"
1368 self.emails[email] = user
3ea2cfd4
LD
1369
1370 s = ''
1371 for (key, val) in self.users.items():
1372 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1373
1374 open(self.getUserCacheFilename(), "wb").write(s)
1375 self.userMapFromPerforceServer = True
1376
1377 def loadUserMapFromCache(self):
1378 self.users = {}
1379 self.userMapFromPerforceServer = False
1380 try:
1381 cache = open(self.getUserCacheFilename(), "rb")
1382 lines = cache.readlines()
1383 cache.close()
1384 for line in lines:
1385 entry = line.strip().split("\t")
1386 self.users[entry[0]] = entry[1]
1387 except IOError:
1388 self.getUserMapFromPerforceServer()
1389
b984733c 1390class P4Debug(Command):
86949eef 1391 def __init__(self):
6ae8de88 1392 Command.__init__(self)
6a10b6aa 1393 self.options = []
c8c39116 1394 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1395 self.needsGit = False
86949eef
SH
1396
1397 def run(self, args):
b1ce9447 1398 j = 0
6de040df 1399 for output in p4CmdList(args):
b1ce9447
HWN
1400 print 'Element: %d' % j
1401 j += 1
86949eef 1402 print output
b984733c 1403 return True
86949eef 1404
5834684d
SH
1405class P4RollBack(Command):
1406 def __init__(self):
1407 Command.__init__(self)
1408 self.options = [
0c66a783 1409 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1410 ]
1411 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1412 self.rollbackLocalBranches = False
5834684d
SH
1413
1414 def run(self, args):
1415 if len(args) != 1:
1416 return False
1417 maxChange = int(args[0])
0c66a783 1418
ad192f28 1419 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1420 die("Problems executing p4");
1421
0c66a783
SH
1422 if self.rollbackLocalBranches:
1423 refPrefix = "refs/heads/"
b016d397 1424 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1425 else:
1426 refPrefix = "refs/remotes/"
b016d397 1427 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1428
1429 for line in lines:
1430 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1431 line = line.strip()
1432 ref = refPrefix + line
5834684d 1433 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1434 settings = extractSettingsGitLog(log)
1435
1436 depotPaths = settings['depot-paths']
1437 change = settings['change']
1438
5834684d 1439 changed = False
52102d47 1440
6326aa58
HWN
1441 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1442 for p in depotPaths]))) == 0:
52102d47
SH
1443 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1444 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1445 continue
1446
bb6e09b2 1447 while change and int(change) > maxChange:
5834684d 1448 changed = True
52102d47
SH
1449 if self.verbose:
1450 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1451 system("git update-ref %s \"%s^\"" % (ref, ref))
1452 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1453 settings = extractSettingsGitLog(log)
1454
1455
1456 depotPaths = settings['depot-paths']
1457 change = settings['change']
5834684d
SH
1458
1459 if changed:
52102d47 1460 print "%s rewound to %s" % (ref, change)
5834684d
SH
1461
1462 return True
1463
3ea2cfd4 1464class P4Submit(Command, P4UserMap):
6bbfd137
PW
1465
1466 conflict_behavior_choices = ("ask", "skip", "quit")
1467
4f5cf76a 1468 def __init__(self):
b984733c 1469 Command.__init__(self)
3ea2cfd4 1470 P4UserMap.__init__(self)
4f5cf76a 1471 self.options = [
4f5cf76a 1472 optparse.make_option("--origin", dest="origin"),
ae901090 1473 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1474 # preserve the user, requires relevant p4 permissions
1475 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1476 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1477 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1478 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1479 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1480 choices=self.conflict_behavior_choices),
1481 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1482 optparse.make_option("--shelve", dest="shelve", action="store_true",
1483 help="Shelve instead of submit. Shelved files are reverted, "
1484 "restoring the workspace to the state before the shelve"),
8cf422db 1485 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1486 metavar="CHANGELIST",
8cf422db 1487 help="update an existing shelved changelist, implies --shelve, "
f55b87c1
RM
1488 "repeat in-order for multiple shelved changelists"),
1489 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1490 help="submit only the specified commit(s), one commit or xxx..xxx"),
1491 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1492 help="Disable rebase after submit is completed. Can be useful if you "
b9d34db9
LD
1493 "work from a local git branch that is not master"),
1494 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1495 help="Skip Perforce sync of p4/master after submit or shelve"),
4f5cf76a
SH
1496 ]
1497 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1498 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1499 self.origin = ""
ae901090 1500 self.detectRenames = False
0d609032 1501 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1502 self.dry_run = False
b34fa577 1503 self.shelve = False
8cf422db 1504 self.update_shelve = list()
f55b87c1 1505 self.commit = ""
3b3477ea 1506 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
b9d34db9 1507 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
728b7ad8 1508 self.prepare_p4_only = False
6bbfd137 1509 self.conflict_behavior = None
f7baba8b 1510 self.isWindows = (platform.system() == "Windows")
06804c76 1511 self.exportLabels = False
249da4c0 1512 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1513 self.branch = None
4f5cf76a 1514
a5db4b12
LS
1515 if gitConfig('git-p4.largeFileSystem'):
1516 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1517
4f5cf76a
SH
1518 def check(self):
1519 if len(p4CmdList("opened ...")) > 0:
1520 die("You have files opened with perforce! Close them before starting the sync.")
1521
f19cb0a0
PW
1522 def separate_jobs_from_description(self, message):
1523 """Extract and return a possible Jobs field in the commit
1524 message. It goes into a separate section in the p4 change
1525 specification.
1526
1527 A jobs line starts with "Jobs:" and looks like a new field
1528 in a form. Values are white-space separated on the same
1529 line or on following lines that start with a tab.
1530
1531 This does not parse and extract the full git commit message
1532 like a p4 form. It just sees the Jobs: line as a marker
1533 to pass everything from then on directly into the p4 form,
1534 but outside the description section.
1535
1536 Return a tuple (stripped log message, jobs string)."""
1537
1538 m = re.search(r'^Jobs:', message, re.MULTILINE)
1539 if m is None:
1540 return (message, None)
1541
1542 jobtext = message[m.start():]
1543 stripped_message = message[:m.start()].rstrip()
1544 return (stripped_message, jobtext)
1545
1546 def prepareLogMessage(self, template, message, jobs):
1547 """Edits the template returned from "p4 change -o" to insert
1548 the message in the Description field, and the jobs text in
1549 the Jobs field."""
4f5cf76a
SH
1550 result = ""
1551
edae1e2f
SH
1552 inDescriptionSection = False
1553
4f5cf76a
SH
1554 for line in template.split("\n"):
1555 if line.startswith("#"):
1556 result += line + "\n"
1557 continue
1558
edae1e2f 1559 if inDescriptionSection:
c9dbab04 1560 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1561 inDescriptionSection = False
f19cb0a0
PW
1562 # insert Jobs section
1563 if jobs:
1564 result += jobs + "\n"
edae1e2f
SH
1565 else:
1566 continue
1567 else:
1568 if line.startswith("Description:"):
1569 inDescriptionSection = True
1570 line += "\n"
1571 for messageLine in message.split("\n"):
1572 line += "\t" + messageLine + "\n"
1573
1574 result += line + "\n"
4f5cf76a
SH
1575
1576 return result
1577
60df071c
LD
1578 def patchRCSKeywords(self, file, pattern):
1579 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1580 (handle, outFileName) = tempfile.mkstemp(dir='.')
1581 try:
1582 outFile = os.fdopen(handle, "w+")
1583 inFile = open(file, "r")
1584 regexp = re.compile(pattern, re.VERBOSE)
1585 for line in inFile.readlines():
1586 line = regexp.sub(r'$\1$', line)
1587 outFile.write(line)
1588 inFile.close()
1589 outFile.close()
1590 # Forcibly overwrite the original file
1591 os.unlink(file)
1592 shutil.move(outFileName, file)
1593 except:
1594 # cleanup our temporary file
1595 os.unlink(outFileName)
1596 print "Failed to strip RCS keywords in %s" % file
1597 raise
1598
1599 print "Patched up RCS keywords in %s" % file
1600
3ea2cfd4
LD
1601 def p4UserForCommit(self,id):
1602 # Return the tuple (perforce user,git email) for a given git commit id
1603 self.getUserMapFromPerforceServer()
9bf28855
PW
1604 gitEmail = read_pipe(["git", "log", "--max-count=1",
1605 "--format=%ae", id])
3ea2cfd4 1606 gitEmail = gitEmail.strip()
dba1c9d9 1607 if gitEmail not in self.emails:
3ea2cfd4
LD
1608 return (None,gitEmail)
1609 else:
1610 return (self.emails[gitEmail],gitEmail)
1611
1612 def checkValidP4Users(self,commits):
1613 # check if any git authors cannot be mapped to p4 users
1614 for id in commits:
1615 (user,email) = self.p4UserForCommit(id)
1616 if not user:
1617 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1618 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1619 print "%s" % msg
1620 else:
1621 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1622
1623 def lastP4Changelist(self):
1624 # Get back the last changelist number submitted in this client spec. This
1625 # then gets used to patch up the username in the change. If the same
1626 # client spec is being used by multiple processes then this might go
1627 # wrong.
1628 results = p4CmdList("client -o") # find the current client
1629 client = None
1630 for r in results:
dba1c9d9 1631 if 'Client' in r:
3ea2cfd4
LD
1632 client = r['Client']
1633 break
1634 if not client:
1635 die("could not get client spec")
6de040df 1636 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4 1637 for r in results:
dba1c9d9 1638 if 'change' in r:
3ea2cfd4
LD
1639 return r['change']
1640 die("Could not get changelist number for last submit - cannot patch up user details")
1641
1642 def modifyChangelistUser(self, changelist, newUser):
1643 # fixup the user field of a changelist after it has been submitted.
1644 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1645 if len(changes) != 1:
1646 die("Bad output from p4 change modifying %s to user %s" %
1647 (changelist, newUser))
1648
1649 c = changes[0]
1650 if c['User'] == newUser: return # nothing to do
1651 c['User'] = newUser
1652 input = marshal.dumps(c)
1653
3ea2cfd4
LD
1654 result = p4CmdList("change -f -i", stdin=input)
1655 for r in result:
dba1c9d9 1656 if 'code' in r:
3ea2cfd4
LD
1657 if r['code'] == 'error':
1658 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
dba1c9d9 1659 if 'data' in r:
3ea2cfd4
LD
1660 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1661 return
1662 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1663
1664 def canChangeChangelists(self):
1665 # check to see if we have p4 admin or super-user permissions, either of
1666 # which are required to modify changelists.
52a4880b 1667 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4 1668 for r in results:
dba1c9d9 1669 if 'perm' in r:
3ea2cfd4
LD
1670 if r['perm'] == 'admin':
1671 return 1
1672 if r['perm'] == 'super':
1673 return 1
1674 return 0
1675
46c609e9 1676 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1677 """Run "p4 change -o" to grab a change specification template.
1678 This does not use "p4 -G", as it is nice to keep the submission
1679 template in original order, since a human might edit it.
1680
1681 Remove lines in the Files section that show changes to files
1682 outside the depot path we're committing into."""
1683
cbc69242
SH
1684 [upstream, settings] = findUpstreamBranchPoint()
1685
b596b3b9
MT
1686 template = """\
1687# A Perforce Change Specification.
1688#
1689# Change: The change number. 'new' on a new changelist.
1690# Date: The date this specification was last modified.
1691# Client: The client on which the changelist was created. Read-only.
1692# User: The user who created the changelist.
1693# Status: Either 'pending' or 'submitted'. Read-only.
1694# Type: Either 'public' or 'restricted'. Default is 'public'.
1695# Description: Comments about the changelist. Required.
1696# Jobs: What opened jobs are to be closed by this changelist.
1697# You may delete jobs from this list. (New changelists only.)
1698# Files: What opened files from the default changelist are to be added
1699# to this changelist. You may delete files from this list.
1700# (New changelists only.)
1701"""
1702 files_list = []
ea99c3ae 1703 inFilesSection = False
b596b3b9 1704 change_entry = None
46c609e9
LD
1705 args = ['change', '-o']
1706 if changelist:
1707 args.append(str(changelist))
b596b3b9 1708 for entry in p4CmdList(args):
dba1c9d9 1709 if 'code' not in entry:
b596b3b9
MT
1710 continue
1711 if entry['code'] == 'stat':
1712 change_entry = entry
1713 break
1714 if not change_entry:
1715 die('Failed to decode output of p4 change -o')
1716 for key, value in change_entry.iteritems():
1717 if key.startswith('File'):
dba1c9d9 1718 if 'depot-paths' in settings:
b596b3b9
MT
1719 if not [p for p in settings['depot-paths']
1720 if p4PathStartsWith(value, p)]:
1721 continue
ea99c3ae 1722 else:
b596b3b9
MT
1723 if not p4PathStartsWith(value, self.depotPath):
1724 continue
1725 files_list.append(value)
1726 continue
1727 # Output in the order expected by prepareLogMessage
1728 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
dba1c9d9 1729 if key not in change_entry:
b596b3b9
MT
1730 continue
1731 template += '\n'
1732 template += key + ':'
1733 if key == 'Description':
1734 template += '\n'
1735 for field_line in change_entry[key].splitlines():
1736 template += '\t'+field_line+'\n'
1737 if len(files_list) > 0:
1738 template += '\n'
1739 template += 'Files:\n'
1740 for path in files_list:
1741 template += '\t'+path+'\n'
ea99c3ae
SH
1742 return template
1743
7c766e57
PW
1744 def edit_template(self, template_file):
1745 """Invoke the editor to let the user change the submission
1746 message. Return true if okay to continue with the submit."""
1747
1748 # if configured to skip the editing part, just submit
0d609032 1749 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1750 return True
1751
1752 # look at the modification time, to check later if the user saved
1753 # the file
1754 mtime = os.stat(template_file).st_mtime
1755
1756 # invoke the editor
dba1c9d9 1757 if "P4EDITOR" in os.environ and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1758 editor = os.environ.get("P4EDITOR")
1759 else:
1760 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1761 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1762
1763 # If the file was not saved, prompt to see if this patch should
1764 # be skipped. But skip this verification step if configured so.
0d609032 1765 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1766 return True
1767
d1652049
PW
1768 # modification time updated means user saved the file
1769 if os.stat(template_file).st_mtime > mtime:
1770 return True
1771
1772 while True:
1773 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1774 if response == 'y':
1775 return True
1776 if response == 'n':
1777 return False
7c766e57 1778
df8a9e86 1779 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3 1780 # diff
dba1c9d9 1781 if "P4DIFF" in os.environ:
b4073bb3
MC
1782 del(os.environ["P4DIFF"])
1783 diff = ""
1784 for editedFile in editedFiles:
1785 diff += p4_read_pipe(['diff', '-du',
1786 wildcard_encode(editedFile)])
1787
1788 # new file diff
1789 newdiff = ""
1790 for newFile in filesToAdd:
1791 newdiff += "==== new file ====\n"
1792 newdiff += "--- /dev/null\n"
1793 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1794
1795 is_link = os.path.islink(newFile)
1796 expect_link = newFile in symlinks
1797
1798 if is_link and expect_link:
1799 newdiff += "+%s\n" % os.readlink(newFile)
1800 else:
1801 f = open(newFile, "r")
1802 for line in f.readlines():
1803 newdiff += "+" + line
1804 f.close()
b4073bb3 1805
e2a892ee 1806 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1807
7cb5cbef 1808 def applyCommit(self, id):
67b0fe2e
PW
1809 """Apply one commit, return True if it succeeded."""
1810
1811 print "Applying", read_pipe(["git", "show", "-s",
1812 "--format=format:%h %s", id])
ae901090 1813
848de9c3 1814 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1815
84cb0003 1816 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1817 filesToAdd = set()
a02b8bc4 1818 filesToChangeType = set()
4f5cf76a 1819 filesToDelete = set()
d336c158 1820 editedFiles = set()
b6ad6dcc 1821 pureRenameCopy = set()
df8a9e86 1822 symlinks = set()
c65b670e 1823 filesToChangeExecBit = {}
46c609e9 1824 all_files = list()
60df071c 1825
4f5cf76a 1826 for line in diff:
b43b0a3c
CP
1827 diff = parseDiffTreeEntry(line)
1828 modifier = diff['status']
1829 path = diff['src']
46c609e9
LD
1830 all_files.append(path)
1831
4f5cf76a 1832 if modifier == "M":
6de040df 1833 p4_edit(path)
c65b670e
CP
1834 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1835 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1836 editedFiles.add(path)
4f5cf76a
SH
1837 elif modifier == "A":
1838 filesToAdd.add(path)
c65b670e 1839 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1840 if path in filesToDelete:
1841 filesToDelete.remove(path)
df8a9e86
LD
1842
1843 dst_mode = int(diff['dst_mode'], 8)
1844 if dst_mode == 0120000:
1845 symlinks.add(path)
1846
4f5cf76a
SH
1847 elif modifier == "D":
1848 filesToDelete.add(path)
1849 if path in filesToAdd:
1850 filesToAdd.remove(path)
4fddb41b
VA
1851 elif modifier == "C":
1852 src, dest = diff['src'], diff['dst']
6de040df 1853 p4_integrate(src, dest)
b6ad6dcc 1854 pureRenameCopy.add(dest)
4fddb41b 1855 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1856 p4_edit(dest)
b6ad6dcc 1857 pureRenameCopy.discard(dest)
4fddb41b 1858 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1859 p4_edit(dest)
b6ad6dcc 1860 pureRenameCopy.discard(dest)
4fddb41b 1861 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1862 if self.isWindows:
1863 # turn off read-only attribute
1864 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1865 os.unlink(dest)
1866 editedFiles.add(dest)
d9a5f25b 1867 elif modifier == "R":
b43b0a3c 1868 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1869 if self.p4HasMoveCommand:
1870 p4_edit(src) # src must be open before move
1871 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1872 else:
8e9497c2
GG
1873 p4_integrate(src, dest)
1874 if diff['src_sha1'] != diff['dst_sha1']:
1875 p4_edit(dest)
1876 else:
1877 pureRenameCopy.add(dest)
c65b670e 1878 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1879 if not self.p4HasMoveCommand:
1880 p4_edit(dest) # with move: already open, writable
c65b670e 1881 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1882 if not self.p4HasMoveCommand:
d20f0f8e
PW
1883 if self.isWindows:
1884 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1885 os.unlink(dest)
1886 filesToDelete.add(src)
d9a5f25b 1887 editedFiles.add(dest)
a02b8bc4
RP
1888 elif modifier == "T":
1889 filesToChangeType.add(path)
4f5cf76a
SH
1890 else:
1891 die("unknown modifier %s for %s" % (modifier, path))
1892
749b668c 1893 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1894 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1895 tryPatchCmd = patchcmd + "--check -"
1896 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1897 patch_succeeded = True
51a2640a 1898
47a130b7 1899 if os.system(tryPatchCmd) != 0:
60df071c
LD
1900 fixed_rcs_keywords = False
1901 patch_succeeded = False
51a2640a 1902 print "Unfortunately applying the change failed!"
60df071c
LD
1903
1904 # Patch failed, maybe it's just RCS keyword woes. Look through
1905 # the patch to see if that's possible.
0d609032 1906 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1907 file = None
1908 pattern = None
1909 kwfiles = {}
1910 for file in editedFiles | filesToDelete:
1911 # did this file's delta contain RCS keywords?
1912 pattern = p4_keywords_regexp_for_file(file)
1913
1914 if pattern:
1915 # this file is a possibility...look for RCS keywords.
1916 regexp = re.compile(pattern, re.VERBOSE)
1917 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1918 if regexp.search(line):
1919 if verbose:
1920 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1921 kwfiles[file] = pattern
1922 break
1923
1924 for file in kwfiles:
1925 if verbose:
1926 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1927 # File is being deleted, so not open in p4. Must
1928 # disable the read-only bit on windows.
1929 if self.isWindows and file not in editedFiles:
1930 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1931 self.patchRCSKeywords(file, kwfiles[file])
1932 fixed_rcs_keywords = True
1933
1934 if fixed_rcs_keywords:
1935 print "Retrying the patch with RCS keywords cleaned up"
1936 if os.system(tryPatchCmd) == 0:
1937 patch_succeeded = True
1938
1939 if not patch_succeeded:
7e5dd9f2
PW
1940 for f in editedFiles:
1941 p4_revert(f)
7e5dd9f2 1942 return False
51a2640a 1943
55ac2ed6
PW
1944 #
1945 # Apply the patch for real, and do add/delete/+x handling.
1946 #
47a130b7 1947 system(applyPatchCmd)
4f5cf76a 1948
a02b8bc4
RP
1949 for f in filesToChangeType:
1950 p4_edit(f, "-t", "auto")
4f5cf76a 1951 for f in filesToAdd:
6de040df 1952 p4_add(f)
4f5cf76a 1953 for f in filesToDelete:
6de040df
LD
1954 p4_revert(f)
1955 p4_delete(f)
4f5cf76a 1956
c65b670e
CP
1957 # Set/clear executable bits
1958 for f in filesToChangeExecBit.keys():
1959 mode = filesToChangeExecBit[f]
1960 setP4ExecBit(f, mode)
1961
8cf422db
LD
1962 update_shelve = 0
1963 if len(self.update_shelve) > 0:
1964 update_shelve = self.update_shelve.pop(0)
1965 p4_reopen_in_change(update_shelve, all_files)
46c609e9 1966
55ac2ed6
PW
1967 #
1968 # Build p4 change description, starting with the contents
1969 # of the git commit message.
1970 #
0e36f2d7 1971 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1972 logMessage = logMessage.strip()
f19cb0a0 1973 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1974
8cf422db 1975 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 1976 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1977
c47178d4 1978 if self.preserveUser:
55ac2ed6 1979 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1980
55ac2ed6
PW
1981 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1982 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1983 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1984 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1985
55ac2ed6 1986 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1987 if not self.prepare_p4_only:
1988 submitTemplate += separatorLine
df8a9e86 1989 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1990
c47178d4 1991 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1992 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1993 if self.isWindows:
1994 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1995 tmpFile.write(submitTemplate)
c47178d4
PW
1996 tmpFile.close()
1997
728b7ad8
PW
1998 if self.prepare_p4_only:
1999 #
2000 # Leave the p4 tree prepared, and the submit template around
2001 # and let the user decide what to do next
2002 #
2003 print
2004 print "P4 workspace prepared for submission."
2005 print "To submit or revert, go to client workspace"
2006 print " " + self.clientPath
2007 print
2008 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 2009 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
2010 " \"git p4\"." % fileName
2011 print "You can delete the file \"%s\" when finished." % fileName
2012
2013 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
2014 print "To preserve change ownership by user %s, you must\n" \
2015 "do \"p4 change -f <change>\" after submitting and\n" \
2016 "edit the User field."
2017 if pureRenameCopy:
2018 print "After submitting, renamed files must be re-synced."
2019 print "Invoke \"p4 sync -f\" on each of these files:"
2020 for f in pureRenameCopy:
2021 print " " + f
2022
2023 print
2024 print "To revert the changes, use \"p4 revert ...\", and delete"
2025 print "the submit template file \"%s\"" % fileName
2026 if filesToAdd:
2027 print "Since the commit adds new files, they must be deleted:"
2028 for f in filesToAdd:
2029 print " " + f
2030 print
2031 return True
2032
55ac2ed6
PW
2033 #
2034 # Let the user edit the change description, then submit it.
2035 #
b7638fed 2036 submitted = False
cdc7e388 2037
b7638fed
GE
2038 try:
2039 if self.edit_template(fileName):
2040 # read the edited message and submit
2041 tmpFile = open(fileName, "rb")
2042 message = tmpFile.read()
2043 tmpFile.close()
2044 if self.isWindows:
2045 message = message.replace("\r\n", "\n")
2046 submitTemplate = message[:message.index(separatorLine)]
46c609e9 2047
8cf422db 2048 if update_shelve:
46c609e9
LD
2049 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2050 elif self.shelve:
b34fa577
VK
2051 p4_write_pipe(['shelve', '-i'], submitTemplate)
2052 else:
2053 p4_write_pipe(['submit', '-i'], submitTemplate)
2054 # The rename/copy happened by applying a patch that created a
2055 # new file. This leaves it writable, which confuses p4.
2056 for f in pureRenameCopy:
2057 p4_sync(f, "-f")
b7638fed
GE
2058
2059 if self.preserveUser:
2060 if p4User:
2061 # Get last changelist number. Cannot easily get it from
2062 # the submit command output as the output is
2063 # unmarshalled.
2064 changelist = self.lastP4Changelist()
2065 self.modifyChangelistUser(changelist, p4User)
2066
b7638fed
GE
2067 submitted = True
2068
2069 finally:
c47178d4 2070 # skip this patch
b34fa577
VK
2071 if not submitted or self.shelve:
2072 if self.shelve:
2073 print ("Reverting shelved files.")
2074 else:
2075 print ("Submission cancelled, undoing p4 changes.")
2076 for f in editedFiles | filesToDelete:
b7638fed
GE
2077 p4_revert(f)
2078 for f in filesToAdd:
2079 p4_revert(f)
2080 os.remove(f)
c47178d4
PW
2081
2082 os.remove(fileName)
b7638fed 2083 return submitted
4f5cf76a 2084
06804c76
LD
2085 # Export git tags as p4 labels. Create a p4 label and then tag
2086 # with that.
2087 def exportGitTags(self, gitTags):
c8942a22
LD
2088 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2089 if len(validLabelRegexp) == 0:
2090 validLabelRegexp = defaultLabelRegexp
2091 m = re.compile(validLabelRegexp)
06804c76
LD
2092
2093 for name in gitTags:
2094
2095 if not m.match(name):
2096 if verbose:
05a3cec5 2097 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
2098 continue
2099
2100 # Get the p4 commit this corresponds to
c8942a22
LD
2101 logMessage = extractLogMessageFromGitCommit(name)
2102 values = extractSettingsGitLog(logMessage)
06804c76 2103
dba1c9d9 2104 if 'change' not in values:
06804c76
LD
2105 # a tag pointing to something not sent to p4; ignore
2106 if verbose:
2107 print "git tag %s does not give a p4 commit" % name
2108 continue
c8942a22
LD
2109 else:
2110 changelist = values['change']
06804c76
LD
2111
2112 # Get the tag details.
2113 inHeader = True
2114 isAnnotated = False
2115 body = []
2116 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2117 l = l.strip()
2118 if inHeader:
2119 if re.match(r'tag\s+', l):
2120 isAnnotated = True
2121 elif re.match(r'\s*$', l):
2122 inHeader = False
2123 continue
2124 else:
2125 body.append(l)
2126
2127 if not isAnnotated:
2128 body = ["lightweight tag imported by git p4\n"]
2129
2130 # Create the label - use the same view as the client spec we are using
2131 clientSpec = getClientSpec()
2132
2133 labelTemplate = "Label: %s\n" % name
2134 labelTemplate += "Description:\n"
2135 for b in body:
2136 labelTemplate += "\t" + b + "\n"
2137 labelTemplate += "View:\n"
9d57c4a6
KS
2138 for depot_side in clientSpec.mappings:
2139 labelTemplate += "\t%s\n" % depot_side
06804c76 2140
ef739f08
PW
2141 if self.dry_run:
2142 print "Would create p4 label %s for tag" % name
728b7ad8
PW
2143 elif self.prepare_p4_only:
2144 print "Not creating p4 label %s for tag due to option" \
2145 " --prepare-p4-only" % name
ef739f08
PW
2146 else:
2147 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2148
ef739f08
PW
2149 # Use the label
2150 p4_system(["tag", "-l", name] +
9d57c4a6 2151 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2152
ef739f08
PW
2153 if verbose:
2154 print "created p4 label for tag %s" % name
06804c76 2155
4f5cf76a 2156 def run(self, args):
c9b50e63
SH
2157 if len(args) == 0:
2158 self.master = currentGitBranch()
c9b50e63
SH
2159 elif len(args) == 1:
2160 self.master = args[0]
28755dba
PW
2161 if not branchExists(self.master):
2162 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2163 else:
2164 return False
2165
8cf422db
LD
2166 for i in self.update_shelve:
2167 if i <= 0:
2168 sys.exit("invalid changelist %d" % i)
2169
00ad6e31
LD
2170 if self.master:
2171 allowSubmit = gitConfig("git-p4.allowSubmit")
2172 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2173 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2174
27d2d811 2175 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 2176 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2177 if len(self.origin) == 0:
2178 self.origin = upstream
a3fdd579 2179
8cf422db 2180 if len(self.update_shelve) > 0:
46c609e9
LD
2181 self.shelve = True
2182
3ea2cfd4
LD
2183 if self.preserveUser:
2184 if not self.canChangeChangelists():
2185 die("Cannot preserve user names without p4 super-user or admin permissions")
2186
6bbfd137
PW
2187 # if not set from the command line, try the config file
2188 if self.conflict_behavior is None:
2189 val = gitConfig("git-p4.conflict")
2190 if val:
2191 if val not in self.conflict_behavior_choices:
2192 die("Invalid value '%s' for config git-p4.conflict" % val)
2193 else:
2194 val = "ask"
2195 self.conflict_behavior = val
2196
a3fdd579
SH
2197 if self.verbose:
2198 print "Origin branch is " + self.origin
9512497b 2199
ea99c3ae 2200 if len(self.depotPath) == 0:
9512497b
SH
2201 print "Internal error: cannot locate perforce depot path from existing branches"
2202 sys.exit(128)
2203
543987bd 2204 self.useClientSpec = False
0d609032 2205 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2206 self.useClientSpec = True
2207 if self.useClientSpec:
2208 self.clientSpecDirs = getClientSpec()
9512497b 2209
2e3a16b2 2210 # Check for the existence of P4 branches
cd884106
VA
2211 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2212
2213 if self.useClientSpec and not branchesDetected:
543987bd
PW
2214 # all files are relative to the client spec
2215 self.clientPath = getClientRoot()
2216 else:
2217 self.clientPath = p4Where(self.depotPath)
9512497b 2218
543987bd
PW
2219 if self.clientPath == "":
2220 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2221
ea99c3ae 2222 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2223 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2224
0591cfa8 2225 # ensure the clientPath exists
8d7ec362 2226 new_client_dir = False
0591cfa8 2227 if not os.path.exists(self.clientPath):
8d7ec362 2228 new_client_dir = True
0591cfa8
GG
2229 os.makedirs(self.clientPath)
2230
bbd84863 2231 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2232 if self.dry_run:
2233 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2234 else:
ef739f08
PW
2235 print "Synchronizing p4 checkout..."
2236 if new_client_dir:
2237 # old one was destroyed, and maybe nobody told p4
2238 p4_sync("...", "-f")
2239 else:
2240 p4_sync("...")
4f5cf76a 2241 self.check()
4f5cf76a 2242
4c750c0d 2243 commits = []
00ad6e31 2244 if self.master:
89f32a92 2245 committish = self.master
00ad6e31 2246 else:
89f32a92 2247 committish = 'HEAD'
00ad6e31 2248
f55b87c1
RM
2249 if self.commit != "":
2250 if self.commit.find("..") != -1:
2251 limits_ish = self.commit.split("..")
2252 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2253 commits.append(line.strip())
2254 commits.reverse()
2255 else:
2256 commits.append(self.commit)
2257 else:
e6388994 2258 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
f55b87c1
RM
2259 commits.append(line.strip())
2260 commits.reverse()
4f5cf76a 2261
0d609032 2262 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2263 self.checkAuthorship = False
2264 else:
2265 self.checkAuthorship = True
2266
3ea2cfd4
LD
2267 if self.preserveUser:
2268 self.checkValidP4Users(commits)
2269
84cb0003
GG
2270 #
2271 # Build up a set of options to be passed to diff when
2272 # submitting each commit to p4.
2273 #
2274 if self.detectRenames:
2275 # command-line -M arg
2276 self.diffOpts = "-M"
2277 else:
2278 # If not explicitly set check the config variable
2279 detectRenames = gitConfig("git-p4.detectRenames")
2280
2281 if detectRenames.lower() == "false" or detectRenames == "":
2282 self.diffOpts = ""
2283 elif detectRenames.lower() == "true":
2284 self.diffOpts = "-M"
2285 else:
2286 self.diffOpts = "-M%s" % detectRenames
2287
2288 # no command-line arg for -C or --find-copies-harder, just
2289 # config variables
2290 detectCopies = gitConfig("git-p4.detectCopies")
2291 if detectCopies.lower() == "false" or detectCopies == "":
2292 pass
2293 elif detectCopies.lower() == "true":
2294 self.diffOpts += " -C"
2295 else:
2296 self.diffOpts += " -C%s" % detectCopies
2297
0d609032 2298 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2299 self.diffOpts += " --find-copies-harder"
2300
8cf422db
LD
2301 num_shelves = len(self.update_shelve)
2302 if num_shelves > 0 and num_shelves != len(commits):
2303 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2304 (len(commits), num_shelves))
2305
7e5dd9f2
PW
2306 #
2307 # Apply the commits, one at a time. On failure, ask if should
2308 # continue to try the rest of the patches, or quit.
2309 #
ef739f08
PW
2310 if self.dry_run:
2311 print "Would apply"
67b0fe2e 2312 applied = []
7e5dd9f2
PW
2313 last = len(commits) - 1
2314 for i, commit in enumerate(commits):
ef739f08
PW
2315 if self.dry_run:
2316 print " ", read_pipe(["git", "show", "-s",
2317 "--format=format:%h %s", commit])
2318 ok = True
2319 else:
2320 ok = self.applyCommit(commit)
67b0fe2e
PW
2321 if ok:
2322 applied.append(commit)
7e5dd9f2 2323 else:
728b7ad8
PW
2324 if self.prepare_p4_only and i < last:
2325 print "Processing only the first commit due to option" \
2326 " --prepare-p4-only"
2327 break
7e5dd9f2
PW
2328 if i < last:
2329 quit = False
2330 while True:
6bbfd137
PW
2331 # prompt for what to do, or use the option/variable
2332 if self.conflict_behavior == "ask":
2333 print "What do you want to do?"
2334 response = raw_input("[s]kip this commit but apply"
2335 " the rest, or [q]uit? ")
2336 if not response:
2337 continue
2338 elif self.conflict_behavior == "skip":
2339 response = "s"
2340 elif self.conflict_behavior == "quit":
2341 response = "q"
2342 else:
2343 die("Unknown conflict_behavior '%s'" %
2344 self.conflict_behavior)
2345
7e5dd9f2
PW
2346 if response[0] == "s":
2347 print "Skipping this commit, but applying the rest"
2348 break
2349 if response[0] == "q":
2350 print "Quitting"
2351 quit = True
2352 break
2353 if quit:
2354 break
4f5cf76a 2355
67b0fe2e 2356 chdir(self.oldWorkingDirectory)
b34fa577 2357 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2358 if self.dry_run:
2359 pass
728b7ad8
PW
2360 elif self.prepare_p4_only:
2361 pass
ef739f08 2362 elif len(commits) == len(applied):
b34fa577 2363 print ("All commits {0}!".format(shelved_applied))
14594f4b 2364
4c750c0d 2365 sync = P4Sync()
44e8d26c
PW
2366 if self.branch:
2367 sync.branch = self.branch
b9d34db9
LD
2368 if self.disable_p4sync:
2369 sync.sync_origin_only()
2370 else:
2371 sync.run([])
14594f4b 2372
b9d34db9
LD
2373 if not self.disable_rebase:
2374 rebase = P4Rebase()
2375 rebase.rebase()
4f5cf76a 2376
67b0fe2e
PW
2377 else:
2378 if len(applied) == 0:
b34fa577 2379 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2380 else:
b34fa577 2381 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2382 for c in commits:
2383 if c in applied:
2384 star = "*"
2385 else:
2386 star = " "
2387 print star, read_pipe(["git", "show", "-s",
2388 "--format=format:%h %s", c])
2389 print "You will have to do 'git p4 sync' and rebase."
2390
0d609032 2391 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2392 self.exportLabels = True
06804c76
LD
2393
2394 if self.exportLabels:
2395 p4Labels = getP4Labels(self.depotPath)
2396 gitTags = getGitTags()
2397
2398 missingGitTags = gitTags - p4Labels
2399 self.exportGitTags(missingGitTags)
2400
98e023de 2401 # exit with error unless everything applied perfectly
67b0fe2e
PW
2402 if len(commits) != len(applied):
2403 sys.exit(1)
2404
b984733c
SH
2405 return True
2406
ecb7cf98
PW
2407class View(object):
2408 """Represent a p4 view ("p4 help views"), and map files in a
2409 repo according to the view."""
2410
9d57c4a6 2411 def __init__(self, client_name):
ecb7cf98 2412 self.mappings = []
9d57c4a6
KS
2413 self.client_prefix = "//%s/" % client_name
2414 # cache results of "p4 where" to lookup client file locations
2415 self.client_spec_path_cache = {}
ecb7cf98
PW
2416
2417 def append(self, view_line):
2418 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2419 sides. Append to self.mappings, preserving order. This
2420 is only needed for tag creation."""
ecb7cf98
PW
2421
2422 # Split the view line into exactly two words. P4 enforces
2423 # structure on these lines that simplifies this quite a bit.
2424 #
2425 # Either or both words may be double-quoted.
2426 # Single quotes do not matter.
2427 # Double-quote marks cannot occur inside the words.
2428 # A + or - prefix is also inside the quotes.
2429 # There are no quotes unless they contain a space.
2430 # The line is already white-space stripped.
2431 # The two words are separated by a single space.
2432 #
2433 if view_line[0] == '"':
2434 # First word is double quoted. Find its end.
2435 close_quote_index = view_line.find('"', 1)
2436 if close_quote_index <= 0:
2437 die("No first-word closing quote found: %s" % view_line)
2438 depot_side = view_line[1:close_quote_index]
2439 # skip closing quote and space
2440 rhs_index = close_quote_index + 1 + 1
2441 else:
2442 space_index = view_line.find(" ")
2443 if space_index <= 0:
2444 die("No word-splitting space found: %s" % view_line)
2445 depot_side = view_line[0:space_index]
2446 rhs_index = space_index + 1
2447
ecb7cf98 2448 # prefix + means overlay on previous mapping
ecb7cf98 2449 if depot_side.startswith("+"):
ecb7cf98
PW
2450 depot_side = depot_side[1:]
2451
9d57c4a6 2452 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2453 exclude = False
2454 if depot_side.startswith("-"):
2455 exclude = True
2456 depot_side = depot_side[1:]
2457
9d57c4a6
KS
2458 if not exclude:
2459 self.mappings.append(depot_side)
ecb7cf98 2460
9d57c4a6
KS
2461 def convert_client_path(self, clientFile):
2462 # chop off //client/ part to make it relative
2463 if not clientFile.startswith(self.client_prefix):
2464 die("No prefix '%s' on clientFile '%s'" %
2465 (self.client_prefix, clientFile))
2466 return clientFile[len(self.client_prefix):]
ecb7cf98 2467
9d57c4a6
KS
2468 def update_client_spec_path_cache(self, files):
2469 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2470
9d57c4a6
KS
2471 # List depot file paths exclude that already cached
2472 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2473
9d57c4a6
KS
2474 if len(fileArgs) == 0:
2475 return # All files in cache
ecb7cf98 2476
9d57c4a6
KS
2477 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2478 for res in where_result:
2479 if "code" in res and res["code"] == "error":
2480 # assume error is "... file(s) not in client view"
2481 continue
2482 if "clientFile" not in res:
20005443 2483 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2484 if "unmap" in res:
2485 # it will list all of them, but only one not unmap-ped
2486 continue
a0a50d87
LS
2487 if gitConfigBool("core.ignorecase"):
2488 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2489 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2490
9d57c4a6
KS
2491 # not found files or unmap files set to ""
2492 for depotFile in fileArgs:
a0a50d87
LS
2493 if gitConfigBool("core.ignorecase"):
2494 depotFile = depotFile.lower()
9d57c4a6
KS
2495 if depotFile not in self.client_spec_path_cache:
2496 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2497
9d57c4a6
KS
2498 def map_in_client(self, depot_path):
2499 """Return the relative location in the client where this
2500 depot file should live. Returns "" if the file should
2501 not be mapped in the client."""
ecb7cf98 2502
a0a50d87
LS
2503 if gitConfigBool("core.ignorecase"):
2504 depot_path = depot_path.lower()
2505
9d57c4a6
KS
2506 if depot_path in self.client_spec_path_cache:
2507 return self.client_spec_path_cache[depot_path]
2508
2509 die( "Error: %s is not found in client spec path" % depot_path )
2510 return ""
ecb7cf98 2511
3ea2cfd4 2512class P4Sync(Command, P4UserMap):
56c09345
PW
2513 delete_actions = ( "delete", "move/delete", "purge" )
2514
b984733c
SH
2515 def __init__(self):
2516 Command.__init__(self)
3ea2cfd4 2517 P4UserMap.__init__(self)
b984733c
SH
2518 self.options = [
2519 optparse.make_option("--branch", dest="branch"),
2520 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2521 optparse.make_option("--changesfile", dest="changesFile"),
2522 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2523 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2524 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2525 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2526 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2527 optparse.make_option("--max-changes", dest="maxChanges",
2528 help="Maximum number of changes to import"),
2529 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2530 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2531 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2532 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2533 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2534 help="Only sync files that are included in the Perforce Client Spec"),
2535 optparse.make_option("-/", dest="cloneExclude",
2536 action="append", type="string",
2537 help="exclude depot path"),
b984733c
SH
2538 ]
2539 self.description = """Imports from Perforce into a git repository.\n
2540 example:
2541 //depot/my/project/ -- to import the current head
2542 //depot/my/project/@all -- to import everything
2543 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2544
2545 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2546
2547 self.usage += " //depot/path[@revRange]"
b984733c 2548 self.silent = False
1d7367dc
RG
2549 self.createdBranches = set()
2550 self.committedChanges = set()
569d1bd4 2551 self.branch = ""
b984733c 2552 self.detectBranches = False
cb53e1f8 2553 self.detectLabels = False
06804c76 2554 self.importLabels = False
b984733c 2555 self.changesFile = ""
01265103 2556 self.syncWithOrigin = True
a028a98e 2557 self.importIntoRemotes = True
01a9c9c5 2558 self.maxChanges = ""
1051ef00 2559 self.changes_block_size = None
8b41a97f 2560 self.keepRepoPath = False
6326aa58 2561 self.depotPaths = None
3c699645 2562 self.p4BranchesInGit = []
354081d5 2563 self.cloneExclude = []
3a70cdfa 2564 self.useClientSpec = False
a93d33ee 2565 self.useClientSpec_from_options = False
ecb7cf98 2566 self.clientSpecDirs = None
fed23693 2567 self.tempBranches = []
d604176d 2568 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12 2569 self.largeFileSystem = None
123f6317 2570 self.suppress_meta_comment = False
a5db4b12
LS
2571
2572 if gitConfig('git-p4.largeFileSystem'):
2573 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2574 self.largeFileSystem = largeFileSystemConstructor(
2575 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2576 )
b984733c 2577
01265103
SH
2578 if gitConfig("git-p4.syncFromOrigin") == "false":
2579 self.syncWithOrigin = False
2580
123f6317
LD
2581 self.depotPaths = []
2582 self.changeRange = ""
2583 self.previousDepotPaths = []
2584 self.hasOrigin = False
2585
2586 # map from branch depot path to parent branch
2587 self.knownBranches = {}
2588 self.initialParents = {}
2589
2590 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2591 self.labels = {}
2592
fed23693
VA
2593 # Force a checkpoint in fast-import and wait for it to finish
2594 def checkpoint(self):
2595 self.gitStream.write("checkpoint\n\n")
2596 self.gitStream.write("progress checkpoint\n\n")
2597 out = self.gitOutput.readline()
2598 if self.verbose:
2599 print "checkpoint finished: " + out
2600
123f6317
LD
2601 def cmp_shelved(self, path, filerev, revision):
2602 """ Determine if a path at revision #filerev is the same as the file
2603 at revision @revision for a shelved changelist. If they don't match,
2604 unshelving won't be safe (we will get other changes mixed in).
2605
2606 This is comparing the revision that the shelved changelist is *based* on, not
2607 the shelved changelist itself.
2608 """
2609 ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2610 if verbose:
2611 print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2612 return ret["status"] == "identical"
2613
2614 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
354081d5
TT
2615 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2616 for path in self.cloneExclude]
b984733c
SH
2617 files = []
2618 fnum = 0
dba1c9d9 2619 while "depotFile%s" % fnum in commit:
b984733c 2620 path = commit["depotFile%s" % fnum]
6326aa58 2621
354081d5 2622 if [p for p in self.cloneExclude
d53de8b9 2623 if p4PathStartsWith(path, p)]:
354081d5
TT
2624 found = False
2625 else:
2626 found = [p for p in self.depotPaths
d53de8b9 2627 if p4PathStartsWith(path, p)]
6326aa58 2628 if not found:
b984733c
SH
2629 fnum = fnum + 1
2630 continue
2631
2632 file = {}
2633 file["path"] = path
2634 file["rev"] = commit["rev%s" % fnum]
2635 file["action"] = commit["action%s" % fnum]
2636 file["type"] = commit["type%s" % fnum]
123f6317
LD
2637 if shelved:
2638 file["shelved_cl"] = int(shelved_cl)
2639
2640 # For shelved changelists, check that the revision of each file that the
2641 # shelve was based on matches the revision that we are using for the
2642 # starting point for git-fast-import (self.initialParent). Otherwise
2643 # the resulting diff will contain deltas from multiple commits.
2644
2645 if file["action"] != "add" and \
2646 not self.cmp_shelved(path, file["rev"], origin_revision):
2647 sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2648 commit["change"], self.initialParent, path))
2649
b984733c
SH
2650 files.append(file)
2651 fnum = fnum + 1
2652 return files
2653
26e6a27d
JD
2654 def extractJobsFromCommit(self, commit):
2655 jobs = []
2656 jnum = 0
dba1c9d9 2657 while "job%s" % jnum in commit:
26e6a27d
JD
2658 job = commit["job%s" % jnum]
2659 jobs.append(job)
2660 jnum = jnum + 1
2661 return jobs
2662
6326aa58 2663 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2664 """When streaming files, this is called to map a p4 depot path
2665 to where it should go in git. The prefixes are either
2666 self.depotPaths, or self.branchPrefixes in the case of
2667 branch detection."""
2668
3952710b 2669 if self.useClientSpec:
21ef5df4
PW
2670 # branch detection moves files up a level (the branch name)
2671 # from what client spec interpretation gives
0d1696ef 2672 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2673 if self.detectBranches:
2674 for b in self.knownBranches:
2675 if path.startswith(b + "/"):
2676 path = path[len(b)+1:]
2677
2678 elif self.keepRepoPath:
2679 # Preserve everything in relative path name except leading
2680 # //depot/; just look at first prefix as they all should
2681 # be in the same depot.
2682 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2683 if p4PathStartsWith(path, depot):
2684 path = path[len(depot):]
3952710b 2685
0d1696ef 2686 else:
0d1696ef
PW
2687 for p in prefixes:
2688 if p4PathStartsWith(path, p):
2689 path = path[len(p):]
21ef5df4 2690 break
8b41a97f 2691
0d1696ef 2692 path = wildcard_decode(path)
6326aa58 2693 return path
6754a299 2694
71b112d4 2695 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2696 """Look at each depotFile in the commit to figure out to what
2697 branch it belongs."""
2698
9d57c4a6
KS
2699 if self.clientSpecDirs:
2700 files = self.extractFilesFromCommit(commit)
2701 self.clientSpecDirs.update_client_spec_path_cache(files)
2702
d5904674 2703 branches = {}
71b112d4 2704 fnum = 0
dba1c9d9 2705 while "depotFile%s" % fnum in commit:
71b112d4 2706 path = commit["depotFile%s" % fnum]
6326aa58 2707 found = [p for p in self.depotPaths
d53de8b9 2708 if p4PathStartsWith(path, p)]
6326aa58 2709 if not found:
71b112d4
SH
2710 fnum = fnum + 1
2711 continue
2712
2713 file = {}
2714 file["path"] = path
2715 file["rev"] = commit["rev%s" % fnum]
2716 file["action"] = commit["action%s" % fnum]
2717 file["type"] = commit["type%s" % fnum]
2718 fnum = fnum + 1
2719
21ef5df4
PW
2720 # start with the full relative path where this file would
2721 # go in a p4 client
2722 if self.useClientSpec:
2723 relPath = self.clientSpecDirs.map_in_client(path)
2724 else:
2725 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2726
4b97ffb1 2727 for branch in self.knownBranches.keys():
21ef5df4
PW
2728 # add a trailing slash so that a commit into qt/4.2foo
2729 # doesn't end up in qt/4.2, e.g.
6754a299 2730 if relPath.startswith(branch + "/"):
d5904674
SH
2731 if branch not in branches:
2732 branches[branch] = []
71b112d4 2733 branches[branch].append(file)
6555b2cc 2734 break
b984733c
SH
2735
2736 return branches
2737
a5db4b12
LS
2738 def writeToGitStream(self, gitMode, relPath, contents):
2739 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2740 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2741 for d in contents:
2742 self.gitStream.write(d)
2743 self.gitStream.write('\n')
2744
a8b05162
LS
2745 def encodeWithUTF8(self, path):
2746 try:
2747 path.decode('ascii')
2748 except:
2749 encoding = 'utf8'
2750 if gitConfig('git-p4.pathEncoding'):
2751 encoding = gitConfig('git-p4.pathEncoding')
2752 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2753 if self.verbose:
2754 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2755 return path
2756
b932705b
LD
2757 # output one file from the P4 stream
2758 # - helper for streamP4Files
2759
2760 def streamOneP4File(self, file, contents):
b932705b 2761 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2762 relPath = self.encodeWithUTF8(relPath)
b932705b 2763 if verbose:
d2176a50
LS
2764 size = int(self.stream_file['fileSize'])
2765 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2766 sys.stdout.flush()
b932705b 2767
9cffb8c8
PW
2768 (type_base, type_mods) = split_p4_type(file["type"])
2769
2770 git_mode = "100644"
2771 if "x" in type_mods:
2772 git_mode = "100755"
2773 if type_base == "symlink":
2774 git_mode = "120000"
1292df11
AJ
2775 # p4 print on a symlink sometimes contains "target\n";
2776 # if it does, remove the newline
b39c3612 2777 data = ''.join(contents)
40f846c3
PW
2778 if not data:
2779 # Some version of p4 allowed creating a symlink that pointed
2780 # to nothing. This causes p4 errors when checking out such
2781 # a change, and errors here too. Work around it by ignoring
2782 # the bad symlink; hopefully a future change fixes it.
2783 print "\nIgnoring empty symlink in %s" % file['depotFile']
2784 return
2785 elif data[-1] == '\n':
1292df11
AJ
2786 contents = [data[:-1]]
2787 else:
2788 contents = [data]
b932705b 2789
9cffb8c8 2790 if type_base == "utf16":
55aa5714
PW
2791 # p4 delivers different text in the python output to -G
2792 # than it does when using "print -o", or normal p4 client
2793 # operations. utf16 is converted to ascii or utf8, perhaps.
2794 # But ascii text saved as -t utf16 is completely mangled.
2795 # Invoke print -o to get the real contents.
7f0e5962
PW
2796 #
2797 # On windows, the newlines will always be mangled by print, so put
2798 # them back too. This is not needed to the cygwin windows version,
2799 # just the native "NT" type.
2800 #
1f5f3907
LS
2801 try:
2802 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2803 except Exception as e:
2804 if 'Translation of file content failed' in str(e):
2805 type_base = 'binary'
2806 else:
2807 raise e
2808 else:
2809 if p4_version_string().find('/NT') >= 0:
2810 text = text.replace('\r\n', '\n')
2811 contents = [ text ]
55aa5714 2812
9f7ef0ea
PW
2813 if type_base == "apple":
2814 # Apple filetype files will be streamed as a concatenation of
2815 # its appledouble header and the contents. This is useless
2816 # on both macs and non-macs. If using "print -q -o xx", it
2817 # will create "xx" with the data, and "%xx" with the header.
2818 # This is also not very useful.
2819 #
2820 # Ideally, someday, this script can learn how to generate
2821 # appledouble files directly and import those to git, but
2822 # non-mac machines can never find a use for apple filetype.
2823 print "\nIgnoring apple filetype file %s" % file['depotFile']
2824 return
2825
55aa5714
PW
2826 # Note that we do not try to de-mangle keywords on utf16 files,
2827 # even though in theory somebody may want that.
60df071c
LD
2828 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2829 if pattern:
2830 regexp = re.compile(pattern, re.VERBOSE)
2831 text = ''.join(contents)
2832 text = regexp.sub(r'$\1$', text)
2833 contents = [ text ]
b932705b 2834
a5db4b12
LS
2835 if self.largeFileSystem:
2836 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2837
a5db4b12 2838 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2839
2840 def streamOneP4Deletion(self, file):
2841 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2842 relPath = self.encodeWithUTF8(relPath)
b932705b 2843 if verbose:
d2176a50
LS
2844 sys.stdout.write("delete %s\n" % relPath)
2845 sys.stdout.flush()
b932705b
LD
2846 self.gitStream.write("D %s\n" % relPath)
2847
a5db4b12
LS
2848 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2849 self.largeFileSystem.removeLargeFile(relPath)
2850
b932705b
LD
2851 # handle another chunk of streaming data
2852 def streamP4FilesCb(self, marshalled):
2853
78189bea
PW
2854 # catch p4 errors and complain
2855 err = None
2856 if "code" in marshalled:
2857 if marshalled["code"] == "error":
2858 if "data" in marshalled:
2859 err = marshalled["data"].rstrip()
4d25dc44
LS
2860
2861 if not err and 'fileSize' in self.stream_file:
2862 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2863 if required_bytes > 0:
2864 err = 'Not enough space left on %s! Free at least %i MB.' % (
2865 os.getcwd(), required_bytes/1024/1024
2866 )
2867
78189bea
PW
2868 if err:
2869 f = None
2870 if self.stream_have_file_info:
2871 if "depotFile" in self.stream_file:
2872 f = self.stream_file["depotFile"]
2873 # force a failure in fast-import, else an empty
2874 # commit will be made
2875 self.gitStream.write("\n")
2876 self.gitStream.write("die-now\n")
2877 self.gitStream.close()
2878 # ignore errors, but make sure it exits first
2879 self.importProcess.wait()
2880 if f:
2881 die("Error from p4 print for %s: %s" % (f, err))
2882 else:
2883 die("Error from p4 print: %s" % err)
2884
dba1c9d9 2885 if 'depotFile' in marshalled and self.stream_have_file_info:
c3f6163b
AG
2886 # start of a new file - output the old one first
2887 self.streamOneP4File(self.stream_file, self.stream_contents)
2888 self.stream_file = {}
2889 self.stream_contents = []
2890 self.stream_have_file_info = False
b932705b 2891
c3f6163b
AG
2892 # pick up the new file information... for the
2893 # 'data' field we need to append to our array
2894 for k in marshalled.keys():
2895 if k == 'data':
d2176a50
LS
2896 if 'streamContentSize' not in self.stream_file:
2897 self.stream_file['streamContentSize'] = 0
2898 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2899 self.stream_contents.append(marshalled['data'])
2900 else:
2901 self.stream_file[k] = marshalled[k]
b932705b 2902
d2176a50
LS
2903 if (verbose and
2904 'streamContentSize' in self.stream_file and
2905 'fileSize' in self.stream_file and
2906 'depotFile' in self.stream_file):
2907 size = int(self.stream_file["fileSize"])
2908 if size > 0:
2909 progress = 100*self.stream_file['streamContentSize']/size
2910 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2911 sys.stdout.flush()
2912
c3f6163b 2913 self.stream_have_file_info = True
b932705b
LD
2914
2915 # Stream directly from "p4 files" into "git fast-import"
2916 def streamP4Files(self, files):
30b5940b
SH
2917 filesForCommit = []
2918 filesToRead = []
b932705b 2919 filesToDelete = []
30b5940b 2920
3a70cdfa 2921 for f in files:
ecb7cf98
PW
2922 filesForCommit.append(f)
2923 if f['action'] in self.delete_actions:
2924 filesToDelete.append(f)
2925 else:
2926 filesToRead.append(f)
6a49f8e2 2927
b932705b
LD
2928 # deleted files...
2929 for f in filesToDelete:
2930 self.streamOneP4Deletion(f)
1b9a4684 2931
b932705b
LD
2932 if len(filesToRead) > 0:
2933 self.stream_file = {}
2934 self.stream_contents = []
2935 self.stream_have_file_info = False
8ff45f2a 2936
c3f6163b
AG
2937 # curry self argument
2938 def streamP4FilesCbSelf(entry):
2939 self.streamP4FilesCb(entry)
6a49f8e2 2940
123f6317
LD
2941 fileArgs = []
2942 for f in filesToRead:
2943 if 'shelved_cl' in f:
2944 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2945 # the contents
2946 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2947 else:
2948 fileArg = '%s#%s' % (f['path'], f['rev'])
2949
2950 fileArgs.append(fileArg)
6de040df
LD
2951
2952 p4CmdList(["-x", "-", "print"],
2953 stdin=fileArgs,
2954 cb=streamP4FilesCbSelf)
30b5940b 2955
b932705b 2956 # do the last chunk
dba1c9d9 2957 if 'depotFile' in self.stream_file:
b932705b 2958 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2959
affb474f
LD
2960 def make_email(self, userid):
2961 if userid in self.users:
2962 return self.users[userid]
2963 else:
2964 return "%s <a@b>" % userid
2965
06804c76 2966 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2967 """ Stream a p4 tag.
2968 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2969 """
2970
06804c76
LD
2971 if verbose:
2972 print "writing tag %s for commit %s" % (labelName, commit)
2973 gitStream.write("tag %s\n" % labelName)
2974 gitStream.write("from %s\n" % commit)
2975
dba1c9d9 2976 if 'Owner' in labelDetails:
06804c76
LD
2977 owner = labelDetails["Owner"]
2978 else:
2979 owner = None
2980
2981 # Try to use the owner of the p4 label, or failing that,
2982 # the current p4 user id.
2983 if owner:
2984 email = self.make_email(owner)
2985 else:
2986 email = self.make_email(self.p4UserId())
2987 tagger = "%s %s %s" % (email, epoch, self.tz)
2988
2989 gitStream.write("tagger %s\n" % tagger)
2990
2991 print "labelDetails=",labelDetails
dba1c9d9 2992 if 'Description' in labelDetails:
06804c76
LD
2993 description = labelDetails['Description']
2994 else:
2995 description = 'Label from git p4'
2996
2997 gitStream.write("data %d\n" % len(description))
2998 gitStream.write(description)
2999 gitStream.write("\n")
3000
4ae048e6
LS
3001 def inClientSpec(self, path):
3002 if not self.clientSpecDirs:
3003 return True
3004 inClientSpec = self.clientSpecDirs.map_in_client(path)
3005 if not inClientSpec and self.verbose:
3006 print('Ignoring file outside of client spec: {0}'.format(path))
3007 return inClientSpec
3008
3009 def hasBranchPrefix(self, path):
3010 if not self.branchPrefixes:
3011 return True
3012 hasPrefix = [p for p in self.branchPrefixes
3013 if p4PathStartsWith(path, p)]
09667d01 3014 if not hasPrefix and self.verbose:
4ae048e6
LS
3015 print('Ignoring file outside of prefix: {0}'.format(path))
3016 return hasPrefix
3017
e63231e5 3018 def commit(self, details, files, branch, parent = ""):
b984733c
SH
3019 epoch = details["time"]
3020 author = details["user"]
26e6a27d 3021 jobs = self.extractJobsFromCommit(details)
b984733c 3022
4b97ffb1 3023 if self.verbose:
4ae048e6 3024 print('commit into {0}'.format(branch))
96e07dd2 3025
9d57c4a6
KS
3026 if self.clientSpecDirs:
3027 self.clientSpecDirs.update_client_spec_path_cache(files)
3028
4ae048e6
LS
3029 files = [f for f in files
3030 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3031
3032 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
3033 print('Ignoring revision {0} as it would produce an empty commit.'
3034 .format(details['change']))
3035 return
3036
b984733c 3037 self.gitStream.write("commit %s\n" % branch)
b43702ac 3038 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
3039 self.committedChanges.add(int(details["change"]))
3040 committer = ""
b607e71e
SH
3041 if author not in self.users:
3042 self.getUserMapFromPerforceServer()
affb474f 3043 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
3044
3045 self.gitStream.write("committer %s\n" % committer)
3046
3047 self.gitStream.write("data <<EOT\n")
3048 self.gitStream.write(details["desc"])
26e6a27d
JD
3049 if len(jobs) > 0:
3050 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
123f6317
LD
3051
3052 if not self.suppress_meta_comment:
3053 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3054 (','.join(self.branchPrefixes), details["change"]))
3055 if len(details['options']) > 0:
3056 self.gitStream.write(": options = %s" % details['options'])
3057 self.gitStream.write("]\n")
3058
3059 self.gitStream.write("EOT\n\n")
b984733c
SH
3060
3061 if len(parent) > 0:
4b97ffb1
SH
3062 if self.verbose:
3063 print "parent %s" % parent
b984733c
SH
3064 self.gitStream.write("from %s\n" % parent)
3065
4ae048e6 3066 self.streamP4Files(files)
b984733c
SH
3067 self.gitStream.write("\n")
3068
1f4ba1cb
SH
3069 change = int(details["change"])
3070
dba1c9d9 3071 if change in self.labels:
1f4ba1cb
SH
3072 label = self.labels[change]
3073 labelDetails = label[0]
3074 labelRevisions = label[1]
71b112d4
SH
3075 if self.verbose:
3076 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 3077
6de040df 3078 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 3079 for p in self.branchPrefixes])
1f4ba1cb
SH
3080
3081 if len(files) == len(labelRevisions):
3082
3083 cleanedFiles = {}
3084 for info in files:
56c09345 3085 if info["action"] in self.delete_actions:
1f4ba1cb
SH
3086 continue
3087 cleanedFiles[info["depotFile"]] = info["rev"]
3088
3089 if cleanedFiles == labelRevisions:
06804c76 3090 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
3091
3092 else:
a46668fa 3093 if not self.silent:
cebdf5af
HWN
3094 print ("Tag %s does not match with change %s: files do not match."
3095 % (labelDetails["label"], change))
1f4ba1cb
SH
3096
3097 else:
a46668fa 3098 if not self.silent:
cebdf5af
HWN
3099 print ("Tag %s does not match with change %s: file count is different."
3100 % (labelDetails["label"], change))
b984733c 3101
06804c76 3102 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
3103 def getLabels(self):
3104 self.labels = {}
3105
52a4880b 3106 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 3107 if len(l) > 0 and not self.silent:
4d88519f 3108 print("Finding files belonging to labels in %s" % self.depotPaths)
01ce1fe9
SH
3109
3110 for output in l:
1f4ba1cb
SH
3111 label = output["label"]
3112 revisions = {}
3113 newestChange = 0
71b112d4
SH
3114 if self.verbose:
3115 print "Querying files for label %s" % label
6de040df
LD
3116 for file in p4CmdList(["files"] +
3117 ["%s...@%s" % (p, label)
3118 for p in self.depotPaths]):
1f4ba1cb
SH
3119 revisions[file["depotFile"]] = file["rev"]
3120 change = int(file["change"])
3121 if change > newestChange:
3122 newestChange = change
3123
9bda3a85
SH
3124 self.labels[newestChange] = [output, revisions]
3125
3126 if self.verbose:
3127 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 3128
06804c76
LD
3129 # Import p4 labels as git tags. A direct mapping does not
3130 # exist, so assume that if all the files are at the same revision
3131 # then we can use that, or it's something more complicated we should
3132 # just ignore.
3133 def importP4Labels(self, stream, p4Labels):
3134 if verbose:
3135 print "import p4 labels: " + ' '.join(p4Labels)
3136
3137 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 3138 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
3139 if len(validLabelRegexp) == 0:
3140 validLabelRegexp = defaultLabelRegexp
3141 m = re.compile(validLabelRegexp)
3142
3143 for name in p4Labels:
3144 commitFound = False
3145
3146 if not m.match(name):
3147 if verbose:
3148 print "label %s does not match regexp %s" % (name,validLabelRegexp)
3149 continue
3150
3151 if name in ignoredP4Labels:
3152 continue
3153
3154 labelDetails = p4CmdList(['label', "-o", name])[0]
3155
3156 # get the most recent changelist for each file in this label
3157 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3158 for p in self.depotPaths])
3159
dba1c9d9 3160 if 'change' in change:
06804c76
LD
3161 # find the corresponding git commit; take the oldest commit
3162 changelist = int(change['change'])
b43702ac
LD
3163 if changelist in self.committedChanges:
3164 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 3165 commitFound = True
b43702ac
LD
3166 else:
3167 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3168 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3169 if len(gitCommit) == 0:
3170 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3171 else:
3172 commitFound = True
3173 gitCommit = gitCommit.strip()
3174
3175 if commitFound:
06804c76
LD
3176 # Convert from p4 time format
3177 try:
3178 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3179 except ValueError:
a4e9054c 3180 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
3181 tmwhen = 1
3182
3183 when = int(time.mktime(tmwhen))
3184 self.streamTag(stream, name, labelDetails, gitCommit, when)
3185 if verbose:
3186 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3187 else:
3188 if verbose:
3189 print "Label %s has no changelists - possibly deleted?" % name
3190
3191 if not commitFound:
3192 # We can't import this label; don't try again as it will get very
3193 # expensive repeatedly fetching all the files for labels that will
3194 # never be imported. If the label is moved in the future, the
3195 # ignore will need to be removed manually.
3196 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3197
86dff6b6
HWN
3198 def guessProjectName(self):
3199 for p in self.depotPaths:
6e5295c4
SH
3200 if p.endswith("/"):
3201 p = p[:-1]
3202 p = p[p.strip().rfind("/") + 1:]
3203 if not p.endswith("/"):
3204 p += "/"
3205 return p
86dff6b6 3206
4b97ffb1 3207 def getBranchMapping(self):
6555b2cc
SH
3208 lostAndFoundBranches = set()
3209
8ace74c0
VA
3210 user = gitConfig("git-p4.branchUser")
3211 if len(user) > 0:
3212 command = "branches -u %s" % user
3213 else:
3214 command = "branches"
3215
3216 for info in p4CmdList(command):
52a4880b 3217 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1 3218 viewIdx = 0
dba1c9d9 3219 while "View%s" % viewIdx in details:
4b97ffb1
SH
3220 paths = details["View%s" % viewIdx].split(" ")
3221 viewIdx = viewIdx + 1
3222 # require standard //depot/foo/... //depot/bar/... mapping
3223 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3224 continue
3225 source = paths[0]
3226 destination = paths[1]
6509e19c 3227 ## HACK
d53de8b9 3228 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3229 source = source[len(self.depotPaths[0]):-4]
3230 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3231
1a2edf4e
SH
3232 if destination in self.knownBranches:
3233 if not self.silent:
3234 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3235 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3236 continue
3237
6555b2cc
SH
3238 self.knownBranches[destination] = source
3239
3240 lostAndFoundBranches.discard(destination)
3241
29bdbac1 3242 if source not in self.knownBranches:
6555b2cc
SH
3243 lostAndFoundBranches.add(source)
3244
7199cf13
VA
3245 # Perforce does not strictly require branches to be defined, so we also
3246 # check git config for a branch list.
3247 #
3248 # Example of branch definition in git config file:
3249 # [git-p4]
3250 # branchList=main:branchA
3251 # branchList=main:branchB
3252 # branchList=branchA:branchC
3253 configBranches = gitConfigList("git-p4.branchList")
3254 for branch in configBranches:
3255 if branch:
3256 (source, destination) = branch.split(":")
3257 self.knownBranches[destination] = source
3258
3259 lostAndFoundBranches.discard(destination)
3260
3261 if source not in self.knownBranches:
3262 lostAndFoundBranches.add(source)
3263
6555b2cc
SH
3264
3265 for branch in lostAndFoundBranches:
3266 self.knownBranches[branch] = branch
29bdbac1 3267
38f9f5ec
SH
3268 def getBranchMappingFromGitBranches(self):
3269 branches = p4BranchesInGit(self.importIntoRemotes)
3270 for branch in branches.keys():
3271 if branch == "master":
3272 branch = "main"
3273 else:
3274 branch = branch[len(self.projectName):]
3275 self.knownBranches[branch] = branch
3276
bb6e09b2
HWN
3277 def updateOptionDict(self, d):
3278 option_keys = {}
3279 if self.keepRepoPath:
3280 option_keys['keepRepoPath'] = 1
3281
3282 d["options"] = ' '.join(sorted(option_keys.keys()))
3283
3284 def readOptions(self, d):
dba1c9d9 3285 self.keepRepoPath = ('options' in d
bb6e09b2 3286 and ('keepRepoPath' in d['options']))
6326aa58 3287
8134f69c
SH
3288 def gitRefForBranch(self, branch):
3289 if branch == "main":
3290 return self.refPrefix + "master"
3291
3292 if len(branch) <= 0:
3293 return branch
3294
3295 return self.refPrefix + self.projectName + branch
3296
1ca3d710
SH
3297 def gitCommitByP4Change(self, ref, change):
3298 if self.verbose:
3299 print "looking in ref " + ref + " for change %s using bisect..." % change
3300
3301 earliestCommit = ""
3302 latestCommit = parseRevision(ref)
3303
3304 while True:
3305 if self.verbose:
3306 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3307 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3308 if len(next) == 0:
3309 if self.verbose:
3310 print "argh"
3311 return ""
3312 log = extractLogMessageFromGitCommit(next)
3313 settings = extractSettingsGitLog(log)
3314 currentChange = int(settings['change'])
3315 if self.verbose:
3316 print "current change %s" % currentChange
3317
3318 if currentChange == change:
3319 if self.verbose:
3320 print "found %s" % next
3321 return next
3322
3323 if currentChange < change:
3324 earliestCommit = "^%s" % next
3325 else:
3326 latestCommit = "%s" % next
3327
3328 return ""
3329
3330 def importNewBranch(self, branch, maxChange):
3331 # make fast-import flush all changes to disk and update the refs using the checkpoint
3332 # command so that we can try to find the branch parent in the git history
3333 self.gitStream.write("checkpoint\n\n");
3334 self.gitStream.flush();
3335 branchPrefix = self.depotPaths[0] + branch + "/"
3336 range = "@1,%s" % maxChange
3337 #print "prefix" + branchPrefix
96b2d54a 3338 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3339 if len(changes) <= 0:
3340 return False
3341 firstChange = changes[0]
3342 #print "first change in branch: %s" % firstChange
3343 sourceBranch = self.knownBranches[branch]
3344 sourceDepotPath = self.depotPaths[0] + sourceBranch
3345 sourceRef = self.gitRefForBranch(sourceBranch)
3346 #print "source " + sourceBranch
3347
52a4880b 3348 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3349 #print "branch parent: %s" % branchParentChange
3350 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3351 if len(gitParent) > 0:
3352 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3353 #print "parent git commit: %s" % gitParent
3354
3355 self.importChanges(changes)
3356 return True
3357
fed23693
VA
3358 def searchParent(self, parent, branch, target):
3359 parentFound = False
c7d34884
PW
3360 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3361 "--no-merges", parent]):
fed23693
VA
3362 blob = blob.strip()
3363 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3364 parentFound = True
3365 if self.verbose:
3366 print "Found parent of %s in commit %s" % (branch, blob)
3367 break
3368 if parentFound:
3369 return blob
3370 else:
3371 return None
3372
123f6317 3373 def importChanges(self, changes, shelved=False, origin_revision=0):
e87f37ae
SH
3374 cnt = 1
3375 for change in changes:
123f6317 3376 description = p4_describe(change, shelved)
e87f37ae
SH
3377 self.updateOptionDict(description)
3378
3379 if not self.silent:
3380 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3381 sys.stdout.flush()
3382 cnt = cnt + 1
3383
3384 try:
3385 if self.detectBranches:
3386 branches = self.splitFilesIntoBranches(description)
3387 for branch in branches.keys():
3388 ## HACK --hwn
3389 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3390 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3391
3392 parent = ""
3393
3394 filesForCommit = branches[branch]
3395
3396 if self.verbose:
3397 print "branch is %s" % branch
3398
3399 self.updatedBranches.add(branch)
3400
3401 if branch not in self.createdBranches:
3402 self.createdBranches.add(branch)
3403 parent = self.knownBranches[branch]
3404 if parent == branch:
3405 parent = ""
1ca3d710
SH
3406 else:
3407 fullBranch = self.projectName + branch
3408 if fullBranch not in self.p4BranchesInGit:
3409 if not self.silent:
3410 print("\n Importing new branch %s" % fullBranch);
3411 if self.importNewBranch(branch, change - 1):
3412 parent = ""
3413 self.p4BranchesInGit.append(fullBranch)
3414 if not self.silent:
3415 print("\n Resuming with change %s" % change);
3416
3417 if self.verbose:
3418 print "parent determined through known branches: %s" % parent
e87f37ae 3419
8134f69c
SH
3420 branch = self.gitRefForBranch(branch)
3421 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3422
3423 if self.verbose:
3424 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3425
3426 if len(parent) == 0 and branch in self.initialParents:
3427 parent = self.initialParents[branch]
3428 del self.initialParents[branch]
3429
fed23693
VA
3430 blob = None
3431 if len(parent) > 0:
4f9273d2 3432 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3433 if self.verbose:
3434 print "Creating temporary branch: " + tempBranch
e63231e5 3435 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3436 self.tempBranches.append(tempBranch)
3437 self.checkpoint()
3438 blob = self.searchParent(parent, branch, tempBranch)
3439 if blob:
e63231e5 3440 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3441 else:
3442 if self.verbose:
3443 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3444 self.commit(description, filesForCommit, branch, parent)
e87f37ae 3445 else:
123f6317 3446 files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
e63231e5 3447 self.commit(description, files, self.branch,
e87f37ae 3448 self.initialParent)
47497844 3449 # only needed once, to connect to the previous commit
e87f37ae
SH
3450 self.initialParent = ""
3451 except IOError:
3452 print self.gitError.read()
3453 sys.exit(1)
3454
b9d34db9
LD
3455 def sync_origin_only(self):
3456 if self.syncWithOrigin:
3457 self.hasOrigin = originP4BranchesExist()
3458 if self.hasOrigin:
3459 if not self.silent:
3460 print 'Syncing with origin first, using "git fetch origin"'
3461 system("git fetch origin")
3462
c208a243
SH
3463 def importHeadRevision(self, revision):
3464 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3465
4e2e6ce4
PW
3466 details = {}
3467 details["user"] = "git perforce import user"
1494fcbb 3468 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3469 % (' '.join(self.depotPaths), revision))
3470 details["change"] = revision