git-p4: narrow the scope of exceptions caught when parsing an int
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
a235e85c
BC
30try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
4addad22 45verbose = False
86949eef 46
06804c76 47# Only labels/tags matching this will be imported/exported
c8942a22 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 49
1051ef00
LD
50# Grab changes in blocks of this many revisions, unless otherwise requested
51defaultBlockSize = 512
52
0ef67acd
LD
53p4_access_checked = False
54
21a50753
AK
55def p4_build_cmd(cmd):
56 """Build a suitable p4 command line.
57
58 This consolidates building and returning a p4 command line into one
59 location. It means that hooking into the environment, or other configuration
60 can be done more easily.
61 """
6de040df 62 real_cmd = ["p4"]
abcaf073
AK
63
64 user = gitConfig("git-p4.user")
65 if len(user) > 0:
6de040df 66 real_cmd += ["-u",user]
abcaf073
AK
67
68 password = gitConfig("git-p4.password")
69 if len(password) > 0:
6de040df 70 real_cmd += ["-P", password]
abcaf073
AK
71
72 port = gitConfig("git-p4.port")
73 if len(port) > 0:
6de040df 74 real_cmd += ["-p", port]
abcaf073
AK
75
76 host = gitConfig("git-p4.host")
77 if len(host) > 0:
41799aa2 78 real_cmd += ["-H", host]
abcaf073
AK
79
80 client = gitConfig("git-p4.client")
81 if len(client) > 0:
6de040df 82 real_cmd += ["-c", client]
abcaf073 83
89a6ecc5
LS
84 retries = gitConfigInt("git-p4.retries")
85 if retries is None:
86 # Perform 3 retries by default
87 retries = 3
bc233524
IK
88 if retries > 0:
89 # Provide a way to not pass this option by setting git-p4.retries to 0
90 real_cmd += ["-r", str(retries)]
6de040df
LD
91
92 if isinstance(cmd,basestring):
93 real_cmd = ' '.join(real_cmd) + ' ' + cmd
94 else:
95 real_cmd += cmd
0ef67acd
LD
96
97 # now check that we can actually talk to the server
98 global p4_access_checked
99 if not p4_access_checked:
100 p4_access_checked = True # suppress access checks in p4_check_access itself
101 p4_check_access()
102
21a50753
AK
103 return real_cmd
104
378f7be1
LD
105def git_dir(path):
106 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
107 This won't automatically add ".git" to a directory.
108 """
109 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
110 if not d or len(d) == 0:
111 return None
112 else:
113 return d
114
bbd84863
MF
115def chdir(path, is_client_path=False):
116 """Do chdir to the given path, and set the PWD environment
117 variable for use by P4. It does not look at getcwd() output.
118 Since we're not using the shell, it is necessary to set the
119 PWD environment variable explicitly.
120
121 Normally, expand the path to force it to be absolute. This
122 addresses the use of relative path names inside P4 settings,
123 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
124 as given; it looks for .p4config using PWD.
125
126 If is_client_path, the path was handed to us directly by p4,
127 and may be a symbolic link. Do not call os.getcwd() in this
128 case, because it will cause p4 to think that PWD is not inside
129 the client path.
130 """
131
132 os.chdir(path)
133 if not is_client_path:
134 path = os.getcwd()
135 os.environ['PWD'] = path
053fd0c1 136
4d25dc44
LS
137def calcDiskFree():
138 """Return free space in bytes on the disk of the given dirname."""
139 if platform.system() == 'Windows':
140 free_bytes = ctypes.c_ulonglong(0)
141 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
142 return free_bytes.value
143 else:
144 st = os.statvfs(os.getcwd())
145 return st.f_bavail * st.f_frsize
146
86dff6b6
HWN
147def die(msg):
148 if verbose:
149 raise Exception(msg)
150 else:
151 sys.stderr.write(msg + "\n")
152 sys.exit(1)
153
6de040df 154def write_pipe(c, stdin):
4addad22 155 if verbose:
6de040df 156 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 157
6de040df
LD
158 expand = isinstance(c,basestring)
159 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
160 pipe = p.stdin
161 val = pipe.write(stdin)
162 pipe.close()
163 if p.wait():
164 die('Command failed: %s' % str(c))
b016d397
HWN
165
166 return val
167
6de040df 168def p4_write_pipe(c, stdin):
d9429194 169 real_cmd = p4_build_cmd(c)
6de040df 170 return write_pipe(real_cmd, stdin)
d9429194 171
78871bf4
LD
172def read_pipe_full(c):
173 """ Read output from command. Returns a tuple
174 of the return status, stdout text and stderr
175 text.
176 """
4addad22 177 if verbose:
6de040df 178 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 179
6de040df 180 expand = isinstance(c,basestring)
1f5f3907
LS
181 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
182 (out, err) = p.communicate()
78871bf4
LD
183 return (p.returncode, out, err)
184
185def read_pipe(c, ignore_error=False):
186 """ Read output from command. Returns the output text on
187 success. On failure, terminates execution, unless
188 ignore_error is True, when it returns an empty string.
189 """
190 (retcode, out, err) = read_pipe_full(c)
191 if retcode != 0:
192 if ignore_error:
193 out = ""
194 else:
195 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 196 return out
b016d397 197
78871bf4
LD
198def read_pipe_text(c):
199 """ Read output from a command with trailing whitespace stripped.
200 On error, returns None.
201 """
202 (retcode, out, err) = read_pipe_full(c)
203 if retcode != 0:
204 return None
205 else:
206 return out.rstrip()
207
d9429194
AK
208def p4_read_pipe(c, ignore_error=False):
209 real_cmd = p4_build_cmd(c)
210 return read_pipe(real_cmd, ignore_error)
b016d397 211
bce4c5fc 212def read_pipe_lines(c):
4addad22 213 if verbose:
6de040df
LD
214 sys.stderr.write('Reading pipe: %s\n' % str(c))
215
216 expand = isinstance(c, basestring)
217 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
218 pipe = p.stdout
b016d397 219 val = pipe.readlines()
6de040df
LD
220 if pipe.close() or p.wait():
221 die('Command failed: %s' % str(c))
b016d397
HWN
222
223 return val
caace111 224
2318121b
AK
225def p4_read_pipe_lines(c):
226 """Specifically invoke p4 on the command supplied. """
155af834 227 real_cmd = p4_build_cmd(c)
2318121b
AK
228 return read_pipe_lines(real_cmd)
229
8e9497c2
GG
230def p4_has_command(cmd):
231 """Ask p4 for help on this command. If it returns an error, the
232 command does not exist in this version of p4."""
233 real_cmd = p4_build_cmd(["help", cmd])
234 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
235 stderr=subprocess.PIPE)
236 p.communicate()
237 return p.returncode == 0
238
249da4c0
PW
239def p4_has_move_command():
240 """See if the move command exists, that it supports -k, and that
241 it has not been administratively disabled. The arguments
242 must be correct, but the filenames do not have to exist. Use
243 ones with wildcards so even if they exist, it will fail."""
244
245 if not p4_has_command("move"):
246 return False
247 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
248 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
249 (out, err) = p.communicate()
250 # return code will be 1 in either case
251 if err.find("Invalid option") >= 0:
252 return False
253 if err.find("disabled") >= 0:
254 return False
255 # assume it failed because @... was invalid changelist
256 return True
257
cbff4b25 258def system(cmd, ignore_error=False):
6de040df 259 expand = isinstance(cmd,basestring)
4addad22 260 if verbose:
6de040df 261 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 262 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 263 if retcode and not ignore_error:
a235e85c 264 raise CalledProcessError(retcode, cmd)
6754a299 265
cbff4b25
LD
266 return retcode
267
bf9320f1
AK
268def p4_system(cmd):
269 """Specifically invoke p4 as the system command. """
155af834 270 real_cmd = p4_build_cmd(cmd)
6de040df 271 expand = isinstance(real_cmd, basestring)
a235e85c
BC
272 retcode = subprocess.call(real_cmd, shell=expand)
273 if retcode:
274 raise CalledProcessError(retcode, real_cmd)
6de040df 275
0ef67acd
LD
276def die_bad_access(s):
277 die("failure accessing depot: {0}".format(s.rstrip()))
278
279def p4_check_access(min_expiration=1):
280 """ Check if we can access Perforce - account still logged in
281 """
282 results = p4CmdList(["login", "-s"])
283
284 if len(results) == 0:
285 # should never get here: always get either some results, or a p4ExitCode
286 assert("could not parse response from perforce")
287
288 result = results[0]
289
290 if 'p4ExitCode' in result:
291 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
292 die_bad_access("could not run p4")
293
294 code = result.get("code")
295 if not code:
296 # we get here if we couldn't connect and there was nothing to unmarshal
297 die_bad_access("could not connect")
298
299 elif code == "stat":
300 expiry = result.get("TicketExpiration")
301 if expiry:
302 expiry = int(expiry)
303 if expiry > min_expiration:
304 # ok to carry on
305 return
306 else:
307 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
308
309 else:
310 # account without a timeout - all ok
311 return
312
313 elif code == "error":
314 data = result.get("data")
315 if data:
316 die_bad_access("p4 error: {0}".format(data))
317 else:
318 die_bad_access("unknown error")
319 else:
320 die_bad_access("unknown error code {0}".format(code))
321
7f0e5962
PW
322_p4_version_string = None
323def p4_version_string():
324 """Read the version string, showing just the last line, which
325 hopefully is the interesting version bit.
326
327 $ p4 -V
328 Perforce - The Fast Software Configuration Management System.
329 Copyright 1995-2011 Perforce Software. All rights reserved.
330 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
331 """
332 global _p4_version_string
333 if not _p4_version_string:
334 a = p4_read_pipe_lines(["-V"])
335 _p4_version_string = a[-1].rstrip()
336 return _p4_version_string
337
6de040df 338def p4_integrate(src, dest):
9d7d446a 339 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 340
8d7ec362 341def p4_sync(f, *options):
9d7d446a 342 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
343
344def p4_add(f):
9d7d446a
PW
345 # forcibly add file names with wildcards
346 if wildcard_present(f):
347 p4_system(["add", "-f", f])
348 else:
349 p4_system(["add", f])
6de040df
LD
350
351def p4_delete(f):
9d7d446a 352 p4_system(["delete", wildcard_encode(f)])
6de040df 353
a02b8bc4
RP
354def p4_edit(f, *options):
355 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
356
357def p4_revert(f):
9d7d446a 358 p4_system(["revert", wildcard_encode(f)])
6de040df 359
9d7d446a
PW
360def p4_reopen(type, f):
361 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 362
46c609e9
LD
363def p4_reopen_in_change(changelist, files):
364 cmd = ["reopen", "-c", str(changelist)] + files
365 p4_system(cmd)
366
8e9497c2
GG
367def p4_move(src, dest):
368 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
369
1051ef00 370def p4_last_change():
1997e91f 371 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
372 return int(results[0]['change'])
373
18fa13d0
PW
374def p4_describe(change):
375 """Make sure it returns a valid result by checking for
376 the presence of field "time". Return a dict of the
377 results."""
378
1997e91f 379 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
18fa13d0
PW
380 if len(ds) != 1:
381 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
382
383 d = ds[0]
384
385 if "p4ExitCode" in d:
386 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
387 str(d)))
388 if "code" in d:
389 if d["code"] == "error":
390 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
391
392 if "time" not in d:
393 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
394
395 return d
396
9cffb8c8
PW
397#
398# Canonicalize the p4 type and return a tuple of the
399# base type, plus any modifiers. See "p4 help filetypes"
400# for a list and explanation.
401#
402def split_p4_type(p4type):
403
404 p4_filetypes_historical = {
405 "ctempobj": "binary+Sw",
406 "ctext": "text+C",
407 "cxtext": "text+Cx",
408 "ktext": "text+k",
409 "kxtext": "text+kx",
410 "ltext": "text+F",
411 "tempobj": "binary+FSw",
412 "ubinary": "binary+F",
413 "uresource": "resource+F",
414 "uxbinary": "binary+Fx",
415 "xbinary": "binary+x",
416 "xltext": "text+Fx",
417 "xtempobj": "binary+Swx",
418 "xtext": "text+x",
419 "xunicode": "unicode+x",
420 "xutf16": "utf16+x",
421 }
422 if p4type in p4_filetypes_historical:
423 p4type = p4_filetypes_historical[p4type]
424 mods = ""
425 s = p4type.split("+")
426 base = s[0]
427 mods = ""
428 if len(s) > 1:
429 mods = s[1]
430 return (base, mods)
b9fc6ea9 431
60df071c
LD
432#
433# return the raw p4 type of a file (text, text+ko, etc)
434#
79467e61
PW
435def p4_type(f):
436 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
437 return results[0]['headType']
438
439#
440# Given a type base and modifier, return a regexp matching
441# the keywords that can be expanded in the file
442#
443def p4_keywords_regexp_for_type(base, type_mods):
444 if base in ("text", "unicode", "binary"):
445 kwords = None
446 if "ko" in type_mods:
447 kwords = 'Id|Header'
448 elif "k" in type_mods:
449 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
450 else:
451 return None
452 pattern = r"""
453 \$ # Starts with a dollar, followed by...
454 (%s) # one of the keywords, followed by...
6b2bf41e 455 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
456 \$ # another dollar
457 """ % kwords
458 return pattern
459 else:
460 return None
461
462#
463# Given a file, return a regexp matching the possible
464# RCS keywords that will be expanded, or None for files
465# with kw expansion turned off.
466#
467def p4_keywords_regexp_for_file(file):
468 if not os.path.exists(file):
469 return None
470 else:
471 (type_base, type_mods) = split_p4_type(p4_type(file))
472 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 473
c65b670e
CP
474def setP4ExecBit(file, mode):
475 # Reopens an already open file and changes the execute bit to match
476 # the execute bit setting in the passed in mode.
477
478 p4Type = "+x"
479
480 if not isModeExec(mode):
481 p4Type = getP4OpenedType(file)
482 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
483 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
484 if p4Type[-1] == "+":
485 p4Type = p4Type[0:-1]
486
6de040df 487 p4_reopen(p4Type, file)
c65b670e
CP
488
489def getP4OpenedType(file):
490 # Returns the perforce file type for the given file.
491
9d7d446a 492 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 493 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
494 if match:
495 return match.group(1)
496 else:
f3e5ae4f 497 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 498
06804c76
LD
499# Return the set of all p4 labels
500def getP4Labels(depotPaths):
501 labels = set()
502 if isinstance(depotPaths,basestring):
503 depotPaths = [depotPaths]
504
505 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
506 label = l['label']
507 labels.add(label)
508
509 return labels
510
511# Return the set of all git tags
512def getGitTags():
513 gitTags = set()
514 for line in read_pipe_lines(["git", "tag"]):
515 tag = line.strip()
516 gitTags.add(tag)
517 return gitTags
518
b43b0a3c
CP
519def diffTreePattern():
520 # This is a simple generator for the diff tree regex pattern. This could be
521 # a class variable if this and parseDiffTreeEntry were a part of a class.
522 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
523 while True:
524 yield pattern
525
526def parseDiffTreeEntry(entry):
527 """Parses a single diff tree entry into its component elements.
528
529 See git-diff-tree(1) manpage for details about the format of the diff
530 output. This method returns a dictionary with the following elements:
531
532 src_mode - The mode of the source file
533 dst_mode - The mode of the destination file
534 src_sha1 - The sha1 for the source file
535 dst_sha1 - The sha1 fr the destination file
536 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
537 status_score - The score for the status (applicable for 'C' and 'R'
538 statuses). This is None if there is no score.
539 src - The path for the source file.
540 dst - The path for the destination file. This is only present for
541 copy or renames. If it is not present, this is None.
542
543 If the pattern is not matched, None is returned."""
544
545 match = diffTreePattern().next().match(entry)
546 if match:
547 return {
548 'src_mode': match.group(1),
549 'dst_mode': match.group(2),
550 'src_sha1': match.group(3),
551 'dst_sha1': match.group(4),
552 'status': match.group(5),
553 'status_score': match.group(6),
554 'src': match.group(7),
555 'dst': match.group(10)
556 }
557 return None
558
c65b670e
CP
559def isModeExec(mode):
560 # Returns True if the given git mode represents an executable file,
561 # otherwise False.
562 return mode[-3:] == "755"
563
55bb3e36
LD
564class P4Exception(Exception):
565 """ Base class for exceptions from the p4 client """
566 def __init__(self, exit_code):
567 self.p4ExitCode = exit_code
568
569class P4ServerException(P4Exception):
570 """ Base class for exceptions where we get some kind of marshalled up result from the server """
571 def __init__(self, exit_code, p4_result):
572 super(P4ServerException, self).__init__(exit_code)
573 self.p4_result = p4_result
574 self.code = p4_result[0]['code']
575 self.data = p4_result[0]['data']
576
577class P4RequestSizeException(P4ServerException):
578 """ One of the maxresults or maxscanrows errors """
579 def __init__(self, exit_code, p4_result, limit):
580 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
581 self.limit = limit
582
c65b670e
CP
583def isModeExecChanged(src_mode, dst_mode):
584 return isModeExec(src_mode) != isModeExec(dst_mode)
585
55bb3e36
LD
586def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
587 errors_as_exceptions=False):
6de040df
LD
588
589 if isinstance(cmd,basestring):
590 cmd = "-G " + cmd
591 expand = True
592 else:
593 cmd = ["-G"] + cmd
594 expand = False
595
596 cmd = p4_build_cmd(cmd)
6a49f8e2 597 if verbose:
6de040df 598 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
599
600 # Use a temporary file to avoid deadlocks without
601 # subprocess.communicate(), which would put another copy
602 # of stdout into memory.
603 stdin_file = None
604 if stdin is not None:
605 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
606 if isinstance(stdin,basestring):
607 stdin_file.write(stdin)
608 else:
609 for i in stdin:
610 stdin_file.write(i + '\n')
9f90c733
SL
611 stdin_file.flush()
612 stdin_file.seek(0)
613
6de040df
LD
614 p4 = subprocess.Popen(cmd,
615 shell=expand,
9f90c733
SL
616 stdin=stdin_file,
617 stdout=subprocess.PIPE)
86949eef
SH
618
619 result = []
620 try:
621 while True:
9f90c733 622 entry = marshal.load(p4.stdout)
1997e91f
MT
623 if skip_info:
624 if 'code' in entry and entry['code'] == 'info':
625 continue
c3f6163b
AG
626 if cb is not None:
627 cb(entry)
628 else:
629 result.append(entry)
86949eef
SH
630 except EOFError:
631 pass
9f90c733
SL
632 exitCode = p4.wait()
633 if exitCode != 0:
55bb3e36
LD
634 if errors_as_exceptions:
635 if len(result) > 0:
636 data = result[0].get('data')
637 if data:
638 m = re.search('Too many rows scanned \(over (\d+)\)', data)
639 if not m:
640 m = re.search('Request too large \(over (\d+)\)', data)
641
642 if m:
643 limit = int(m.group(1))
644 raise P4RequestSizeException(exitCode, result, limit)
645
646 raise P4ServerException(exitCode, result)
647 else:
648 raise P4Exception(exitCode)
649 else:
650 entry = {}
651 entry["p4ExitCode"] = exitCode
652 result.append(entry)
86949eef
SH
653
654 return result
655
656def p4Cmd(cmd):
657 list = p4CmdList(cmd)
658 result = {}
659 for entry in list:
660 result.update(entry)
661 return result;
662
cb2c9db5
SH
663def p4Where(depotPath):
664 if not depotPath.endswith("/"):
665 depotPath += "/"
cd884106
VA
666 depotPathLong = depotPath + "..."
667 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
668 output = None
669 for entry in outputList:
75bc9573 670 if "depotFile" in entry:
cd884106
VA
671 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
672 # The base path always ends with "/...".
673 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
674 output = entry
675 break
676 elif "data" in entry:
677 data = entry.get("data")
678 space = data.find(" ")
679 if data[:space] == depotPath:
680 output = entry
681 break
7f705dc3
TAL
682 if output == None:
683 return ""
dc524036
SH
684 if output["code"] == "error":
685 return ""
cb2c9db5
SH
686 clientPath = ""
687 if "path" in output:
688 clientPath = output.get("path")
689 elif "data" in output:
690 data = output.get("data")
691 lastSpace = data.rfind(" ")
692 clientPath = data[lastSpace + 1:]
693
694 if clientPath.endswith("..."):
695 clientPath = clientPath[:-3]
696 return clientPath
697
86949eef 698def currentGitBranch():
eff45110 699 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 700
4f5cf76a 701def isValidGitDir(path):
378f7be1 702 return git_dir(path) != None
4f5cf76a 703
463e8af6 704def parseRevision(ref):
b25b2065 705 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 706
28755dba
PW
707def branchExists(ref):
708 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
709 ignore_error=True)
710 return len(rev) > 0
711
6ae8de88
SH
712def extractLogMessageFromGitCommit(commit):
713 logMessage = ""
b016d397
HWN
714
715 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 716 foundTitle = False
b016d397 717 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
718 if not foundTitle:
719 if len(log) == 1:
1c094184 720 foundTitle = True
6ae8de88
SH
721 continue
722
723 logMessage += log
724 return logMessage
725
bb6e09b2 726def extractSettingsGitLog(log):
6ae8de88
SH
727 values = {}
728 for line in log.split("\n"):
729 line = line.strip()
6326aa58
HWN
730 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
731 if not m:
732 continue
733
734 assignments = m.group(1).split (':')
735 for a in assignments:
736 vals = a.split ('=')
737 key = vals[0].strip()
738 val = ('='.join (vals[1:])).strip()
739 if val.endswith ('\"') and val.startswith('"'):
740 val = val[1:-1]
741
742 values[key] = val
743
845b42cb
SH
744 paths = values.get("depot-paths")
745 if not paths:
746 paths = values.get("depot-path")
a3fdd579
SH
747 if paths:
748 values['depot-paths'] = paths.split(',')
bb6e09b2 749 return values
6ae8de88 750
8136a639 751def gitBranchExists(branch):
bb6e09b2
HWN
752 proc = subprocess.Popen(["git", "rev-parse", branch],
753 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 754 return proc.wait() == 0;
8136a639 755
36bd8446 756_gitConfig = {}
b345d6c3 757
692e1796 758def gitConfig(key, typeSpecifier=None):
36bd8446 759 if not _gitConfig.has_key(key):
692e1796
LS
760 cmd = [ "git", "config" ]
761 if typeSpecifier:
762 cmd += [ typeSpecifier ]
763 cmd += [ key ]
b345d6c3
PW
764 s = read_pipe(cmd, ignore_error=True)
765 _gitConfig[key] = s.strip()
36bd8446 766 return _gitConfig[key]
01265103 767
0d609032
PW
768def gitConfigBool(key):
769 """Return a bool, using git config --bool. It is True only if the
770 variable is set to true, and False if set to false or not present
771 in the config."""
772
36bd8446 773 if not _gitConfig.has_key(key):
692e1796 774 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 775 return _gitConfig[key]
01265103 776
cb1dafdf
LS
777def gitConfigInt(key):
778 if not _gitConfig.has_key(key):
779 cmd = [ "git", "config", "--int", key ]
0d609032
PW
780 s = read_pipe(cmd, ignore_error=True)
781 v = s.strip()
cb1dafdf
LS
782 try:
783 _gitConfig[key] = int(gitConfig(key, '--int'))
784 except ValueError:
785 _gitConfig[key] = None
36bd8446 786 return _gitConfig[key]
01265103 787
7199cf13
VA
788def gitConfigList(key):
789 if not _gitConfig.has_key(key):
2abba301 790 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 791 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
792 if _gitConfig[key] == ['']:
793 _gitConfig[key] = []
7199cf13
VA
794 return _gitConfig[key]
795
2c8037ed
PW
796def p4BranchesInGit(branchesAreInRemotes=True):
797 """Find all the branches whose names start with "p4/", looking
798 in remotes or heads as specified by the argument. Return
799 a dictionary of { branch: revision } for each one found.
800 The branch names are the short names, without any
801 "p4/" prefix."""
802
062410bb
SH
803 branches = {}
804
805 cmdline = "git rev-parse --symbolic "
806 if branchesAreInRemotes:
2c8037ed 807 cmdline += "--remotes"
062410bb 808 else:
2c8037ed 809 cmdline += "--branches"
062410bb
SH
810
811 for line in read_pipe_lines(cmdline):
812 line = line.strip()
813
2c8037ed
PW
814 # only import to p4/
815 if not line.startswith('p4/'):
816 continue
817 # special symbolic ref to p4/master
818 if line == "p4/HEAD":
062410bb 819 continue
062410bb 820
2c8037ed
PW
821 # strip off p4/ prefix
822 branch = line[len("p4/"):]
062410bb
SH
823
824 branches[branch] = parseRevision(line)
2c8037ed 825
062410bb
SH
826 return branches
827
5a8e84cd
PW
828def branch_exists(branch):
829 """Make sure that the given ref name really exists."""
830
831 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
832 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
833 out, _ = p.communicate()
834 if p.returncode:
835 return False
836 # expect exactly one line of output: the branch name
837 return out.rstrip() == branch
838
9ceab363 839def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
840 branches = p4BranchesInGit()
841 # map from depot-path to branch name
842 branchByDepotPath = {}
843 for branch in branches.keys():
844 tip = branches[branch]
845 log = extractLogMessageFromGitCommit(tip)
846 settings = extractSettingsGitLog(log)
847 if settings.has_key("depot-paths"):
848 paths = ",".join(settings["depot-paths"])
849 branchByDepotPath[paths] = "remotes/p4/" + branch
850
27d2d811 851 settings = None
27d2d811
SH
852 parent = 0
853 while parent < 65535:
9ceab363 854 commit = head + "~%s" % parent
27d2d811
SH
855 log = extractLogMessageFromGitCommit(commit)
856 settings = extractSettingsGitLog(log)
86506fe5
SH
857 if settings.has_key("depot-paths"):
858 paths = ",".join(settings["depot-paths"])
859 if branchByDepotPath.has_key(paths):
860 return [branchByDepotPath[paths], settings]
27d2d811 861
86506fe5 862 parent = parent + 1
27d2d811 863
86506fe5 864 return ["", settings]
27d2d811 865
5ca44617
SH
866def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
867 if not silent:
868 print ("Creating/updating branch(es) in %s based on origin branch(es)"
869 % localRefPrefix)
870
871 originPrefix = "origin/p4/"
872
873 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
874 line = line.strip()
875 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
876 continue
877
878 headName = line[len(originPrefix):]
879 remoteHead = localRefPrefix + headName
880 originHead = line
881
882 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
883 if (not original.has_key('depot-paths')
884 or not original.has_key('change')):
885 continue
886
887 update = False
888 if not gitBranchExists(remoteHead):
889 if verbose:
890 print "creating %s" % remoteHead
891 update = True
892 else:
893 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
894 if settings.has_key('change') > 0:
895 if settings['depot-paths'] == original['depot-paths']:
896 originP4Change = int(original['change'])
897 p4Change = int(settings['change'])
898 if originP4Change > p4Change:
899 print ("%s (%s) is newer than %s (%s). "
900 "Updating p4 branch from origin."
901 % (originHead, originP4Change,
902 remoteHead, p4Change))
903 update = True
904 else:
905 print ("Ignoring: %s was imported from %s while "
906 "%s was imported from %s"
907 % (originHead, ','.join(original['depot-paths']),
908 remoteHead, ','.join(settings['depot-paths'])))
909
910 if update:
911 system("git update-ref %s %s" % (remoteHead, originHead))
912
913def originP4BranchesExist():
914 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
915
1051ef00
LD
916
917def p4ParseNumericChangeRange(parts):
918 changeStart = int(parts[0][1:])
919 if parts[1] == '#head':
920 changeEnd = p4_last_change()
921 else:
922 changeEnd = int(parts[1])
923
924 return (changeStart, changeEnd)
925
926def chooseBlockSize(blockSize):
927 if blockSize:
928 return blockSize
929 else:
930 return defaultBlockSize
931
932def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 933 assert depotPaths
96b2d54a 934
1051ef00
LD
935 # Parse the change range into start and end. Try to find integer
936 # revision ranges as these can be broken up into blocks to avoid
937 # hitting server-side limits (maxrows, maxscanresults). But if
938 # that doesn't work, fall back to using the raw revision specifier
939 # strings, without using block mode.
940
96b2d54a 941 if changeRange is None or changeRange == '':
1051ef00
LD
942 changeStart = 1
943 changeEnd = p4_last_change()
944 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
945 else:
946 parts = changeRange.split(',')
947 assert len(parts) == 2
1051ef00
LD
948 try:
949 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
950 block_size = chooseBlockSize(requestedBlockSize)
8fa0abf8 951 except ValueError:
1051ef00
LD
952 changeStart = parts[0][1:]
953 changeEnd = parts[1]
954 if requestedBlockSize:
955 die("cannot use --changes-block-size with non-numeric revisions")
956 block_size = None
4f6432d8 957
9943e5b9 958 changes = set()
96b2d54a 959
1f90a648
SH
960 # Retrieve changes a block at a time, to prevent running
961 # into a MaxResults/MaxScanRows error from the server.
1051ef00 962
1f90a648
SH
963 while True:
964 cmd = ['changes']
1051ef00 965
1f90a648
SH
966 if block_size:
967 end = min(changeEnd, changeStart + block_size)
968 revisionRange = "%d,%d" % (changeStart, end)
969 else:
970 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 971
1f90a648 972 for p in depotPaths:
1051ef00
LD
973 cmd += ["%s...@%s" % (p, revisionRange)]
974
1f90a648 975 # Insert changes in chronological order
b596b3b9
MT
976 for entry in reversed(p4CmdList(cmd)):
977 if entry.has_key('p4ExitCode'):
978 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
979 if not entry.has_key('change'):
980 continue
981 changes.add(int(entry['change']))
1051ef00 982
1f90a648
SH
983 if not block_size:
984 break
1051ef00 985
1f90a648
SH
986 if end >= changeEnd:
987 break
1051ef00 988
1f90a648 989 changeStart = end + 1
4f6432d8 990
1f90a648
SH
991 changes = sorted(changes)
992 return changes
4f6432d8 993
d53de8b9
TAL
994def p4PathStartsWith(path, prefix):
995 # This method tries to remedy a potential mixed-case issue:
996 #
997 # If UserA adds //depot/DirA/file1
998 # and UserB adds //depot/dira/file2
999 #
1000 # we may or may not have a problem. If you have core.ignorecase=true,
1001 # we treat DirA and dira as the same directory
0d609032 1002 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
1003 return path.lower().startswith(prefix.lower())
1004 return path.startswith(prefix)
1005
543987bd
PW
1006def getClientSpec():
1007 """Look at the p4 client spec, create a View() object that contains
1008 all the mappings, and return it."""
1009
1010 specList = p4CmdList("client -o")
1011 if len(specList) != 1:
1012 die('Output from "client -o" is %d lines, expecting 1' %
1013 len(specList))
1014
1015 # dictionary of all client parameters
1016 entry = specList[0]
1017
9d57c4a6
KS
1018 # the //client/ name
1019 client_name = entry["Client"]
1020
543987bd
PW
1021 # just the keys that start with "View"
1022 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1023
1024 # hold this new View
9d57c4a6 1025 view = View(client_name)
543987bd
PW
1026
1027 # append the lines, in order, to the view
1028 for view_num in range(len(view_keys)):
1029 k = "View%d" % view_num
1030 if k not in view_keys:
1031 die("Expected view key %s missing" % k)
1032 view.append(entry[k])
1033
1034 return view
1035
1036def getClientRoot():
1037 """Grab the client directory."""
1038
1039 output = p4CmdList("client -o")
1040 if len(output) != 1:
1041 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1042
1043 entry = output[0]
1044 if "Root" not in entry:
1045 die('Client has no "Root"')
1046
1047 return entry["Root"]
1048
9d7d446a
PW
1049#
1050# P4 wildcards are not allowed in filenames. P4 complains
1051# if you simply add them, but you can force it with "-f", in
1052# which case it translates them into %xx encoding internally.
1053#
1054def wildcard_decode(path):
1055 # Search for and fix just these four characters. Do % last so
1056 # that fixing it does not inadvertently create new %-escapes.
1057 # Cannot have * in a filename in windows; untested as to
1058 # what p4 would do in such a case.
1059 if not platform.system() == "Windows":
1060 path = path.replace("%2A", "*")
1061 path = path.replace("%23", "#") \
1062 .replace("%40", "@") \
1063 .replace("%25", "%")
1064 return path
1065
1066def wildcard_encode(path):
1067 # do % first to avoid double-encoding the %s introduced here
1068 path = path.replace("%", "%25") \
1069 .replace("*", "%2A") \
1070 .replace("#", "%23") \
1071 .replace("@", "%40")
1072 return path
1073
1074def wildcard_present(path):
598354c0
BC
1075 m = re.search("[*#@%]", path)
1076 return m is not None
9d7d446a 1077
a5db4b12
LS
1078class LargeFileSystem(object):
1079 """Base class for large file system support."""
1080
1081 def __init__(self, writeToGitStream):
1082 self.largeFiles = set()
1083 self.writeToGitStream = writeToGitStream
1084
1085 def generatePointer(self, cloneDestination, contentFile):
1086 """Return the content of a pointer file that is stored in Git instead of
1087 the actual content."""
1088 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1089
1090 def pushFile(self, localLargeFile):
1091 """Push the actual content which is not stored in the Git repository to
1092 a server."""
1093 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1094
1095 def hasLargeFileExtension(self, relPath):
1096 return reduce(
1097 lambda a, b: a or b,
1098 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1099 False
1100 )
1101
1102 def generateTempFile(self, contents):
1103 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1104 for d in contents:
1105 contentFile.write(d)
1106 contentFile.close()
1107 return contentFile.name
1108
1109 def exceedsLargeFileThreshold(self, relPath, contents):
1110 if gitConfigInt('git-p4.largeFileThreshold'):
1111 contentsSize = sum(len(d) for d in contents)
1112 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1113 return True
1114 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1115 contentsSize = sum(len(d) for d in contents)
1116 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1117 return False
1118 contentTempFile = self.generateTempFile(contents)
1119 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1120 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1121 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1122 zf.close()
1123 compressedContentsSize = zf.infolist()[0].compress_size
1124 os.remove(contentTempFile)
1125 os.remove(compressedContentFile.name)
1126 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1127 return True
1128 return False
1129
1130 def addLargeFile(self, relPath):
1131 self.largeFiles.add(relPath)
1132
1133 def removeLargeFile(self, relPath):
1134 self.largeFiles.remove(relPath)
1135
1136 def isLargeFile(self, relPath):
1137 return relPath in self.largeFiles
1138
1139 def processContent(self, git_mode, relPath, contents):
1140 """Processes the content of git fast import. This method decides if a
1141 file is stored in the large file system and handles all necessary
1142 steps."""
1143 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1144 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1145 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1146 if pointer_git_mode:
1147 git_mode = pointer_git_mode
1148 if localLargeFile:
1149 # Move temp file to final location in large file system
1150 largeFileDir = os.path.dirname(localLargeFile)
1151 if not os.path.isdir(largeFileDir):
1152 os.makedirs(largeFileDir)
1153 shutil.move(contentTempFile, localLargeFile)
1154 self.addLargeFile(relPath)
1155 if gitConfigBool('git-p4.largeFilePush'):
1156 self.pushFile(localLargeFile)
1157 if verbose:
1158 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1159 return (git_mode, contents)
1160
1161class MockLFS(LargeFileSystem):
1162 """Mock large file system for testing."""
1163
1164 def generatePointer(self, contentFile):
1165 """The pointer content is the original content prefixed with "pointer-".
1166 The local filename of the large file storage is derived from the file content.
1167 """
1168 with open(contentFile, 'r') as f:
1169 content = next(f)
1170 gitMode = '100644'
1171 pointerContents = 'pointer-' + content
1172 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1173 return (gitMode, pointerContents, localLargeFile)
1174
1175 def pushFile(self, localLargeFile):
1176 """The remote filename of the large file storage is the same as the local
1177 one but in a different directory.
1178 """
1179 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1180 if not os.path.exists(remotePath):
1181 os.makedirs(remotePath)
1182 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1183
b47d807d
LS
1184class GitLFS(LargeFileSystem):
1185 """Git LFS as backend for the git-p4 large file system.
1186 See https://git-lfs.github.com/ for details."""
1187
1188 def __init__(self, *args):
1189 LargeFileSystem.__init__(self, *args)
1190 self.baseGitAttributes = []
1191
1192 def generatePointer(self, contentFile):
1193 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1194 mode and content which is stored in the Git repository instead of
1195 the actual content. Return also the new location of the actual
1196 content.
1197 """
d5eb3cf5
LS
1198 if os.path.getsize(contentFile) == 0:
1199 return (None, '', None)
1200
b47d807d
LS
1201 pointerProcess = subprocess.Popen(
1202 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1203 stdout=subprocess.PIPE
1204 )
1205 pointerFile = pointerProcess.stdout.read()
1206 if pointerProcess.wait():
1207 os.remove(contentFile)
1208 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1209
1210 # Git LFS removed the preamble in the output of the 'pointer' command
1211 # starting from version 1.2.0. Check for the preamble here to support
1212 # earlier versions.
1213 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1214 if pointerFile.startswith('Git LFS pointer for'):
1215 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1216
1217 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1218 localLargeFile = os.path.join(
1219 os.getcwd(),
1220 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1221 oid,
1222 )
1223 # LFS Spec states that pointer files should not have the executable bit set.
1224 gitMode = '100644'
82f2567e 1225 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1226
1227 def pushFile(self, localLargeFile):
1228 uploadProcess = subprocess.Popen(
1229 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1230 )
1231 if uploadProcess.wait():
1232 die('git-lfs push command failed. Did you define a remote?')
1233
1234 def generateGitAttributes(self):
1235 return (
1236 self.baseGitAttributes +
1237 [
1238 '\n',
1239 '#\n',
1240 '# Git LFS (see https://git-lfs.github.com/)\n',
1241 '#\n',
1242 ] +
862f9312 1243 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1244 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1245 ] +
862f9312 1246 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1247 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1248 ]
1249 )
1250
1251 def addLargeFile(self, relPath):
1252 LargeFileSystem.addLargeFile(self, relPath)
1253 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1254
1255 def removeLargeFile(self, relPath):
1256 LargeFileSystem.removeLargeFile(self, relPath)
1257 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1258
1259 def processContent(self, git_mode, relPath, contents):
1260 if relPath == '.gitattributes':
1261 self.baseGitAttributes = contents
1262 return (git_mode, self.generateGitAttributes())
1263 else:
1264 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1265
b984733c
SH
1266class Command:
1267 def __init__(self):
1268 self.usage = "usage: %prog [options]"
8910ac0e 1269 self.needsGit = True
6a10b6aa 1270 self.verbose = False
b984733c 1271
8cf422db
LD
1272 # This is required for the "append" cloneExclude action
1273 def ensure_value(self, attr, value):
1274 if not hasattr(self, attr) or getattr(self, attr) is None:
1275 setattr(self, attr, value)
1276 return getattr(self, attr)
1277
3ea2cfd4
LD
1278class P4UserMap:
1279 def __init__(self):
1280 self.userMapFromPerforceServer = False
affb474f
LD
1281 self.myP4UserId = None
1282
1283 def p4UserId(self):
1284 if self.myP4UserId:
1285 return self.myP4UserId
1286
1287 results = p4CmdList("user -o")
1288 for r in results:
1289 if r.has_key('User'):
1290 self.myP4UserId = r['User']
1291 return r['User']
1292 die("Could not find your p4 user id")
1293
1294 def p4UserIsMe(self, p4User):
1295 # return True if the given p4 user is actually me
1296 me = self.p4UserId()
1297 if not p4User or p4User != me:
1298 return False
1299 else:
1300 return True
3ea2cfd4
LD
1301
1302 def getUserCacheFilename(self):
1303 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1304 return home + "/.gitp4-usercache.txt"
1305
1306 def getUserMapFromPerforceServer(self):
1307 if self.userMapFromPerforceServer:
1308 return
1309 self.users = {}
1310 self.emails = {}
1311
1312 for output in p4CmdList("users"):
1313 if not output.has_key("User"):
1314 continue
1315 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1316 self.emails[output["Email"]] = output["User"]
1317
10d08a14
LS
1318 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1319 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1320 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1321 if mapUser and len(mapUser[0]) == 3:
1322 user = mapUser[0][0]
1323 fullname = mapUser[0][1]
1324 email = mapUser[0][2]
1325 self.users[user] = fullname + " <" + email + ">"
1326 self.emails[email] = user
3ea2cfd4
LD
1327
1328 s = ''
1329 for (key, val) in self.users.items():
1330 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1331
1332 open(self.getUserCacheFilename(), "wb").write(s)
1333 self.userMapFromPerforceServer = True
1334
1335 def loadUserMapFromCache(self):
1336 self.users = {}
1337 self.userMapFromPerforceServer = False
1338 try:
1339 cache = open(self.getUserCacheFilename(), "rb")
1340 lines = cache.readlines()
1341 cache.close()
1342 for line in lines:
1343 entry = line.strip().split("\t")
1344 self.users[entry[0]] = entry[1]
1345 except IOError:
1346 self.getUserMapFromPerforceServer()
1347
b984733c 1348class P4Debug(Command):
86949eef 1349 def __init__(self):
6ae8de88 1350 Command.__init__(self)
6a10b6aa 1351 self.options = []
c8c39116 1352 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1353 self.needsGit = False
86949eef
SH
1354
1355 def run(self, args):
b1ce9447 1356 j = 0
6de040df 1357 for output in p4CmdList(args):
b1ce9447
HWN
1358 print 'Element: %d' % j
1359 j += 1
86949eef 1360 print output
b984733c 1361 return True
86949eef 1362
5834684d
SH
1363class P4RollBack(Command):
1364 def __init__(self):
1365 Command.__init__(self)
1366 self.options = [
0c66a783 1367 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1368 ]
1369 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1370 self.rollbackLocalBranches = False
5834684d
SH
1371
1372 def run(self, args):
1373 if len(args) != 1:
1374 return False
1375 maxChange = int(args[0])
0c66a783 1376
ad192f28 1377 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1378 die("Problems executing p4");
1379
0c66a783
SH
1380 if self.rollbackLocalBranches:
1381 refPrefix = "refs/heads/"
b016d397 1382 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1383 else:
1384 refPrefix = "refs/remotes/"
b016d397 1385 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1386
1387 for line in lines:
1388 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1389 line = line.strip()
1390 ref = refPrefix + line
5834684d 1391 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1392 settings = extractSettingsGitLog(log)
1393
1394 depotPaths = settings['depot-paths']
1395 change = settings['change']
1396
5834684d 1397 changed = False
52102d47 1398
6326aa58
HWN
1399 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1400 for p in depotPaths]))) == 0:
52102d47
SH
1401 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1402 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1403 continue
1404
bb6e09b2 1405 while change and int(change) > maxChange:
5834684d 1406 changed = True
52102d47
SH
1407 if self.verbose:
1408 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1409 system("git update-ref %s \"%s^\"" % (ref, ref))
1410 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1411 settings = extractSettingsGitLog(log)
1412
1413
1414 depotPaths = settings['depot-paths']
1415 change = settings['change']
5834684d
SH
1416
1417 if changed:
52102d47 1418 print "%s rewound to %s" % (ref, change)
5834684d
SH
1419
1420 return True
1421
3ea2cfd4 1422class P4Submit(Command, P4UserMap):
6bbfd137
PW
1423
1424 conflict_behavior_choices = ("ask", "skip", "quit")
1425
4f5cf76a 1426 def __init__(self):
b984733c 1427 Command.__init__(self)
3ea2cfd4 1428 P4UserMap.__init__(self)
4f5cf76a 1429 self.options = [
4f5cf76a 1430 optparse.make_option("--origin", dest="origin"),
ae901090 1431 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1432 # preserve the user, requires relevant p4 permissions
1433 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1434 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1435 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1436 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1437 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1438 choices=self.conflict_behavior_choices),
1439 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1440 optparse.make_option("--shelve", dest="shelve", action="store_true",
1441 help="Shelve instead of submit. Shelved files are reverted, "
1442 "restoring the workspace to the state before the shelve"),
8cf422db 1443 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1444 metavar="CHANGELIST",
8cf422db 1445 help="update an existing shelved changelist, implies --shelve, "
f55b87c1
RM
1446 "repeat in-order for multiple shelved changelists"),
1447 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1448 help="submit only the specified commit(s), one commit or xxx..xxx"),
1449 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1450 help="Disable rebase after submit is completed. Can be useful if you "
b9d34db9
LD
1451 "work from a local git branch that is not master"),
1452 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1453 help="Skip Perforce sync of p4/master after submit or shelve"),
4f5cf76a
SH
1454 ]
1455 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1456 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1457 self.origin = ""
ae901090 1458 self.detectRenames = False
0d609032 1459 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1460 self.dry_run = False
b34fa577 1461 self.shelve = False
8cf422db 1462 self.update_shelve = list()
f55b87c1 1463 self.commit = ""
3b3477ea 1464 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
b9d34db9 1465 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
728b7ad8 1466 self.prepare_p4_only = False
6bbfd137 1467 self.conflict_behavior = None
f7baba8b 1468 self.isWindows = (platform.system() == "Windows")
06804c76 1469 self.exportLabels = False
249da4c0 1470 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1471 self.branch = None
4f5cf76a 1472
a5db4b12
LS
1473 if gitConfig('git-p4.largeFileSystem'):
1474 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1475
4f5cf76a
SH
1476 def check(self):
1477 if len(p4CmdList("opened ...")) > 0:
1478 die("You have files opened with perforce! Close them before starting the sync.")
1479
f19cb0a0
PW
1480 def separate_jobs_from_description(self, message):
1481 """Extract and return a possible Jobs field in the commit
1482 message. It goes into a separate section in the p4 change
1483 specification.
1484
1485 A jobs line starts with "Jobs:" and looks like a new field
1486 in a form. Values are white-space separated on the same
1487 line or on following lines that start with a tab.
1488
1489 This does not parse and extract the full git commit message
1490 like a p4 form. It just sees the Jobs: line as a marker
1491 to pass everything from then on directly into the p4 form,
1492 but outside the description section.
1493
1494 Return a tuple (stripped log message, jobs string)."""
1495
1496 m = re.search(r'^Jobs:', message, re.MULTILINE)
1497 if m is None:
1498 return (message, None)
1499
1500 jobtext = message[m.start():]
1501 stripped_message = message[:m.start()].rstrip()
1502 return (stripped_message, jobtext)
1503
1504 def prepareLogMessage(self, template, message, jobs):
1505 """Edits the template returned from "p4 change -o" to insert
1506 the message in the Description field, and the jobs text in
1507 the Jobs field."""
4f5cf76a
SH
1508 result = ""
1509
edae1e2f
SH
1510 inDescriptionSection = False
1511
4f5cf76a
SH
1512 for line in template.split("\n"):
1513 if line.startswith("#"):
1514 result += line + "\n"
1515 continue
1516
edae1e2f 1517 if inDescriptionSection:
c9dbab04 1518 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1519 inDescriptionSection = False
f19cb0a0
PW
1520 # insert Jobs section
1521 if jobs:
1522 result += jobs + "\n"
edae1e2f
SH
1523 else:
1524 continue
1525 else:
1526 if line.startswith("Description:"):
1527 inDescriptionSection = True
1528 line += "\n"
1529 for messageLine in message.split("\n"):
1530 line += "\t" + messageLine + "\n"
1531
1532 result += line + "\n"
4f5cf76a
SH
1533
1534 return result
1535
60df071c
LD
1536 def patchRCSKeywords(self, file, pattern):
1537 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1538 (handle, outFileName) = tempfile.mkstemp(dir='.')
1539 try:
1540 outFile = os.fdopen(handle, "w+")
1541 inFile = open(file, "r")
1542 regexp = re.compile(pattern, re.VERBOSE)
1543 for line in inFile.readlines():
1544 line = regexp.sub(r'$\1$', line)
1545 outFile.write(line)
1546 inFile.close()
1547 outFile.close()
1548 # Forcibly overwrite the original file
1549 os.unlink(file)
1550 shutil.move(outFileName, file)
1551 except:
1552 # cleanup our temporary file
1553 os.unlink(outFileName)
1554 print "Failed to strip RCS keywords in %s" % file
1555 raise
1556
1557 print "Patched up RCS keywords in %s" % file
1558
3ea2cfd4
LD
1559 def p4UserForCommit(self,id):
1560 # Return the tuple (perforce user,git email) for a given git commit id
1561 self.getUserMapFromPerforceServer()
9bf28855
PW
1562 gitEmail = read_pipe(["git", "log", "--max-count=1",
1563 "--format=%ae", id])
3ea2cfd4
LD
1564 gitEmail = gitEmail.strip()
1565 if not self.emails.has_key(gitEmail):
1566 return (None,gitEmail)
1567 else:
1568 return (self.emails[gitEmail],gitEmail)
1569
1570 def checkValidP4Users(self,commits):
1571 # check if any git authors cannot be mapped to p4 users
1572 for id in commits:
1573 (user,email) = self.p4UserForCommit(id)
1574 if not user:
1575 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1576 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1577 print "%s" % msg
1578 else:
1579 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1580
1581 def lastP4Changelist(self):
1582 # Get back the last changelist number submitted in this client spec. This
1583 # then gets used to patch up the username in the change. If the same
1584 # client spec is being used by multiple processes then this might go
1585 # wrong.
1586 results = p4CmdList("client -o") # find the current client
1587 client = None
1588 for r in results:
1589 if r.has_key('Client'):
1590 client = r['Client']
1591 break
1592 if not client:
1593 die("could not get client spec")
6de040df 1594 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1595 for r in results:
1596 if r.has_key('change'):
1597 return r['change']
1598 die("Could not get changelist number for last submit - cannot patch up user details")
1599
1600 def modifyChangelistUser(self, changelist, newUser):
1601 # fixup the user field of a changelist after it has been submitted.
1602 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1603 if len(changes) != 1:
1604 die("Bad output from p4 change modifying %s to user %s" %
1605 (changelist, newUser))
1606
1607 c = changes[0]
1608 if c['User'] == newUser: return # nothing to do
1609 c['User'] = newUser
1610 input = marshal.dumps(c)
1611
3ea2cfd4
LD
1612 result = p4CmdList("change -f -i", stdin=input)
1613 for r in result:
1614 if r.has_key('code'):
1615 if r['code'] == 'error':
1616 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1617 if r.has_key('data'):
1618 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1619 return
1620 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1621
1622 def canChangeChangelists(self):
1623 # check to see if we have p4 admin or super-user permissions, either of
1624 # which are required to modify changelists.
52a4880b 1625 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1626 for r in results:
1627 if r.has_key('perm'):
1628 if r['perm'] == 'admin':
1629 return 1
1630 if r['perm'] == 'super':
1631 return 1
1632 return 0
1633
46c609e9 1634 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1635 """Run "p4 change -o" to grab a change specification template.
1636 This does not use "p4 -G", as it is nice to keep the submission
1637 template in original order, since a human might edit it.
1638
1639 Remove lines in the Files section that show changes to files
1640 outside the depot path we're committing into."""
1641
cbc69242
SH
1642 [upstream, settings] = findUpstreamBranchPoint()
1643
b596b3b9
MT
1644 template = """\
1645# A Perforce Change Specification.
1646#
1647# Change: The change number. 'new' on a new changelist.
1648# Date: The date this specification was last modified.
1649# Client: The client on which the changelist was created. Read-only.
1650# User: The user who created the changelist.
1651# Status: Either 'pending' or 'submitted'. Read-only.
1652# Type: Either 'public' or 'restricted'. Default is 'public'.
1653# Description: Comments about the changelist. Required.
1654# Jobs: What opened jobs are to be closed by this changelist.
1655# You may delete jobs from this list. (New changelists only.)
1656# Files: What opened files from the default changelist are to be added
1657# to this changelist. You may delete files from this list.
1658# (New changelists only.)
1659"""
1660 files_list = []
ea99c3ae 1661 inFilesSection = False
b596b3b9 1662 change_entry = None
46c609e9
LD
1663 args = ['change', '-o']
1664 if changelist:
1665 args.append(str(changelist))
b596b3b9
MT
1666 for entry in p4CmdList(args):
1667 if not entry.has_key('code'):
1668 continue
1669 if entry['code'] == 'stat':
1670 change_entry = entry
1671 break
1672 if not change_entry:
1673 die('Failed to decode output of p4 change -o')
1674 for key, value in change_entry.iteritems():
1675 if key.startswith('File'):
1676 if settings.has_key('depot-paths'):
1677 if not [p for p in settings['depot-paths']
1678 if p4PathStartsWith(value, p)]:
1679 continue
ea99c3ae 1680 else:
b596b3b9
MT
1681 if not p4PathStartsWith(value, self.depotPath):
1682 continue
1683 files_list.append(value)
1684 continue
1685 # Output in the order expected by prepareLogMessage
1686 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1687 if not change_entry.has_key(key):
1688 continue
1689 template += '\n'
1690 template += key + ':'
1691 if key == 'Description':
1692 template += '\n'
1693 for field_line in change_entry[key].splitlines():
1694 template += '\t'+field_line+'\n'
1695 if len(files_list) > 0:
1696 template += '\n'
1697 template += 'Files:\n'
1698 for path in files_list:
1699 template += '\t'+path+'\n'
ea99c3ae
SH
1700 return template
1701
7c766e57
PW
1702 def edit_template(self, template_file):
1703 """Invoke the editor to let the user change the submission
1704 message. Return true if okay to continue with the submit."""
1705
1706 # if configured to skip the editing part, just submit
0d609032 1707 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1708 return True
1709
1710 # look at the modification time, to check later if the user saved
1711 # the file
1712 mtime = os.stat(template_file).st_mtime
1713
1714 # invoke the editor
f95ceaf0 1715 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1716 editor = os.environ.get("P4EDITOR")
1717 else:
1718 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1719 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1720
1721 # If the file was not saved, prompt to see if this patch should
1722 # be skipped. But skip this verification step if configured so.
0d609032 1723 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1724 return True
1725
d1652049
PW
1726 # modification time updated means user saved the file
1727 if os.stat(template_file).st_mtime > mtime:
1728 return True
1729
1730 while True:
1731 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1732 if response == 'y':
1733 return True
1734 if response == 'n':
1735 return False
7c766e57 1736
df8a9e86 1737 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3
MC
1738 # diff
1739 if os.environ.has_key("P4DIFF"):
1740 del(os.environ["P4DIFF"])
1741 diff = ""
1742 for editedFile in editedFiles:
1743 diff += p4_read_pipe(['diff', '-du',
1744 wildcard_encode(editedFile)])
1745
1746 # new file diff
1747 newdiff = ""
1748 for newFile in filesToAdd:
1749 newdiff += "==== new file ====\n"
1750 newdiff += "--- /dev/null\n"
1751 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1752
1753 is_link = os.path.islink(newFile)
1754 expect_link = newFile in symlinks
1755
1756 if is_link and expect_link:
1757 newdiff += "+%s\n" % os.readlink(newFile)
1758 else:
1759 f = open(newFile, "r")
1760 for line in f.readlines():
1761 newdiff += "+" + line
1762 f.close()
b4073bb3 1763
e2a892ee 1764 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1765
7cb5cbef 1766 def applyCommit(self, id):
67b0fe2e
PW
1767 """Apply one commit, return True if it succeeded."""
1768
1769 print "Applying", read_pipe(["git", "show", "-s",
1770 "--format=format:%h %s", id])
ae901090 1771
848de9c3 1772 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1773
84cb0003 1774 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1775 filesToAdd = set()
a02b8bc4 1776 filesToChangeType = set()
4f5cf76a 1777 filesToDelete = set()
d336c158 1778 editedFiles = set()
b6ad6dcc 1779 pureRenameCopy = set()
df8a9e86 1780 symlinks = set()
c65b670e 1781 filesToChangeExecBit = {}
46c609e9 1782 all_files = list()
60df071c 1783
4f5cf76a 1784 for line in diff:
b43b0a3c
CP
1785 diff = parseDiffTreeEntry(line)
1786 modifier = diff['status']
1787 path = diff['src']
46c609e9
LD
1788 all_files.append(path)
1789
4f5cf76a 1790 if modifier == "M":
6de040df 1791 p4_edit(path)
c65b670e
CP
1792 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1793 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1794 editedFiles.add(path)
4f5cf76a
SH
1795 elif modifier == "A":
1796 filesToAdd.add(path)
c65b670e 1797 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1798 if path in filesToDelete:
1799 filesToDelete.remove(path)
df8a9e86
LD
1800
1801 dst_mode = int(diff['dst_mode'], 8)
1802 if dst_mode == 0120000:
1803 symlinks.add(path)
1804
4f5cf76a
SH
1805 elif modifier == "D":
1806 filesToDelete.add(path)
1807 if path in filesToAdd:
1808 filesToAdd.remove(path)
4fddb41b
VA
1809 elif modifier == "C":
1810 src, dest = diff['src'], diff['dst']
6de040df 1811 p4_integrate(src, dest)
b6ad6dcc 1812 pureRenameCopy.add(dest)
4fddb41b 1813 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1814 p4_edit(dest)
b6ad6dcc 1815 pureRenameCopy.discard(dest)
4fddb41b 1816 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1817 p4_edit(dest)
b6ad6dcc 1818 pureRenameCopy.discard(dest)
4fddb41b 1819 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1820 if self.isWindows:
1821 # turn off read-only attribute
1822 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1823 os.unlink(dest)
1824 editedFiles.add(dest)
d9a5f25b 1825 elif modifier == "R":
b43b0a3c 1826 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1827 if self.p4HasMoveCommand:
1828 p4_edit(src) # src must be open before move
1829 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1830 else:
8e9497c2
GG
1831 p4_integrate(src, dest)
1832 if diff['src_sha1'] != diff['dst_sha1']:
1833 p4_edit(dest)
1834 else:
1835 pureRenameCopy.add(dest)
c65b670e 1836 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1837 if not self.p4HasMoveCommand:
1838 p4_edit(dest) # with move: already open, writable
c65b670e 1839 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1840 if not self.p4HasMoveCommand:
d20f0f8e
PW
1841 if self.isWindows:
1842 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1843 os.unlink(dest)
1844 filesToDelete.add(src)
d9a5f25b 1845 editedFiles.add(dest)
a02b8bc4
RP
1846 elif modifier == "T":
1847 filesToChangeType.add(path)
4f5cf76a
SH
1848 else:
1849 die("unknown modifier %s for %s" % (modifier, path))
1850
749b668c 1851 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1852 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1853 tryPatchCmd = patchcmd + "--check -"
1854 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1855 patch_succeeded = True
51a2640a 1856
47a130b7 1857 if os.system(tryPatchCmd) != 0:
60df071c
LD
1858 fixed_rcs_keywords = False
1859 patch_succeeded = False
51a2640a 1860 print "Unfortunately applying the change failed!"
60df071c
LD
1861
1862 # Patch failed, maybe it's just RCS keyword woes. Look through
1863 # the patch to see if that's possible.
0d609032 1864 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1865 file = None
1866 pattern = None
1867 kwfiles = {}
1868 for file in editedFiles | filesToDelete:
1869 # did this file's delta contain RCS keywords?
1870 pattern = p4_keywords_regexp_for_file(file)
1871
1872 if pattern:
1873 # this file is a possibility...look for RCS keywords.
1874 regexp = re.compile(pattern, re.VERBOSE)
1875 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1876 if regexp.search(line):
1877 if verbose:
1878 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1879 kwfiles[file] = pattern
1880 break
1881
1882 for file in kwfiles:
1883 if verbose:
1884 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1885 # File is being deleted, so not open in p4. Must
1886 # disable the read-only bit on windows.
1887 if self.isWindows and file not in editedFiles:
1888 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1889 self.patchRCSKeywords(file, kwfiles[file])
1890 fixed_rcs_keywords = True
1891
1892 if fixed_rcs_keywords:
1893 print "Retrying the patch with RCS keywords cleaned up"
1894 if os.system(tryPatchCmd) == 0:
1895 patch_succeeded = True
1896
1897 if not patch_succeeded:
7e5dd9f2
PW
1898 for f in editedFiles:
1899 p4_revert(f)
7e5dd9f2 1900 return False
51a2640a 1901
55ac2ed6
PW
1902 #
1903 # Apply the patch for real, and do add/delete/+x handling.
1904 #
47a130b7 1905 system(applyPatchCmd)
4f5cf76a 1906
a02b8bc4
RP
1907 for f in filesToChangeType:
1908 p4_edit(f, "-t", "auto")
4f5cf76a 1909 for f in filesToAdd:
6de040df 1910 p4_add(f)
4f5cf76a 1911 for f in filesToDelete:
6de040df
LD
1912 p4_revert(f)
1913 p4_delete(f)
4f5cf76a 1914
c65b670e
CP
1915 # Set/clear executable bits
1916 for f in filesToChangeExecBit.keys():
1917 mode = filesToChangeExecBit[f]
1918 setP4ExecBit(f, mode)
1919
8cf422db
LD
1920 update_shelve = 0
1921 if len(self.update_shelve) > 0:
1922 update_shelve = self.update_shelve.pop(0)
1923 p4_reopen_in_change(update_shelve, all_files)
46c609e9 1924
55ac2ed6
PW
1925 #
1926 # Build p4 change description, starting with the contents
1927 # of the git commit message.
1928 #
0e36f2d7 1929 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1930 logMessage = logMessage.strip()
f19cb0a0 1931 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1932
8cf422db 1933 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 1934 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1935
c47178d4 1936 if self.preserveUser:
55ac2ed6 1937 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1938
55ac2ed6
PW
1939 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1940 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1941 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1942 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1943
55ac2ed6 1944 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1945 if not self.prepare_p4_only:
1946 submitTemplate += separatorLine
df8a9e86 1947 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1948
c47178d4 1949 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1950 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1951 if self.isWindows:
1952 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1953 tmpFile.write(submitTemplate)
c47178d4
PW
1954 tmpFile.close()
1955
728b7ad8
PW
1956 if self.prepare_p4_only:
1957 #
1958 # Leave the p4 tree prepared, and the submit template around
1959 # and let the user decide what to do next
1960 #
1961 print
1962 print "P4 workspace prepared for submission."
1963 print "To submit or revert, go to client workspace"
1964 print " " + self.clientPath
1965 print
1966 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1967 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1968 " \"git p4\"." % fileName
1969 print "You can delete the file \"%s\" when finished." % fileName
1970
1971 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1972 print "To preserve change ownership by user %s, you must\n" \
1973 "do \"p4 change -f <change>\" after submitting and\n" \
1974 "edit the User field."
1975 if pureRenameCopy:
1976 print "After submitting, renamed files must be re-synced."
1977 print "Invoke \"p4 sync -f\" on each of these files:"
1978 for f in pureRenameCopy:
1979 print " " + f
1980
1981 print
1982 print "To revert the changes, use \"p4 revert ...\", and delete"
1983 print "the submit template file \"%s\"" % fileName
1984 if filesToAdd:
1985 print "Since the commit adds new files, they must be deleted:"
1986 for f in filesToAdd:
1987 print " " + f
1988 print
1989 return True
1990
55ac2ed6
PW
1991 #
1992 # Let the user edit the change description, then submit it.
1993 #
b7638fed 1994 submitted = False
cdc7e388 1995
b7638fed
GE
1996 try:
1997 if self.edit_template(fileName):
1998 # read the edited message and submit
1999 tmpFile = open(fileName, "rb")
2000 message = tmpFile.read()
2001 tmpFile.close()
2002 if self.isWindows:
2003 message = message.replace("\r\n", "\n")
2004 submitTemplate = message[:message.index(separatorLine)]
46c609e9 2005
8cf422db 2006 if update_shelve:
46c609e9
LD
2007 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2008 elif self.shelve:
b34fa577
VK
2009 p4_write_pipe(['shelve', '-i'], submitTemplate)
2010 else:
2011 p4_write_pipe(['submit', '-i'], submitTemplate)
2012 # The rename/copy happened by applying a patch that created a
2013 # new file. This leaves it writable, which confuses p4.
2014 for f in pureRenameCopy:
2015 p4_sync(f, "-f")
b7638fed
GE
2016
2017 if self.preserveUser:
2018 if p4User:
2019 # Get last changelist number. Cannot easily get it from
2020 # the submit command output as the output is
2021 # unmarshalled.
2022 changelist = self.lastP4Changelist()
2023 self.modifyChangelistUser(changelist, p4User)
2024
b7638fed
GE
2025 submitted = True
2026
2027 finally:
c47178d4 2028 # skip this patch
b34fa577
VK
2029 if not submitted or self.shelve:
2030 if self.shelve:
2031 print ("Reverting shelved files.")
2032 else:
2033 print ("Submission cancelled, undoing p4 changes.")
2034 for f in editedFiles | filesToDelete:
b7638fed
GE
2035 p4_revert(f)
2036 for f in filesToAdd:
2037 p4_revert(f)
2038 os.remove(f)
c47178d4
PW
2039
2040 os.remove(fileName)
b7638fed 2041 return submitted
4f5cf76a 2042
06804c76
LD
2043 # Export git tags as p4 labels. Create a p4 label and then tag
2044 # with that.
2045 def exportGitTags(self, gitTags):
c8942a22
LD
2046 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2047 if len(validLabelRegexp) == 0:
2048 validLabelRegexp = defaultLabelRegexp
2049 m = re.compile(validLabelRegexp)
06804c76
LD
2050
2051 for name in gitTags:
2052
2053 if not m.match(name):
2054 if verbose:
05a3cec5 2055 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
2056 continue
2057
2058 # Get the p4 commit this corresponds to
c8942a22
LD
2059 logMessage = extractLogMessageFromGitCommit(name)
2060 values = extractSettingsGitLog(logMessage)
06804c76 2061
c8942a22 2062 if not values.has_key('change'):
06804c76
LD
2063 # a tag pointing to something not sent to p4; ignore
2064 if verbose:
2065 print "git tag %s does not give a p4 commit" % name
2066 continue
c8942a22
LD
2067 else:
2068 changelist = values['change']
06804c76
LD
2069
2070 # Get the tag details.
2071 inHeader = True
2072 isAnnotated = False
2073 body = []
2074 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2075 l = l.strip()
2076 if inHeader:
2077 if re.match(r'tag\s+', l):
2078 isAnnotated = True
2079 elif re.match(r'\s*$', l):
2080 inHeader = False
2081 continue
2082 else:
2083 body.append(l)
2084
2085 if not isAnnotated:
2086 body = ["lightweight tag imported by git p4\n"]
2087
2088 # Create the label - use the same view as the client spec we are using
2089 clientSpec = getClientSpec()
2090
2091 labelTemplate = "Label: %s\n" % name
2092 labelTemplate += "Description:\n"
2093 for b in body:
2094 labelTemplate += "\t" + b + "\n"
2095 labelTemplate += "View:\n"
9d57c4a6
KS
2096 for depot_side in clientSpec.mappings:
2097 labelTemplate += "\t%s\n" % depot_side
06804c76 2098
ef739f08
PW
2099 if self.dry_run:
2100 print "Would create p4 label %s for tag" % name
728b7ad8
PW
2101 elif self.prepare_p4_only:
2102 print "Not creating p4 label %s for tag due to option" \
2103 " --prepare-p4-only" % name
ef739f08
PW
2104 else:
2105 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2106
ef739f08
PW
2107 # Use the label
2108 p4_system(["tag", "-l", name] +
9d57c4a6 2109 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2110
ef739f08
PW
2111 if verbose:
2112 print "created p4 label for tag %s" % name
06804c76 2113
4f5cf76a 2114 def run(self, args):
c9b50e63
SH
2115 if len(args) == 0:
2116 self.master = currentGitBranch()
c9b50e63
SH
2117 elif len(args) == 1:
2118 self.master = args[0]
28755dba
PW
2119 if not branchExists(self.master):
2120 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2121 else:
2122 return False
2123
8cf422db
LD
2124 for i in self.update_shelve:
2125 if i <= 0:
2126 sys.exit("invalid changelist %d" % i)
2127
00ad6e31
LD
2128 if self.master:
2129 allowSubmit = gitConfig("git-p4.allowSubmit")
2130 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2131 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2132
27d2d811 2133 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 2134 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2135 if len(self.origin) == 0:
2136 self.origin = upstream
a3fdd579 2137
8cf422db 2138 if len(self.update_shelve) > 0:
46c609e9
LD
2139 self.shelve = True
2140
3ea2cfd4
LD
2141 if self.preserveUser:
2142 if not self.canChangeChangelists():
2143 die("Cannot preserve user names without p4 super-user or admin permissions")
2144
6bbfd137
PW
2145 # if not set from the command line, try the config file
2146 if self.conflict_behavior is None:
2147 val = gitConfig("git-p4.conflict")
2148 if val:
2149 if val not in self.conflict_behavior_choices:
2150 die("Invalid value '%s' for config git-p4.conflict" % val)
2151 else:
2152 val = "ask"
2153 self.conflict_behavior = val
2154
a3fdd579
SH
2155 if self.verbose:
2156 print "Origin branch is " + self.origin
9512497b 2157
ea99c3ae 2158 if len(self.depotPath) == 0:
9512497b
SH
2159 print "Internal error: cannot locate perforce depot path from existing branches"
2160 sys.exit(128)
2161
543987bd 2162 self.useClientSpec = False
0d609032 2163 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2164 self.useClientSpec = True
2165 if self.useClientSpec:
2166 self.clientSpecDirs = getClientSpec()
9512497b 2167
2e3a16b2 2168 # Check for the existence of P4 branches
cd884106
VA
2169 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2170
2171 if self.useClientSpec and not branchesDetected:
543987bd
PW
2172 # all files are relative to the client spec
2173 self.clientPath = getClientRoot()
2174 else:
2175 self.clientPath = p4Where(self.depotPath)
9512497b 2176
543987bd
PW
2177 if self.clientPath == "":
2178 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2179
ea99c3ae 2180 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2181 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2182
0591cfa8 2183 # ensure the clientPath exists
8d7ec362 2184 new_client_dir = False
0591cfa8 2185 if not os.path.exists(self.clientPath):
8d7ec362 2186 new_client_dir = True
0591cfa8
GG
2187 os.makedirs(self.clientPath)
2188
bbd84863 2189 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2190 if self.dry_run:
2191 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2192 else:
ef739f08
PW
2193 print "Synchronizing p4 checkout..."
2194 if new_client_dir:
2195 # old one was destroyed, and maybe nobody told p4
2196 p4_sync("...", "-f")
2197 else:
2198 p4_sync("...")
4f5cf76a 2199 self.check()
4f5cf76a 2200
4c750c0d 2201 commits = []
00ad6e31
LD
2202 if self.master:
2203 commitish = self.master
2204 else:
2205 commitish = 'HEAD'
2206
f55b87c1
RM
2207 if self.commit != "":
2208 if self.commit.find("..") != -1:
2209 limits_ish = self.commit.split("..")
2210 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2211 commits.append(line.strip())
2212 commits.reverse()
2213 else:
2214 commits.append(self.commit)
2215 else:
2216 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2217 commits.append(line.strip())
2218 commits.reverse()
4f5cf76a 2219
0d609032 2220 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2221 self.checkAuthorship = False
2222 else:
2223 self.checkAuthorship = True
2224
3ea2cfd4
LD
2225 if self.preserveUser:
2226 self.checkValidP4Users(commits)
2227
84cb0003
GG
2228 #
2229 # Build up a set of options to be passed to diff when
2230 # submitting each commit to p4.
2231 #
2232 if self.detectRenames:
2233 # command-line -M arg
2234 self.diffOpts = "-M"
2235 else:
2236 # If not explicitly set check the config variable
2237 detectRenames = gitConfig("git-p4.detectRenames")
2238
2239 if detectRenames.lower() == "false" or detectRenames == "":
2240 self.diffOpts = ""
2241 elif detectRenames.lower() == "true":
2242 self.diffOpts = "-M"
2243 else:
2244 self.diffOpts = "-M%s" % detectRenames
2245
2246 # no command-line arg for -C or --find-copies-harder, just
2247 # config variables
2248 detectCopies = gitConfig("git-p4.detectCopies")
2249 if detectCopies.lower() == "false" or detectCopies == "":
2250 pass
2251 elif detectCopies.lower() == "true":
2252 self.diffOpts += " -C"
2253 else:
2254 self.diffOpts += " -C%s" % detectCopies
2255
0d609032 2256 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2257 self.diffOpts += " --find-copies-harder"
2258
8cf422db
LD
2259 num_shelves = len(self.update_shelve)
2260 if num_shelves > 0 and num_shelves != len(commits):
2261 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2262 (len(commits), num_shelves))
2263
7e5dd9f2
PW
2264 #
2265 # Apply the commits, one at a time. On failure, ask if should
2266 # continue to try the rest of the patches, or quit.
2267 #
ef739f08
PW
2268 if self.dry_run:
2269 print "Would apply"
67b0fe2e 2270 applied = []
7e5dd9f2
PW
2271 last = len(commits) - 1
2272 for i, commit in enumerate(commits):
ef739f08
PW
2273 if self.dry_run:
2274 print " ", read_pipe(["git", "show", "-s",
2275 "--format=format:%h %s", commit])
2276 ok = True
2277 else:
2278 ok = self.applyCommit(commit)
67b0fe2e
PW
2279 if ok:
2280 applied.append(commit)
7e5dd9f2 2281 else:
728b7ad8
PW
2282 if self.prepare_p4_only and i < last:
2283 print "Processing only the first commit due to option" \
2284 " --prepare-p4-only"
2285 break
7e5dd9f2
PW
2286 if i < last:
2287 quit = False
2288 while True:
6bbfd137
PW
2289 # prompt for what to do, or use the option/variable
2290 if self.conflict_behavior == "ask":
2291 print "What do you want to do?"
2292 response = raw_input("[s]kip this commit but apply"
2293 " the rest, or [q]uit? ")
2294 if not response:
2295 continue
2296 elif self.conflict_behavior == "skip":
2297 response = "s"
2298 elif self.conflict_behavior == "quit":
2299 response = "q"
2300 else:
2301 die("Unknown conflict_behavior '%s'" %
2302 self.conflict_behavior)
2303
7e5dd9f2
PW
2304 if response[0] == "s":
2305 print "Skipping this commit, but applying the rest"
2306 break
2307 if response[0] == "q":
2308 print "Quitting"
2309 quit = True
2310 break
2311 if quit:
2312 break
4f5cf76a 2313
67b0fe2e 2314 chdir(self.oldWorkingDirectory)
b34fa577 2315 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2316 if self.dry_run:
2317 pass
728b7ad8
PW
2318 elif self.prepare_p4_only:
2319 pass
ef739f08 2320 elif len(commits) == len(applied):
b34fa577 2321 print ("All commits {0}!".format(shelved_applied))
14594f4b 2322
4c750c0d 2323 sync = P4Sync()
44e8d26c
PW
2324 if self.branch:
2325 sync.branch = self.branch
b9d34db9
LD
2326 if self.disable_p4sync:
2327 sync.sync_origin_only()
2328 else:
2329 sync.run([])
14594f4b 2330
b9d34db9
LD
2331 if not self.disable_rebase:
2332 rebase = P4Rebase()
2333 rebase.rebase()
4f5cf76a 2334
67b0fe2e
PW
2335 else:
2336 if len(applied) == 0:
b34fa577 2337 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2338 else:
b34fa577 2339 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2340 for c in commits:
2341 if c in applied:
2342 star = "*"
2343 else:
2344 star = " "
2345 print star, read_pipe(["git", "show", "-s",
2346 "--format=format:%h %s", c])
2347 print "You will have to do 'git p4 sync' and rebase."
2348
0d609032 2349 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2350 self.exportLabels = True
06804c76
LD
2351
2352 if self.exportLabels:
2353 p4Labels = getP4Labels(self.depotPath)
2354 gitTags = getGitTags()
2355
2356 missingGitTags = gitTags - p4Labels
2357 self.exportGitTags(missingGitTags)
2358
98e023de 2359 # exit with error unless everything applied perfectly
67b0fe2e
PW
2360 if len(commits) != len(applied):
2361 sys.exit(1)
2362
b984733c
SH
2363 return True
2364
ecb7cf98
PW
2365class View(object):
2366 """Represent a p4 view ("p4 help views"), and map files in a
2367 repo according to the view."""
2368
9d57c4a6 2369 def __init__(self, client_name):
ecb7cf98 2370 self.mappings = []
9d57c4a6
KS
2371 self.client_prefix = "//%s/" % client_name
2372 # cache results of "p4 where" to lookup client file locations
2373 self.client_spec_path_cache = {}
ecb7cf98
PW
2374
2375 def append(self, view_line):
2376 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2377 sides. Append to self.mappings, preserving order. This
2378 is only needed for tag creation."""
ecb7cf98
PW
2379
2380 # Split the view line into exactly two words. P4 enforces
2381 # structure on these lines that simplifies this quite a bit.
2382 #
2383 # Either or both words may be double-quoted.
2384 # Single quotes do not matter.
2385 # Double-quote marks cannot occur inside the words.
2386 # A + or - prefix is also inside the quotes.
2387 # There are no quotes unless they contain a space.
2388 # The line is already white-space stripped.
2389 # The two words are separated by a single space.
2390 #
2391 if view_line[0] == '"':
2392 # First word is double quoted. Find its end.
2393 close_quote_index = view_line.find('"', 1)
2394 if close_quote_index <= 0:
2395 die("No first-word closing quote found: %s" % view_line)
2396 depot_side = view_line[1:close_quote_index]
2397 # skip closing quote and space
2398 rhs_index = close_quote_index + 1 + 1
2399 else:
2400 space_index = view_line.find(" ")
2401 if space_index <= 0:
2402 die("No word-splitting space found: %s" % view_line)
2403 depot_side = view_line[0:space_index]
2404 rhs_index = space_index + 1
2405
ecb7cf98 2406 # prefix + means overlay on previous mapping
ecb7cf98 2407 if depot_side.startswith("+"):
ecb7cf98
PW
2408 depot_side = depot_side[1:]
2409
9d57c4a6 2410 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2411 exclude = False
2412 if depot_side.startswith("-"):
2413 exclude = True
2414 depot_side = depot_side[1:]
2415
9d57c4a6
KS
2416 if not exclude:
2417 self.mappings.append(depot_side)
ecb7cf98 2418
9d57c4a6
KS
2419 def convert_client_path(self, clientFile):
2420 # chop off //client/ part to make it relative
2421 if not clientFile.startswith(self.client_prefix):
2422 die("No prefix '%s' on clientFile '%s'" %
2423 (self.client_prefix, clientFile))
2424 return clientFile[len(self.client_prefix):]
ecb7cf98 2425
9d57c4a6
KS
2426 def update_client_spec_path_cache(self, files):
2427 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2428
9d57c4a6
KS
2429 # List depot file paths exclude that already cached
2430 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2431
9d57c4a6
KS
2432 if len(fileArgs) == 0:
2433 return # All files in cache
ecb7cf98 2434
9d57c4a6
KS
2435 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2436 for res in where_result:
2437 if "code" in res and res["code"] == "error":
2438 # assume error is "... file(s) not in client view"
2439 continue
2440 if "clientFile" not in res:
20005443 2441 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2442 if "unmap" in res:
2443 # it will list all of them, but only one not unmap-ped
2444 continue
a0a50d87
LS
2445 if gitConfigBool("core.ignorecase"):
2446 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2447 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2448
9d57c4a6
KS
2449 # not found files or unmap files set to ""
2450 for depotFile in fileArgs:
a0a50d87
LS
2451 if gitConfigBool("core.ignorecase"):
2452 depotFile = depotFile.lower()
9d57c4a6
KS
2453 if depotFile not in self.client_spec_path_cache:
2454 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2455
9d57c4a6
KS
2456 def map_in_client(self, depot_path):
2457 """Return the relative location in the client where this
2458 depot file should live. Returns "" if the file should
2459 not be mapped in the client."""
ecb7cf98 2460
a0a50d87
LS
2461 if gitConfigBool("core.ignorecase"):
2462 depot_path = depot_path.lower()
2463
9d57c4a6
KS
2464 if depot_path in self.client_spec_path_cache:
2465 return self.client_spec_path_cache[depot_path]
2466
2467 die( "Error: %s is not found in client spec path" % depot_path )
2468 return ""
ecb7cf98 2469
3ea2cfd4 2470class P4Sync(Command, P4UserMap):
56c09345
PW
2471 delete_actions = ( "delete", "move/delete", "purge" )
2472
b984733c
SH
2473 def __init__(self):
2474 Command.__init__(self)
3ea2cfd4 2475 P4UserMap.__init__(self)
b984733c
SH
2476 self.options = [
2477 optparse.make_option("--branch", dest="branch"),
2478 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2479 optparse.make_option("--changesfile", dest="changesFile"),
2480 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2481 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2482 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2483 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2484 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2485 optparse.make_option("--max-changes", dest="maxChanges",
2486 help="Maximum number of changes to import"),
2487 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2488 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2489 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2490 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2491 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2492 help="Only sync files that are included in the Perforce Client Spec"),
2493 optparse.make_option("-/", dest="cloneExclude",
2494 action="append", type="string",
2495 help="exclude depot path"),
b984733c
SH
2496 ]
2497 self.description = """Imports from Perforce into a git repository.\n
2498 example:
2499 //depot/my/project/ -- to import the current head
2500 //depot/my/project/@all -- to import everything
2501 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2502
2503 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2504
2505 self.usage += " //depot/path[@revRange]"
b984733c 2506 self.silent = False
1d7367dc
RG
2507 self.createdBranches = set()
2508 self.committedChanges = set()
569d1bd4 2509 self.branch = ""
b984733c 2510 self.detectBranches = False
cb53e1f8 2511 self.detectLabels = False
06804c76 2512 self.importLabels = False
b984733c 2513 self.changesFile = ""
01265103 2514 self.syncWithOrigin = True
a028a98e 2515 self.importIntoRemotes = True
01a9c9c5 2516 self.maxChanges = ""
1051ef00 2517 self.changes_block_size = None
8b41a97f 2518 self.keepRepoPath = False
6326aa58 2519 self.depotPaths = None
3c699645 2520 self.p4BranchesInGit = []
354081d5 2521 self.cloneExclude = []
3a70cdfa 2522 self.useClientSpec = False
a93d33ee 2523 self.useClientSpec_from_options = False
ecb7cf98 2524 self.clientSpecDirs = None
fed23693 2525 self.tempBranches = []
d604176d 2526 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12
LS
2527 self.largeFileSystem = None
2528
2529 if gitConfig('git-p4.largeFileSystem'):
2530 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2531 self.largeFileSystem = largeFileSystemConstructor(
2532 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2533 )
b984733c 2534
01265103
SH
2535 if gitConfig("git-p4.syncFromOrigin") == "false":
2536 self.syncWithOrigin = False
2537
fed23693
VA
2538 # Force a checkpoint in fast-import and wait for it to finish
2539 def checkpoint(self):
2540 self.gitStream.write("checkpoint\n\n")
2541 self.gitStream.write("progress checkpoint\n\n")
2542 out = self.gitOutput.readline()
2543 if self.verbose:
2544 print "checkpoint finished: " + out
2545
b984733c 2546 def extractFilesFromCommit(self, commit):
354081d5
TT
2547 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2548 for path in self.cloneExclude]
b984733c
SH
2549 files = []
2550 fnum = 0
2551 while commit.has_key("depotFile%s" % fnum):
2552 path = commit["depotFile%s" % fnum]
6326aa58 2553
354081d5 2554 if [p for p in self.cloneExclude
d53de8b9 2555 if p4PathStartsWith(path, p)]:
354081d5
TT
2556 found = False
2557 else:
2558 found = [p for p in self.depotPaths
d53de8b9 2559 if p4PathStartsWith(path, p)]
6326aa58 2560 if not found:
b984733c
SH
2561 fnum = fnum + 1
2562 continue
2563
2564 file = {}
2565 file["path"] = path
2566 file["rev"] = commit["rev%s" % fnum]
2567 file["action"] = commit["action%s" % fnum]
2568 file["type"] = commit["type%s" % fnum]
2569 files.append(file)
2570 fnum = fnum + 1
2571 return files
2572
26e6a27d
JD
2573 def extractJobsFromCommit(self, commit):
2574 jobs = []
2575 jnum = 0
2576 while commit.has_key("job%s" % jnum):
2577 job = commit["job%s" % jnum]
2578 jobs.append(job)
2579 jnum = jnum + 1
2580 return jobs
2581
6326aa58 2582 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2583 """When streaming files, this is called to map a p4 depot path
2584 to where it should go in git. The prefixes are either
2585 self.depotPaths, or self.branchPrefixes in the case of
2586 branch detection."""
2587
3952710b 2588 if self.useClientSpec:
21ef5df4
PW
2589 # branch detection moves files up a level (the branch name)
2590 # from what client spec interpretation gives
0d1696ef 2591 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2592 if self.detectBranches:
2593 for b in self.knownBranches:
2594 if path.startswith(b + "/"):
2595 path = path[len(b)+1:]
2596
2597 elif self.keepRepoPath:
2598 # Preserve everything in relative path name except leading
2599 # //depot/; just look at first prefix as they all should
2600 # be in the same depot.
2601 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2602 if p4PathStartsWith(path, depot):
2603 path = path[len(depot):]
3952710b 2604
0d1696ef 2605 else:
0d1696ef
PW
2606 for p in prefixes:
2607 if p4PathStartsWith(path, p):
2608 path = path[len(p):]
21ef5df4 2609 break
8b41a97f 2610
0d1696ef 2611 path = wildcard_decode(path)
6326aa58 2612 return path
6754a299 2613
71b112d4 2614 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2615 """Look at each depotFile in the commit to figure out to what
2616 branch it belongs."""
2617
9d57c4a6
KS
2618 if self.clientSpecDirs:
2619 files = self.extractFilesFromCommit(commit)
2620 self.clientSpecDirs.update_client_spec_path_cache(files)
2621
d5904674 2622 branches = {}
71b112d4
SH
2623 fnum = 0
2624 while commit.has_key("depotFile%s" % fnum):
2625 path = commit["depotFile%s" % fnum]
6326aa58 2626 found = [p for p in self.depotPaths
d53de8b9 2627 if p4PathStartsWith(path, p)]
6326aa58 2628 if not found:
71b112d4
SH
2629 fnum = fnum + 1
2630 continue
2631
2632 file = {}
2633 file["path"] = path
2634 file["rev"] = commit["rev%s" % fnum]
2635 file["action"] = commit["action%s" % fnum]
2636 file["type"] = commit["type%s" % fnum]
2637 fnum = fnum + 1
2638
21ef5df4
PW
2639 # start with the full relative path where this file would
2640 # go in a p4 client
2641 if self.useClientSpec:
2642 relPath = self.clientSpecDirs.map_in_client(path)
2643 else:
2644 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2645
4b97ffb1 2646 for branch in self.knownBranches.keys():
21ef5df4
PW
2647 # add a trailing slash so that a commit into qt/4.2foo
2648 # doesn't end up in qt/4.2, e.g.
6754a299 2649 if relPath.startswith(branch + "/"):
d5904674
SH
2650 if branch not in branches:
2651 branches[branch] = []
71b112d4 2652 branches[branch].append(file)
6555b2cc 2653 break
b984733c
SH
2654
2655 return branches
2656
a5db4b12
LS
2657 def writeToGitStream(self, gitMode, relPath, contents):
2658 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2659 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2660 for d in contents:
2661 self.gitStream.write(d)
2662 self.gitStream.write('\n')
2663
a8b05162
LS
2664 def encodeWithUTF8(self, path):
2665 try:
2666 path.decode('ascii')
2667 except:
2668 encoding = 'utf8'
2669 if gitConfig('git-p4.pathEncoding'):
2670 encoding = gitConfig('git-p4.pathEncoding')
2671 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2672 if self.verbose:
2673 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2674 return path
2675
b932705b
LD
2676 # output one file from the P4 stream
2677 # - helper for streamP4Files
2678
2679 def streamOneP4File(self, file, contents):
b932705b 2680 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2681 relPath = self.encodeWithUTF8(relPath)
b932705b 2682 if verbose:
d2176a50
LS
2683 size = int(self.stream_file['fileSize'])
2684 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2685 sys.stdout.flush()
b932705b 2686
9cffb8c8
PW
2687 (type_base, type_mods) = split_p4_type(file["type"])
2688
2689 git_mode = "100644"
2690 if "x" in type_mods:
2691 git_mode = "100755"
2692 if type_base == "symlink":
2693 git_mode = "120000"
1292df11
AJ
2694 # p4 print on a symlink sometimes contains "target\n";
2695 # if it does, remove the newline
b39c3612 2696 data = ''.join(contents)
40f846c3
PW
2697 if not data:
2698 # Some version of p4 allowed creating a symlink that pointed
2699 # to nothing. This causes p4 errors when checking out such
2700 # a change, and errors here too. Work around it by ignoring
2701 # the bad symlink; hopefully a future change fixes it.
2702 print "\nIgnoring empty symlink in %s" % file['depotFile']
2703 return
2704 elif data[-1] == '\n':
1292df11
AJ
2705 contents = [data[:-1]]
2706 else:
2707 contents = [data]
b932705b 2708
9cffb8c8 2709 if type_base == "utf16":
55aa5714
PW
2710 # p4 delivers different text in the python output to -G
2711 # than it does when using "print -o", or normal p4 client
2712 # operations. utf16 is converted to ascii or utf8, perhaps.
2713 # But ascii text saved as -t utf16 is completely mangled.
2714 # Invoke print -o to get the real contents.
7f0e5962
PW
2715 #
2716 # On windows, the newlines will always be mangled by print, so put
2717 # them back too. This is not needed to the cygwin windows version,
2718 # just the native "NT" type.
2719 #
1f5f3907
LS
2720 try:
2721 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2722 except Exception as e:
2723 if 'Translation of file content failed' in str(e):
2724 type_base = 'binary'
2725 else:
2726 raise e
2727 else:
2728 if p4_version_string().find('/NT') >= 0:
2729 text = text.replace('\r\n', '\n')
2730 contents = [ text ]
55aa5714 2731
9f7ef0ea
PW
2732 if type_base == "apple":
2733 # Apple filetype files will be streamed as a concatenation of
2734 # its appledouble header and the contents. This is useless
2735 # on both macs and non-macs. If using "print -q -o xx", it
2736 # will create "xx" with the data, and "%xx" with the header.
2737 # This is also not very useful.
2738 #
2739 # Ideally, someday, this script can learn how to generate
2740 # appledouble files directly and import those to git, but
2741 # non-mac machines can never find a use for apple filetype.
2742 print "\nIgnoring apple filetype file %s" % file['depotFile']
2743 return
2744
55aa5714
PW
2745 # Note that we do not try to de-mangle keywords on utf16 files,
2746 # even though in theory somebody may want that.
60df071c
LD
2747 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2748 if pattern:
2749 regexp = re.compile(pattern, re.VERBOSE)
2750 text = ''.join(contents)
2751 text = regexp.sub(r'$\1$', text)
2752 contents = [ text ]
b932705b 2753
a5db4b12
LS
2754 if self.largeFileSystem:
2755 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2756
a5db4b12 2757 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2758
2759 def streamOneP4Deletion(self, file):
2760 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2761 relPath = self.encodeWithUTF8(relPath)
b932705b 2762 if verbose:
d2176a50
LS
2763 sys.stdout.write("delete %s\n" % relPath)
2764 sys.stdout.flush()
b932705b
LD
2765 self.gitStream.write("D %s\n" % relPath)
2766
a5db4b12
LS
2767 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2768 self.largeFileSystem.removeLargeFile(relPath)
2769
b932705b
LD
2770 # handle another chunk of streaming data
2771 def streamP4FilesCb(self, marshalled):
2772
78189bea
PW
2773 # catch p4 errors and complain
2774 err = None
2775 if "code" in marshalled:
2776 if marshalled["code"] == "error":
2777 if "data" in marshalled:
2778 err = marshalled["data"].rstrip()
4d25dc44
LS
2779
2780 if not err and 'fileSize' in self.stream_file:
2781 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2782 if required_bytes > 0:
2783 err = 'Not enough space left on %s! Free at least %i MB.' % (
2784 os.getcwd(), required_bytes/1024/1024
2785 )
2786
78189bea
PW
2787 if err:
2788 f = None
2789 if self.stream_have_file_info:
2790 if "depotFile" in self.stream_file:
2791 f = self.stream_file["depotFile"]
2792 # force a failure in fast-import, else an empty
2793 # commit will be made
2794 self.gitStream.write("\n")
2795 self.gitStream.write("die-now\n")
2796 self.gitStream.close()
2797 # ignore errors, but make sure it exits first
2798 self.importProcess.wait()
2799 if f:
2800 die("Error from p4 print for %s: %s" % (f, err))
2801 else:
2802 die("Error from p4 print: %s" % err)
2803
c3f6163b
AG
2804 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2805 # start of a new file - output the old one first
2806 self.streamOneP4File(self.stream_file, self.stream_contents)
2807 self.stream_file = {}
2808 self.stream_contents = []
2809 self.stream_have_file_info = False
b932705b 2810
c3f6163b
AG
2811 # pick up the new file information... for the
2812 # 'data' field we need to append to our array
2813 for k in marshalled.keys():
2814 if k == 'data':
d2176a50
LS
2815 if 'streamContentSize' not in self.stream_file:
2816 self.stream_file['streamContentSize'] = 0
2817 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2818 self.stream_contents.append(marshalled['data'])
2819 else:
2820 self.stream_file[k] = marshalled[k]
b932705b 2821
d2176a50
LS
2822 if (verbose and
2823 'streamContentSize' in self.stream_file and
2824 'fileSize' in self.stream_file and
2825 'depotFile' in self.stream_file):
2826 size = int(self.stream_file["fileSize"])
2827 if size > 0:
2828 progress = 100*self.stream_file['streamContentSize']/size
2829 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2830 sys.stdout.flush()
2831
c3f6163b 2832 self.stream_have_file_info = True
b932705b
LD
2833
2834 # Stream directly from "p4 files" into "git fast-import"
2835 def streamP4Files(self, files):
30b5940b
SH
2836 filesForCommit = []
2837 filesToRead = []
b932705b 2838 filesToDelete = []
30b5940b 2839
3a70cdfa 2840 for f in files:
ecb7cf98
PW
2841 filesForCommit.append(f)
2842 if f['action'] in self.delete_actions:
2843 filesToDelete.append(f)
2844 else:
2845 filesToRead.append(f)
6a49f8e2 2846
b932705b
LD
2847 # deleted files...
2848 for f in filesToDelete:
2849 self.streamOneP4Deletion(f)
1b9a4684 2850
b932705b
LD
2851 if len(filesToRead) > 0:
2852 self.stream_file = {}
2853 self.stream_contents = []
2854 self.stream_have_file_info = False
8ff45f2a 2855
c3f6163b
AG
2856 # curry self argument
2857 def streamP4FilesCbSelf(entry):
2858 self.streamP4FilesCb(entry)
6a49f8e2 2859
6de040df
LD
2860 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2861
2862 p4CmdList(["-x", "-", "print"],
2863 stdin=fileArgs,
2864 cb=streamP4FilesCbSelf)
30b5940b 2865
b932705b
LD
2866 # do the last chunk
2867 if self.stream_file.has_key('depotFile'):
2868 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2869
affb474f
LD
2870 def make_email(self, userid):
2871 if userid in self.users:
2872 return self.users[userid]
2873 else:
2874 return "%s <a@b>" % userid
2875
06804c76 2876 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2877 """ Stream a p4 tag.
2878 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2879 """
2880
06804c76
LD
2881 if verbose:
2882 print "writing tag %s for commit %s" % (labelName, commit)
2883 gitStream.write("tag %s\n" % labelName)
2884 gitStream.write("from %s\n" % commit)
2885
2886 if labelDetails.has_key('Owner'):
2887 owner = labelDetails["Owner"]
2888 else:
2889 owner = None
2890
2891 # Try to use the owner of the p4 label, or failing that,
2892 # the current p4 user id.
2893 if owner:
2894 email = self.make_email(owner)
2895 else:
2896 email = self.make_email(self.p4UserId())
2897 tagger = "%s %s %s" % (email, epoch, self.tz)
2898
2899 gitStream.write("tagger %s\n" % tagger)
2900
2901 print "labelDetails=",labelDetails
2902 if labelDetails.has_key('Description'):
2903 description = labelDetails['Description']
2904 else:
2905 description = 'Label from git p4'
2906
2907 gitStream.write("data %d\n" % len(description))
2908 gitStream.write(description)
2909 gitStream.write("\n")
2910
4ae048e6
LS
2911 def inClientSpec(self, path):
2912 if not self.clientSpecDirs:
2913 return True
2914 inClientSpec = self.clientSpecDirs.map_in_client(path)
2915 if not inClientSpec and self.verbose:
2916 print('Ignoring file outside of client spec: {0}'.format(path))
2917 return inClientSpec
2918
2919 def hasBranchPrefix(self, path):
2920 if not self.branchPrefixes:
2921 return True
2922 hasPrefix = [p for p in self.branchPrefixes
2923 if p4PathStartsWith(path, p)]
09667d01 2924 if not hasPrefix and self.verbose:
4ae048e6
LS
2925 print('Ignoring file outside of prefix: {0}'.format(path))
2926 return hasPrefix
2927
e63231e5 2928 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2929 epoch = details["time"]
2930 author = details["user"]
26e6a27d 2931 jobs = self.extractJobsFromCommit(details)
b984733c 2932
4b97ffb1 2933 if self.verbose:
4ae048e6 2934 print('commit into {0}'.format(branch))
96e07dd2 2935
9d57c4a6
KS
2936 if self.clientSpecDirs:
2937 self.clientSpecDirs.update_client_spec_path_cache(files)
2938
4ae048e6
LS
2939 files = [f for f in files
2940 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2941
2942 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2943 print('Ignoring revision {0} as it would produce an empty commit.'
2944 .format(details['change']))
2945 return
2946
b984733c 2947 self.gitStream.write("commit %s\n" % branch)
b43702ac 2948 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2949 self.committedChanges.add(int(details["change"]))
2950 committer = ""
b607e71e
SH
2951 if author not in self.users:
2952 self.getUserMapFromPerforceServer()
affb474f 2953 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2954
2955 self.gitStream.write("committer %s\n" % committer)
2956
2957 self.gitStream.write("data <<EOT\n")
2958 self.gitStream.write(details["desc"])
26e6a27d
JD
2959 if len(jobs) > 0:
2960 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
e63231e5
PW
2961 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2962 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2963 if len(details['options']) > 0:
2964 self.gitStream.write(": options = %s" % details['options'])
2965 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2966
2967 if len(parent) > 0:
4b97ffb1
SH
2968 if self.verbose:
2969 print "parent %s" % parent
b984733c
SH
2970 self.gitStream.write("from %s\n" % parent)
2971
4ae048e6 2972 self.streamP4Files(files)
b984733c
SH
2973 self.gitStream.write("\n")
2974
1f4ba1cb
SH
2975 change = int(details["change"])
2976
9bda3a85 2977 if self.labels.has_key(change):
1f4ba1cb
SH
2978 label = self.labels[change]
2979 labelDetails = label[0]
2980 labelRevisions = label[1]
71b112d4
SH
2981 if self.verbose:
2982 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2983
6de040df 2984 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2985 for p in self.branchPrefixes])
1f4ba1cb
SH
2986
2987 if len(files) == len(labelRevisions):
2988
2989 cleanedFiles = {}
2990 for info in files:
56c09345 2991 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2992 continue
2993 cleanedFiles[info["depotFile"]] = info["rev"]
2994
2995 if cleanedFiles == labelRevisions:
06804c76 2996 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2997
2998 else:
a46668fa 2999 if not self.silent:
cebdf5af
HWN
3000 print ("Tag %s does not match with change %s: files do not match."
3001 % (labelDetails["label"], change))
1f4ba1cb
SH
3002
3003 else:
a46668fa 3004 if not self.silent:
cebdf5af
HWN
3005 print ("Tag %s does not match with change %s: file count is different."
3006 % (labelDetails["label"], change))
b984733c 3007
06804c76 3008 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
3009 def getLabels(self):
3010 self.labels = {}
3011
52a4880b 3012 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 3013 if len(l) > 0 and not self.silent:
183f8436 3014 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
3015
3016 for output in l:
1f4ba1cb
SH
3017 label = output["label"]
3018 revisions = {}
3019 newestChange = 0
71b112d4
SH
3020 if self.verbose:
3021 print "Querying files for label %s" % label
6de040df
LD
3022 for file in p4CmdList(["files"] +
3023 ["%s...@%s" % (p, label)
3024 for p in self.depotPaths]):
1f4ba1cb
SH
3025 revisions[file["depotFile"]] = file["rev"]
3026 change = int(file["change"])
3027 if change > newestChange:
3028 newestChange = change
3029
9bda3a85
SH
3030 self.labels[newestChange] = [output, revisions]
3031
3032 if self.verbose:
3033 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 3034
06804c76
LD
3035 # Import p4 labels as git tags. A direct mapping does not
3036 # exist, so assume that if all the files are at the same revision
3037 # then we can use that, or it's something more complicated we should
3038 # just ignore.
3039 def importP4Labels(self, stream, p4Labels):
3040 if verbose:
3041 print "import p4 labels: " + ' '.join(p4Labels)
3042
3043 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 3044 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
3045 if len(validLabelRegexp) == 0:
3046 validLabelRegexp = defaultLabelRegexp
3047 m = re.compile(validLabelRegexp)
3048
3049 for name in p4Labels:
3050 commitFound = False
3051
3052 if not m.match(name):
3053 if verbose:
3054 print "label %s does not match regexp %s" % (name,validLabelRegexp)
3055 continue
3056
3057 if name in ignoredP4Labels:
3058 continue
3059
3060 labelDetails = p4CmdList(['label', "-o", name])[0]
3061
3062 # get the most recent changelist for each file in this label
3063 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3064 for p in self.depotPaths])
3065
3066 if change.has_key('change'):
3067 # find the corresponding git commit; take the oldest commit
3068 changelist = int(change['change'])
b43702ac
LD
3069 if changelist in self.committedChanges:
3070 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 3071 commitFound = True
b43702ac
LD
3072 else:
3073 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3074 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3075 if len(gitCommit) == 0:
3076 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3077 else:
3078 commitFound = True
3079 gitCommit = gitCommit.strip()
3080
3081 if commitFound:
06804c76
LD
3082 # Convert from p4 time format
3083 try:
3084 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3085 except ValueError:
a4e9054c 3086 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
3087 tmwhen = 1
3088
3089 when = int(time.mktime(tmwhen))
3090 self.streamTag(stream, name, labelDetails, gitCommit, when)
3091 if verbose:
3092 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3093 else:
3094 if verbose:
3095 print "Label %s has no changelists - possibly deleted?" % name
3096
3097 if not commitFound:
3098 # We can't import this label; don't try again as it will get very
3099 # expensive repeatedly fetching all the files for labels that will
3100 # never be imported. If the label is moved in the future, the
3101 # ignore will need to be removed manually.
3102 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3103
86dff6b6
HWN
3104 def guessProjectName(self):
3105 for p in self.depotPaths:
6e5295c4
SH
3106 if p.endswith("/"):
3107 p = p[:-1]
3108 p = p[p.strip().rfind("/") + 1:]
3109 if not p.endswith("/"):
3110 p += "/"
3111 return p
86dff6b6 3112
4b97ffb1 3113 def getBranchMapping(self):
6555b2cc
SH
3114 lostAndFoundBranches = set()
3115
8ace74c0
VA
3116 user = gitConfig("git-p4.branchUser")
3117 if len(user) > 0:
3118 command = "branches -u %s" % user
3119 else:
3120 command = "branches"
3121
3122 for info in p4CmdList(command):
52a4880b 3123 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
3124 viewIdx = 0
3125 while details.has_key("View%s" % viewIdx):
3126 paths = details["View%s" % viewIdx].split(" ")
3127 viewIdx = viewIdx + 1
3128 # require standard //depot/foo/... //depot/bar/... mapping
3129 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3130 continue
3131 source = paths[0]
3132 destination = paths[1]
6509e19c 3133 ## HACK
d53de8b9 3134 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3135 source = source[len(self.depotPaths[0]):-4]
3136 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3137
1a2edf4e
SH
3138 if destination in self.knownBranches:
3139 if not self.silent:
3140 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3141 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3142 continue
3143
6555b2cc
SH
3144 self.knownBranches[destination] = source
3145
3146 lostAndFoundBranches.discard(destination)
3147
29bdbac1 3148 if source not in self.knownBranches:
6555b2cc
SH
3149 lostAndFoundBranches.add(source)
3150
7199cf13
VA
3151 # Perforce does not strictly require branches to be defined, so we also
3152 # check git config for a branch list.
3153 #
3154 # Example of branch definition in git config file:
3155 # [git-p4]
3156 # branchList=main:branchA
3157 # branchList=main:branchB
3158 # branchList=branchA:branchC
3159 configBranches = gitConfigList("git-p4.branchList")
3160 for branch in configBranches:
3161 if branch:
3162 (source, destination) = branch.split(":")
3163 self.knownBranches[destination] = source
3164
3165 lostAndFoundBranches.discard(destination)
3166
3167 if source not in self.knownBranches:
3168 lostAndFoundBranches.add(source)
3169
6555b2cc
SH
3170
3171 for branch in lostAndFoundBranches:
3172 self.knownBranches[branch] = branch
29bdbac1 3173
38f9f5ec
SH
3174 def getBranchMappingFromGitBranches(self):
3175 branches = p4BranchesInGit(self.importIntoRemotes)
3176 for branch in branches.keys():
3177 if branch == "master":
3178 branch = "main"
3179 else:
3180 branch = branch[len(self.projectName):]
3181 self.knownBranches[branch] = branch
3182
bb6e09b2
HWN
3183 def updateOptionDict(self, d):
3184 option_keys = {}
3185 if self.keepRepoPath:
3186 option_keys['keepRepoPath'] = 1
3187
3188 d["options"] = ' '.join(sorted(option_keys.keys()))
3189
3190 def readOptions(self, d):
3191 self.keepRepoPath = (d.has_key('options')
3192 and ('keepRepoPath' in d['options']))
6326aa58 3193
8134f69c
SH
3194 def gitRefForBranch(self, branch):
3195 if branch == "main":
3196 return self.refPrefix + "master"
3197
3198 if len(branch) <= 0:
3199 return branch
3200
3201 return self.refPrefix + self.projectName + branch
3202
1ca3d710
SH
3203 def gitCommitByP4Change(self, ref, change):
3204 if self.verbose:
3205 print "looking in ref " + ref + " for change %s using bisect..." % change
3206
3207 earliestCommit = ""
3208 latestCommit = parseRevision(ref)
3209
3210 while True:
3211 if self.verbose:
3212 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3213 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3214 if len(next) == 0:
3215 if self.verbose:
3216 print "argh"
3217 return ""
3218 log = extractLogMessageFromGitCommit(next)
3219 settings = extractSettingsGitLog(log)
3220 currentChange = int(settings['change'])
3221 if self.verbose:
3222 print "current change %s" % currentChange
3223
3224 if currentChange == change:
3225 if self.verbose:
3226 print "found %s" % next
3227 return next
3228
3229 if currentChange < change:
3230 earliestCommit = "^%s" % next
3231 else:
3232 latestCommit = "%s" % next
3233
3234 return ""
3235
3236 def importNewBranch(self, branch, maxChange):
3237 # make fast-import flush all changes to disk and update the refs using the checkpoint
3238 # command so that we can try to find the branch parent in the git history
3239 self.gitStream.write("checkpoint\n\n");
3240 self.gitStream.flush();
3241 branchPrefix = self.depotPaths[0] + branch + "/"
3242 range = "@1,%s" % maxChange
3243 #print "prefix" + branchPrefix
96b2d54a 3244 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3245 if len(changes) <= 0:
3246 return False
3247 firstChange = changes[0]
3248 #print "first change in branch: %s" % firstChange
3249 sourceBranch = self.knownBranches[branch]
3250 sourceDepotPath = self.depotPaths[0] + sourceBranch
3251 sourceRef = self.gitRefForBranch(sourceBranch)
3252 #print "source " + sourceBranch
3253
52a4880b 3254 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3255 #print "branch parent: %s" % branchParentChange
3256 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3257 if len(gitParent) > 0:
3258 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3259 #print "parent git commit: %s" % gitParent
3260
3261 self.importChanges(changes)
3262 return True
3263
fed23693
VA
3264 def searchParent(self, parent, branch, target):
3265 parentFound = False
c7d34884
PW
3266 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3267 "--no-merges", parent]):
fed23693
VA
3268 blob = blob.strip()
3269 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3270 parentFound = True
3271 if self.verbose:
3272 print "Found parent of %s in commit %s" % (branch, blob)
3273 break
3274 if parentFound:
3275 return blob
3276 else:
3277 return None
3278
e87f37ae
SH
3279 def importChanges(self, changes):
3280 cnt = 1
3281 for change in changes:
18fa13d0 3282 description = p4_describe(change)
e87f37ae
SH
3283 self.updateOptionDict(description)
3284
3285 if not self.silent:
3286 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3287 sys.stdout.flush()
3288 cnt = cnt + 1
3289
3290 try:
3291 if self.detectBranches:
3292 branches = self.splitFilesIntoBranches(description)
3293 for branch in branches.keys():
3294 ## HACK --hwn
3295 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3296 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3297
3298 parent = ""
3299
3300 filesForCommit = branches[branch]
3301
3302 if self.verbose:
3303 print "branch is %s" % branch
3304
3305 self.updatedBranches.add(branch)
3306
3307 if branch not in self.createdBranches:
3308 self.createdBranches.add(branch)
3309 parent = self.knownBranches[branch]
3310 if parent == branch:
3311 parent = ""
1ca3d710
SH
3312 else:
3313 fullBranch = self.projectName + branch
3314 if fullBranch not in self.p4BranchesInGit:
3315 if not self.silent:
3316 print("\n Importing new branch %s" % fullBranch);
3317 if self.importNewBranch(branch, change - 1):
3318 parent = ""
3319 self.p4BranchesInGit.append(fullBranch)
3320 if not self.silent:
3321 print("\n Resuming with change %s" % change);
3322
3323 if self.verbose:
3324 print "parent determined through known branches: %s" % parent
e87f37ae 3325
8134f69c
SH
3326 branch = self.gitRefForBranch(branch)
3327 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3328
3329 if self.verbose:
3330 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3331
3332 if len(parent) == 0 and branch in self.initialParents:
3333 parent = self.initialParents[branch]
3334 del self.initialParents[branch]
3335
fed23693
VA
3336 blob = None
3337 if len(parent) > 0:
4f9273d2 3338 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3339 if self.verbose:
3340 print "Creating temporary branch: " + tempBranch
e63231e5 3341 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3342 self.tempBranches.append(tempBranch)
3343 self.checkpoint()
3344 blob = self.searchParent(parent, branch, tempBranch)
3345 if blob:
e63231e5 3346 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3347 else:
3348 if self.verbose:
3349 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3350 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3351 else:
3352 files = self.extractFilesFromCommit(description)
e63231e5 3353 self.commit(description, files, self.branch,
e87f37ae 3354 self.initialParent)
47497844 3355 # only needed once, to connect to the previous commit
e87f37ae
SH
3356 self.initialParent = ""
3357 except IOError:
3358 print self.gitError.read()
3359 sys.exit(1)
3360
b9d34db9
LD
3361 def sync_origin_only(self):
3362 if self.syncWithOrigin:
3363 self.hasOrigin = originP4BranchesExist()
3364 if self.hasOrigin:
3365 if not self.silent:
3366 print 'Syncing with origin first, using "git fetch origin"'
3367 system("git fetch origin")
3368
c208a243
SH
3369 def importHeadRevision(self, revision):
3370 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3371
4e2e6ce4
PW
3372 details = {}
3373 details["user"] = "git perforce import user"
1494fcbb 3374 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3375 % (' '.join(self.depotPaths), revision))
3376 details["change"] = revision
3377 newestRevision = 0
3378
3379 fileCnt = 0
6de040df
LD
3380 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3381
3382 for info in p4CmdList(["files"] + fileArgs):
c208a243 3383
68b28593 3384 if 'code' in info and info['code'] == 'error':
c208a243
SH
3385 sys.stderr.write("p4 returned an error: %s\n"
3386 % info['data'])
d88e707f
PW
3387 if info['data'].find("must refer to client") >= 0:
3388 sys.stderr.write("This particular p4 error is misleading.\n")
3389 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3390 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3391 sys.exit(1)
68b28593
PW
3392 if 'p4ExitCode' in info:
3393 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3394 sys.exit(1)
3395
3396
3397 change = int(info["change"])
3398 if change > newestRevision:
3399 newestRevision = change
3400
56c09345 3401 if info["action"] in self.delete_actions:
c208a243
SH
3402 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3403 #fileCnt = fileCnt + 1
3404 continue
3405
3406 for prop in ["depotFile", "rev", "action", "type" ]:
3407 details["%s%s" % (prop, fileCnt)] = info[prop]
3408
3409 fileCnt = fileCnt + 1
3410
3411 details["change"] = newestRevision
4e2e6ce4 3412
9dcb9f24 3413 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3414 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3415 res = p4_describe(newestRevision)
3416 details["time"] = res["time"]
4e2e6ce4 3417
c208a243
SH
3418 self.updateOptionDict(details)
3419 try:
e63231e5 3420 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3421 except IOError:
3422 print "IO error with git fast-import. Is your git version recent enough?"
3423 print self.gitError.read()
3424
3425
b984733c 3426 def run(self, args):
6326aa58 3427 self.depotPaths = []
179caebf 3428 self.changeRange = ""
6326aa58 3429 self.previousDepotPaths = []
991a2de4 3430 self.hasOrigin = False
ce6f33c8 3431
29bdbac1
SH
3432 # map from branch depot path to parent branch
3433 self.knownBranches = {}
3434 self.initialParents = {}
3435
a028a98e
SH
3436 if self.importIntoRemotes:
3437 self.refPrefix = "refs/remotes/p4/"
3438 else:
db775559 3439 self.refPrefix = "refs/heads/p4/"
a028a98e 3440
b9d34db9 3441 self.sync_origin_only()
10f880f8 3442
5a8e84cd 3443 branch_arg_given = bool(self.branch)
569d1bd4 3444 if len(self.branch) == 0:
db775559 3445 self.branch = self.refPrefix + "master"
a028a98e 3446 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3447 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3448 system("git branch -D p4")
967f72e2 3449
a93d33ee
PW
3450 # accept either the command-line option, or the configuration variable
3451 if self.useClientSpec:
3452 # will use this after clone to set the variable
3453 self.useClientSpec_from_options = True
3454 else:
0d609032 3455 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3456 self.useClientSpec = True
3457 if self.useClientSpec:
543987bd 3458 self.clientSpecDirs = getClientSpec()
3a70cdfa 3459
6a49f8e2
HWN
3460 # TODO: should always look at previous commits,
3461 # merge with previous imports, if possible.
3462 if args == []:
d414c74a 3463 if self.hasOrigin:
5ca44617 3464 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3465
3466 # branches holds mapping from branch name to sha1
3467 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3468
3469 # restrict to just this one, disabling detect-branches
3470 if branch_arg_given:
3471 short = self.branch.split("/")[-1]
3472 if short in branches:
3473 self.p4BranchesInGit = [ short ]
3474 else:
3475 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3476
3477 if len(self.p4BranchesInGit) > 1:
3478 if not self.silent:
3479 print "Importing from/into multiple branches"
3480 self.detectBranches = True
8c9e8b6e
PW
3481 for branch in branches.keys():