git-p4: python3: replace <> with !=
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
a235e85c
BC
30try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
4addad22 45verbose = False
86949eef 46
06804c76 47# Only labels/tags matching this will be imported/exported
c8942a22 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 49
3deed5e0
LD
50# The block size is reduced automatically if required
51defaultBlockSize = 1<<20
1051ef00 52
0ef67acd 53p4_access_checked = False
1051ef00 54
21a50753
AK
55def p4_build_cmd(cmd):
56 """Build a suitable p4 command line.
57
58 This consolidates building and returning a p4 command line into one
59 location. It means that hooking into the environment, or other configuration
60 can be done more easily.
61 """
6de040df 62 real_cmd = ["p4"]
abcaf073
AK
63
64 user = gitConfig("git-p4.user")
65 if len(user) > 0:
6de040df 66 real_cmd += ["-u",user]
abcaf073
AK
67
68 password = gitConfig("git-p4.password")
69 if len(password) > 0:
6de040df 70 real_cmd += ["-P", password]
abcaf073
AK
71
72 port = gitConfig("git-p4.port")
73 if len(port) > 0:
6de040df 74 real_cmd += ["-p", port]
abcaf073
AK
75
76 host = gitConfig("git-p4.host")
77 if len(host) > 0:
41799aa2 78 real_cmd += ["-H", host]
abcaf073
AK
79
80 client = gitConfig("git-p4.client")
81 if len(client) > 0:
6de040df 82 real_cmd += ["-c", client]
abcaf073 83
89a6ecc5
LS
84 retries = gitConfigInt("git-p4.retries")
85 if retries is None:
86 # Perform 3 retries by default
87 retries = 3
bc233524
IK
88 if retries > 0:
89 # Provide a way to not pass this option by setting git-p4.retries to 0
90 real_cmd += ["-r", str(retries)]
6de040df
LD
91
92 if isinstance(cmd,basestring):
93 real_cmd = ' '.join(real_cmd) + ' ' + cmd
94 else:
95 real_cmd += cmd
0ef67acd
LD
96
97 # now check that we can actually talk to the server
98 global p4_access_checked
99 if not p4_access_checked:
100 p4_access_checked = True # suppress access checks in p4_check_access itself
101 p4_check_access()
102
21a50753
AK
103 return real_cmd
104
378f7be1
LD
105def git_dir(path):
106 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
107 This won't automatically add ".git" to a directory.
108 """
109 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
110 if not d or len(d) == 0:
111 return None
112 else:
113 return d
114
bbd84863
MF
115def chdir(path, is_client_path=False):
116 """Do chdir to the given path, and set the PWD environment
117 variable for use by P4. It does not look at getcwd() output.
118 Since we're not using the shell, it is necessary to set the
119 PWD environment variable explicitly.
120
121 Normally, expand the path to force it to be absolute. This
122 addresses the use of relative path names inside P4 settings,
123 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
124 as given; it looks for .p4config using PWD.
125
126 If is_client_path, the path was handed to us directly by p4,
127 and may be a symbolic link. Do not call os.getcwd() in this
128 case, because it will cause p4 to think that PWD is not inside
129 the client path.
130 """
131
132 os.chdir(path)
133 if not is_client_path:
134 path = os.getcwd()
135 os.environ['PWD'] = path
053fd0c1 136
4d25dc44
LS
137def calcDiskFree():
138 """Return free space in bytes on the disk of the given dirname."""
139 if platform.system() == 'Windows':
140 free_bytes = ctypes.c_ulonglong(0)
141 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
142 return free_bytes.value
143 else:
144 st = os.statvfs(os.getcwd())
145 return st.f_bavail * st.f_frsize
146
86dff6b6
HWN
147def die(msg):
148 if verbose:
149 raise Exception(msg)
150 else:
151 sys.stderr.write(msg + "\n")
152 sys.exit(1)
153
6de040df 154def write_pipe(c, stdin):
4addad22 155 if verbose:
6de040df 156 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 157
6de040df
LD
158 expand = isinstance(c,basestring)
159 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
160 pipe = p.stdin
161 val = pipe.write(stdin)
162 pipe.close()
163 if p.wait():
164 die('Command failed: %s' % str(c))
b016d397
HWN
165
166 return val
167
6de040df 168def p4_write_pipe(c, stdin):
d9429194 169 real_cmd = p4_build_cmd(c)
6de040df 170 return write_pipe(real_cmd, stdin)
d9429194 171
78871bf4
LD
172def read_pipe_full(c):
173 """ Read output from command. Returns a tuple
174 of the return status, stdout text and stderr
175 text.
176 """
4addad22 177 if verbose:
6de040df 178 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 179
6de040df 180 expand = isinstance(c,basestring)
1f5f3907
LS
181 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
182 (out, err) = p.communicate()
78871bf4
LD
183 return (p.returncode, out, err)
184
185def read_pipe(c, ignore_error=False):
186 """ Read output from command. Returns the output text on
187 success. On failure, terminates execution, unless
188 ignore_error is True, when it returns an empty string.
189 """
190 (retcode, out, err) = read_pipe_full(c)
191 if retcode != 0:
192 if ignore_error:
193 out = ""
194 else:
195 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 196 return out
b016d397 197
78871bf4
LD
198def read_pipe_text(c):
199 """ Read output from a command with trailing whitespace stripped.
200 On error, returns None.
201 """
202 (retcode, out, err) = read_pipe_full(c)
203 if retcode != 0:
204 return None
205 else:
206 return out.rstrip()
207
d9429194
AK
208def p4_read_pipe(c, ignore_error=False):
209 real_cmd = p4_build_cmd(c)
210 return read_pipe(real_cmd, ignore_error)
b016d397 211
bce4c5fc 212def read_pipe_lines(c):
4addad22 213 if verbose:
6de040df
LD
214 sys.stderr.write('Reading pipe: %s\n' % str(c))
215
216 expand = isinstance(c, basestring)
217 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
218 pipe = p.stdout
b016d397 219 val = pipe.readlines()
6de040df
LD
220 if pipe.close() or p.wait():
221 die('Command failed: %s' % str(c))
b016d397
HWN
222
223 return val
caace111 224
2318121b
AK
225def p4_read_pipe_lines(c):
226 """Specifically invoke p4 on the command supplied. """
155af834 227 real_cmd = p4_build_cmd(c)
2318121b
AK
228 return read_pipe_lines(real_cmd)
229
8e9497c2
GG
230def p4_has_command(cmd):
231 """Ask p4 for help on this command. If it returns an error, the
232 command does not exist in this version of p4."""
233 real_cmd = p4_build_cmd(["help", cmd])
234 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
235 stderr=subprocess.PIPE)
236 p.communicate()
237 return p.returncode == 0
238
249da4c0
PW
239def p4_has_move_command():
240 """See if the move command exists, that it supports -k, and that
241 it has not been administratively disabled. The arguments
242 must be correct, but the filenames do not have to exist. Use
243 ones with wildcards so even if they exist, it will fail."""
244
245 if not p4_has_command("move"):
246 return False
247 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
248 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
249 (out, err) = p.communicate()
250 # return code will be 1 in either case
251 if err.find("Invalid option") >= 0:
252 return False
253 if err.find("disabled") >= 0:
254 return False
255 # assume it failed because @... was invalid changelist
256 return True
257
cbff4b25 258def system(cmd, ignore_error=False):
6de040df 259 expand = isinstance(cmd,basestring)
4addad22 260 if verbose:
6de040df 261 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 262 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 263 if retcode and not ignore_error:
a235e85c 264 raise CalledProcessError(retcode, cmd)
6754a299 265
cbff4b25
LD
266 return retcode
267
bf9320f1
AK
268def p4_system(cmd):
269 """Specifically invoke p4 as the system command. """
155af834 270 real_cmd = p4_build_cmd(cmd)
6de040df 271 expand = isinstance(real_cmd, basestring)
a235e85c
BC
272 retcode = subprocess.call(real_cmd, shell=expand)
273 if retcode:
274 raise CalledProcessError(retcode, real_cmd)
6de040df 275
0ef67acd
LD
276def die_bad_access(s):
277 die("failure accessing depot: {0}".format(s.rstrip()))
278
279def p4_check_access(min_expiration=1):
280 """ Check if we can access Perforce - account still logged in
281 """
282 results = p4CmdList(["login", "-s"])
283
284 if len(results) == 0:
285 # should never get here: always get either some results, or a p4ExitCode
286 assert("could not parse response from perforce")
287
288 result = results[0]
289
290 if 'p4ExitCode' in result:
291 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
292 die_bad_access("could not run p4")
293
294 code = result.get("code")
295 if not code:
296 # we get here if we couldn't connect and there was nothing to unmarshal
297 die_bad_access("could not connect")
298
299 elif code == "stat":
300 expiry = result.get("TicketExpiration")
301 if expiry:
302 expiry = int(expiry)
303 if expiry > min_expiration:
304 # ok to carry on
305 return
306 else:
307 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
308
309 else:
310 # account without a timeout - all ok
311 return
312
313 elif code == "error":
314 data = result.get("data")
315 if data:
316 die_bad_access("p4 error: {0}".format(data))
317 else:
318 die_bad_access("unknown error")
319 else:
320 die_bad_access("unknown error code {0}".format(code))
321
7f0e5962
PW
322_p4_version_string = None
323def p4_version_string():
324 """Read the version string, showing just the last line, which
325 hopefully is the interesting version bit.
326
327 $ p4 -V
328 Perforce - The Fast Software Configuration Management System.
329 Copyright 1995-2011 Perforce Software. All rights reserved.
330 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
331 """
332 global _p4_version_string
333 if not _p4_version_string:
334 a = p4_read_pipe_lines(["-V"])
335 _p4_version_string = a[-1].rstrip()
336 return _p4_version_string
337
6de040df 338def p4_integrate(src, dest):
9d7d446a 339 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 340
8d7ec362 341def p4_sync(f, *options):
9d7d446a 342 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
343
344def p4_add(f):
9d7d446a
PW
345 # forcibly add file names with wildcards
346 if wildcard_present(f):
347 p4_system(["add", "-f", f])
348 else:
349 p4_system(["add", f])
6de040df
LD
350
351def p4_delete(f):
9d7d446a 352 p4_system(["delete", wildcard_encode(f)])
6de040df 353
a02b8bc4
RP
354def p4_edit(f, *options):
355 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
356
357def p4_revert(f):
9d7d446a 358 p4_system(["revert", wildcard_encode(f)])
6de040df 359
9d7d446a
PW
360def p4_reopen(type, f):
361 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 362
46c609e9
LD
363def p4_reopen_in_change(changelist, files):
364 cmd = ["reopen", "-c", str(changelist)] + files
365 p4_system(cmd)
366
8e9497c2
GG
367def p4_move(src, dest):
368 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
369
1051ef00 370def p4_last_change():
1997e91f 371 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
372 return int(results[0]['change'])
373
123f6317 374def p4_describe(change, shelved=False):
18fa13d0
PW
375 """Make sure it returns a valid result by checking for
376 the presence of field "time". Return a dict of the
377 results."""
378
123f6317
LD
379 cmd = ["describe", "-s"]
380 if shelved:
381 cmd += ["-S"]
382 cmd += [str(change)]
383
384 ds = p4CmdList(cmd, skip_info=True)
18fa13d0
PW
385 if len(ds) != 1:
386 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
387
388 d = ds[0]
389
390 if "p4ExitCode" in d:
391 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
392 str(d)))
393 if "code" in d:
394 if d["code"] == "error":
395 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
396
397 if "time" not in d:
398 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
399
400 return d
401
9cffb8c8
PW
402#
403# Canonicalize the p4 type and return a tuple of the
404# base type, plus any modifiers. See "p4 help filetypes"
405# for a list and explanation.
406#
407def split_p4_type(p4type):
408
409 p4_filetypes_historical = {
410 "ctempobj": "binary+Sw",
411 "ctext": "text+C",
412 "cxtext": "text+Cx",
413 "ktext": "text+k",
414 "kxtext": "text+kx",
415 "ltext": "text+F",
416 "tempobj": "binary+FSw",
417 "ubinary": "binary+F",
418 "uresource": "resource+F",
419 "uxbinary": "binary+Fx",
420 "xbinary": "binary+x",
421 "xltext": "text+Fx",
422 "xtempobj": "binary+Swx",
423 "xtext": "text+x",
424 "xunicode": "unicode+x",
425 "xutf16": "utf16+x",
426 }
427 if p4type in p4_filetypes_historical:
428 p4type = p4_filetypes_historical[p4type]
429 mods = ""
430 s = p4type.split("+")
431 base = s[0]
432 mods = ""
433 if len(s) > 1:
434 mods = s[1]
435 return (base, mods)
b9fc6ea9 436
60df071c
LD
437#
438# return the raw p4 type of a file (text, text+ko, etc)
439#
79467e61
PW
440def p4_type(f):
441 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
442 return results[0]['headType']
443
444#
445# Given a type base and modifier, return a regexp matching
446# the keywords that can be expanded in the file
447#
448def p4_keywords_regexp_for_type(base, type_mods):
449 if base in ("text", "unicode", "binary"):
450 kwords = None
451 if "ko" in type_mods:
452 kwords = 'Id|Header'
453 elif "k" in type_mods:
454 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
455 else:
456 return None
457 pattern = r"""
458 \$ # Starts with a dollar, followed by...
459 (%s) # one of the keywords, followed by...
6b2bf41e 460 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
461 \$ # another dollar
462 """ % kwords
463 return pattern
464 else:
465 return None
466
467#
468# Given a file, return a regexp matching the possible
469# RCS keywords that will be expanded, or None for files
470# with kw expansion turned off.
471#
472def p4_keywords_regexp_for_file(file):
473 if not os.path.exists(file):
474 return None
475 else:
476 (type_base, type_mods) = split_p4_type(p4_type(file))
477 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 478
c65b670e
CP
479def setP4ExecBit(file, mode):
480 # Reopens an already open file and changes the execute bit to match
481 # the execute bit setting in the passed in mode.
482
483 p4Type = "+x"
484
485 if not isModeExec(mode):
486 p4Type = getP4OpenedType(file)
487 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
488 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
489 if p4Type[-1] == "+":
490 p4Type = p4Type[0:-1]
491
6de040df 492 p4_reopen(p4Type, file)
c65b670e
CP
493
494def getP4OpenedType(file):
495 # Returns the perforce file type for the given file.
496
9d7d446a 497 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 498 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
499 if match:
500 return match.group(1)
501 else:
f3e5ae4f 502 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 503
06804c76
LD
504# Return the set of all p4 labels
505def getP4Labels(depotPaths):
506 labels = set()
507 if isinstance(depotPaths,basestring):
508 depotPaths = [depotPaths]
509
510 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
511 label = l['label']
512 labels.add(label)
513
514 return labels
515
516# Return the set of all git tags
517def getGitTags():
518 gitTags = set()
519 for line in read_pipe_lines(["git", "tag"]):
520 tag = line.strip()
521 gitTags.add(tag)
522 return gitTags
523
b43b0a3c
CP
524def diffTreePattern():
525 # This is a simple generator for the diff tree regex pattern. This could be
526 # a class variable if this and parseDiffTreeEntry were a part of a class.
527 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
528 while True:
529 yield pattern
530
531def parseDiffTreeEntry(entry):
532 """Parses a single diff tree entry into its component elements.
533
534 See git-diff-tree(1) manpage for details about the format of the diff
535 output. This method returns a dictionary with the following elements:
536
537 src_mode - The mode of the source file
538 dst_mode - The mode of the destination file
539 src_sha1 - The sha1 for the source file
540 dst_sha1 - The sha1 fr the destination file
541 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
542 status_score - The score for the status (applicable for 'C' and 'R'
543 statuses). This is None if there is no score.
544 src - The path for the source file.
545 dst - The path for the destination file. This is only present for
546 copy or renames. If it is not present, this is None.
547
548 If the pattern is not matched, None is returned."""
549
550 match = diffTreePattern().next().match(entry)
551 if match:
552 return {
553 'src_mode': match.group(1),
554 'dst_mode': match.group(2),
555 'src_sha1': match.group(3),
556 'dst_sha1': match.group(4),
557 'status': match.group(5),
558 'status_score': match.group(6),
559 'src': match.group(7),
560 'dst': match.group(10)
561 }
562 return None
563
c65b670e
CP
564def isModeExec(mode):
565 # Returns True if the given git mode represents an executable file,
566 # otherwise False.
567 return mode[-3:] == "755"
568
55bb3e36
LD
569class P4Exception(Exception):
570 """ Base class for exceptions from the p4 client """
571 def __init__(self, exit_code):
572 self.p4ExitCode = exit_code
573
574class P4ServerException(P4Exception):
575 """ Base class for exceptions where we get some kind of marshalled up result from the server """
576 def __init__(self, exit_code, p4_result):
577 super(P4ServerException, self).__init__(exit_code)
578 self.p4_result = p4_result
579 self.code = p4_result[0]['code']
580 self.data = p4_result[0]['data']
581
582class P4RequestSizeException(P4ServerException):
583 """ One of the maxresults or maxscanrows errors """
584 def __init__(self, exit_code, p4_result, limit):
585 super(P4RequestSizeException, self).__init__(exit_code, p4_result)
586 self.limit = limit
587
c65b670e
CP
588def isModeExecChanged(src_mode, dst_mode):
589 return isModeExec(src_mode) != isModeExec(dst_mode)
590
55bb3e36
LD
591def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
592 errors_as_exceptions=False):
6de040df
LD
593
594 if isinstance(cmd,basestring):
595 cmd = "-G " + cmd
596 expand = True
597 else:
598 cmd = ["-G"] + cmd
599 expand = False
600
601 cmd = p4_build_cmd(cmd)
6a49f8e2 602 if verbose:
6de040df 603 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
604
605 # Use a temporary file to avoid deadlocks without
606 # subprocess.communicate(), which would put another copy
607 # of stdout into memory.
608 stdin_file = None
609 if stdin is not None:
610 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
611 if isinstance(stdin,basestring):
612 stdin_file.write(stdin)
613 else:
614 for i in stdin:
615 stdin_file.write(i + '\n')
9f90c733
SL
616 stdin_file.flush()
617 stdin_file.seek(0)
618
6de040df
LD
619 p4 = subprocess.Popen(cmd,
620 shell=expand,
9f90c733
SL
621 stdin=stdin_file,
622 stdout=subprocess.PIPE)
86949eef
SH
623
624 result = []
625 try:
626 while True:
9f90c733 627 entry = marshal.load(p4.stdout)
1997e91f
MT
628 if skip_info:
629 if 'code' in entry and entry['code'] == 'info':
630 continue
c3f6163b
AG
631 if cb is not None:
632 cb(entry)
633 else:
634 result.append(entry)
86949eef
SH
635 except EOFError:
636 pass
9f90c733
SL
637 exitCode = p4.wait()
638 if exitCode != 0:
55bb3e36
LD
639 if errors_as_exceptions:
640 if len(result) > 0:
641 data = result[0].get('data')
642 if data:
643 m = re.search('Too many rows scanned \(over (\d+)\)', data)
644 if not m:
645 m = re.search('Request too large \(over (\d+)\)', data)
646
647 if m:
648 limit = int(m.group(1))
649 raise P4RequestSizeException(exitCode, result, limit)
650
651 raise P4ServerException(exitCode, result)
652 else:
653 raise P4Exception(exitCode)
654 else:
655 entry = {}
656 entry["p4ExitCode"] = exitCode
657 result.append(entry)
86949eef
SH
658
659 return result
660
661def p4Cmd(cmd):
662 list = p4CmdList(cmd)
663 result = {}
664 for entry in list:
665 result.update(entry)
666 return result;
667
cb2c9db5
SH
668def p4Where(depotPath):
669 if not depotPath.endswith("/"):
670 depotPath += "/"
cd884106
VA
671 depotPathLong = depotPath + "..."
672 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
673 output = None
674 for entry in outputList:
75bc9573 675 if "depotFile" in entry:
cd884106
VA
676 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
677 # The base path always ends with "/...".
678 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
679 output = entry
680 break
681 elif "data" in entry:
682 data = entry.get("data")
683 space = data.find(" ")
684 if data[:space] == depotPath:
685 output = entry
686 break
7f705dc3
TAL
687 if output == None:
688 return ""
dc524036
SH
689 if output["code"] == "error":
690 return ""
cb2c9db5
SH
691 clientPath = ""
692 if "path" in output:
693 clientPath = output.get("path")
694 elif "data" in output:
695 data = output.get("data")
696 lastSpace = data.rfind(" ")
697 clientPath = data[lastSpace + 1:]
698
699 if clientPath.endswith("..."):
700 clientPath = clientPath[:-3]
701 return clientPath
702
86949eef 703def currentGitBranch():
eff45110 704 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 705
4f5cf76a 706def isValidGitDir(path):
378f7be1 707 return git_dir(path) != None
4f5cf76a 708
463e8af6 709def parseRevision(ref):
b25b2065 710 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 711
28755dba
PW
712def branchExists(ref):
713 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
714 ignore_error=True)
715 return len(rev) > 0
716
6ae8de88
SH
717def extractLogMessageFromGitCommit(commit):
718 logMessage = ""
b016d397
HWN
719
720 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 721 foundTitle = False
b016d397 722 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
723 if not foundTitle:
724 if len(log) == 1:
1c094184 725 foundTitle = True
6ae8de88
SH
726 continue
727
728 logMessage += log
729 return logMessage
730
bb6e09b2 731def extractSettingsGitLog(log):
6ae8de88
SH
732 values = {}
733 for line in log.split("\n"):
734 line = line.strip()
6326aa58
HWN
735 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
736 if not m:
737 continue
738
739 assignments = m.group(1).split (':')
740 for a in assignments:
741 vals = a.split ('=')
742 key = vals[0].strip()
743 val = ('='.join (vals[1:])).strip()
744 if val.endswith ('\"') and val.startswith('"'):
745 val = val[1:-1]
746
747 values[key] = val
748
845b42cb
SH
749 paths = values.get("depot-paths")
750 if not paths:
751 paths = values.get("depot-path")
a3fdd579
SH
752 if paths:
753 values['depot-paths'] = paths.split(',')
bb6e09b2 754 return values
6ae8de88 755
8136a639 756def gitBranchExists(branch):
bb6e09b2
HWN
757 proc = subprocess.Popen(["git", "rev-parse", branch],
758 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 759 return proc.wait() == 0;
8136a639 760
123f6317
LD
761def gitUpdateRef(ref, newvalue):
762 subprocess.check_call(["git", "update-ref", ref, newvalue])
763
764def gitDeleteRef(ref):
765 subprocess.check_call(["git", "update-ref", "-d", ref])
766
36bd8446 767_gitConfig = {}
b345d6c3 768
692e1796 769def gitConfig(key, typeSpecifier=None):
36bd8446 770 if not _gitConfig.has_key(key):
692e1796
LS
771 cmd = [ "git", "config" ]
772 if typeSpecifier:
773 cmd += [ typeSpecifier ]
774 cmd += [ key ]
b345d6c3
PW
775 s = read_pipe(cmd, ignore_error=True)
776 _gitConfig[key] = s.strip()
36bd8446 777 return _gitConfig[key]
01265103 778
0d609032
PW
779def gitConfigBool(key):
780 """Return a bool, using git config --bool. It is True only if the
781 variable is set to true, and False if set to false or not present
782 in the config."""
783
36bd8446 784 if not _gitConfig.has_key(key):
692e1796 785 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 786 return _gitConfig[key]
01265103 787
cb1dafdf
LS
788def gitConfigInt(key):
789 if not _gitConfig.has_key(key):
790 cmd = [ "git", "config", "--int", key ]
0d609032
PW
791 s = read_pipe(cmd, ignore_error=True)
792 v = s.strip()
cb1dafdf
LS
793 try:
794 _gitConfig[key] = int(gitConfig(key, '--int'))
795 except ValueError:
796 _gitConfig[key] = None
36bd8446 797 return _gitConfig[key]
01265103 798
7199cf13
VA
799def gitConfigList(key):
800 if not _gitConfig.has_key(key):
2abba301 801 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 802 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
803 if _gitConfig[key] == ['']:
804 _gitConfig[key] = []
7199cf13
VA
805 return _gitConfig[key]
806
2c8037ed
PW
807def p4BranchesInGit(branchesAreInRemotes=True):
808 """Find all the branches whose names start with "p4/", looking
809 in remotes or heads as specified by the argument. Return
810 a dictionary of { branch: revision } for each one found.
811 The branch names are the short names, without any
812 "p4/" prefix."""
813
062410bb
SH
814 branches = {}
815
816 cmdline = "git rev-parse --symbolic "
817 if branchesAreInRemotes:
2c8037ed 818 cmdline += "--remotes"
062410bb 819 else:
2c8037ed 820 cmdline += "--branches"
062410bb
SH
821
822 for line in read_pipe_lines(cmdline):
823 line = line.strip()
824
2c8037ed
PW
825 # only import to p4/
826 if not line.startswith('p4/'):
827 continue
828 # special symbolic ref to p4/master
829 if line == "p4/HEAD":
062410bb 830 continue
062410bb 831
2c8037ed
PW
832 # strip off p4/ prefix
833 branch = line[len("p4/"):]
062410bb
SH
834
835 branches[branch] = parseRevision(line)
2c8037ed 836
062410bb
SH
837 return branches
838
5a8e84cd
PW
839def branch_exists(branch):
840 """Make sure that the given ref name really exists."""
841
842 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
843 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
844 out, _ = p.communicate()
845 if p.returncode:
846 return False
847 # expect exactly one line of output: the branch name
848 return out.rstrip() == branch
849
9ceab363 850def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
851 branches = p4BranchesInGit()
852 # map from depot-path to branch name
853 branchByDepotPath = {}
854 for branch in branches.keys():
855 tip = branches[branch]
856 log = extractLogMessageFromGitCommit(tip)
857 settings = extractSettingsGitLog(log)
858 if settings.has_key("depot-paths"):
859 paths = ",".join(settings["depot-paths"])
860 branchByDepotPath[paths] = "remotes/p4/" + branch
861
27d2d811 862 settings = None
27d2d811
SH
863 parent = 0
864 while parent < 65535:
9ceab363 865 commit = head + "~%s" % parent
27d2d811
SH
866 log = extractLogMessageFromGitCommit(commit)
867 settings = extractSettingsGitLog(log)
86506fe5
SH
868 if settings.has_key("depot-paths"):
869 paths = ",".join(settings["depot-paths"])
870 if branchByDepotPath.has_key(paths):
871 return [branchByDepotPath[paths], settings]
27d2d811 872
86506fe5 873 parent = parent + 1
27d2d811 874
86506fe5 875 return ["", settings]
27d2d811 876
5ca44617
SH
877def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
878 if not silent:
879 print ("Creating/updating branch(es) in %s based on origin branch(es)"
880 % localRefPrefix)
881
882 originPrefix = "origin/p4/"
883
884 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
885 line = line.strip()
886 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
887 continue
888
889 headName = line[len(originPrefix):]
890 remoteHead = localRefPrefix + headName
891 originHead = line
892
893 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
894 if (not original.has_key('depot-paths')
895 or not original.has_key('change')):
896 continue
897
898 update = False
899 if not gitBranchExists(remoteHead):
900 if verbose:
901 print "creating %s" % remoteHead
902 update = True
903 else:
904 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
905 if settings.has_key('change') > 0:
906 if settings['depot-paths'] == original['depot-paths']:
907 originP4Change = int(original['change'])
908 p4Change = int(settings['change'])
909 if originP4Change > p4Change:
910 print ("%s (%s) is newer than %s (%s). "
911 "Updating p4 branch from origin."
912 % (originHead, originP4Change,
913 remoteHead, p4Change))
914 update = True
915 else:
916 print ("Ignoring: %s was imported from %s while "
917 "%s was imported from %s"
918 % (originHead, ','.join(original['depot-paths']),
919 remoteHead, ','.join(settings['depot-paths'])))
920
921 if update:
922 system("git update-ref %s %s" % (remoteHead, originHead))
923
924def originP4BranchesExist():
925 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
926
1051ef00
LD
927
928def p4ParseNumericChangeRange(parts):
929 changeStart = int(parts[0][1:])
930 if parts[1] == '#head':
931 changeEnd = p4_last_change()
932 else:
933 changeEnd = int(parts[1])
934
935 return (changeStart, changeEnd)
936
937def chooseBlockSize(blockSize):
938 if blockSize:
939 return blockSize
940 else:
941 return defaultBlockSize
942
943def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 944 assert depotPaths
96b2d54a 945
1051ef00
LD
946 # Parse the change range into start and end. Try to find integer
947 # revision ranges as these can be broken up into blocks to avoid
948 # hitting server-side limits (maxrows, maxscanresults). But if
949 # that doesn't work, fall back to using the raw revision specifier
950 # strings, without using block mode.
951
96b2d54a 952 if changeRange is None or changeRange == '':
1051ef00
LD
953 changeStart = 1
954 changeEnd = p4_last_change()
955 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
956 else:
957 parts = changeRange.split(',')
958 assert len(parts) == 2
1051ef00
LD
959 try:
960 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
961 block_size = chooseBlockSize(requestedBlockSize)
8fa0abf8 962 except ValueError:
1051ef00
LD
963 changeStart = parts[0][1:]
964 changeEnd = parts[1]
965 if requestedBlockSize:
966 die("cannot use --changes-block-size with non-numeric revisions")
967 block_size = None
4f6432d8 968
9943e5b9 969 changes = set()
96b2d54a 970
1f90a648 971 # Retrieve changes a block at a time, to prevent running
3deed5e0
LD
972 # into a MaxResults/MaxScanRows error from the server. If
973 # we _do_ hit one of those errors, turn down the block size
1051ef00 974
1f90a648
SH
975 while True:
976 cmd = ['changes']
1051ef00 977
1f90a648
SH
978 if block_size:
979 end = min(changeEnd, changeStart + block_size)
980 revisionRange = "%d,%d" % (changeStart, end)
981 else:
982 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 983
1f90a648 984 for p in depotPaths:
1051ef00
LD
985 cmd += ["%s...@%s" % (p, revisionRange)]
986
3deed5e0
LD
987 # fetch the changes
988 try:
989 result = p4CmdList(cmd, errors_as_exceptions=True)
990 except P4RequestSizeException as e:
991 if not block_size:
992 block_size = e.limit
993 elif block_size > e.limit:
994 block_size = e.limit
995 else:
996 block_size = max(2, block_size // 2)
997
998 if verbose: print("block size error, retrying with block size {0}".format(block_size))
999 continue
1000 except P4Exception as e:
1001 die('Error retrieving changes description ({0})'.format(e.p4ExitCode))
1002
1f90a648 1003 # Insert changes in chronological order
3deed5e0 1004 for entry in reversed(result):
b596b3b9
MT
1005 if not entry.has_key('change'):
1006 continue
1007 changes.add(int(entry['change']))
1051ef00 1008
1f90a648
SH
1009 if not block_size:
1010 break
1051ef00 1011
1f90a648
SH
1012 if end >= changeEnd:
1013 break
1051ef00 1014
1f90a648 1015 changeStart = end + 1
4f6432d8 1016
1f90a648
SH
1017 changes = sorted(changes)
1018 return changes
4f6432d8 1019
d53de8b9
TAL
1020def p4PathStartsWith(path, prefix):
1021 # This method tries to remedy a potential mixed-case issue:
1022 #
1023 # If UserA adds //depot/DirA/file1
1024 # and UserB adds //depot/dira/file2
1025 #
1026 # we may or may not have a problem. If you have core.ignorecase=true,
1027 # we treat DirA and dira as the same directory
0d609032 1028 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
1029 return path.lower().startswith(prefix.lower())
1030 return path.startswith(prefix)
1031
543987bd
PW
1032def getClientSpec():
1033 """Look at the p4 client spec, create a View() object that contains
1034 all the mappings, and return it."""
1035
1036 specList = p4CmdList("client -o")
1037 if len(specList) != 1:
1038 die('Output from "client -o" is %d lines, expecting 1' %
1039 len(specList))
1040
1041 # dictionary of all client parameters
1042 entry = specList[0]
1043
9d57c4a6
KS
1044 # the //client/ name
1045 client_name = entry["Client"]
1046
543987bd
PW
1047 # just the keys that start with "View"
1048 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
1049
1050 # hold this new View
9d57c4a6 1051 view = View(client_name)
543987bd
PW
1052
1053 # append the lines, in order, to the view
1054 for view_num in range(len(view_keys)):
1055 k = "View%d" % view_num
1056 if k not in view_keys:
1057 die("Expected view key %s missing" % k)
1058 view.append(entry[k])
1059
1060 return view
1061
1062def getClientRoot():
1063 """Grab the client directory."""
1064
1065 output = p4CmdList("client -o")
1066 if len(output) != 1:
1067 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1068
1069 entry = output[0]
1070 if "Root" not in entry:
1071 die('Client has no "Root"')
1072
1073 return entry["Root"]
1074
9d7d446a
PW
1075#
1076# P4 wildcards are not allowed in filenames. P4 complains
1077# if you simply add them, but you can force it with "-f", in
1078# which case it translates them into %xx encoding internally.
1079#
1080def wildcard_decode(path):
1081 # Search for and fix just these four characters. Do % last so
1082 # that fixing it does not inadvertently create new %-escapes.
1083 # Cannot have * in a filename in windows; untested as to
1084 # what p4 would do in such a case.
1085 if not platform.system() == "Windows":
1086 path = path.replace("%2A", "*")
1087 path = path.replace("%23", "#") \
1088 .replace("%40", "@") \
1089 .replace("%25", "%")
1090 return path
1091
1092def wildcard_encode(path):
1093 # do % first to avoid double-encoding the %s introduced here
1094 path = path.replace("%", "%25") \
1095 .replace("*", "%2A") \
1096 .replace("#", "%23") \
1097 .replace("@", "%40")
1098 return path
1099
1100def wildcard_present(path):
598354c0
BC
1101 m = re.search("[*#@%]", path)
1102 return m is not None
9d7d446a 1103
a5db4b12
LS
1104class LargeFileSystem(object):
1105 """Base class for large file system support."""
1106
1107 def __init__(self, writeToGitStream):
1108 self.largeFiles = set()
1109 self.writeToGitStream = writeToGitStream
1110
1111 def generatePointer(self, cloneDestination, contentFile):
1112 """Return the content of a pointer file that is stored in Git instead of
1113 the actual content."""
1114 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1115
1116 def pushFile(self, localLargeFile):
1117 """Push the actual content which is not stored in the Git repository to
1118 a server."""
1119 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1120
1121 def hasLargeFileExtension(self, relPath):
1122 return reduce(
1123 lambda a, b: a or b,
1124 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1125 False
1126 )
1127
1128 def generateTempFile(self, contents):
1129 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1130 for d in contents:
1131 contentFile.write(d)
1132 contentFile.close()
1133 return contentFile.name
1134
1135 def exceedsLargeFileThreshold(self, relPath, contents):
1136 if gitConfigInt('git-p4.largeFileThreshold'):
1137 contentsSize = sum(len(d) for d in contents)
1138 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1139 return True
1140 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1141 contentsSize = sum(len(d) for d in contents)
1142 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1143 return False
1144 contentTempFile = self.generateTempFile(contents)
1145 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1146 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1147 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1148 zf.close()
1149 compressedContentsSize = zf.infolist()[0].compress_size
1150 os.remove(contentTempFile)
1151 os.remove(compressedContentFile.name)
1152 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1153 return True
1154 return False
1155
1156 def addLargeFile(self, relPath):
1157 self.largeFiles.add(relPath)
1158
1159 def removeLargeFile(self, relPath):
1160 self.largeFiles.remove(relPath)
1161
1162 def isLargeFile(self, relPath):
1163 return relPath in self.largeFiles
1164
1165 def processContent(self, git_mode, relPath, contents):
1166 """Processes the content of git fast import. This method decides if a
1167 file is stored in the large file system and handles all necessary
1168 steps."""
1169 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1170 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1171 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1172 if pointer_git_mode:
1173 git_mode = pointer_git_mode
1174 if localLargeFile:
1175 # Move temp file to final location in large file system
1176 largeFileDir = os.path.dirname(localLargeFile)
1177 if not os.path.isdir(largeFileDir):
1178 os.makedirs(largeFileDir)
1179 shutil.move(contentTempFile, localLargeFile)
1180 self.addLargeFile(relPath)
1181 if gitConfigBool('git-p4.largeFilePush'):
1182 self.pushFile(localLargeFile)
1183 if verbose:
1184 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1185 return (git_mode, contents)
1186
1187class MockLFS(LargeFileSystem):
1188 """Mock large file system for testing."""
1189
1190 def generatePointer(self, contentFile):
1191 """The pointer content is the original content prefixed with "pointer-".
1192 The local filename of the large file storage is derived from the file content.
1193 """
1194 with open(contentFile, 'r') as f:
1195 content = next(f)
1196 gitMode = '100644'
1197 pointerContents = 'pointer-' + content
1198 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1199 return (gitMode, pointerContents, localLargeFile)
1200
1201 def pushFile(self, localLargeFile):
1202 """The remote filename of the large file storage is the same as the local
1203 one but in a different directory.
1204 """
1205 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1206 if not os.path.exists(remotePath):
1207 os.makedirs(remotePath)
1208 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1209
b47d807d
LS
1210class GitLFS(LargeFileSystem):
1211 """Git LFS as backend for the git-p4 large file system.
1212 See https://git-lfs.github.com/ for details."""
1213
1214 def __init__(self, *args):
1215 LargeFileSystem.__init__(self, *args)
1216 self.baseGitAttributes = []
1217
1218 def generatePointer(self, contentFile):
1219 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1220 mode and content which is stored in the Git repository instead of
1221 the actual content. Return also the new location of the actual
1222 content.
1223 """
d5eb3cf5
LS
1224 if os.path.getsize(contentFile) == 0:
1225 return (None, '', None)
1226
b47d807d
LS
1227 pointerProcess = subprocess.Popen(
1228 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1229 stdout=subprocess.PIPE
1230 )
1231 pointerFile = pointerProcess.stdout.read()
1232 if pointerProcess.wait():
1233 os.remove(contentFile)
1234 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1235
1236 # Git LFS removed the preamble in the output of the 'pointer' command
1237 # starting from version 1.2.0. Check for the preamble here to support
1238 # earlier versions.
1239 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1240 if pointerFile.startswith('Git LFS pointer for'):
1241 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1242
1243 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1244 localLargeFile = os.path.join(
1245 os.getcwd(),
1246 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1247 oid,
1248 )
1249 # LFS Spec states that pointer files should not have the executable bit set.
1250 gitMode = '100644'
82f2567e 1251 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1252
1253 def pushFile(self, localLargeFile):
1254 uploadProcess = subprocess.Popen(
1255 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1256 )
1257 if uploadProcess.wait():
1258 die('git-lfs push command failed. Did you define a remote?')
1259
1260 def generateGitAttributes(self):
1261 return (
1262 self.baseGitAttributes +
1263 [
1264 '\n',
1265 '#\n',
1266 '# Git LFS (see https://git-lfs.github.com/)\n',
1267 '#\n',
1268 ] +
862f9312 1269 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1270 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1271 ] +
862f9312 1272 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1273 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1274 ]
1275 )
1276
1277 def addLargeFile(self, relPath):
1278 LargeFileSystem.addLargeFile(self, relPath)
1279 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1280
1281 def removeLargeFile(self, relPath):
1282 LargeFileSystem.removeLargeFile(self, relPath)
1283 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1284
1285 def processContent(self, git_mode, relPath, contents):
1286 if relPath == '.gitattributes':
1287 self.baseGitAttributes = contents
1288 return (git_mode, self.generateGitAttributes())
1289 else:
1290 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1291
b984733c
SH
1292class Command:
1293 def __init__(self):
1294 self.usage = "usage: %prog [options]"
8910ac0e 1295 self.needsGit = True
6a10b6aa 1296 self.verbose = False
b984733c 1297
8cf422db
LD
1298 # This is required for the "append" cloneExclude action
1299 def ensure_value(self, attr, value):
1300 if not hasattr(self, attr) or getattr(self, attr) is None:
1301 setattr(self, attr, value)
1302 return getattr(self, attr)
1303
3ea2cfd4
LD
1304class P4UserMap:
1305 def __init__(self):
1306 self.userMapFromPerforceServer = False
affb474f
LD
1307 self.myP4UserId = None
1308
1309 def p4UserId(self):
1310 if self.myP4UserId:
1311 return self.myP4UserId
1312
1313 results = p4CmdList("user -o")
1314 for r in results:
1315 if r.has_key('User'):
1316 self.myP4UserId = r['User']
1317 return r['User']
1318 die("Could not find your p4 user id")
1319
1320 def p4UserIsMe(self, p4User):
1321 # return True if the given p4 user is actually me
1322 me = self.p4UserId()
1323 if not p4User or p4User != me:
1324 return False
1325 else:
1326 return True
3ea2cfd4
LD
1327
1328 def getUserCacheFilename(self):
1329 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1330 return home + "/.gitp4-usercache.txt"
1331
1332 def getUserMapFromPerforceServer(self):
1333 if self.userMapFromPerforceServer:
1334 return
1335 self.users = {}
1336 self.emails = {}
1337
1338 for output in p4CmdList("users"):
1339 if not output.has_key("User"):
1340 continue
1341 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1342 self.emails[output["Email"]] = output["User"]
1343
10d08a14
LS
1344 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1345 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1346 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1347 if mapUser and len(mapUser[0]) == 3:
1348 user = mapUser[0][0]
1349 fullname = mapUser[0][1]
1350 email = mapUser[0][2]
1351 self.users[user] = fullname + " <" + email + ">"
1352 self.emails[email] = user
3ea2cfd4
LD
1353
1354 s = ''
1355 for (key, val) in self.users.items():
1356 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1357
1358 open(self.getUserCacheFilename(), "wb").write(s)
1359 self.userMapFromPerforceServer = True
1360
1361 def loadUserMapFromCache(self):
1362 self.users = {}
1363 self.userMapFromPerforceServer = False
1364 try:
1365 cache = open(self.getUserCacheFilename(), "rb")
1366 lines = cache.readlines()
1367 cache.close()
1368 for line in lines:
1369 entry = line.strip().split("\t")
1370 self.users[entry[0]] = entry[1]
1371 except IOError:
1372 self.getUserMapFromPerforceServer()
1373
b984733c 1374class P4Debug(Command):
86949eef 1375 def __init__(self):
6ae8de88 1376 Command.__init__(self)
6a10b6aa 1377 self.options = []
c8c39116 1378 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1379 self.needsGit = False
86949eef
SH
1380
1381 def run(self, args):
b1ce9447 1382 j = 0
6de040df 1383 for output in p4CmdList(args):
b1ce9447
HWN
1384 print 'Element: %d' % j
1385 j += 1
86949eef 1386 print output
b984733c 1387 return True
86949eef 1388
5834684d
SH
1389class P4RollBack(Command):
1390 def __init__(self):
1391 Command.__init__(self)
1392 self.options = [
0c66a783 1393 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1394 ]
1395 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1396 self.rollbackLocalBranches = False
5834684d
SH
1397
1398 def run(self, args):
1399 if len(args) != 1:
1400 return False
1401 maxChange = int(args[0])
0c66a783 1402
ad192f28 1403 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1404 die("Problems executing p4");
1405
0c66a783
SH
1406 if self.rollbackLocalBranches:
1407 refPrefix = "refs/heads/"
b016d397 1408 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1409 else:
1410 refPrefix = "refs/remotes/"
b016d397 1411 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1412
1413 for line in lines:
1414 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1415 line = line.strip()
1416 ref = refPrefix + line
5834684d 1417 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1418 settings = extractSettingsGitLog(log)
1419
1420 depotPaths = settings['depot-paths']
1421 change = settings['change']
1422
5834684d 1423 changed = False
52102d47 1424
6326aa58
HWN
1425 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1426 for p in depotPaths]))) == 0:
52102d47
SH
1427 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1428 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1429 continue
1430
bb6e09b2 1431 while change and int(change) > maxChange:
5834684d 1432 changed = True
52102d47
SH
1433 if self.verbose:
1434 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1435 system("git update-ref %s \"%s^\"" % (ref, ref))
1436 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1437 settings = extractSettingsGitLog(log)
1438
1439
1440 depotPaths = settings['depot-paths']
1441 change = settings['change']
5834684d
SH
1442
1443 if changed:
52102d47 1444 print "%s rewound to %s" % (ref, change)
5834684d
SH
1445
1446 return True
1447
3ea2cfd4 1448class P4Submit(Command, P4UserMap):
6bbfd137
PW
1449
1450 conflict_behavior_choices = ("ask", "skip", "quit")
1451
4f5cf76a 1452 def __init__(self):
b984733c 1453 Command.__init__(self)
3ea2cfd4 1454 P4UserMap.__init__(self)
4f5cf76a 1455 self.options = [
4f5cf76a 1456 optparse.make_option("--origin", dest="origin"),
ae901090 1457 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1458 # preserve the user, requires relevant p4 permissions
1459 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1460 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1461 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1462 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1463 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1464 choices=self.conflict_behavior_choices),
1465 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1466 optparse.make_option("--shelve", dest="shelve", action="store_true",
1467 help="Shelve instead of submit. Shelved files are reverted, "
1468 "restoring the workspace to the state before the shelve"),
8cf422db 1469 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1470 metavar="CHANGELIST",
8cf422db 1471 help="update an existing shelved changelist, implies --shelve, "
f55b87c1
RM
1472 "repeat in-order for multiple shelved changelists"),
1473 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1474 help="submit only the specified commit(s), one commit or xxx..xxx"),
1475 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1476 help="Disable rebase after submit is completed. Can be useful if you "
b9d34db9
LD
1477 "work from a local git branch that is not master"),
1478 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1479 help="Skip Perforce sync of p4/master after submit or shelve"),
4f5cf76a
SH
1480 ]
1481 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1482 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1483 self.origin = ""
ae901090 1484 self.detectRenames = False
0d609032 1485 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1486 self.dry_run = False
b34fa577 1487 self.shelve = False
8cf422db 1488 self.update_shelve = list()
f55b87c1 1489 self.commit = ""
3b3477ea 1490 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
b9d34db9 1491 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
728b7ad8 1492 self.prepare_p4_only = False
6bbfd137 1493 self.conflict_behavior = None
f7baba8b 1494 self.isWindows = (platform.system() == "Windows")
06804c76 1495 self.exportLabels = False
249da4c0 1496 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1497 self.branch = None
4f5cf76a 1498
a5db4b12
LS
1499 if gitConfig('git-p4.largeFileSystem'):
1500 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1501
4f5cf76a
SH
1502 def check(self):
1503 if len(p4CmdList("opened ...")) > 0:
1504 die("You have files opened with perforce! Close them before starting the sync.")
1505
f19cb0a0
PW
1506 def separate_jobs_from_description(self, message):
1507 """Extract and return a possible Jobs field in the commit
1508 message. It goes into a separate section in the p4 change
1509 specification.
1510
1511 A jobs line starts with "Jobs:" and looks like a new field
1512 in a form. Values are white-space separated on the same
1513 line or on following lines that start with a tab.
1514
1515 This does not parse and extract the full git commit message
1516 like a p4 form. It just sees the Jobs: line as a marker
1517 to pass everything from then on directly into the p4 form,
1518 but outside the description section.
1519
1520 Return a tuple (stripped log message, jobs string)."""
1521
1522 m = re.search(r'^Jobs:', message, re.MULTILINE)
1523 if m is None:
1524 return (message, None)
1525
1526 jobtext = message[m.start():]
1527 stripped_message = message[:m.start()].rstrip()
1528 return (stripped_message, jobtext)
1529
1530 def prepareLogMessage(self, template, message, jobs):
1531 """Edits the template returned from "p4 change -o" to insert
1532 the message in the Description field, and the jobs text in
1533 the Jobs field."""
4f5cf76a
SH
1534 result = ""
1535
edae1e2f
SH
1536 inDescriptionSection = False
1537
4f5cf76a
SH
1538 for line in template.split("\n"):
1539 if line.startswith("#"):
1540 result += line + "\n"
1541 continue
1542
edae1e2f 1543 if inDescriptionSection:
c9dbab04 1544 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1545 inDescriptionSection = False
f19cb0a0
PW
1546 # insert Jobs section
1547 if jobs:
1548 result += jobs + "\n"
edae1e2f
SH
1549 else:
1550 continue
1551 else:
1552 if line.startswith("Description:"):
1553 inDescriptionSection = True
1554 line += "\n"
1555 for messageLine in message.split("\n"):
1556 line += "\t" + messageLine + "\n"
1557
1558 result += line + "\n"
4f5cf76a
SH
1559
1560 return result
1561
60df071c
LD
1562 def patchRCSKeywords(self, file, pattern):
1563 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1564 (handle, outFileName) = tempfile.mkstemp(dir='.')
1565 try:
1566 outFile = os.fdopen(handle, "w+")
1567 inFile = open(file, "r")
1568 regexp = re.compile(pattern, re.VERBOSE)
1569 for line in inFile.readlines():
1570 line = regexp.sub(r'$\1$', line)
1571 outFile.write(line)
1572 inFile.close()
1573 outFile.close()
1574 # Forcibly overwrite the original file
1575 os.unlink(file)
1576 shutil.move(outFileName, file)
1577 except:
1578 # cleanup our temporary file
1579 os.unlink(outFileName)
1580 print "Failed to strip RCS keywords in %s" % file
1581 raise
1582
1583 print "Patched up RCS keywords in %s" % file
1584
3ea2cfd4
LD
1585 def p4UserForCommit(self,id):
1586 # Return the tuple (perforce user,git email) for a given git commit id
1587 self.getUserMapFromPerforceServer()
9bf28855
PW
1588 gitEmail = read_pipe(["git", "log", "--max-count=1",
1589 "--format=%ae", id])
3ea2cfd4
LD
1590 gitEmail = gitEmail.strip()
1591 if not self.emails.has_key(gitEmail):
1592 return (None,gitEmail)
1593 else:
1594 return (self.emails[gitEmail],gitEmail)
1595
1596 def checkValidP4Users(self,commits):
1597 # check if any git authors cannot be mapped to p4 users
1598 for id in commits:
1599 (user,email) = self.p4UserForCommit(id)
1600 if not user:
1601 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1602 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1603 print "%s" % msg
1604 else:
1605 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1606
1607 def lastP4Changelist(self):
1608 # Get back the last changelist number submitted in this client spec. This
1609 # then gets used to patch up the username in the change. If the same
1610 # client spec is being used by multiple processes then this might go
1611 # wrong.
1612 results = p4CmdList("client -o") # find the current client
1613 client = None
1614 for r in results:
1615 if r.has_key('Client'):
1616 client = r['Client']
1617 break
1618 if not client:
1619 die("could not get client spec")
6de040df 1620 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1621 for r in results:
1622 if r.has_key('change'):
1623 return r['change']
1624 die("Could not get changelist number for last submit - cannot patch up user details")
1625
1626 def modifyChangelistUser(self, changelist, newUser):
1627 # fixup the user field of a changelist after it has been submitted.
1628 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1629 if len(changes) != 1:
1630 die("Bad output from p4 change modifying %s to user %s" %
1631 (changelist, newUser))
1632
1633 c = changes[0]
1634 if c['User'] == newUser: return # nothing to do
1635 c['User'] = newUser
1636 input = marshal.dumps(c)
1637
3ea2cfd4
LD
1638 result = p4CmdList("change -f -i", stdin=input)
1639 for r in result:
1640 if r.has_key('code'):
1641 if r['code'] == 'error':
1642 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1643 if r.has_key('data'):
1644 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1645 return
1646 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1647
1648 def canChangeChangelists(self):
1649 # check to see if we have p4 admin or super-user permissions, either of
1650 # which are required to modify changelists.
52a4880b 1651 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1652 for r in results:
1653 if r.has_key('perm'):
1654 if r['perm'] == 'admin':
1655 return 1
1656 if r['perm'] == 'super':
1657 return 1
1658 return 0
1659
46c609e9 1660 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1661 """Run "p4 change -o" to grab a change specification template.
1662 This does not use "p4 -G", as it is nice to keep the submission
1663 template in original order, since a human might edit it.
1664
1665 Remove lines in the Files section that show changes to files
1666 outside the depot path we're committing into."""
1667
cbc69242
SH
1668 [upstream, settings] = findUpstreamBranchPoint()
1669
b596b3b9
MT
1670 template = """\
1671# A Perforce Change Specification.
1672#
1673# Change: The change number. 'new' on a new changelist.
1674# Date: The date this specification was last modified.
1675# Client: The client on which the changelist was created. Read-only.
1676# User: The user who created the changelist.
1677# Status: Either 'pending' or 'submitted'. Read-only.
1678# Type: Either 'public' or 'restricted'. Default is 'public'.
1679# Description: Comments about the changelist. Required.
1680# Jobs: What opened jobs are to be closed by this changelist.
1681# You may delete jobs from this list. (New changelists only.)
1682# Files: What opened files from the default changelist are to be added
1683# to this changelist. You may delete files from this list.
1684# (New changelists only.)
1685"""
1686 files_list = []
ea99c3ae 1687 inFilesSection = False
b596b3b9 1688 change_entry = None
46c609e9
LD
1689 args = ['change', '-o']
1690 if changelist:
1691 args.append(str(changelist))
b596b3b9
MT
1692 for entry in p4CmdList(args):
1693 if not entry.has_key('code'):
1694 continue
1695 if entry['code'] == 'stat':
1696 change_entry = entry
1697 break
1698 if not change_entry:
1699 die('Failed to decode output of p4 change -o')
1700 for key, value in change_entry.iteritems():
1701 if key.startswith('File'):
1702 if settings.has_key('depot-paths'):
1703 if not [p for p in settings['depot-paths']
1704 if p4PathStartsWith(value, p)]:
1705 continue
ea99c3ae 1706 else:
b596b3b9
MT
1707 if not p4PathStartsWith(value, self.depotPath):
1708 continue
1709 files_list.append(value)
1710 continue
1711 # Output in the order expected by prepareLogMessage
1712 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1713 if not change_entry.has_key(key):
1714 continue
1715 template += '\n'
1716 template += key + ':'
1717 if key == 'Description':
1718 template += '\n'
1719 for field_line in change_entry[key].splitlines():
1720 template += '\t'+field_line+'\n'
1721 if len(files_list) > 0:
1722 template += '\n'
1723 template += 'Files:\n'
1724 for path in files_list:
1725 template += '\t'+path+'\n'
ea99c3ae
SH
1726 return template
1727
7c766e57
PW
1728 def edit_template(self, template_file):
1729 """Invoke the editor to let the user change the submission
1730 message. Return true if okay to continue with the submit."""
1731
1732 # if configured to skip the editing part, just submit
0d609032 1733 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1734 return True
1735
1736 # look at the modification time, to check later if the user saved
1737 # the file
1738 mtime = os.stat(template_file).st_mtime
1739
1740 # invoke the editor
f95ceaf0 1741 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1742 editor = os.environ.get("P4EDITOR")
1743 else:
1744 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1745 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1746
1747 # If the file was not saved, prompt to see if this patch should
1748 # be skipped. But skip this verification step if configured so.
0d609032 1749 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1750 return True
1751
d1652049
PW
1752 # modification time updated means user saved the file
1753 if os.stat(template_file).st_mtime > mtime:
1754 return True
1755
1756 while True:
1757 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1758 if response == 'y':
1759 return True
1760 if response == 'n':
1761 return False
7c766e57 1762
df8a9e86 1763 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3
MC
1764 # diff
1765 if os.environ.has_key("P4DIFF"):
1766 del(os.environ["P4DIFF"])
1767 diff = ""
1768 for editedFile in editedFiles:
1769 diff += p4_read_pipe(['diff', '-du',
1770 wildcard_encode(editedFile)])
1771
1772 # new file diff
1773 newdiff = ""
1774 for newFile in filesToAdd:
1775 newdiff += "==== new file ====\n"
1776 newdiff += "--- /dev/null\n"
1777 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1778
1779 is_link = os.path.islink(newFile)
1780 expect_link = newFile in symlinks
1781
1782 if is_link and expect_link:
1783 newdiff += "+%s\n" % os.readlink(newFile)
1784 else:
1785 f = open(newFile, "r")
1786 for line in f.readlines():
1787 newdiff += "+" + line
1788 f.close()
b4073bb3 1789
e2a892ee 1790 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1791
7cb5cbef 1792 def applyCommit(self, id):
67b0fe2e
PW
1793 """Apply one commit, return True if it succeeded."""
1794
1795 print "Applying", read_pipe(["git", "show", "-s",
1796 "--format=format:%h %s", id])
ae901090 1797
848de9c3 1798 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1799
84cb0003 1800 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1801 filesToAdd = set()
a02b8bc4 1802 filesToChangeType = set()
4f5cf76a 1803 filesToDelete = set()
d336c158 1804 editedFiles = set()
b6ad6dcc 1805 pureRenameCopy = set()
df8a9e86 1806 symlinks = set()
c65b670e 1807 filesToChangeExecBit = {}
46c609e9 1808 all_files = list()
60df071c 1809
4f5cf76a 1810 for line in diff:
b43b0a3c
CP
1811 diff = parseDiffTreeEntry(line)
1812 modifier = diff['status']
1813 path = diff['src']
46c609e9
LD
1814 all_files.append(path)
1815
4f5cf76a 1816 if modifier == "M":
6de040df 1817 p4_edit(path)
c65b670e
CP
1818 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1819 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1820 editedFiles.add(path)
4f5cf76a
SH
1821 elif modifier == "A":
1822 filesToAdd.add(path)
c65b670e 1823 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1824 if path in filesToDelete:
1825 filesToDelete.remove(path)
df8a9e86
LD
1826
1827 dst_mode = int(diff['dst_mode'], 8)
1828 if dst_mode == 0120000:
1829 symlinks.add(path)
1830
4f5cf76a
SH
1831 elif modifier == "D":
1832 filesToDelete.add(path)
1833 if path in filesToAdd:
1834 filesToAdd.remove(path)
4fddb41b
VA
1835 elif modifier == "C":
1836 src, dest = diff['src'], diff['dst']
6de040df 1837 p4_integrate(src, dest)
b6ad6dcc 1838 pureRenameCopy.add(dest)
4fddb41b 1839 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1840 p4_edit(dest)
b6ad6dcc 1841 pureRenameCopy.discard(dest)
4fddb41b 1842 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1843 p4_edit(dest)
b6ad6dcc 1844 pureRenameCopy.discard(dest)
4fddb41b 1845 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1846 if self.isWindows:
1847 # turn off read-only attribute
1848 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1849 os.unlink(dest)
1850 editedFiles.add(dest)
d9a5f25b 1851 elif modifier == "R":
b43b0a3c 1852 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1853 if self.p4HasMoveCommand:
1854 p4_edit(src) # src must be open before move
1855 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1856 else:
8e9497c2
GG
1857 p4_integrate(src, dest)
1858 if diff['src_sha1'] != diff['dst_sha1']:
1859 p4_edit(dest)
1860 else:
1861 pureRenameCopy.add(dest)
c65b670e 1862 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1863 if not self.p4HasMoveCommand:
1864 p4_edit(dest) # with move: already open, writable
c65b670e 1865 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1866 if not self.p4HasMoveCommand:
d20f0f8e
PW
1867 if self.isWindows:
1868 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1869 os.unlink(dest)
1870 filesToDelete.add(src)
d9a5f25b 1871 editedFiles.add(dest)
a02b8bc4
RP
1872 elif modifier == "T":
1873 filesToChangeType.add(path)
4f5cf76a
SH
1874 else:
1875 die("unknown modifier %s for %s" % (modifier, path))
1876
749b668c 1877 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1878 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1879 tryPatchCmd = patchcmd + "--check -"
1880 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1881 patch_succeeded = True
51a2640a 1882
47a130b7 1883 if os.system(tryPatchCmd) != 0:
60df071c
LD
1884 fixed_rcs_keywords = False
1885 patch_succeeded = False
51a2640a 1886 print "Unfortunately applying the change failed!"
60df071c
LD
1887
1888 # Patch failed, maybe it's just RCS keyword woes. Look through
1889 # the patch to see if that's possible.
0d609032 1890 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1891 file = None
1892 pattern = None
1893 kwfiles = {}
1894 for file in editedFiles | filesToDelete:
1895 # did this file's delta contain RCS keywords?
1896 pattern = p4_keywords_regexp_for_file(file)
1897
1898 if pattern:
1899 # this file is a possibility...look for RCS keywords.
1900 regexp = re.compile(pattern, re.VERBOSE)
1901 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1902 if regexp.search(line):
1903 if verbose:
1904 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1905 kwfiles[file] = pattern
1906 break
1907
1908 for file in kwfiles:
1909 if verbose:
1910 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1911 # File is being deleted, so not open in p4. Must
1912 # disable the read-only bit on windows.
1913 if self.isWindows and file not in editedFiles:
1914 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1915 self.patchRCSKeywords(file, kwfiles[file])
1916 fixed_rcs_keywords = True
1917
1918 if fixed_rcs_keywords:
1919 print "Retrying the patch with RCS keywords cleaned up"
1920 if os.system(tryPatchCmd) == 0:
1921 patch_succeeded = True
1922
1923 if not patch_succeeded:
7e5dd9f2
PW
1924 for f in editedFiles:
1925 p4_revert(f)
7e5dd9f2 1926 return False
51a2640a 1927
55ac2ed6
PW
1928 #
1929 # Apply the patch for real, and do add/delete/+x handling.
1930 #
47a130b7 1931 system(applyPatchCmd)
4f5cf76a 1932
a02b8bc4
RP
1933 for f in filesToChangeType:
1934 p4_edit(f, "-t", "auto")
4f5cf76a 1935 for f in filesToAdd:
6de040df 1936 p4_add(f)
4f5cf76a 1937 for f in filesToDelete:
6de040df
LD
1938 p4_revert(f)
1939 p4_delete(f)
4f5cf76a 1940
c65b670e
CP
1941 # Set/clear executable bits
1942 for f in filesToChangeExecBit.keys():
1943 mode = filesToChangeExecBit[f]
1944 setP4ExecBit(f, mode)
1945
8cf422db
LD
1946 update_shelve = 0
1947 if len(self.update_shelve) > 0:
1948 update_shelve = self.update_shelve.pop(0)
1949 p4_reopen_in_change(update_shelve, all_files)
46c609e9 1950
55ac2ed6
PW
1951 #
1952 # Build p4 change description, starting with the contents
1953 # of the git commit message.
1954 #
0e36f2d7 1955 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1956 logMessage = logMessage.strip()
f19cb0a0 1957 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1958
8cf422db 1959 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 1960 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1961
c47178d4 1962 if self.preserveUser:
55ac2ed6 1963 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1964
55ac2ed6
PW
1965 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1966 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1967 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1968 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1969
55ac2ed6 1970 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1971 if not self.prepare_p4_only:
1972 submitTemplate += separatorLine
df8a9e86 1973 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1974
c47178d4 1975 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1976 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1977 if self.isWindows:
1978 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1979 tmpFile.write(submitTemplate)
c47178d4
PW
1980 tmpFile.close()
1981
728b7ad8
PW
1982 if self.prepare_p4_only:
1983 #
1984 # Leave the p4 tree prepared, and the submit template around
1985 # and let the user decide what to do next
1986 #
1987 print
1988 print "P4 workspace prepared for submission."
1989 print "To submit or revert, go to client workspace"
1990 print " " + self.clientPath
1991 print
1992 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1993 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1994 " \"git p4\"." % fileName
1995 print "You can delete the file \"%s\" when finished." % fileName
1996
1997 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1998 print "To preserve change ownership by user %s, you must\n" \
1999 "do \"p4 change -f <change>\" after submitting and\n" \
2000 "edit the User field."
2001 if pureRenameCopy:
2002 print "After submitting, renamed files must be re-synced."
2003 print "Invoke \"p4 sync -f\" on each of these files:"
2004 for f in pureRenameCopy:
2005 print " " + f
2006
2007 print
2008 print "To revert the changes, use \"p4 revert ...\", and delete"
2009 print "the submit template file \"%s\"" % fileName
2010 if filesToAdd:
2011 print "Since the commit adds new files, they must be deleted:"
2012 for f in filesToAdd:
2013 print " " + f
2014 print
2015 return True
2016
55ac2ed6
PW
2017 #
2018 # Let the user edit the change description, then submit it.
2019 #
b7638fed 2020 submitted = False
cdc7e388 2021
b7638fed
GE
2022 try:
2023 if self.edit_template(fileName):
2024 # read the edited message and submit
2025 tmpFile = open(fileName, "rb")
2026 message = tmpFile.read()
2027 tmpFile.close()
2028 if self.isWindows:
2029 message = message.replace("\r\n", "\n")
2030 submitTemplate = message[:message.index(separatorLine)]
46c609e9 2031
8cf422db 2032 if update_shelve:
46c609e9
LD
2033 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
2034 elif self.shelve:
b34fa577
VK
2035 p4_write_pipe(['shelve', '-i'], submitTemplate)
2036 else:
2037 p4_write_pipe(['submit', '-i'], submitTemplate)
2038 # The rename/copy happened by applying a patch that created a
2039 # new file. This leaves it writable, which confuses p4.
2040 for f in pureRenameCopy:
2041 p4_sync(f, "-f")
b7638fed
GE
2042
2043 if self.preserveUser:
2044 if p4User:
2045 # Get last changelist number. Cannot easily get it from
2046 # the submit command output as the output is
2047 # unmarshalled.
2048 changelist = self.lastP4Changelist()
2049 self.modifyChangelistUser(changelist, p4User)
2050
b7638fed
GE
2051 submitted = True
2052
2053 finally:
c47178d4 2054 # skip this patch
b34fa577
VK
2055 if not submitted or self.shelve:
2056 if self.shelve:
2057 print ("Reverting shelved files.")
2058 else:
2059 print ("Submission cancelled, undoing p4 changes.")
2060 for f in editedFiles | filesToDelete:
b7638fed
GE
2061 p4_revert(f)
2062 for f in filesToAdd:
2063 p4_revert(f)
2064 os.remove(f)
c47178d4
PW
2065
2066 os.remove(fileName)
b7638fed 2067 return submitted
4f5cf76a 2068
06804c76
LD
2069 # Export git tags as p4 labels. Create a p4 label and then tag
2070 # with that.
2071 def exportGitTags(self, gitTags):
c8942a22
LD
2072 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2073 if len(validLabelRegexp) == 0:
2074 validLabelRegexp = defaultLabelRegexp
2075 m = re.compile(validLabelRegexp)
06804c76
LD
2076
2077 for name in gitTags:
2078
2079 if not m.match(name):
2080 if verbose:
05a3cec5 2081 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
2082 continue
2083
2084 # Get the p4 commit this corresponds to
c8942a22
LD
2085 logMessage = extractLogMessageFromGitCommit(name)
2086 values = extractSettingsGitLog(logMessage)
06804c76 2087
c8942a22 2088 if not values.has_key('change'):
06804c76
LD
2089 # a tag pointing to something not sent to p4; ignore
2090 if verbose:
2091 print "git tag %s does not give a p4 commit" % name
2092 continue
c8942a22
LD
2093 else:
2094 changelist = values['change']
06804c76
LD
2095
2096 # Get the tag details.
2097 inHeader = True
2098 isAnnotated = False
2099 body = []
2100 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2101 l = l.strip()
2102 if inHeader:
2103 if re.match(r'tag\s+', l):
2104 isAnnotated = True
2105 elif re.match(r'\s*$', l):
2106 inHeader = False
2107 continue
2108 else:
2109 body.append(l)
2110
2111 if not isAnnotated:
2112 body = ["lightweight tag imported by git p4\n"]
2113
2114 # Create the label - use the same view as the client spec we are using
2115 clientSpec = getClientSpec()
2116
2117 labelTemplate = "Label: %s\n" % name
2118 labelTemplate += "Description:\n"
2119 for b in body:
2120 labelTemplate += "\t" + b + "\n"
2121 labelTemplate += "View:\n"
9d57c4a6
KS
2122 for depot_side in clientSpec.mappings:
2123 labelTemplate += "\t%s\n" % depot_side
06804c76 2124
ef739f08
PW
2125 if self.dry_run:
2126 print "Would create p4 label %s for tag" % name
728b7ad8
PW
2127 elif self.prepare_p4_only:
2128 print "Not creating p4 label %s for tag due to option" \
2129 " --prepare-p4-only" % name
ef739f08
PW
2130 else:
2131 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2132
ef739f08
PW
2133 # Use the label
2134 p4_system(["tag", "-l", name] +
9d57c4a6 2135 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2136
ef739f08
PW
2137 if verbose:
2138 print "created p4 label for tag %s" % name
06804c76 2139
4f5cf76a 2140 def run(self, args):
c9b50e63
SH
2141 if len(args) == 0:
2142 self.master = currentGitBranch()
c9b50e63
SH
2143 elif len(args) == 1:
2144 self.master = args[0]
28755dba
PW
2145 if not branchExists(self.master):
2146 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2147 else:
2148 return False
2149
8cf422db
LD
2150 for i in self.update_shelve:
2151 if i <= 0:
2152 sys.exit("invalid changelist %d" % i)
2153
00ad6e31
LD
2154 if self.master:
2155 allowSubmit = gitConfig("git-p4.allowSubmit")
2156 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2157 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2158
27d2d811 2159 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 2160 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2161 if len(self.origin) == 0:
2162 self.origin = upstream
a3fdd579 2163
8cf422db 2164 if len(self.update_shelve) > 0:
46c609e9
LD
2165 self.shelve = True
2166
3ea2cfd4
LD
2167 if self.preserveUser:
2168 if not self.canChangeChangelists():
2169 die("Cannot preserve user names without p4 super-user or admin permissions")
2170
6bbfd137
PW
2171 # if not set from the command line, try the config file
2172 if self.conflict_behavior is None:
2173 val = gitConfig("git-p4.conflict")
2174 if val:
2175 if val not in self.conflict_behavior_choices:
2176 die("Invalid value '%s' for config git-p4.conflict" % val)
2177 else:
2178 val = "ask"
2179 self.conflict_behavior = val
2180
a3fdd579
SH
2181 if self.verbose:
2182 print "Origin branch is " + self.origin
9512497b 2183
ea99c3ae 2184 if len(self.depotPath) == 0:
9512497b
SH
2185 print "Internal error: cannot locate perforce depot path from existing branches"
2186 sys.exit(128)
2187
543987bd 2188 self.useClientSpec = False
0d609032 2189 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2190 self.useClientSpec = True
2191 if self.useClientSpec:
2192 self.clientSpecDirs = getClientSpec()
9512497b 2193
2e3a16b2 2194 # Check for the existence of P4 branches
cd884106
VA
2195 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2196
2197 if self.useClientSpec and not branchesDetected:
543987bd
PW
2198 # all files are relative to the client spec
2199 self.clientPath = getClientRoot()
2200 else:
2201 self.clientPath = p4Where(self.depotPath)
9512497b 2202
543987bd
PW
2203 if self.clientPath == "":
2204 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2205
ea99c3ae 2206 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2207 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2208
0591cfa8 2209 # ensure the clientPath exists
8d7ec362 2210 new_client_dir = False
0591cfa8 2211 if not os.path.exists(self.clientPath):
8d7ec362 2212 new_client_dir = True
0591cfa8
GG
2213 os.makedirs(self.clientPath)
2214
bbd84863 2215 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2216 if self.dry_run:
2217 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2218 else:
ef739f08
PW
2219 print "Synchronizing p4 checkout..."
2220 if new_client_dir:
2221 # old one was destroyed, and maybe nobody told p4
2222 p4_sync("...", "-f")
2223 else:
2224 p4_sync("...")
4f5cf76a 2225 self.check()
4f5cf76a 2226
4c750c0d 2227 commits = []
00ad6e31 2228 if self.master:
89f32a92 2229 committish = self.master
00ad6e31 2230 else:
89f32a92 2231 committish = 'HEAD'
00ad6e31 2232
f55b87c1
RM
2233 if self.commit != "":
2234 if self.commit.find("..") != -1:
2235 limits_ish = self.commit.split("..")
2236 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2237 commits.append(line.strip())
2238 commits.reverse()
2239 else:
2240 commits.append(self.commit)
2241 else:
e6388994 2242 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
f55b87c1
RM
2243 commits.append(line.strip())
2244 commits.reverse()
4f5cf76a 2245
0d609032 2246 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2247 self.checkAuthorship = False
2248 else:
2249 self.checkAuthorship = True
2250
3ea2cfd4
LD
2251 if self.preserveUser:
2252 self.checkValidP4Users(commits)
2253
84cb0003
GG
2254 #
2255 # Build up a set of options to be passed to diff when
2256 # submitting each commit to p4.
2257 #
2258 if self.detectRenames:
2259 # command-line -M arg
2260 self.diffOpts = "-M"
2261 else:
2262 # If not explicitly set check the config variable
2263 detectRenames = gitConfig("git-p4.detectRenames")
2264
2265 if detectRenames.lower() == "false" or detectRenames == "":
2266 self.diffOpts = ""
2267 elif detectRenames.lower() == "true":
2268 self.diffOpts = "-M"
2269 else:
2270 self.diffOpts = "-M%s" % detectRenames
2271
2272 # no command-line arg for -C or --find-copies-harder, just
2273 # config variables
2274 detectCopies = gitConfig("git-p4.detectCopies")
2275 if detectCopies.lower() == "false" or detectCopies == "":
2276 pass
2277 elif detectCopies.lower() == "true":
2278 self.diffOpts += " -C"
2279 else:
2280 self.diffOpts += " -C%s" % detectCopies
2281
0d609032 2282 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2283 self.diffOpts += " --find-copies-harder"
2284
8cf422db
LD
2285 num_shelves = len(self.update_shelve)
2286 if num_shelves > 0 and num_shelves != len(commits):
2287 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2288 (len(commits), num_shelves))
2289
7e5dd9f2
PW
2290 #
2291 # Apply the commits, one at a time. On failure, ask if should
2292 # continue to try the rest of the patches, or quit.
2293 #
ef739f08
PW
2294 if self.dry_run:
2295 print "Would apply"
67b0fe2e 2296 applied = []
7e5dd9f2
PW
2297 last = len(commits) - 1
2298 for i, commit in enumerate(commits):
ef739f08
PW
2299 if self.dry_run:
2300 print " ", read_pipe(["git", "show", "-s",
2301 "--format=format:%h %s", commit])
2302 ok = True
2303 else:
2304 ok = self.applyCommit(commit)
67b0fe2e
PW
2305 if ok:
2306 applied.append(commit)
7e5dd9f2 2307 else:
728b7ad8
PW
2308 if self.prepare_p4_only and i < last:
2309 print "Processing only the first commit due to option" \
2310 " --prepare-p4-only"
2311 break
7e5dd9f2
PW
2312 if i < last:
2313 quit = False
2314 while True:
6bbfd137
PW
2315 # prompt for what to do, or use the option/variable
2316 if self.conflict_behavior == "ask":
2317 print "What do you want to do?"
2318 response = raw_input("[s]kip this commit but apply"
2319 " the rest, or [q]uit? ")
2320 if not response:
2321 continue
2322 elif self.conflict_behavior == "skip":
2323 response = "s"
2324 elif self.conflict_behavior == "quit":
2325 response = "q"
2326 else:
2327 die("Unknown conflict_behavior '%s'" %
2328 self.conflict_behavior)
2329
7e5dd9f2
PW
2330 if response[0] == "s":
2331 print "Skipping this commit, but applying the rest"
2332 break
2333 if response[0] == "q":
2334 print "Quitting"
2335 quit = True
2336 break
2337 if quit:
2338 break
4f5cf76a 2339
67b0fe2e 2340 chdir(self.oldWorkingDirectory)
b34fa577 2341 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2342 if self.dry_run:
2343 pass
728b7ad8
PW
2344 elif self.prepare_p4_only:
2345 pass
ef739f08 2346 elif len(commits) == len(applied):
b34fa577 2347 print ("All commits {0}!".format(shelved_applied))
14594f4b 2348
4c750c0d 2349 sync = P4Sync()
44e8d26c
PW
2350 if self.branch:
2351 sync.branch = self.branch
b9d34db9
LD
2352 if self.disable_p4sync:
2353 sync.sync_origin_only()
2354 else:
2355 sync.run([])
14594f4b 2356
b9d34db9
LD
2357 if not self.disable_rebase:
2358 rebase = P4Rebase()
2359 rebase.rebase()
4f5cf76a 2360
67b0fe2e
PW
2361 else:
2362 if len(applied) == 0:
b34fa577 2363 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2364 else:
b34fa577 2365 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2366 for c in commits:
2367 if c in applied:
2368 star = "*"
2369 else:
2370 star = " "
2371 print star, read_pipe(["git", "show", "-s",
2372 "--format=format:%h %s", c])
2373 print "You will have to do 'git p4 sync' and rebase."
2374
0d609032 2375 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2376 self.exportLabels = True
06804c76
LD
2377
2378 if self.exportLabels:
2379 p4Labels = getP4Labels(self.depotPath)
2380 gitTags = getGitTags()
2381
2382 missingGitTags = gitTags - p4Labels
2383 self.exportGitTags(missingGitTags)
2384
98e023de 2385 # exit with error unless everything applied perfectly
67b0fe2e
PW
2386 if len(commits) != len(applied):
2387 sys.exit(1)
2388
b984733c
SH
2389 return True
2390
ecb7cf98
PW
2391class View(object):
2392 """Represent a p4 view ("p4 help views"), and map files in a
2393 repo according to the view."""
2394
9d57c4a6 2395 def __init__(self, client_name):
ecb7cf98 2396 self.mappings = []
9d57c4a6
KS
2397 self.client_prefix = "//%s/" % client_name
2398 # cache results of "p4 where" to lookup client file locations
2399 self.client_spec_path_cache = {}
ecb7cf98
PW
2400
2401 def append(self, view_line):
2402 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2403 sides. Append to self.mappings, preserving order. This
2404 is only needed for tag creation."""
ecb7cf98
PW
2405
2406 # Split the view line into exactly two words. P4 enforces
2407 # structure on these lines that simplifies this quite a bit.
2408 #
2409 # Either or both words may be double-quoted.
2410 # Single quotes do not matter.
2411 # Double-quote marks cannot occur inside the words.
2412 # A + or - prefix is also inside the quotes.
2413 # There are no quotes unless they contain a space.
2414 # The line is already white-space stripped.
2415 # The two words are separated by a single space.
2416 #
2417 if view_line[0] == '"':
2418 # First word is double quoted. Find its end.
2419 close_quote_index = view_line.find('"', 1)
2420 if close_quote_index <= 0:
2421 die("No first-word closing quote found: %s" % view_line)
2422 depot_side = view_line[1:close_quote_index]
2423 # skip closing quote and space
2424 rhs_index = close_quote_index + 1 + 1
2425 else:
2426 space_index = view_line.find(" ")
2427 if space_index <= 0:
2428 die("No word-splitting space found: %s" % view_line)
2429 depot_side = view_line[0:space_index]
2430 rhs_index = space_index + 1
2431
ecb7cf98 2432 # prefix + means overlay on previous mapping
ecb7cf98 2433 if depot_side.startswith("+"):
ecb7cf98
PW
2434 depot_side = depot_side[1:]
2435
9d57c4a6 2436 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2437 exclude = False
2438 if depot_side.startswith("-"):
2439 exclude = True
2440 depot_side = depot_side[1:]
2441
9d57c4a6
KS
2442 if not exclude:
2443 self.mappings.append(depot_side)
ecb7cf98 2444
9d57c4a6
KS
2445 def convert_client_path(self, clientFile):
2446 # chop off //client/ part to make it relative
2447 if not clientFile.startswith(self.client_prefix):
2448 die("No prefix '%s' on clientFile '%s'" %
2449 (self.client_prefix, clientFile))
2450 return clientFile[len(self.client_prefix):]
ecb7cf98 2451
9d57c4a6
KS
2452 def update_client_spec_path_cache(self, files):
2453 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2454
9d57c4a6
KS
2455 # List depot file paths exclude that already cached
2456 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2457
9d57c4a6
KS
2458 if len(fileArgs) == 0:
2459 return # All files in cache
ecb7cf98 2460
9d57c4a6
KS
2461 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2462 for res in where_result:
2463 if "code" in res and res["code"] == "error":
2464 # assume error is "... file(s) not in client view"
2465 continue
2466 if "clientFile" not in res:
20005443 2467 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2468 if "unmap" in res:
2469 # it will list all of them, but only one not unmap-ped
2470 continue
a0a50d87
LS
2471 if gitConfigBool("core.ignorecase"):
2472 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2473 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2474
9d57c4a6
KS
2475 # not found files or unmap files set to ""
2476 for depotFile in fileArgs:
a0a50d87
LS
2477 if gitConfigBool("core.ignorecase"):
2478 depotFile = depotFile.lower()
9d57c4a6
KS
2479 if depotFile not in self.client_spec_path_cache:
2480 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2481
9d57c4a6
KS
2482 def map_in_client(self, depot_path):
2483 """Return the relative location in the client where this
2484 depot file should live. Returns "" if the file should
2485 not be mapped in the client."""
ecb7cf98 2486
a0a50d87
LS
2487 if gitConfigBool("core.ignorecase"):
2488 depot_path = depot_path.lower()
2489
9d57c4a6
KS
2490 if depot_path in self.client_spec_path_cache:
2491 return self.client_spec_path_cache[depot_path]
2492
2493 die( "Error: %s is not found in client spec path" % depot_path )
2494 return ""
ecb7cf98 2495
3ea2cfd4 2496class P4Sync(Command, P4UserMap):
56c09345
PW
2497 delete_actions = ( "delete", "move/delete", "purge" )
2498
b984733c
SH
2499 def __init__(self):
2500 Command.__init__(self)
3ea2cfd4 2501 P4UserMap.__init__(self)
b984733c
SH
2502 self.options = [
2503 optparse.make_option("--branch", dest="branch"),
2504 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2505 optparse.make_option("--changesfile", dest="changesFile"),
2506 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2507 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2508 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2509 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2510 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2511 optparse.make_option("--max-changes", dest="maxChanges",
2512 help="Maximum number of changes to import"),
2513 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2514 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2515 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2516 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2517 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2518 help="Only sync files that are included in the Perforce Client Spec"),
2519 optparse.make_option("-/", dest="cloneExclude",
2520 action="append", type="string",
2521 help="exclude depot path"),
b984733c
SH
2522 ]
2523 self.description = """Imports from Perforce into a git repository.\n
2524 example:
2525 //depot/my/project/ -- to import the current head
2526 //depot/my/project/@all -- to import everything
2527 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2528
2529 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2530
2531 self.usage += " //depot/path[@revRange]"
b984733c 2532 self.silent = False
1d7367dc
RG
2533 self.createdBranches = set()
2534 self.committedChanges = set()
569d1bd4 2535 self.branch = ""
b984733c 2536 self.detectBranches = False
cb53e1f8 2537 self.detectLabels = False
06804c76 2538 self.importLabels = False
b984733c 2539 self.changesFile = ""
01265103 2540 self.syncWithOrigin = True
a028a98e 2541 self.importIntoRemotes = True
01a9c9c5 2542 self.maxChanges = ""
1051ef00 2543 self.changes_block_size = None
8b41a97f 2544 self.keepRepoPath = False
6326aa58 2545 self.depotPaths = None
3c699645 2546 self.p4BranchesInGit = []
354081d5 2547 self.cloneExclude = []
3a70cdfa 2548 self.useClientSpec = False
a93d33ee 2549 self.useClientSpec_from_options = False
ecb7cf98 2550 self.clientSpecDirs = None
fed23693 2551 self.tempBranches = []
d604176d 2552 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12 2553 self.largeFileSystem = None
123f6317 2554 self.suppress_meta_comment = False
a5db4b12
LS
2555
2556 if gitConfig('git-p4.largeFileSystem'):
2557 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2558 self.largeFileSystem = largeFileSystemConstructor(
2559 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2560 )
b984733c 2561
01265103
SH
2562 if gitConfig("git-p4.syncFromOrigin") == "false":
2563 self.syncWithOrigin = False
2564
123f6317
LD
2565 self.depotPaths = []
2566 self.changeRange = ""
2567 self.previousDepotPaths = []
2568 self.hasOrigin = False
2569
2570 # map from branch depot path to parent branch
2571 self.knownBranches = {}
2572 self.initialParents = {}
2573
2574 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2575 self.labels = {}
2576
fed23693
VA
2577 # Force a checkpoint in fast-import and wait for it to finish
2578 def checkpoint(self):
2579 self.gitStream.write("checkpoint\n\n")
2580 self.gitStream.write("progress checkpoint\n\n")
2581 out = self.gitOutput.readline()
2582 if self.verbose:
2583 print "checkpoint finished: " + out
2584
123f6317
LD
2585 def cmp_shelved(self, path, filerev, revision):
2586 """ Determine if a path at revision #filerev is the same as the file
2587 at revision @revision for a shelved changelist. If they don't match,
2588 unshelving won't be safe (we will get other changes mixed in).
2589
2590 This is comparing the revision that the shelved changelist is *based* on, not
2591 the shelved changelist itself.
2592 """
2593 ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2594 if verbose:
2595 print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2596 return ret["status"] == "identical"
2597
2598 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
354081d5
TT
2599 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2600 for path in self.cloneExclude]
b984733c
SH
2601 files = []
2602 fnum = 0
2603 while commit.has_key("depotFile%s" % fnum):
2604 path = commit["depotFile%s" % fnum]
6326aa58 2605
354081d5 2606 if [p for p in self.cloneExclude
d53de8b9 2607 if p4PathStartsWith(path, p)]:
354081d5
TT
2608 found = False
2609 else:
2610 found = [p for p in self.depotPaths
d53de8b9 2611 if p4PathStartsWith(path, p)]
6326aa58 2612 if not found:
b984733c
SH
2613 fnum = fnum + 1
2614 continue
2615
2616 file = {}
2617 file["path"] = path
2618 file["rev"] = commit["rev%s" % fnum]
2619 file["action"] = commit["action%s" % fnum]
2620 file["type"] = commit["type%s" % fnum]
123f6317
LD
2621 if shelved:
2622 file["shelved_cl"] = int(shelved_cl)
2623
2624 # For shelved changelists, check that the revision of each file that the
2625 # shelve was based on matches the revision that we are using for the
2626 # starting point for git-fast-import (self.initialParent). Otherwise
2627 # the resulting diff will contain deltas from multiple commits.
2628
2629 if file["action"] != "add" and \
2630 not self.cmp_shelved(path, file["rev"], origin_revision):
2631 sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2632 commit["change"], self.initialParent, path))
2633
b984733c
SH
2634 files.append(file)
2635 fnum = fnum + 1
2636 return files
2637
26e6a27d
JD
2638 def extractJobsFromCommit(self, commit):
2639 jobs = []
2640 jnum = 0
2641 while commit.has_key("job%s" % jnum):
2642 job = commit["job%s" % jnum]
2643 jobs.append(job)
2644 jnum = jnum + 1
2645 return jobs
2646
6326aa58 2647 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2648 """When streaming files, this is called to map a p4 depot path
2649 to where it should go in git. The prefixes are either
2650 self.depotPaths, or self.branchPrefixes in the case of
2651 branch detection."""
2652
3952710b 2653 if self.useClientSpec:
21ef5df4
PW
2654 # branch detection moves files up a level (the branch name)
2655 # from what client spec interpretation gives
0d1696ef 2656 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2657 if self.detectBranches:
2658 for b in self.knownBranches:
2659 if path.startswith(b + "/"):
2660 path = path[len(b)+1:]
2661
2662 elif self.keepRepoPath:
2663 # Preserve everything in relative path name except leading
2664 # //depot/; just look at first prefix as they all should
2665 # be in the same depot.
2666 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2667 if p4PathStartsWith(path, depot):
2668 path = path[len(depot):]
3952710b 2669
0d1696ef 2670 else:
0d1696ef
PW
2671 for p in prefixes:
2672 if p4PathStartsWith(path, p):
2673 path = path[len(p):]
21ef5df4 2674 break
8b41a97f 2675
0d1696ef 2676 path = wildcard_decode(path)
6326aa58 2677 return path
6754a299 2678
71b112d4 2679 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2680 """Look at each depotFile in the commit to figure out to what
2681 branch it belongs."""
2682
9d57c4a6
KS
2683 if self.clientSpecDirs:
2684 files = self.extractFilesFromCommit(commit)
2685 self.clientSpecDirs.update_client_spec_path_cache(files)
2686
d5904674 2687 branches = {}
71b112d4
SH
2688 fnum = 0
2689 while commit.has_key("depotFile%s" % fnum):
2690 path = commit["depotFile%s" % fnum]
6326aa58 2691 found = [p for p in self.depotPaths
d53de8b9 2692 if p4PathStartsWith(path, p)]
6326aa58 2693 if not found:
71b112d4
SH
2694 fnum = fnum + 1
2695 continue
2696
2697 file = {}
2698 file["path"] = path
2699 file["rev"] = commit["rev%s" % fnum]
2700 file["action"] = commit["action%s" % fnum]
2701 file["type"] = commit["type%s" % fnum]
2702 fnum = fnum + 1
2703
21ef5df4
PW
2704 # start with the full relative path where this file would
2705 # go in a p4 client
2706 if self.useClientSpec:
2707 relPath = self.clientSpecDirs.map_in_client(path)
2708 else:
2709 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2710
4b97ffb1 2711 for branch in self.knownBranches.keys():
21ef5df4
PW
2712 # add a trailing slash so that a commit into qt/4.2foo
2713 # doesn't end up in qt/4.2, e.g.
6754a299 2714 if relPath.startswith(branch + "/"):
d5904674
SH
2715 if branch not in branches:
2716 branches[branch] = []
71b112d4 2717 branches[branch].append(file)
6555b2cc 2718 break
b984733c
SH
2719
2720 return branches
2721
a5db4b12
LS
2722 def writeToGitStream(self, gitMode, relPath, contents):
2723 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2724 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2725 for d in contents:
2726 self.gitStream.write(d)
2727 self.gitStream.write('\n')
2728
a8b05162
LS
2729 def encodeWithUTF8(self, path):
2730 try:
2731 path.decode('ascii')
2732 except:
2733 encoding = 'utf8'
2734 if gitConfig('git-p4.pathEncoding'):
2735 encoding = gitConfig('git-p4.pathEncoding')
2736 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2737 if self.verbose:
2738 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2739 return path
2740
b932705b
LD
2741 # output one file from the P4 stream
2742 # - helper for streamP4Files
2743
2744 def streamOneP4File(self, file, contents):
b932705b 2745 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2746 relPath = self.encodeWithUTF8(relPath)
b932705b 2747 if verbose:
d2176a50
LS
2748 size = int(self.stream_file['fileSize'])
2749 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2750 sys.stdout.flush()
b932705b 2751
9cffb8c8
PW
2752 (type_base, type_mods) = split_p4_type(file["type"])
2753
2754 git_mode = "100644"
2755 if "x" in type_mods:
2756 git_mode = "100755"
2757 if type_base == "symlink":
2758 git_mode = "120000"
1292df11
AJ
2759 # p4 print on a symlink sometimes contains "target\n";
2760 # if it does, remove the newline
b39c3612 2761 data = ''.join(contents)
40f846c3
PW
2762 if not data:
2763 # Some version of p4 allowed creating a symlink that pointed
2764 # to nothing. This causes p4 errors when checking out such
2765 # a change, and errors here too. Work around it by ignoring
2766 # the bad symlink; hopefully a future change fixes it.
2767 print "\nIgnoring empty symlink in %s" % file['depotFile']
2768 return
2769 elif data[-1] == '\n':
1292df11
AJ
2770 contents = [data[:-1]]
2771 else:
2772 contents = [data]
b932705b 2773
9cffb8c8 2774 if type_base == "utf16":
55aa5714
PW
2775 # p4 delivers different text in the python output to -G
2776 # than it does when using "print -o", or normal p4 client
2777 # operations. utf16 is converted to ascii or utf8, perhaps.
2778 # But ascii text saved as -t utf16 is completely mangled.
2779 # Invoke print -o to get the real contents.
7f0e5962
PW
2780 #
2781 # On windows, the newlines will always be mangled by print, so put
2782 # them back too. This is not needed to the cygwin windows version,
2783 # just the native "NT" type.
2784 #
1f5f3907
LS
2785 try:
2786 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2787 except Exception as e:
2788 if 'Translation of file content failed' in str(e):
2789 type_base = 'binary'
2790 else:
2791 raise e
2792 else:
2793 if p4_version_string().find('/NT') >= 0:
2794 text = text.replace('\r\n', '\n')
2795 contents = [ text ]
55aa5714 2796
9f7ef0ea
PW
2797 if type_base == "apple":
2798 # Apple filetype files will be streamed as a concatenation of
2799 # its appledouble header and the contents. This is useless
2800 # on both macs and non-macs. If using "print -q -o xx", it
2801 # will create "xx" with the data, and "%xx" with the header.
2802 # This is also not very useful.
2803 #
2804 # Ideally, someday, this script can learn how to generate
2805 # appledouble files directly and import those to git, but
2806 # non-mac machines can never find a use for apple filetype.
2807 print "\nIgnoring apple filetype file %s" % file['depotFile']
2808 return
2809
55aa5714
PW
2810 # Note that we do not try to de-mangle keywords on utf16 files,
2811 # even though in theory somebody may want that.
60df071c
LD
2812 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2813 if pattern:
2814 regexp = re.compile(pattern, re.VERBOSE)
2815 text = ''.join(contents)
2816 text = regexp.sub(r'$\1$', text)
2817 contents = [ text ]
b932705b 2818
a5db4b12
LS
2819 if self.largeFileSystem:
2820 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2821
a5db4b12 2822 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2823
2824 def streamOneP4Deletion(self, file):
2825 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2826 relPath = self.encodeWithUTF8(relPath)
b932705b 2827 if verbose:
d2176a50
LS
2828 sys.stdout.write("delete %s\n" % relPath)
2829 sys.stdout.flush()
b932705b
LD
2830 self.gitStream.write("D %s\n" % relPath)
2831
a5db4b12
LS
2832 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2833 self.largeFileSystem.removeLargeFile(relPath)
2834
b932705b
LD
2835 # handle another chunk of streaming data
2836 def streamP4FilesCb(self, marshalled):
2837
78189bea
PW
2838 # catch p4 errors and complain
2839 err = None
2840 if "code" in marshalled:
2841 if marshalled["code"] == "error":
2842 if "data" in marshalled:
2843 err = marshalled["data"].rstrip()
4d25dc44
LS
2844
2845 if not err and 'fileSize' in self.stream_file:
2846 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2847 if required_bytes > 0:
2848 err = 'Not enough space left on %s! Free at least %i MB.' % (
2849 os.getcwd(), required_bytes/1024/1024
2850 )
2851
78189bea
PW
2852 if err:
2853 f = None
2854 if self.stream_have_file_info:
2855 if "depotFile" in self.stream_file:
2856 f = self.stream_file["depotFile"]
2857 # force a failure in fast-import, else an empty
2858 # commit will be made
2859 self.gitStream.write("\n")
2860 self.gitStream.write("die-now\n")
2861 self.gitStream.close()
2862 # ignore errors, but make sure it exits first
2863 self.importProcess.wait()
2864 if f:
2865 die("Error from p4 print for %s: %s" % (f, err))
2866 else:
2867 die("Error from p4 print: %s" % err)
2868
c3f6163b
AG
2869 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2870 # start of a new file - output the old one first
2871 self.streamOneP4File(self.stream_file, self.stream_contents)
2872 self.stream_file = {}
2873 self.stream_contents = []
2874 self.stream_have_file_info = False
b932705b 2875
c3f6163b
AG
2876 # pick up the new file information... for the
2877 # 'data' field we need to append to our array
2878 for k in marshalled.keys():
2879 if k == 'data':
d2176a50
LS
2880 if 'streamContentSize' not in self.stream_file:
2881 self.stream_file['streamContentSize'] = 0
2882 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2883 self.stream_contents.append(marshalled['data'])
2884 else:
2885 self.stream_file[k] = marshalled[k]
b932705b 2886
d2176a50
LS
2887 if (verbose and
2888 'streamContentSize' in self.stream_file and
2889 'fileSize' in self.stream_file and
2890 'depotFile' in self.stream_file):
2891 size = int(self.stream_file["fileSize"])
2892 if size > 0:
2893 progress = 100*self.stream_file['streamContentSize']/size
2894 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2895 sys.stdout.flush()
2896
c3f6163b 2897 self.stream_have_file_info = True
b932705b
LD
2898
2899 # Stream directly from "p4 files" into "git fast-import"
2900 def streamP4Files(self, files):
30b5940b
SH
2901 filesForCommit = []
2902 filesToRead = []
b932705b 2903 filesToDelete = []
30b5940b 2904
3a70cdfa 2905 for f in files:
ecb7cf98
PW
2906 filesForCommit.append(f)
2907 if f['action'] in self.delete_actions:
2908 filesToDelete.append(f)
2909 else:
2910 filesToRead.append(f)
6a49f8e2 2911
b932705b
LD
2912 # deleted files...
2913 for f in filesToDelete:
2914 self.streamOneP4Deletion(f)
1b9a4684 2915
b932705b
LD
2916 if len(filesToRead) > 0:
2917 self.stream_file = {}
2918 self.stream_contents = []
2919 self.stream_have_file_info = False
8ff45f2a 2920
c3f6163b
AG
2921 # curry self argument
2922 def streamP4FilesCbSelf(entry):
2923 self.streamP4FilesCb(entry)
6a49f8e2 2924
123f6317
LD
2925 fileArgs = []
2926 for f in filesToRead:
2927 if 'shelved_cl' in f:
2928 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2929 # the contents
2930 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2931 else:
2932 fileArg = '%s#%s' % (f['path'], f['rev'])
2933
2934 fileArgs.append(fileArg)
6de040df
LD
2935
2936 p4CmdList(["-x", "-", "print"],
2937 stdin=fileArgs,
2938 cb=streamP4FilesCbSelf)
30b5940b 2939
b932705b
LD
2940 # do the last chunk
2941 if self.stream_file.has_key('depotFile'):
2942 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2943
affb474f
LD
2944 def make_email(self, userid):
2945 if userid in self.users:
2946 return self.users[userid]
2947 else:
2948 return "%s <a@b>" % userid
2949
06804c76 2950 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2951 """ Stream a p4 tag.
2952 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2953 """
2954
06804c76
LD
2955 if verbose:
2956 print "writing tag %s for commit %s" % (labelName, commit)
2957 gitStream.write("tag %s\n" % labelName)
2958 gitStream.write("from %s\n" % commit)
2959
2960 if labelDetails.has_key('Owner'):
2961 owner = labelDetails["Owner"]
2962 else:
2963 owner = None
2964
2965 # Try to use the owner of the p4 label, or failing that,
2966 # the current p4 user id.
2967 if owner:
2968 email = self.make_email(owner)
2969 else:
2970 email = self.make_email(self.p4UserId())
2971 tagger = "%s %s %s" % (email, epoch, self.tz)
2972
2973 gitStream.write("tagger %s\n" % tagger)
2974
2975 print "labelDetails=",labelDetails
2976 if labelDetails.has_key('Description'):
2977 description = labelDetails['Description']
2978 else:
2979 description = 'Label from git p4'
2980
2981 gitStream.write("data %d\n" % len(description))
2982 gitStream.write(description)
2983 gitStream.write("\n")
2984
4ae048e6
LS
2985 def inClientSpec(self, path):
2986 if not self.clientSpecDirs:
2987 return True
2988 inClientSpec = self.clientSpecDirs.map_in_client(path)
2989 if not inClientSpec and self.verbose:
2990 print('Ignoring file outside of client spec: {0}'.format(path))
2991 return inClientSpec
2992
2993 def hasBranchPrefix(self, path):
2994 if not self.branchPrefixes:
2995 return True
2996 hasPrefix = [p for p in self.branchPrefixes
2997 if p4PathStartsWith(path, p)]
09667d01 2998 if not hasPrefix and self.verbose:
4ae048e6
LS
2999 print('Ignoring file outside of prefix: {0}'.format(path))
3000 return hasPrefix
3001
e63231e5 3002 def commit(self, details, files, branch, parent = ""):
b984733c
SH
3003 epoch = details["time"]
3004 author = details["user"]
26e6a27d 3005 jobs = self.extractJobsFromCommit(details)
b984733c 3006
4b97ffb1 3007 if self.verbose:
4ae048e6 3008 print('commit into {0}'.format(branch))
96e07dd2 3009
9d57c4a6
KS
3010 if self.clientSpecDirs:
3011 self.clientSpecDirs.update_client_spec_path_cache(files)
3012
4ae048e6
LS
3013 files = [f for f in files
3014 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
3015
3016 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
3017 print('Ignoring revision {0} as it would produce an empty commit.'
3018 .format(details['change']))
3019 return
3020
b984733c 3021 self.gitStream.write("commit %s\n" % branch)
b43702ac 3022 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
3023 self.committedChanges.add(int(details["change"]))
3024 committer = ""
b607e71e
SH
3025 if author not in self.users:
3026 self.getUserMapFromPerforceServer()
affb474f 3027 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
3028
3029 self.gitStream.write("committer %s\n" % committer)
3030
3031 self.gitStream.write("data <<EOT\n")
3032 self.gitStream.write(details["desc"])
26e6a27d
JD
3033 if len(jobs) > 0:
3034 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
123f6317
LD
3035
3036 if not self.suppress_meta_comment:
3037 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
3038 (','.join(self.branchPrefixes), details["change"]))
3039 if len(details['options']) > 0:
3040 self.gitStream.write(": options = %s" % details['options'])
3041 self.gitStream.write("]\n")
3042
3043 self.gitStream.write("EOT\n\n")
b984733c
SH
3044
3045 if len(parent) > 0:
4b97ffb1
SH
3046 if self.verbose:
3047 print "parent %s" % parent
b984733c
SH
3048 self.gitStream.write("from %s\n" % parent)
3049
4ae048e6 3050 self.streamP4Files(files)
b984733c
SH
3051 self.gitStream.write("\n")
3052
1f4ba1cb
SH
3053 change = int(details["change"])
3054
9bda3a85 3055 if self.labels.has_key(change):
1f4ba1cb
SH
3056 label = self.labels[change]
3057 labelDetails = label[0]
3058 labelRevisions = label[1]
71b112d4
SH
3059 if self.verbose:
3060 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 3061
6de040df 3062 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 3063 for p in self.branchPrefixes])
1f4ba1cb
SH
3064
3065 if len(files) == len(labelRevisions):
3066
3067 cleanedFiles = {}
3068 for info in files:
56c09345 3069 if info["action"] in self.delete_actions:
1f4ba1cb
SH
3070 continue
3071 cleanedFiles[info["depotFile"]] = info["rev"]
3072
3073 if cleanedFiles == labelRevisions:
06804c76 3074 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
3075
3076 else:
a46668fa 3077 if not self.silent:
cebdf5af
HWN
3078 print ("Tag %s does not match with change %s: files do not match."
3079 % (labelDetails["label"], change))
1f4ba1cb
SH
3080
3081 else:
a46668fa 3082 if not self.silent:
cebdf5af
HWN
3083 print ("Tag %s does not match with change %s: file count is different."
3084 % (labelDetails["label"], change))
b984733c 3085
06804c76 3086 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
3087 def getLabels(self):
3088 self.labels = {}
3089
52a4880b 3090 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 3091 if len(l) > 0 and not self.silent:
183f8436 3092 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
3093
3094 for output in l:
1f4ba1cb
SH
3095 label = output["label"]
3096 revisions = {}
3097 newestChange = 0
71b112d4
SH
3098 if self.verbose:
3099 print "Querying files for label %s" % label
6de040df
LD
3100 for file in p4CmdList(["files"] +
3101 ["%s...@%s" % (p, label)
3102 for p in self.depotPaths]):
1f4ba1cb
SH
3103 revisions[file["depotFile"]] = file["rev"]
3104 change = int(file["change"])
3105 if change > newestChange:
3106 newestChange = change
3107
9bda3a85
SH
3108 self.labels[newestChange] = [output, revisions]
3109
3110 if self.verbose:
3111 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 3112
06804c76
LD
3113 # Import p4 labels as git tags. A direct mapping does not
3114 # exist, so assume that if all the files are at the same revision
3115 # then we can use that, or it's something more complicated we should
3116 # just ignore.
3117 def importP4Labels(self, stream, p4Labels):
3118 if verbose:
3119 print "import p4 labels: " + ' '.join(p4Labels)
3120
3121 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 3122 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
3123 if len(validLabelRegexp) == 0:
3124 validLabelRegexp = defaultLabelRegexp
3125 m = re.compile(validLabelRegexp)
3126
3127 for name in p4Labels:
3128 commitFound = False
3129
3130 if not m.match(name):
3131 if verbose:
3132 print "label %s does not match regexp %s" % (name,validLabelRegexp)
3133 continue
3134
3135 if name in ignoredP4Labels:
3136 continue
3137
3138 labelDetails = p4CmdList(['label', "-o", name])[0]
3139
3140 # get the most recent changelist for each file in this label
3141 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3142 for p in self.depotPaths])
3143
3144 if change.has_key('change'):
3145 # find the corresponding git commit; take the oldest commit
3146 changelist = int(change['change'])
b43702ac
LD
3147 if changelist in self.committedChanges:
3148 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 3149 commitFound = True
b43702ac
LD
3150 else:
3151 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3152 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3153 if len(gitCommit) == 0:
3154 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3155 else:
3156 commitFound = True
3157 gitCommit = gitCommit.strip()
3158
3159 if commitFound:
06804c76
LD
3160 # Convert from p4 time format
3161 try:
3162 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3163 except ValueError:
a4e9054c 3164 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
3165 tmwhen = 1
3166
3167 when = int(time.mktime(tmwhen))
3168 self.streamTag(stream, name, labelDetails, gitCommit, when)
3169 if verbose:
3170 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3171 else:
3172 if verbose:
3173 print "Label %s has no changelists - possibly deleted?" % name
3174
3175 if not commitFound:
3176 # We can't import this label; don't try again as it will get very
3177 # expensive repeatedly fetching all the files for labels that will
3178 # never be imported. If the label is moved in the future, the
3179 # ignore will need to be removed manually.
3180 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3181
86dff6b6
HWN
3182 def guessProjectName(self):
3183 for p in self.depotPaths:
6e5295c4
SH
3184 if p.endswith("/"):
3185 p = p[:-1]
3186 p = p[p.strip().rfind("/") + 1:]
3187 if not p.endswith("/"):
3188 p += "/"
3189 return p
86dff6b6 3190
4b97ffb1 3191 def getBranchMapping(self):
6555b2cc
SH
3192 lostAndFoundBranches = set()
3193
8ace74c0
VA
3194 user = gitConfig("git-p4.branchUser")
3195 if len(user) > 0:
3196 command = "branches -u %s" % user
3197 else:
3198 command = "branches"
3199
3200 for info in p4CmdList(command):
52a4880b 3201 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
3202 viewIdx = 0
3203 while details.has_key("View%s" % viewIdx):
3204 paths = details["View%s" % viewIdx].split(" ")
3205 viewIdx = viewIdx + 1
3206 # require standard //depot/foo/... //depot/bar/... mapping
3207 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3208 continue
3209 source = paths[0]
3210 destination = paths[1]
6509e19c 3211 ## HACK
d53de8b9 3212 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3213 source = source[len(self.depotPaths[0]):-4]
3214 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3215
1a2edf4e
SH
3216 if destination in self.knownBranches:
3217 if not self.silent:
3218 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3219 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3220 continue
3221
6555b2cc
SH
3222 self.knownBranches[destination] = source
3223
3224 lostAndFoundBranches.discard(destination)
3225
29bdbac1 3226 if source not in self.knownBranches:
6555b2cc
SH
3227 lostAndFoundBranches.add(source)
3228
7199cf13
VA
3229 # Perforce does not strictly require branches to be defined, so we also
3230 # check git config for a branch list.
3231 #
3232 # Example of branch definition in git config file:
3233 # [git-p4]
3234 # branchList=main:branchA
3235 # branchList=main:branchB
3236 # branchList=branchA:branchC
3237 configBranches = gitConfigList("git-p4.branchList")
3238 for branch in configBranches:
3239 if branch:
3240 (source, destination) = branch.split(":")
3241 self.knownBranches[destination] = source
3242
3243 lostAndFoundBranches.discard(destination)
3244
3245 if source not in self.knownBranches:
3246 lostAndFoundBranches.add(source)
3247
6555b2cc
SH
3248
3249 for branch in lostAndFoundBranches:
3250 self.knownBranches[branch] = branch
29bdbac1 3251
38f9f5ec
SH
3252 def getBranchMappingFromGitBranches(self):
3253 branches = p4BranchesInGit(self.importIntoRemotes)
3254 for branch in branches.keys():
3255 if branch == "master":
3256 branch = "main"
3257 else:
3258 branch = branch[len(self.projectName):]
3259 self.knownBranches[branch] = branch
3260
bb6e09b2
HWN
3261 def updateOptionDict(self, d):
3262 option_keys = {}
3263 if self.keepRepoPath:
3264 option_keys['keepRepoPath'] = 1
3265
3266 d["options"] = ' '.join(sorted(option_keys.keys()))
3267
3268 def readOptions(self, d):
3269 self.keepRepoPath = (d.has_key('options')
3270 and ('keepRepoPath' in d['options']))
6326aa58 3271
8134f69c
SH
3272 def gitRefForBranch(self, branch):
3273 if branch == "main":
3274 return self.refPrefix + "master"
3275
3276 if len(branch) <= 0:
3277 return branch
3278
3279 return self.refPrefix + self.projectName + branch
3280
1ca3d710
SH
3281 def gitCommitByP4Change(self, ref, change):
3282 if self.verbose:
3283 print "looking in ref " + ref + " for change %s using bisect..." % change
3284
3285 earliestCommit = ""
3286 latestCommit = parseRevision(ref)
3287
3288 while True:
3289 if self.verbose:
3290 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3291 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3292 if len(next) == 0:
3293 if self.verbose:
3294 print "argh"
3295 return ""
3296 log = extractLogMessageFromGitCommit(next)
3297 settings = extractSettingsGitLog(log)
3298 currentChange = int(settings['change'])
3299 if self.verbose:
3300 print "current change %s" % currentChange
3301
3302 if currentChange == change:
3303 if self.verbose:
3304 print "found %s" % next
3305 return next
3306
3307 if currentChange < change:
3308 earliestCommit = "^%s" % next
3309 else:
3310 latestCommit = "%s" % next
3311
3312 return ""
3313
3314 def importNewBranch(self, branch, maxChange):
3315 # make fast-import flush all changes to disk and update the refs using the checkpoint
3316 # command so that we can try to find the branch parent in the git history
3317 self.gitStream.write("checkpoint\n\n");
3318 self.gitStream.flush();
3319 branchPrefix = self.depotPaths[0] + branch + "/"
3320 range = "@1,%s" % maxChange
3321 #print "prefix" + branchPrefix
96b2d54a 3322 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3323 if len(changes) <= 0:
3324 return False
3325 firstChange = changes[0]
3326 #print "first change in branch: %s" % firstChange
3327 sourceBranch = self.knownBranches[branch]
3328 sourceDepotPath = self.depotPaths[0] + sourceBranch
3329 sourceRef = self.gitRefForBranch(sourceBranch)
3330 #print "source " + sourceBranch
3331
52a4880b 3332 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3333 #print "branch parent: %s" % branchParentChange
3334 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3335 if len(gitParent) > 0:
3336 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3337 #print "parent git commit: %s" % gitParent
3338
3339 self.importChanges(changes)
3340 return True
3341
fed23693
VA
3342 def searchParent(self, parent, branch, target):
3343 parentFound = False
c7d34884
PW
3344 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3345 "--no-merges", parent]):
fed23693
VA
3346 blob = blob.strip()
3347 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3348 parentFound = True
3349 if self.verbose:
3350 print "Found parent of %s in commit %s" % (branch, blob)
3351 break
3352 if parentFound:
3353 return blob
3354 else:
3355 return None
3356
123f6317 3357 def importChanges(self, changes, shelved=False, origin_revision=0):
e87f37ae
SH
3358 cnt = 1
3359 for change in changes:
123f6317 3360 description = p4_describe(change, shelved)
e87f37ae
SH
3361 self.updateOptionDict(description)
3362
3363 if not self.silent:
3364 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3365 sys.stdout.flush()
3366 cnt = cnt + 1
3367
3368 try:
3369 if self.detectBranches:
3370 branches = self.splitFilesIntoBranches(description)
3371 for branch in branches.keys():
3372 ## HACK --hwn
3373 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3374 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3375
3376 parent = ""
3377
3378 filesForCommit = branches[branch]
3379
3380 if self.verbose:
3381 print "branch is %s" % branch
3382
3383 self.updatedBranches.add(branch)
3384
3385 if branch not in self.createdBranches:
3386 self.createdBranches.add(branch)
3387 parent = self.knownBranches[branch]
3388 if parent == branch:
3389 parent = ""
1ca3d710
SH
3390 else:
3391 fullBranch = self.projectName + branch
3392 if fullBranch not in self.p4BranchesInGit:
3393 if not self.silent:
3394 print("\n Importing new branch %s" % fullBranch);
3395 if self.importNewBranch(branch, change - 1):
3396 parent = ""
3397 self.p4BranchesInGit.append(fullBranch)
3398 if not self.silent:
3399 print("\n Resuming with change %s" % change);
3400
3401 if self.verbose:
3402 print "parent determined through known branches: %s" % parent
e87f37ae 3403
8134f69c
SH
3404 branch = self.gitRefForBranch(branch)
3405 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3406
3407 if self.verbose:
3408 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3409
3410 if len(parent) == 0 and branch in self.initialParents:
3411 parent = self.initialParents[branch]
3412 del self.initialParents[branch]
3413
fed23693
VA
3414 blob = None
3415 if len(parent) > 0:
4f9273d2 3416 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3417 if self.verbose:
3418 print "Creating temporary branch: " + tempBranch
e63231e5 3419 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3420 self.tempBranches.append(tempBranch)
3421 self.checkpoint()
3422 blob = self.searchParent(parent, branch, tempBranch)
3423 if blob:
e63231e5 3424 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3425 else:
3426 if self.verbose:
3427 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3428 self.commit(description, filesForCommit, branch, parent)
e87f37ae 3429 else:
123f6317 3430 files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
e63231e5 3431 self.commit(description, files, self.branch,
e87f37ae 3432 self.initialParent)
47497844 3433 # only needed once, to connect to the previous commit
e87f37ae
SH
3434 self.initialParent = ""
3435 except IOError:
3436 print self.gitError.read()
3437 sys.exit(1)
3438
b9d34db9
LD
3439 def sync_origin_only(self):
3440 if self.syncWithOrigin:
3441 self.hasOrigin = originP4BranchesExist()
3442 if self.hasOrigin:
3443 if not self.silent:
3444 print 'Syncing with origin first, using "git fetch origin"'
3445 system("git fetch origin")
3446
c208a243
SH
3447 def importHeadRevision(self, revision):
3448 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3449
4e2e6ce4
PW
3450 details = {}
3451 details["user"] = "git perforce import user"
1494fcbb 3452 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3453 % (' '.join(self.depotPaths), revision))
3454 details["change"] = revision
3455 newestRevision = 0
3456
3457 fileCnt = 0
6de040df
LD
3458 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3459
3460 for info in p4CmdList(["files"] + fileArgs):
c208a243 3461
68b28593 3462 if 'code' in info and info['code'] == 'error':
c208a243
SH
3463 sys.stderr.write("p4 returned an error: %s\n"
3464 % info['data'])
d88e707f
PW
3465 if info['data'].find("must refer to client") >= 0:
3466 sys.stderr.write("This particular p4 error is misleading.\n")
3467 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3468 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3469 sys.exit(1)
68b28593
PW
3470 if 'p4ExitCode' in info:
3471 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3472 sys.exit(1)
3473
3474
3475 change = int(info["change"])
3476 if change > newestRevision:
3477 newestRevision = change
3478
56c09345 3479 if info["action"] in self.delete_actions:
c208a243
SH
3480 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3481 #fileCnt = fileCnt + 1
3482 continue
3483
3484 for prop in ["depotFile", "rev", "action", "type" ]:
3485 details["%s%s" % (prop, fileCnt)] = info[prop]
3486
3487 fileCnt = fileCnt + 1
3488
3489 details["change"] = newestRevision
4e2e6ce4 3490
9dcb9f24 3491 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3492 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3493 res = p4_describe(newestRevision)
3494 details["time"] = res["time"]