git-p4: better error reporting when p4 fails
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
a235e85c
BC
30try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
4addad22 45verbose = False
86949eef 46
06804c76 47# Only labels/tags matching this will be imported/exported
c8942a22 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 49
1051ef00
LD
50# Grab changes in blocks of this many revisions, unless otherwise requested
51defaultBlockSize = 512
52
0ef67acd
LD
53p4_access_checked = False
54
21a50753
AK
55def p4_build_cmd(cmd):
56 """Build a suitable p4 command line.
57
58 This consolidates building and returning a p4 command line into one
59 location. It means that hooking into the environment, or other configuration
60 can be done more easily.
61 """
6de040df 62 real_cmd = ["p4"]
abcaf073
AK
63
64 user = gitConfig("git-p4.user")
65 if len(user) > 0:
6de040df 66 real_cmd += ["-u",user]
abcaf073
AK
67
68 password = gitConfig("git-p4.password")
69 if len(password) > 0:
6de040df 70 real_cmd += ["-P", password]
abcaf073
AK
71
72 port = gitConfig("git-p4.port")
73 if len(port) > 0:
6de040df 74 real_cmd += ["-p", port]
abcaf073
AK
75
76 host = gitConfig("git-p4.host")
77 if len(host) > 0:
41799aa2 78 real_cmd += ["-H", host]
abcaf073
AK
79
80 client = gitConfig("git-p4.client")
81 if len(client) > 0:
6de040df 82 real_cmd += ["-c", client]
abcaf073 83
89a6ecc5
LS
84 retries = gitConfigInt("git-p4.retries")
85 if retries is None:
86 # Perform 3 retries by default
87 retries = 3
bc233524
IK
88 if retries > 0:
89 # Provide a way to not pass this option by setting git-p4.retries to 0
90 real_cmd += ["-r", str(retries)]
6de040df
LD
91
92 if isinstance(cmd,basestring):
93 real_cmd = ' '.join(real_cmd) + ' ' + cmd
94 else:
95 real_cmd += cmd
0ef67acd
LD
96
97 # now check that we can actually talk to the server
98 global p4_access_checked
99 if not p4_access_checked:
100 p4_access_checked = True # suppress access checks in p4_check_access itself
101 p4_check_access()
102
21a50753
AK
103 return real_cmd
104
378f7be1
LD
105def git_dir(path):
106 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
107 This won't automatically add ".git" to a directory.
108 """
109 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
110 if not d or len(d) == 0:
111 return None
112 else:
113 return d
114
bbd84863
MF
115def chdir(path, is_client_path=False):
116 """Do chdir to the given path, and set the PWD environment
117 variable for use by P4. It does not look at getcwd() output.
118 Since we're not using the shell, it is necessary to set the
119 PWD environment variable explicitly.
120
121 Normally, expand the path to force it to be absolute. This
122 addresses the use of relative path names inside P4 settings,
123 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
124 as given; it looks for .p4config using PWD.
125
126 If is_client_path, the path was handed to us directly by p4,
127 and may be a symbolic link. Do not call os.getcwd() in this
128 case, because it will cause p4 to think that PWD is not inside
129 the client path.
130 """
131
132 os.chdir(path)
133 if not is_client_path:
134 path = os.getcwd()
135 os.environ['PWD'] = path
053fd0c1 136
4d25dc44
LS
137def calcDiskFree():
138 """Return free space in bytes on the disk of the given dirname."""
139 if platform.system() == 'Windows':
140 free_bytes = ctypes.c_ulonglong(0)
141 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
142 return free_bytes.value
143 else:
144 st = os.statvfs(os.getcwd())
145 return st.f_bavail * st.f_frsize
146
86dff6b6
HWN
147def die(msg):
148 if verbose:
149 raise Exception(msg)
150 else:
151 sys.stderr.write(msg + "\n")
152 sys.exit(1)
153
6de040df 154def write_pipe(c, stdin):
4addad22 155 if verbose:
6de040df 156 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 157
6de040df
LD
158 expand = isinstance(c,basestring)
159 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
160 pipe = p.stdin
161 val = pipe.write(stdin)
162 pipe.close()
163 if p.wait():
164 die('Command failed: %s' % str(c))
b016d397
HWN
165
166 return val
167
6de040df 168def p4_write_pipe(c, stdin):
d9429194 169 real_cmd = p4_build_cmd(c)
6de040df 170 return write_pipe(real_cmd, stdin)
d9429194 171
78871bf4
LD
172def read_pipe_full(c):
173 """ Read output from command. Returns a tuple
174 of the return status, stdout text and stderr
175 text.
176 """
4addad22 177 if verbose:
6de040df 178 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 179
6de040df 180 expand = isinstance(c,basestring)
1f5f3907
LS
181 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
182 (out, err) = p.communicate()
78871bf4
LD
183 return (p.returncode, out, err)
184
185def read_pipe(c, ignore_error=False):
186 """ Read output from command. Returns the output text on
187 success. On failure, terminates execution, unless
188 ignore_error is True, when it returns an empty string.
189 """
190 (retcode, out, err) = read_pipe_full(c)
191 if retcode != 0:
192 if ignore_error:
193 out = ""
194 else:
195 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 196 return out
b016d397 197
78871bf4
LD
198def read_pipe_text(c):
199 """ Read output from a command with trailing whitespace stripped.
200 On error, returns None.
201 """
202 (retcode, out, err) = read_pipe_full(c)
203 if retcode != 0:
204 return None
205 else:
206 return out.rstrip()
207
d9429194
AK
208def p4_read_pipe(c, ignore_error=False):
209 real_cmd = p4_build_cmd(c)
210 return read_pipe(real_cmd, ignore_error)
b016d397 211
bce4c5fc 212def read_pipe_lines(c):
4addad22 213 if verbose:
6de040df
LD
214 sys.stderr.write('Reading pipe: %s\n' % str(c))
215
216 expand = isinstance(c, basestring)
217 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
218 pipe = p.stdout
b016d397 219 val = pipe.readlines()
6de040df
LD
220 if pipe.close() or p.wait():
221 die('Command failed: %s' % str(c))
b016d397
HWN
222
223 return val
caace111 224
2318121b
AK
225def p4_read_pipe_lines(c):
226 """Specifically invoke p4 on the command supplied. """
155af834 227 real_cmd = p4_build_cmd(c)
2318121b
AK
228 return read_pipe_lines(real_cmd)
229
8e9497c2
GG
230def p4_has_command(cmd):
231 """Ask p4 for help on this command. If it returns an error, the
232 command does not exist in this version of p4."""
233 real_cmd = p4_build_cmd(["help", cmd])
234 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
235 stderr=subprocess.PIPE)
236 p.communicate()
237 return p.returncode == 0
238
249da4c0
PW
239def p4_has_move_command():
240 """See if the move command exists, that it supports -k, and that
241 it has not been administratively disabled. The arguments
242 must be correct, but the filenames do not have to exist. Use
243 ones with wildcards so even if they exist, it will fail."""
244
245 if not p4_has_command("move"):
246 return False
247 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
248 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
249 (out, err) = p.communicate()
250 # return code will be 1 in either case
251 if err.find("Invalid option") >= 0:
252 return False
253 if err.find("disabled") >= 0:
254 return False
255 # assume it failed because @... was invalid changelist
256 return True
257
cbff4b25 258def system(cmd, ignore_error=False):
6de040df 259 expand = isinstance(cmd,basestring)
4addad22 260 if verbose:
6de040df 261 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 262 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 263 if retcode and not ignore_error:
a235e85c 264 raise CalledProcessError(retcode, cmd)
6754a299 265
cbff4b25
LD
266 return retcode
267
bf9320f1
AK
268def p4_system(cmd):
269 """Specifically invoke p4 as the system command. """
155af834 270 real_cmd = p4_build_cmd(cmd)
6de040df 271 expand = isinstance(real_cmd, basestring)
a235e85c
BC
272 retcode = subprocess.call(real_cmd, shell=expand)
273 if retcode:
274 raise CalledProcessError(retcode, real_cmd)
6de040df 275
0ef67acd
LD
276def die_bad_access(s):
277 die("failure accessing depot: {0}".format(s.rstrip()))
278
279def p4_check_access(min_expiration=1):
280 """ Check if we can access Perforce - account still logged in
281 """
282 results = p4CmdList(["login", "-s"])
283
284 if len(results) == 0:
285 # should never get here: always get either some results, or a p4ExitCode
286 assert("could not parse response from perforce")
287
288 result = results[0]
289
290 if 'p4ExitCode' in result:
291 # p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path
292 die_bad_access("could not run p4")
293
294 code = result.get("code")
295 if not code:
296 # we get here if we couldn't connect and there was nothing to unmarshal
297 die_bad_access("could not connect")
298
299 elif code == "stat":
300 expiry = result.get("TicketExpiration")
301 if expiry:
302 expiry = int(expiry)
303 if expiry > min_expiration:
304 # ok to carry on
305 return
306 else:
307 die_bad_access("perforce ticket expires in {0} seconds".format(expiry))
308
309 else:
310 # account without a timeout - all ok
311 return
312
313 elif code == "error":
314 data = result.get("data")
315 if data:
316 die_bad_access("p4 error: {0}".format(data))
317 else:
318 die_bad_access("unknown error")
319 else:
320 die_bad_access("unknown error code {0}".format(code))
321
7f0e5962
PW
322_p4_version_string = None
323def p4_version_string():
324 """Read the version string, showing just the last line, which
325 hopefully is the interesting version bit.
326
327 $ p4 -V
328 Perforce - The Fast Software Configuration Management System.
329 Copyright 1995-2011 Perforce Software. All rights reserved.
330 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
331 """
332 global _p4_version_string
333 if not _p4_version_string:
334 a = p4_read_pipe_lines(["-V"])
335 _p4_version_string = a[-1].rstrip()
336 return _p4_version_string
337
6de040df 338def p4_integrate(src, dest):
9d7d446a 339 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 340
8d7ec362 341def p4_sync(f, *options):
9d7d446a 342 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
343
344def p4_add(f):
9d7d446a
PW
345 # forcibly add file names with wildcards
346 if wildcard_present(f):
347 p4_system(["add", "-f", f])
348 else:
349 p4_system(["add", f])
6de040df
LD
350
351def p4_delete(f):
9d7d446a 352 p4_system(["delete", wildcard_encode(f)])
6de040df 353
a02b8bc4
RP
354def p4_edit(f, *options):
355 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
356
357def p4_revert(f):
9d7d446a 358 p4_system(["revert", wildcard_encode(f)])
6de040df 359
9d7d446a
PW
360def p4_reopen(type, f):
361 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 362
46c609e9
LD
363def p4_reopen_in_change(changelist, files):
364 cmd = ["reopen", "-c", str(changelist)] + files
365 p4_system(cmd)
366
8e9497c2
GG
367def p4_move(src, dest):
368 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
369
1051ef00 370def p4_last_change():
1997e91f 371 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
372 return int(results[0]['change'])
373
18fa13d0
PW
374def p4_describe(change):
375 """Make sure it returns a valid result by checking for
376 the presence of field "time". Return a dict of the
377 results."""
378
1997e91f 379 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
18fa13d0
PW
380 if len(ds) != 1:
381 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
382
383 d = ds[0]
384
385 if "p4ExitCode" in d:
386 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
387 str(d)))
388 if "code" in d:
389 if d["code"] == "error":
390 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
391
392 if "time" not in d:
393 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
394
395 return d
396
9cffb8c8
PW
397#
398# Canonicalize the p4 type and return a tuple of the
399# base type, plus any modifiers. See "p4 help filetypes"
400# for a list and explanation.
401#
402def split_p4_type(p4type):
403
404 p4_filetypes_historical = {
405 "ctempobj": "binary+Sw",
406 "ctext": "text+C",
407 "cxtext": "text+Cx",
408 "ktext": "text+k",
409 "kxtext": "text+kx",
410 "ltext": "text+F",
411 "tempobj": "binary+FSw",
412 "ubinary": "binary+F",
413 "uresource": "resource+F",
414 "uxbinary": "binary+Fx",
415 "xbinary": "binary+x",
416 "xltext": "text+Fx",
417 "xtempobj": "binary+Swx",
418 "xtext": "text+x",
419 "xunicode": "unicode+x",
420 "xutf16": "utf16+x",
421 }
422 if p4type in p4_filetypes_historical:
423 p4type = p4_filetypes_historical[p4type]
424 mods = ""
425 s = p4type.split("+")
426 base = s[0]
427 mods = ""
428 if len(s) > 1:
429 mods = s[1]
430 return (base, mods)
b9fc6ea9 431
60df071c
LD
432#
433# return the raw p4 type of a file (text, text+ko, etc)
434#
79467e61
PW
435def p4_type(f):
436 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
437 return results[0]['headType']
438
439#
440# Given a type base and modifier, return a regexp matching
441# the keywords that can be expanded in the file
442#
443def p4_keywords_regexp_for_type(base, type_mods):
444 if base in ("text", "unicode", "binary"):
445 kwords = None
446 if "ko" in type_mods:
447 kwords = 'Id|Header'
448 elif "k" in type_mods:
449 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
450 else:
451 return None
452 pattern = r"""
453 \$ # Starts with a dollar, followed by...
454 (%s) # one of the keywords, followed by...
6b2bf41e 455 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
456 \$ # another dollar
457 """ % kwords
458 return pattern
459 else:
460 return None
461
462#
463# Given a file, return a regexp matching the possible
464# RCS keywords that will be expanded, or None for files
465# with kw expansion turned off.
466#
467def p4_keywords_regexp_for_file(file):
468 if not os.path.exists(file):
469 return None
470 else:
471 (type_base, type_mods) = split_p4_type(p4_type(file))
472 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 473
c65b670e
CP
474def setP4ExecBit(file, mode):
475 # Reopens an already open file and changes the execute bit to match
476 # the execute bit setting in the passed in mode.
477
478 p4Type = "+x"
479
480 if not isModeExec(mode):
481 p4Type = getP4OpenedType(file)
482 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
483 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
484 if p4Type[-1] == "+":
485 p4Type = p4Type[0:-1]
486
6de040df 487 p4_reopen(p4Type, file)
c65b670e
CP
488
489def getP4OpenedType(file):
490 # Returns the perforce file type for the given file.
491
9d7d446a 492 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 493 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
494 if match:
495 return match.group(1)
496 else:
f3e5ae4f 497 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 498
06804c76
LD
499# Return the set of all p4 labels
500def getP4Labels(depotPaths):
501 labels = set()
502 if isinstance(depotPaths,basestring):
503 depotPaths = [depotPaths]
504
505 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
506 label = l['label']
507 labels.add(label)
508
509 return labels
510
511# Return the set of all git tags
512def getGitTags():
513 gitTags = set()
514 for line in read_pipe_lines(["git", "tag"]):
515 tag = line.strip()
516 gitTags.add(tag)
517 return gitTags
518
b43b0a3c
CP
519def diffTreePattern():
520 # This is a simple generator for the diff tree regex pattern. This could be
521 # a class variable if this and parseDiffTreeEntry were a part of a class.
522 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
523 while True:
524 yield pattern
525
526def parseDiffTreeEntry(entry):
527 """Parses a single diff tree entry into its component elements.
528
529 See git-diff-tree(1) manpage for details about the format of the diff
530 output. This method returns a dictionary with the following elements:
531
532 src_mode - The mode of the source file
533 dst_mode - The mode of the destination file
534 src_sha1 - The sha1 for the source file
535 dst_sha1 - The sha1 fr the destination file
536 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
537 status_score - The score for the status (applicable for 'C' and 'R'
538 statuses). This is None if there is no score.
539 src - The path for the source file.
540 dst - The path for the destination file. This is only present for
541 copy or renames. If it is not present, this is None.
542
543 If the pattern is not matched, None is returned."""
544
545 match = diffTreePattern().next().match(entry)
546 if match:
547 return {
548 'src_mode': match.group(1),
549 'dst_mode': match.group(2),
550 'src_sha1': match.group(3),
551 'dst_sha1': match.group(4),
552 'status': match.group(5),
553 'status_score': match.group(6),
554 'src': match.group(7),
555 'dst': match.group(10)
556 }
557 return None
558
c65b670e
CP
559def isModeExec(mode):
560 # Returns True if the given git mode represents an executable file,
561 # otherwise False.
562 return mode[-3:] == "755"
563
564def isModeExecChanged(src_mode, dst_mode):
565 return isModeExec(src_mode) != isModeExec(dst_mode)
566
1997e91f 567def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
6de040df
LD
568
569 if isinstance(cmd,basestring):
570 cmd = "-G " + cmd
571 expand = True
572 else:
573 cmd = ["-G"] + cmd
574 expand = False
575
576 cmd = p4_build_cmd(cmd)
6a49f8e2 577 if verbose:
6de040df 578 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
579
580 # Use a temporary file to avoid deadlocks without
581 # subprocess.communicate(), which would put another copy
582 # of stdout into memory.
583 stdin_file = None
584 if stdin is not None:
585 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
586 if isinstance(stdin,basestring):
587 stdin_file.write(stdin)
588 else:
589 for i in stdin:
590 stdin_file.write(i + '\n')
9f90c733
SL
591 stdin_file.flush()
592 stdin_file.seek(0)
593
6de040df
LD
594 p4 = subprocess.Popen(cmd,
595 shell=expand,
9f90c733
SL
596 stdin=stdin_file,
597 stdout=subprocess.PIPE)
86949eef
SH
598
599 result = []
600 try:
601 while True:
9f90c733 602 entry = marshal.load(p4.stdout)
1997e91f
MT
603 if skip_info:
604 if 'code' in entry and entry['code'] == 'info':
605 continue
c3f6163b
AG
606 if cb is not None:
607 cb(entry)
608 else:
609 result.append(entry)
86949eef
SH
610 except EOFError:
611 pass
9f90c733
SL
612 exitCode = p4.wait()
613 if exitCode != 0:
ac3e0d79
SH
614 entry = {}
615 entry["p4ExitCode"] = exitCode
616 result.append(entry)
86949eef
SH
617
618 return result
619
620def p4Cmd(cmd):
621 list = p4CmdList(cmd)
622 result = {}
623 for entry in list:
624 result.update(entry)
625 return result;
626
cb2c9db5
SH
627def p4Where(depotPath):
628 if not depotPath.endswith("/"):
629 depotPath += "/"
cd884106
VA
630 depotPathLong = depotPath + "..."
631 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
632 output = None
633 for entry in outputList:
75bc9573 634 if "depotFile" in entry:
cd884106
VA
635 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
636 # The base path always ends with "/...".
637 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
638 output = entry
639 break
640 elif "data" in entry:
641 data = entry.get("data")
642 space = data.find(" ")
643 if data[:space] == depotPath:
644 output = entry
645 break
7f705dc3
TAL
646 if output == None:
647 return ""
dc524036
SH
648 if output["code"] == "error":
649 return ""
cb2c9db5
SH
650 clientPath = ""
651 if "path" in output:
652 clientPath = output.get("path")
653 elif "data" in output:
654 data = output.get("data")
655 lastSpace = data.rfind(" ")
656 clientPath = data[lastSpace + 1:]
657
658 if clientPath.endswith("..."):
659 clientPath = clientPath[:-3]
660 return clientPath
661
86949eef 662def currentGitBranch():
eff45110 663 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 664
4f5cf76a 665def isValidGitDir(path):
378f7be1 666 return git_dir(path) != None
4f5cf76a 667
463e8af6 668def parseRevision(ref):
b25b2065 669 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 670
28755dba
PW
671def branchExists(ref):
672 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
673 ignore_error=True)
674 return len(rev) > 0
675
6ae8de88
SH
676def extractLogMessageFromGitCommit(commit):
677 logMessage = ""
b016d397
HWN
678
679 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 680 foundTitle = False
b016d397 681 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
682 if not foundTitle:
683 if len(log) == 1:
1c094184 684 foundTitle = True
6ae8de88
SH
685 continue
686
687 logMessage += log
688 return logMessage
689
bb6e09b2 690def extractSettingsGitLog(log):
6ae8de88
SH
691 values = {}
692 for line in log.split("\n"):
693 line = line.strip()
6326aa58
HWN
694 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
695 if not m:
696 continue
697
698 assignments = m.group(1).split (':')
699 for a in assignments:
700 vals = a.split ('=')
701 key = vals[0].strip()
702 val = ('='.join (vals[1:])).strip()
703 if val.endswith ('\"') and val.startswith('"'):
704 val = val[1:-1]
705
706 values[key] = val
707
845b42cb
SH
708 paths = values.get("depot-paths")
709 if not paths:
710 paths = values.get("depot-path")
a3fdd579
SH
711 if paths:
712 values['depot-paths'] = paths.split(',')
bb6e09b2 713 return values
6ae8de88 714
8136a639 715def gitBranchExists(branch):
bb6e09b2
HWN
716 proc = subprocess.Popen(["git", "rev-parse", branch],
717 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 718 return proc.wait() == 0;
8136a639 719
36bd8446 720_gitConfig = {}
b345d6c3 721
692e1796 722def gitConfig(key, typeSpecifier=None):
36bd8446 723 if not _gitConfig.has_key(key):
692e1796
LS
724 cmd = [ "git", "config" ]
725 if typeSpecifier:
726 cmd += [ typeSpecifier ]
727 cmd += [ key ]
b345d6c3
PW
728 s = read_pipe(cmd, ignore_error=True)
729 _gitConfig[key] = s.strip()
36bd8446 730 return _gitConfig[key]
01265103 731
0d609032
PW
732def gitConfigBool(key):
733 """Return a bool, using git config --bool. It is True only if the
734 variable is set to true, and False if set to false or not present
735 in the config."""
736
36bd8446 737 if not _gitConfig.has_key(key):
692e1796 738 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 739 return _gitConfig[key]
01265103 740
cb1dafdf
LS
741def gitConfigInt(key):
742 if not _gitConfig.has_key(key):
743 cmd = [ "git", "config", "--int", key ]
0d609032
PW
744 s = read_pipe(cmd, ignore_error=True)
745 v = s.strip()
cb1dafdf
LS
746 try:
747 _gitConfig[key] = int(gitConfig(key, '--int'))
748 except ValueError:
749 _gitConfig[key] = None
36bd8446 750 return _gitConfig[key]
01265103 751
7199cf13
VA
752def gitConfigList(key):
753 if not _gitConfig.has_key(key):
2abba301 754 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 755 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
756 if _gitConfig[key] == ['']:
757 _gitConfig[key] = []
7199cf13
VA
758 return _gitConfig[key]
759
2c8037ed
PW
760def p4BranchesInGit(branchesAreInRemotes=True):
761 """Find all the branches whose names start with "p4/", looking
762 in remotes or heads as specified by the argument. Return
763 a dictionary of { branch: revision } for each one found.
764 The branch names are the short names, without any
765 "p4/" prefix."""
766
062410bb
SH
767 branches = {}
768
769 cmdline = "git rev-parse --symbolic "
770 if branchesAreInRemotes:
2c8037ed 771 cmdline += "--remotes"
062410bb 772 else:
2c8037ed 773 cmdline += "--branches"
062410bb
SH
774
775 for line in read_pipe_lines(cmdline):
776 line = line.strip()
777
2c8037ed
PW
778 # only import to p4/
779 if not line.startswith('p4/'):
780 continue
781 # special symbolic ref to p4/master
782 if line == "p4/HEAD":
062410bb 783 continue
062410bb 784
2c8037ed
PW
785 # strip off p4/ prefix
786 branch = line[len("p4/"):]
062410bb
SH
787
788 branches[branch] = parseRevision(line)
2c8037ed 789
062410bb
SH
790 return branches
791
5a8e84cd
PW
792def branch_exists(branch):
793 """Make sure that the given ref name really exists."""
794
795 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
796 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
797 out, _ = p.communicate()
798 if p.returncode:
799 return False
800 # expect exactly one line of output: the branch name
801 return out.rstrip() == branch
802
9ceab363 803def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
804 branches = p4BranchesInGit()
805 # map from depot-path to branch name
806 branchByDepotPath = {}
807 for branch in branches.keys():
808 tip = branches[branch]
809 log = extractLogMessageFromGitCommit(tip)
810 settings = extractSettingsGitLog(log)
811 if settings.has_key("depot-paths"):
812 paths = ",".join(settings["depot-paths"])
813 branchByDepotPath[paths] = "remotes/p4/" + branch
814
27d2d811 815 settings = None
27d2d811
SH
816 parent = 0
817 while parent < 65535:
9ceab363 818 commit = head + "~%s" % parent
27d2d811
SH
819 log = extractLogMessageFromGitCommit(commit)
820 settings = extractSettingsGitLog(log)
86506fe5
SH
821 if settings.has_key("depot-paths"):
822 paths = ",".join(settings["depot-paths"])
823 if branchByDepotPath.has_key(paths):
824 return [branchByDepotPath[paths], settings]
27d2d811 825
86506fe5 826 parent = parent + 1
27d2d811 827
86506fe5 828 return ["", settings]
27d2d811 829
5ca44617
SH
830def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
831 if not silent:
832 print ("Creating/updating branch(es) in %s based on origin branch(es)"
833 % localRefPrefix)
834
835 originPrefix = "origin/p4/"
836
837 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
838 line = line.strip()
839 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
840 continue
841
842 headName = line[len(originPrefix):]
843 remoteHead = localRefPrefix + headName
844 originHead = line
845
846 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
847 if (not original.has_key('depot-paths')
848 or not original.has_key('change')):
849 continue
850
851 update = False
852 if not gitBranchExists(remoteHead):
853 if verbose:
854 print "creating %s" % remoteHead
855 update = True
856 else:
857 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
858 if settings.has_key('change') > 0:
859 if settings['depot-paths'] == original['depot-paths']:
860 originP4Change = int(original['change'])
861 p4Change = int(settings['change'])
862 if originP4Change > p4Change:
863 print ("%s (%s) is newer than %s (%s). "
864 "Updating p4 branch from origin."
865 % (originHead, originP4Change,
866 remoteHead, p4Change))
867 update = True
868 else:
869 print ("Ignoring: %s was imported from %s while "
870 "%s was imported from %s"
871 % (originHead, ','.join(original['depot-paths']),
872 remoteHead, ','.join(settings['depot-paths'])))
873
874 if update:
875 system("git update-ref %s %s" % (remoteHead, originHead))
876
877def originP4BranchesExist():
878 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
879
1051ef00
LD
880
881def p4ParseNumericChangeRange(parts):
882 changeStart = int(parts[0][1:])
883 if parts[1] == '#head':
884 changeEnd = p4_last_change()
885 else:
886 changeEnd = int(parts[1])
887
888 return (changeStart, changeEnd)
889
890def chooseBlockSize(blockSize):
891 if blockSize:
892 return blockSize
893 else:
894 return defaultBlockSize
895
896def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 897 assert depotPaths
96b2d54a 898
1051ef00
LD
899 # Parse the change range into start and end. Try to find integer
900 # revision ranges as these can be broken up into blocks to avoid
901 # hitting server-side limits (maxrows, maxscanresults). But if
902 # that doesn't work, fall back to using the raw revision specifier
903 # strings, without using block mode.
904
96b2d54a 905 if changeRange is None or changeRange == '':
1051ef00
LD
906 changeStart = 1
907 changeEnd = p4_last_change()
908 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
909 else:
910 parts = changeRange.split(',')
911 assert len(parts) == 2
1051ef00
LD
912 try:
913 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
914 block_size = chooseBlockSize(requestedBlockSize)
915 except:
916 changeStart = parts[0][1:]
917 changeEnd = parts[1]
918 if requestedBlockSize:
919 die("cannot use --changes-block-size with non-numeric revisions")
920 block_size = None
4f6432d8 921
9943e5b9 922 changes = set()
96b2d54a 923
1f90a648
SH
924 # Retrieve changes a block at a time, to prevent running
925 # into a MaxResults/MaxScanRows error from the server.
1051ef00 926
1f90a648
SH
927 while True:
928 cmd = ['changes']
1051ef00 929
1f90a648
SH
930 if block_size:
931 end = min(changeEnd, changeStart + block_size)
932 revisionRange = "%d,%d" % (changeStart, end)
933 else:
934 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 935
1f90a648 936 for p in depotPaths:
1051ef00
LD
937 cmd += ["%s...@%s" % (p, revisionRange)]
938
1f90a648 939 # Insert changes in chronological order
b596b3b9
MT
940 for entry in reversed(p4CmdList(cmd)):
941 if entry.has_key('p4ExitCode'):
942 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
943 if not entry.has_key('change'):
944 continue
945 changes.add(int(entry['change']))
1051ef00 946
1f90a648
SH
947 if not block_size:
948 break
1051ef00 949
1f90a648
SH
950 if end >= changeEnd:
951 break
1051ef00 952
1f90a648 953 changeStart = end + 1
4f6432d8 954
1f90a648
SH
955 changes = sorted(changes)
956 return changes
4f6432d8 957
d53de8b9
TAL
958def p4PathStartsWith(path, prefix):
959 # This method tries to remedy a potential mixed-case issue:
960 #
961 # If UserA adds //depot/DirA/file1
962 # and UserB adds //depot/dira/file2
963 #
964 # we may or may not have a problem. If you have core.ignorecase=true,
965 # we treat DirA and dira as the same directory
0d609032 966 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
967 return path.lower().startswith(prefix.lower())
968 return path.startswith(prefix)
969
543987bd
PW
970def getClientSpec():
971 """Look at the p4 client spec, create a View() object that contains
972 all the mappings, and return it."""
973
974 specList = p4CmdList("client -o")
975 if len(specList) != 1:
976 die('Output from "client -o" is %d lines, expecting 1' %
977 len(specList))
978
979 # dictionary of all client parameters
980 entry = specList[0]
981
9d57c4a6
KS
982 # the //client/ name
983 client_name = entry["Client"]
984
543987bd
PW
985 # just the keys that start with "View"
986 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
987
988 # hold this new View
9d57c4a6 989 view = View(client_name)
543987bd
PW
990
991 # append the lines, in order, to the view
992 for view_num in range(len(view_keys)):
993 k = "View%d" % view_num
994 if k not in view_keys:
995 die("Expected view key %s missing" % k)
996 view.append(entry[k])
997
998 return view
999
1000def getClientRoot():
1001 """Grab the client directory."""
1002
1003 output = p4CmdList("client -o")
1004 if len(output) != 1:
1005 die('Output from "client -o" is %d lines, expecting 1' % len(output))
1006
1007 entry = output[0]
1008 if "Root" not in entry:
1009 die('Client has no "Root"')
1010
1011 return entry["Root"]
1012
9d7d446a
PW
1013#
1014# P4 wildcards are not allowed in filenames. P4 complains
1015# if you simply add them, but you can force it with "-f", in
1016# which case it translates them into %xx encoding internally.
1017#
1018def wildcard_decode(path):
1019 # Search for and fix just these four characters. Do % last so
1020 # that fixing it does not inadvertently create new %-escapes.
1021 # Cannot have * in a filename in windows; untested as to
1022 # what p4 would do in such a case.
1023 if not platform.system() == "Windows":
1024 path = path.replace("%2A", "*")
1025 path = path.replace("%23", "#") \
1026 .replace("%40", "@") \
1027 .replace("%25", "%")
1028 return path
1029
1030def wildcard_encode(path):
1031 # do % first to avoid double-encoding the %s introduced here
1032 path = path.replace("%", "%25") \
1033 .replace("*", "%2A") \
1034 .replace("#", "%23") \
1035 .replace("@", "%40")
1036 return path
1037
1038def wildcard_present(path):
598354c0
BC
1039 m = re.search("[*#@%]", path)
1040 return m is not None
9d7d446a 1041
a5db4b12
LS
1042class LargeFileSystem(object):
1043 """Base class for large file system support."""
1044
1045 def __init__(self, writeToGitStream):
1046 self.largeFiles = set()
1047 self.writeToGitStream = writeToGitStream
1048
1049 def generatePointer(self, cloneDestination, contentFile):
1050 """Return the content of a pointer file that is stored in Git instead of
1051 the actual content."""
1052 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1053
1054 def pushFile(self, localLargeFile):
1055 """Push the actual content which is not stored in the Git repository to
1056 a server."""
1057 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1058
1059 def hasLargeFileExtension(self, relPath):
1060 return reduce(
1061 lambda a, b: a or b,
1062 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1063 False
1064 )
1065
1066 def generateTempFile(self, contents):
1067 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1068 for d in contents:
1069 contentFile.write(d)
1070 contentFile.close()
1071 return contentFile.name
1072
1073 def exceedsLargeFileThreshold(self, relPath, contents):
1074 if gitConfigInt('git-p4.largeFileThreshold'):
1075 contentsSize = sum(len(d) for d in contents)
1076 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1077 return True
1078 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1079 contentsSize = sum(len(d) for d in contents)
1080 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1081 return False
1082 contentTempFile = self.generateTempFile(contents)
1083 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1084 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1085 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1086 zf.close()
1087 compressedContentsSize = zf.infolist()[0].compress_size
1088 os.remove(contentTempFile)
1089 os.remove(compressedContentFile.name)
1090 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1091 return True
1092 return False
1093
1094 def addLargeFile(self, relPath):
1095 self.largeFiles.add(relPath)
1096
1097 def removeLargeFile(self, relPath):
1098 self.largeFiles.remove(relPath)
1099
1100 def isLargeFile(self, relPath):
1101 return relPath in self.largeFiles
1102
1103 def processContent(self, git_mode, relPath, contents):
1104 """Processes the content of git fast import. This method decides if a
1105 file is stored in the large file system and handles all necessary
1106 steps."""
1107 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1108 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1109 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1110 if pointer_git_mode:
1111 git_mode = pointer_git_mode
1112 if localLargeFile:
1113 # Move temp file to final location in large file system
1114 largeFileDir = os.path.dirname(localLargeFile)
1115 if not os.path.isdir(largeFileDir):
1116 os.makedirs(largeFileDir)
1117 shutil.move(contentTempFile, localLargeFile)
1118 self.addLargeFile(relPath)
1119 if gitConfigBool('git-p4.largeFilePush'):
1120 self.pushFile(localLargeFile)
1121 if verbose:
1122 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1123 return (git_mode, contents)
1124
1125class MockLFS(LargeFileSystem):
1126 """Mock large file system for testing."""
1127
1128 def generatePointer(self, contentFile):
1129 """The pointer content is the original content prefixed with "pointer-".
1130 The local filename of the large file storage is derived from the file content.
1131 """
1132 with open(contentFile, 'r') as f:
1133 content = next(f)
1134 gitMode = '100644'
1135 pointerContents = 'pointer-' + content
1136 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1137 return (gitMode, pointerContents, localLargeFile)
1138
1139 def pushFile(self, localLargeFile):
1140 """The remote filename of the large file storage is the same as the local
1141 one but in a different directory.
1142 """
1143 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1144 if not os.path.exists(remotePath):
1145 os.makedirs(remotePath)
1146 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1147
b47d807d
LS
1148class GitLFS(LargeFileSystem):
1149 """Git LFS as backend for the git-p4 large file system.
1150 See https://git-lfs.github.com/ for details."""
1151
1152 def __init__(self, *args):
1153 LargeFileSystem.__init__(self, *args)
1154 self.baseGitAttributes = []
1155
1156 def generatePointer(self, contentFile):
1157 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1158 mode and content which is stored in the Git repository instead of
1159 the actual content. Return also the new location of the actual
1160 content.
1161 """
d5eb3cf5
LS
1162 if os.path.getsize(contentFile) == 0:
1163 return (None, '', None)
1164
b47d807d
LS
1165 pointerProcess = subprocess.Popen(
1166 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1167 stdout=subprocess.PIPE
1168 )
1169 pointerFile = pointerProcess.stdout.read()
1170 if pointerProcess.wait():
1171 os.remove(contentFile)
1172 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1173
1174 # Git LFS removed the preamble in the output of the 'pointer' command
1175 # starting from version 1.2.0. Check for the preamble here to support
1176 # earlier versions.
1177 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1178 if pointerFile.startswith('Git LFS pointer for'):
1179 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1180
1181 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1182 localLargeFile = os.path.join(
1183 os.getcwd(),
1184 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1185 oid,
1186 )
1187 # LFS Spec states that pointer files should not have the executable bit set.
1188 gitMode = '100644'
82f2567e 1189 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1190
1191 def pushFile(self, localLargeFile):
1192 uploadProcess = subprocess.Popen(
1193 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1194 )
1195 if uploadProcess.wait():
1196 die('git-lfs push command failed. Did you define a remote?')
1197
1198 def generateGitAttributes(self):
1199 return (
1200 self.baseGitAttributes +
1201 [
1202 '\n',
1203 '#\n',
1204 '# Git LFS (see https://git-lfs.github.com/)\n',
1205 '#\n',
1206 ] +
862f9312 1207 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1208 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1209 ] +
862f9312 1210 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1211 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1212 ]
1213 )
1214
1215 def addLargeFile(self, relPath):
1216 LargeFileSystem.addLargeFile(self, relPath)
1217 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1218
1219 def removeLargeFile(self, relPath):
1220 LargeFileSystem.removeLargeFile(self, relPath)
1221 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1222
1223 def processContent(self, git_mode, relPath, contents):
1224 if relPath == '.gitattributes':
1225 self.baseGitAttributes = contents
1226 return (git_mode, self.generateGitAttributes())
1227 else:
1228 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1229
b984733c
SH
1230class Command:
1231 def __init__(self):
1232 self.usage = "usage: %prog [options]"
8910ac0e 1233 self.needsGit = True
6a10b6aa 1234 self.verbose = False
b984733c 1235
8cf422db
LD
1236 # This is required for the "append" cloneExclude action
1237 def ensure_value(self, attr, value):
1238 if not hasattr(self, attr) or getattr(self, attr) is None:
1239 setattr(self, attr, value)
1240 return getattr(self, attr)
1241
3ea2cfd4
LD
1242class P4UserMap:
1243 def __init__(self):
1244 self.userMapFromPerforceServer = False
affb474f
LD
1245 self.myP4UserId = None
1246
1247 def p4UserId(self):
1248 if self.myP4UserId:
1249 return self.myP4UserId
1250
1251 results = p4CmdList("user -o")
1252 for r in results:
1253 if r.has_key('User'):
1254 self.myP4UserId = r['User']
1255 return r['User']
1256 die("Could not find your p4 user id")
1257
1258 def p4UserIsMe(self, p4User):
1259 # return True if the given p4 user is actually me
1260 me = self.p4UserId()
1261 if not p4User or p4User != me:
1262 return False
1263 else:
1264 return True
3ea2cfd4
LD
1265
1266 def getUserCacheFilename(self):
1267 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1268 return home + "/.gitp4-usercache.txt"
1269
1270 def getUserMapFromPerforceServer(self):
1271 if self.userMapFromPerforceServer:
1272 return
1273 self.users = {}
1274 self.emails = {}
1275
1276 for output in p4CmdList("users"):
1277 if not output.has_key("User"):
1278 continue
1279 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1280 self.emails[output["Email"]] = output["User"]
1281
10d08a14
LS
1282 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1283 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1284 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1285 if mapUser and len(mapUser[0]) == 3:
1286 user = mapUser[0][0]
1287 fullname = mapUser[0][1]
1288 email = mapUser[0][2]
1289 self.users[user] = fullname + " <" + email + ">"
1290 self.emails[email] = user
3ea2cfd4
LD
1291
1292 s = ''
1293 for (key, val) in self.users.items():
1294 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1295
1296 open(self.getUserCacheFilename(), "wb").write(s)
1297 self.userMapFromPerforceServer = True
1298
1299 def loadUserMapFromCache(self):
1300 self.users = {}
1301 self.userMapFromPerforceServer = False
1302 try:
1303 cache = open(self.getUserCacheFilename(), "rb")
1304 lines = cache.readlines()
1305 cache.close()
1306 for line in lines:
1307 entry = line.strip().split("\t")
1308 self.users[entry[0]] = entry[1]
1309 except IOError:
1310 self.getUserMapFromPerforceServer()
1311
b984733c 1312class P4Debug(Command):
86949eef 1313 def __init__(self):
6ae8de88 1314 Command.__init__(self)
6a10b6aa 1315 self.options = []
c8c39116 1316 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1317 self.needsGit = False
86949eef
SH
1318
1319 def run(self, args):
b1ce9447 1320 j = 0
6de040df 1321 for output in p4CmdList(args):
b1ce9447
HWN
1322 print 'Element: %d' % j
1323 j += 1
86949eef 1324 print output
b984733c 1325 return True
86949eef 1326
5834684d
SH
1327class P4RollBack(Command):
1328 def __init__(self):
1329 Command.__init__(self)
1330 self.options = [
0c66a783 1331 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1332 ]
1333 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1334 self.rollbackLocalBranches = False
5834684d
SH
1335
1336 def run(self, args):
1337 if len(args) != 1:
1338 return False
1339 maxChange = int(args[0])
0c66a783 1340
ad192f28 1341 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1342 die("Problems executing p4");
1343
0c66a783
SH
1344 if self.rollbackLocalBranches:
1345 refPrefix = "refs/heads/"
b016d397 1346 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1347 else:
1348 refPrefix = "refs/remotes/"
b016d397 1349 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1350
1351 for line in lines:
1352 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1353 line = line.strip()
1354 ref = refPrefix + line
5834684d 1355 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1356 settings = extractSettingsGitLog(log)
1357
1358 depotPaths = settings['depot-paths']
1359 change = settings['change']
1360
5834684d 1361 changed = False
52102d47 1362
6326aa58
HWN
1363 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1364 for p in depotPaths]))) == 0:
52102d47
SH
1365 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1366 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1367 continue
1368
bb6e09b2 1369 while change and int(change) > maxChange:
5834684d 1370 changed = True
52102d47
SH
1371 if self.verbose:
1372 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1373 system("git update-ref %s \"%s^\"" % (ref, ref))
1374 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1375 settings = extractSettingsGitLog(log)
1376
1377
1378 depotPaths = settings['depot-paths']
1379 change = settings['change']
5834684d
SH
1380
1381 if changed:
52102d47 1382 print "%s rewound to %s" % (ref, change)
5834684d
SH
1383
1384 return True
1385
3ea2cfd4 1386class P4Submit(Command, P4UserMap):
6bbfd137
PW
1387
1388 conflict_behavior_choices = ("ask", "skip", "quit")
1389
4f5cf76a 1390 def __init__(self):
b984733c 1391 Command.__init__(self)
3ea2cfd4 1392 P4UserMap.__init__(self)
4f5cf76a 1393 self.options = [
4f5cf76a 1394 optparse.make_option("--origin", dest="origin"),
ae901090 1395 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1396 # preserve the user, requires relevant p4 permissions
1397 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1398 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1399 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1400 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1401 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1402 choices=self.conflict_behavior_choices),
1403 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1404 optparse.make_option("--shelve", dest="shelve", action="store_true",
1405 help="Shelve instead of submit. Shelved files are reverted, "
1406 "restoring the workspace to the state before the shelve"),
8cf422db 1407 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1408 metavar="CHANGELIST",
8cf422db 1409 help="update an existing shelved changelist, implies --shelve, "
f55b87c1
RM
1410 "repeat in-order for multiple shelved changelists"),
1411 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1412 help="submit only the specified commit(s), one commit or xxx..xxx"),
1413 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1414 help="Disable rebase after submit is completed. Can be useful if you "
b9d34db9
LD
1415 "work from a local git branch that is not master"),
1416 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1417 help="Skip Perforce sync of p4/master after submit or shelve"),
4f5cf76a
SH
1418 ]
1419 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1420 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1421 self.origin = ""
ae901090 1422 self.detectRenames = False
0d609032 1423 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1424 self.dry_run = False
b34fa577 1425 self.shelve = False
8cf422db 1426 self.update_shelve = list()
f55b87c1 1427 self.commit = ""
3b3477ea 1428 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
b9d34db9 1429 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
728b7ad8 1430 self.prepare_p4_only = False
6bbfd137 1431 self.conflict_behavior = None
f7baba8b 1432 self.isWindows = (platform.system() == "Windows")
06804c76 1433 self.exportLabels = False
249da4c0 1434 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1435 self.branch = None
4f5cf76a 1436
a5db4b12
LS
1437 if gitConfig('git-p4.largeFileSystem'):
1438 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1439
4f5cf76a
SH
1440 def check(self):
1441 if len(p4CmdList("opened ...")) > 0:
1442 die("You have files opened with perforce! Close them before starting the sync.")
1443
f19cb0a0
PW
1444 def separate_jobs_from_description(self, message):
1445 """Extract and return a possible Jobs field in the commit
1446 message. It goes into a separate section in the p4 change
1447 specification.
1448
1449 A jobs line starts with "Jobs:" and looks like a new field
1450 in a form. Values are white-space separated on the same
1451 line or on following lines that start with a tab.
1452
1453 This does not parse and extract the full git commit message
1454 like a p4 form. It just sees the Jobs: line as a marker
1455 to pass everything from then on directly into the p4 form,
1456 but outside the description section.
1457
1458 Return a tuple (stripped log message, jobs string)."""
1459
1460 m = re.search(r'^Jobs:', message, re.MULTILINE)
1461 if m is None:
1462 return (message, None)
1463
1464 jobtext = message[m.start():]
1465 stripped_message = message[:m.start()].rstrip()
1466 return (stripped_message, jobtext)
1467
1468 def prepareLogMessage(self, template, message, jobs):
1469 """Edits the template returned from "p4 change -o" to insert
1470 the message in the Description field, and the jobs text in
1471 the Jobs field."""
4f5cf76a
SH
1472 result = ""
1473
edae1e2f
SH
1474 inDescriptionSection = False
1475
4f5cf76a
SH
1476 for line in template.split("\n"):
1477 if line.startswith("#"):
1478 result += line + "\n"
1479 continue
1480
edae1e2f 1481 if inDescriptionSection:
c9dbab04 1482 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1483 inDescriptionSection = False
f19cb0a0
PW
1484 # insert Jobs section
1485 if jobs:
1486 result += jobs + "\n"
edae1e2f
SH
1487 else:
1488 continue
1489 else:
1490 if line.startswith("Description:"):
1491 inDescriptionSection = True
1492 line += "\n"
1493 for messageLine in message.split("\n"):
1494 line += "\t" + messageLine + "\n"
1495
1496 result += line + "\n"
4f5cf76a
SH
1497
1498 return result
1499
60df071c
LD
1500 def patchRCSKeywords(self, file, pattern):
1501 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1502 (handle, outFileName) = tempfile.mkstemp(dir='.')
1503 try:
1504 outFile = os.fdopen(handle, "w+")
1505 inFile = open(file, "r")
1506 regexp = re.compile(pattern, re.VERBOSE)
1507 for line in inFile.readlines():
1508 line = regexp.sub(r'$\1$', line)
1509 outFile.write(line)
1510 inFile.close()
1511 outFile.close()
1512 # Forcibly overwrite the original file
1513 os.unlink(file)
1514 shutil.move(outFileName, file)
1515 except:
1516 # cleanup our temporary file
1517 os.unlink(outFileName)
1518 print "Failed to strip RCS keywords in %s" % file
1519 raise
1520
1521 print "Patched up RCS keywords in %s" % file
1522
3ea2cfd4
LD
1523 def p4UserForCommit(self,id):
1524 # Return the tuple (perforce user,git email) for a given git commit id
1525 self.getUserMapFromPerforceServer()
9bf28855
PW
1526 gitEmail = read_pipe(["git", "log", "--max-count=1",
1527 "--format=%ae", id])
3ea2cfd4
LD
1528 gitEmail = gitEmail.strip()
1529 if not self.emails.has_key(gitEmail):
1530 return (None,gitEmail)
1531 else:
1532 return (self.emails[gitEmail],gitEmail)
1533
1534 def checkValidP4Users(self,commits):
1535 # check if any git authors cannot be mapped to p4 users
1536 for id in commits:
1537 (user,email) = self.p4UserForCommit(id)
1538 if not user:
1539 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1540 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1541 print "%s" % msg
1542 else:
1543 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1544
1545 def lastP4Changelist(self):
1546 # Get back the last changelist number submitted in this client spec. This
1547 # then gets used to patch up the username in the change. If the same
1548 # client spec is being used by multiple processes then this might go
1549 # wrong.
1550 results = p4CmdList("client -o") # find the current client
1551 client = None
1552 for r in results:
1553 if r.has_key('Client'):
1554 client = r['Client']
1555 break
1556 if not client:
1557 die("could not get client spec")
6de040df 1558 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1559 for r in results:
1560 if r.has_key('change'):
1561 return r['change']
1562 die("Could not get changelist number for last submit - cannot patch up user details")
1563
1564 def modifyChangelistUser(self, changelist, newUser):
1565 # fixup the user field of a changelist after it has been submitted.
1566 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1567 if len(changes) != 1:
1568 die("Bad output from p4 change modifying %s to user %s" %
1569 (changelist, newUser))
1570
1571 c = changes[0]
1572 if c['User'] == newUser: return # nothing to do
1573 c['User'] = newUser
1574 input = marshal.dumps(c)
1575
3ea2cfd4
LD
1576 result = p4CmdList("change -f -i", stdin=input)
1577 for r in result:
1578 if r.has_key('code'):
1579 if r['code'] == 'error':
1580 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1581 if r.has_key('data'):
1582 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1583 return
1584 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1585
1586 def canChangeChangelists(self):
1587 # check to see if we have p4 admin or super-user permissions, either of
1588 # which are required to modify changelists.
52a4880b 1589 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1590 for r in results:
1591 if r.has_key('perm'):
1592 if r['perm'] == 'admin':
1593 return 1
1594 if r['perm'] == 'super':
1595 return 1
1596 return 0
1597
46c609e9 1598 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1599 """Run "p4 change -o" to grab a change specification template.
1600 This does not use "p4 -G", as it is nice to keep the submission
1601 template in original order, since a human might edit it.
1602
1603 Remove lines in the Files section that show changes to files
1604 outside the depot path we're committing into."""
1605
cbc69242
SH
1606 [upstream, settings] = findUpstreamBranchPoint()
1607
b596b3b9
MT
1608 template = """\
1609# A Perforce Change Specification.
1610#
1611# Change: The change number. 'new' on a new changelist.
1612# Date: The date this specification was last modified.
1613# Client: The client on which the changelist was created. Read-only.
1614# User: The user who created the changelist.
1615# Status: Either 'pending' or 'submitted'. Read-only.
1616# Type: Either 'public' or 'restricted'. Default is 'public'.
1617# Description: Comments about the changelist. Required.
1618# Jobs: What opened jobs are to be closed by this changelist.
1619# You may delete jobs from this list. (New changelists only.)
1620# Files: What opened files from the default changelist are to be added
1621# to this changelist. You may delete files from this list.
1622# (New changelists only.)
1623"""
1624 files_list = []
ea99c3ae 1625 inFilesSection = False
b596b3b9 1626 change_entry = None
46c609e9
LD
1627 args = ['change', '-o']
1628 if changelist:
1629 args.append(str(changelist))
b596b3b9
MT
1630 for entry in p4CmdList(args):
1631 if not entry.has_key('code'):
1632 continue
1633 if entry['code'] == 'stat':
1634 change_entry = entry
1635 break
1636 if not change_entry:
1637 die('Failed to decode output of p4 change -o')
1638 for key, value in change_entry.iteritems():
1639 if key.startswith('File'):
1640 if settings.has_key('depot-paths'):
1641 if not [p for p in settings['depot-paths']
1642 if p4PathStartsWith(value, p)]:
1643 continue
ea99c3ae 1644 else:
b596b3b9
MT
1645 if not p4PathStartsWith(value, self.depotPath):
1646 continue
1647 files_list.append(value)
1648 continue
1649 # Output in the order expected by prepareLogMessage
1650 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1651 if not change_entry.has_key(key):
1652 continue
1653 template += '\n'
1654 template += key + ':'
1655 if key == 'Description':
1656 template += '\n'
1657 for field_line in change_entry[key].splitlines():
1658 template += '\t'+field_line+'\n'
1659 if len(files_list) > 0:
1660 template += '\n'
1661 template += 'Files:\n'
1662 for path in files_list:
1663 template += '\t'+path+'\n'
ea99c3ae
SH
1664 return template
1665
7c766e57
PW
1666 def edit_template(self, template_file):
1667 """Invoke the editor to let the user change the submission
1668 message. Return true if okay to continue with the submit."""
1669
1670 # if configured to skip the editing part, just submit
0d609032 1671 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1672 return True
1673
1674 # look at the modification time, to check later if the user saved
1675 # the file
1676 mtime = os.stat(template_file).st_mtime
1677
1678 # invoke the editor
f95ceaf0 1679 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1680 editor = os.environ.get("P4EDITOR")
1681 else:
1682 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1683 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1684
1685 # If the file was not saved, prompt to see if this patch should
1686 # be skipped. But skip this verification step if configured so.
0d609032 1687 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1688 return True
1689
d1652049
PW
1690 # modification time updated means user saved the file
1691 if os.stat(template_file).st_mtime > mtime:
1692 return True
1693
1694 while True:
1695 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1696 if response == 'y':
1697 return True
1698 if response == 'n':
1699 return False
7c766e57 1700
df8a9e86 1701 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3
MC
1702 # diff
1703 if os.environ.has_key("P4DIFF"):
1704 del(os.environ["P4DIFF"])
1705 diff = ""
1706 for editedFile in editedFiles:
1707 diff += p4_read_pipe(['diff', '-du',
1708 wildcard_encode(editedFile)])
1709
1710 # new file diff
1711 newdiff = ""
1712 for newFile in filesToAdd:
1713 newdiff += "==== new file ====\n"
1714 newdiff += "--- /dev/null\n"
1715 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1716
1717 is_link = os.path.islink(newFile)
1718 expect_link = newFile in symlinks
1719
1720 if is_link and expect_link:
1721 newdiff += "+%s\n" % os.readlink(newFile)
1722 else:
1723 f = open(newFile, "r")
1724 for line in f.readlines():
1725 newdiff += "+" + line
1726 f.close()
b4073bb3 1727
e2a892ee 1728 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1729
7cb5cbef 1730 def applyCommit(self, id):
67b0fe2e
PW
1731 """Apply one commit, return True if it succeeded."""
1732
1733 print "Applying", read_pipe(["git", "show", "-s",
1734 "--format=format:%h %s", id])
ae901090 1735
848de9c3 1736 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1737
84cb0003 1738 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1739 filesToAdd = set()
a02b8bc4 1740 filesToChangeType = set()
4f5cf76a 1741 filesToDelete = set()
d336c158 1742 editedFiles = set()
b6ad6dcc 1743 pureRenameCopy = set()
df8a9e86 1744 symlinks = set()
c65b670e 1745 filesToChangeExecBit = {}
46c609e9 1746 all_files = list()
60df071c 1747
4f5cf76a 1748 for line in diff:
b43b0a3c
CP
1749 diff = parseDiffTreeEntry(line)
1750 modifier = diff['status']
1751 path = diff['src']
46c609e9
LD
1752 all_files.append(path)
1753
4f5cf76a 1754 if modifier == "M":
6de040df 1755 p4_edit(path)
c65b670e
CP
1756 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1757 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1758 editedFiles.add(path)
4f5cf76a
SH
1759 elif modifier == "A":
1760 filesToAdd.add(path)
c65b670e 1761 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1762 if path in filesToDelete:
1763 filesToDelete.remove(path)
df8a9e86
LD
1764
1765 dst_mode = int(diff['dst_mode'], 8)
1766 if dst_mode == 0120000:
1767 symlinks.add(path)
1768
4f5cf76a
SH
1769 elif modifier == "D":
1770 filesToDelete.add(path)
1771 if path in filesToAdd:
1772 filesToAdd.remove(path)
4fddb41b
VA
1773 elif modifier == "C":
1774 src, dest = diff['src'], diff['dst']
6de040df 1775 p4_integrate(src, dest)
b6ad6dcc 1776 pureRenameCopy.add(dest)
4fddb41b 1777 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1778 p4_edit(dest)
b6ad6dcc 1779 pureRenameCopy.discard(dest)
4fddb41b 1780 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1781 p4_edit(dest)
b6ad6dcc 1782 pureRenameCopy.discard(dest)
4fddb41b 1783 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1784 if self.isWindows:
1785 # turn off read-only attribute
1786 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1787 os.unlink(dest)
1788 editedFiles.add(dest)
d9a5f25b 1789 elif modifier == "R":
b43b0a3c 1790 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1791 if self.p4HasMoveCommand:
1792 p4_edit(src) # src must be open before move
1793 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1794 else:
8e9497c2
GG
1795 p4_integrate(src, dest)
1796 if diff['src_sha1'] != diff['dst_sha1']:
1797 p4_edit(dest)
1798 else:
1799 pureRenameCopy.add(dest)
c65b670e 1800 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1801 if not self.p4HasMoveCommand:
1802 p4_edit(dest) # with move: already open, writable
c65b670e 1803 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1804 if not self.p4HasMoveCommand:
d20f0f8e
PW
1805 if self.isWindows:
1806 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1807 os.unlink(dest)
1808 filesToDelete.add(src)
d9a5f25b 1809 editedFiles.add(dest)
a02b8bc4
RP
1810 elif modifier == "T":
1811 filesToChangeType.add(path)
4f5cf76a
SH
1812 else:
1813 die("unknown modifier %s for %s" % (modifier, path))
1814
749b668c 1815 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1816 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1817 tryPatchCmd = patchcmd + "--check -"
1818 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1819 patch_succeeded = True
51a2640a 1820
47a130b7 1821 if os.system(tryPatchCmd) != 0:
60df071c
LD
1822 fixed_rcs_keywords = False
1823 patch_succeeded = False
51a2640a 1824 print "Unfortunately applying the change failed!"
60df071c
LD
1825
1826 # Patch failed, maybe it's just RCS keyword woes. Look through
1827 # the patch to see if that's possible.
0d609032 1828 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1829 file = None
1830 pattern = None
1831 kwfiles = {}
1832 for file in editedFiles | filesToDelete:
1833 # did this file's delta contain RCS keywords?
1834 pattern = p4_keywords_regexp_for_file(file)
1835
1836 if pattern:
1837 # this file is a possibility...look for RCS keywords.
1838 regexp = re.compile(pattern, re.VERBOSE)
1839 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1840 if regexp.search(line):
1841 if verbose:
1842 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1843 kwfiles[file] = pattern
1844 break
1845
1846 for file in kwfiles:
1847 if verbose:
1848 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1849 # File is being deleted, so not open in p4. Must
1850 # disable the read-only bit on windows.
1851 if self.isWindows and file not in editedFiles:
1852 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1853 self.patchRCSKeywords(file, kwfiles[file])
1854 fixed_rcs_keywords = True
1855
1856 if fixed_rcs_keywords:
1857 print "Retrying the patch with RCS keywords cleaned up"
1858 if os.system(tryPatchCmd) == 0:
1859 patch_succeeded = True
1860
1861 if not patch_succeeded:
7e5dd9f2
PW
1862 for f in editedFiles:
1863 p4_revert(f)
7e5dd9f2 1864 return False
51a2640a 1865
55ac2ed6
PW
1866 #
1867 # Apply the patch for real, and do add/delete/+x handling.
1868 #
47a130b7 1869 system(applyPatchCmd)
4f5cf76a 1870
a02b8bc4
RP
1871 for f in filesToChangeType:
1872 p4_edit(f, "-t", "auto")
4f5cf76a 1873 for f in filesToAdd:
6de040df 1874 p4_add(f)
4f5cf76a 1875 for f in filesToDelete:
6de040df
LD
1876 p4_revert(f)
1877 p4_delete(f)
4f5cf76a 1878
c65b670e
CP
1879 # Set/clear executable bits
1880 for f in filesToChangeExecBit.keys():
1881 mode = filesToChangeExecBit[f]
1882 setP4ExecBit(f, mode)
1883
8cf422db
LD
1884 update_shelve = 0
1885 if len(self.update_shelve) > 0:
1886 update_shelve = self.update_shelve.pop(0)
1887 p4_reopen_in_change(update_shelve, all_files)
46c609e9 1888
55ac2ed6
PW
1889 #
1890 # Build p4 change description, starting with the contents
1891 # of the git commit message.
1892 #
0e36f2d7 1893 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1894 logMessage = logMessage.strip()
f19cb0a0 1895 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1896
8cf422db 1897 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 1898 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1899
c47178d4 1900 if self.preserveUser:
55ac2ed6 1901 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1902
55ac2ed6
PW
1903 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1904 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1905 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1906 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1907
55ac2ed6 1908 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1909 if not self.prepare_p4_only:
1910 submitTemplate += separatorLine
df8a9e86 1911 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1912
c47178d4 1913 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1914 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1915 if self.isWindows:
1916 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1917 tmpFile.write(submitTemplate)
c47178d4
PW
1918 tmpFile.close()
1919
728b7ad8
PW
1920 if self.prepare_p4_only:
1921 #
1922 # Leave the p4 tree prepared, and the submit template around
1923 # and let the user decide what to do next
1924 #
1925 print
1926 print "P4 workspace prepared for submission."
1927 print "To submit or revert, go to client workspace"
1928 print " " + self.clientPath
1929 print
1930 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1931 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1932 " \"git p4\"." % fileName
1933 print "You can delete the file \"%s\" when finished." % fileName
1934
1935 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1936 print "To preserve change ownership by user %s, you must\n" \
1937 "do \"p4 change -f <change>\" after submitting and\n" \
1938 "edit the User field."
1939 if pureRenameCopy:
1940 print "After submitting, renamed files must be re-synced."
1941 print "Invoke \"p4 sync -f\" on each of these files:"
1942 for f in pureRenameCopy:
1943 print " " + f
1944
1945 print
1946 print "To revert the changes, use \"p4 revert ...\", and delete"
1947 print "the submit template file \"%s\"" % fileName
1948 if filesToAdd:
1949 print "Since the commit adds new files, they must be deleted:"
1950 for f in filesToAdd:
1951 print " " + f
1952 print
1953 return True
1954
55ac2ed6
PW
1955 #
1956 # Let the user edit the change description, then submit it.
1957 #
b7638fed 1958 submitted = False
cdc7e388 1959
b7638fed
GE
1960 try:
1961 if self.edit_template(fileName):
1962 # read the edited message and submit
1963 tmpFile = open(fileName, "rb")
1964 message = tmpFile.read()
1965 tmpFile.close()
1966 if self.isWindows:
1967 message = message.replace("\r\n", "\n")
1968 submitTemplate = message[:message.index(separatorLine)]
46c609e9 1969
8cf422db 1970 if update_shelve:
46c609e9
LD
1971 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1972 elif self.shelve:
b34fa577
VK
1973 p4_write_pipe(['shelve', '-i'], submitTemplate)
1974 else:
1975 p4_write_pipe(['submit', '-i'], submitTemplate)
1976 # The rename/copy happened by applying a patch that created a
1977 # new file. This leaves it writable, which confuses p4.
1978 for f in pureRenameCopy:
1979 p4_sync(f, "-f")
b7638fed
GE
1980
1981 if self.preserveUser:
1982 if p4User:
1983 # Get last changelist number. Cannot easily get it from
1984 # the submit command output as the output is
1985 # unmarshalled.
1986 changelist = self.lastP4Changelist()
1987 self.modifyChangelistUser(changelist, p4User)
1988
b7638fed
GE
1989 submitted = True
1990
1991 finally:
c47178d4 1992 # skip this patch
b34fa577
VK
1993 if not submitted or self.shelve:
1994 if self.shelve:
1995 print ("Reverting shelved files.")
1996 else:
1997 print ("Submission cancelled, undoing p4 changes.")
1998 for f in editedFiles | filesToDelete:
b7638fed
GE
1999 p4_revert(f)
2000 for f in filesToAdd:
2001 p4_revert(f)
2002 os.remove(f)
c47178d4
PW
2003
2004 os.remove(fileName)
b7638fed 2005 return submitted
4f5cf76a 2006
06804c76
LD
2007 # Export git tags as p4 labels. Create a p4 label and then tag
2008 # with that.
2009 def exportGitTags(self, gitTags):
c8942a22
LD
2010 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
2011 if len(validLabelRegexp) == 0:
2012 validLabelRegexp = defaultLabelRegexp
2013 m = re.compile(validLabelRegexp)
06804c76
LD
2014
2015 for name in gitTags:
2016
2017 if not m.match(name):
2018 if verbose:
05a3cec5 2019 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
2020 continue
2021
2022 # Get the p4 commit this corresponds to
c8942a22
LD
2023 logMessage = extractLogMessageFromGitCommit(name)
2024 values = extractSettingsGitLog(logMessage)
06804c76 2025
c8942a22 2026 if not values.has_key('change'):
06804c76
LD
2027 # a tag pointing to something not sent to p4; ignore
2028 if verbose:
2029 print "git tag %s does not give a p4 commit" % name
2030 continue
c8942a22
LD
2031 else:
2032 changelist = values['change']
06804c76
LD
2033
2034 # Get the tag details.
2035 inHeader = True
2036 isAnnotated = False
2037 body = []
2038 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
2039 l = l.strip()
2040 if inHeader:
2041 if re.match(r'tag\s+', l):
2042 isAnnotated = True
2043 elif re.match(r'\s*$', l):
2044 inHeader = False
2045 continue
2046 else:
2047 body.append(l)
2048
2049 if not isAnnotated:
2050 body = ["lightweight tag imported by git p4\n"]
2051
2052 # Create the label - use the same view as the client spec we are using
2053 clientSpec = getClientSpec()
2054
2055 labelTemplate = "Label: %s\n" % name
2056 labelTemplate += "Description:\n"
2057 for b in body:
2058 labelTemplate += "\t" + b + "\n"
2059 labelTemplate += "View:\n"
9d57c4a6
KS
2060 for depot_side in clientSpec.mappings:
2061 labelTemplate += "\t%s\n" % depot_side
06804c76 2062
ef739f08
PW
2063 if self.dry_run:
2064 print "Would create p4 label %s for tag" % name
728b7ad8
PW
2065 elif self.prepare_p4_only:
2066 print "Not creating p4 label %s for tag due to option" \
2067 " --prepare-p4-only" % name
ef739f08
PW
2068 else:
2069 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2070
ef739f08
PW
2071 # Use the label
2072 p4_system(["tag", "-l", name] +
9d57c4a6 2073 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2074
ef739f08
PW
2075 if verbose:
2076 print "created p4 label for tag %s" % name
06804c76 2077
4f5cf76a 2078 def run(self, args):
c9b50e63
SH
2079 if len(args) == 0:
2080 self.master = currentGitBranch()
c9b50e63
SH
2081 elif len(args) == 1:
2082 self.master = args[0]
28755dba
PW
2083 if not branchExists(self.master):
2084 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2085 else:
2086 return False
2087
8cf422db
LD
2088 for i in self.update_shelve:
2089 if i <= 0:
2090 sys.exit("invalid changelist %d" % i)
2091
00ad6e31
LD
2092 if self.master:
2093 allowSubmit = gitConfig("git-p4.allowSubmit")
2094 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2095 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2096
27d2d811 2097 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 2098 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2099 if len(self.origin) == 0:
2100 self.origin = upstream
a3fdd579 2101
8cf422db 2102 if len(self.update_shelve) > 0:
46c609e9
LD
2103 self.shelve = True
2104
3ea2cfd4
LD
2105 if self.preserveUser:
2106 if not self.canChangeChangelists():
2107 die("Cannot preserve user names without p4 super-user or admin permissions")
2108
6bbfd137
PW
2109 # if not set from the command line, try the config file
2110 if self.conflict_behavior is None:
2111 val = gitConfig("git-p4.conflict")
2112 if val:
2113 if val not in self.conflict_behavior_choices:
2114 die("Invalid value '%s' for config git-p4.conflict" % val)
2115 else:
2116 val = "ask"
2117 self.conflict_behavior = val
2118
a3fdd579
SH
2119 if self.verbose:
2120 print "Origin branch is " + self.origin
9512497b 2121
ea99c3ae 2122 if len(self.depotPath) == 0:
9512497b
SH
2123 print "Internal error: cannot locate perforce depot path from existing branches"
2124 sys.exit(128)
2125
543987bd 2126 self.useClientSpec = False
0d609032 2127 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2128 self.useClientSpec = True
2129 if self.useClientSpec:
2130 self.clientSpecDirs = getClientSpec()
9512497b 2131
2e3a16b2 2132 # Check for the existence of P4 branches
cd884106
VA
2133 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2134
2135 if self.useClientSpec and not branchesDetected:
543987bd
PW
2136 # all files are relative to the client spec
2137 self.clientPath = getClientRoot()
2138 else:
2139 self.clientPath = p4Where(self.depotPath)
9512497b 2140
543987bd
PW
2141 if self.clientPath == "":
2142 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2143
ea99c3ae 2144 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2145 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2146
0591cfa8 2147 # ensure the clientPath exists
8d7ec362 2148 new_client_dir = False
0591cfa8 2149 if not os.path.exists(self.clientPath):
8d7ec362 2150 new_client_dir = True
0591cfa8
GG
2151 os.makedirs(self.clientPath)
2152
bbd84863 2153 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2154 if self.dry_run:
2155 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2156 else:
ef739f08
PW
2157 print "Synchronizing p4 checkout..."
2158 if new_client_dir:
2159 # old one was destroyed, and maybe nobody told p4
2160 p4_sync("...", "-f")
2161 else:
2162 p4_sync("...")
4f5cf76a 2163 self.check()
4f5cf76a 2164
4c750c0d 2165 commits = []
00ad6e31
LD
2166 if self.master:
2167 commitish = self.master
2168 else:
2169 commitish = 'HEAD'
2170
f55b87c1
RM
2171 if self.commit != "":
2172 if self.commit.find("..") != -1:
2173 limits_ish = self.commit.split("..")
2174 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2175 commits.append(line.strip())
2176 commits.reverse()
2177 else:
2178 commits.append(self.commit)
2179 else:
2180 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2181 commits.append(line.strip())
2182 commits.reverse()
4f5cf76a 2183
0d609032 2184 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2185 self.checkAuthorship = False
2186 else:
2187 self.checkAuthorship = True
2188
3ea2cfd4
LD
2189 if self.preserveUser:
2190 self.checkValidP4Users(commits)
2191
84cb0003
GG
2192 #
2193 # Build up a set of options to be passed to diff when
2194 # submitting each commit to p4.
2195 #
2196 if self.detectRenames:
2197 # command-line -M arg
2198 self.diffOpts = "-M"
2199 else:
2200 # If not explicitly set check the config variable
2201 detectRenames = gitConfig("git-p4.detectRenames")
2202
2203 if detectRenames.lower() == "false" or detectRenames == "":
2204 self.diffOpts = ""
2205 elif detectRenames.lower() == "true":
2206 self.diffOpts = "-M"
2207 else:
2208 self.diffOpts = "-M%s" % detectRenames
2209
2210 # no command-line arg for -C or --find-copies-harder, just
2211 # config variables
2212 detectCopies = gitConfig("git-p4.detectCopies")
2213 if detectCopies.lower() == "false" or detectCopies == "":
2214 pass
2215 elif detectCopies.lower() == "true":
2216 self.diffOpts += " -C"
2217 else:
2218 self.diffOpts += " -C%s" % detectCopies
2219
0d609032 2220 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2221 self.diffOpts += " --find-copies-harder"
2222
8cf422db
LD
2223 num_shelves = len(self.update_shelve)
2224 if num_shelves > 0 and num_shelves != len(commits):
2225 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2226 (len(commits), num_shelves))
2227
7e5dd9f2
PW
2228 #
2229 # Apply the commits, one at a time. On failure, ask if should
2230 # continue to try the rest of the patches, or quit.
2231 #
ef739f08
PW
2232 if self.dry_run:
2233 print "Would apply"
67b0fe2e 2234 applied = []
7e5dd9f2
PW
2235 last = len(commits) - 1
2236 for i, commit in enumerate(commits):
ef739f08
PW
2237 if self.dry_run:
2238 print " ", read_pipe(["git", "show", "-s",
2239 "--format=format:%h %s", commit])
2240 ok = True
2241 else:
2242 ok = self.applyCommit(commit)
67b0fe2e
PW
2243 if ok:
2244 applied.append(commit)
7e5dd9f2 2245 else:
728b7ad8
PW
2246 if self.prepare_p4_only and i < last:
2247 print "Processing only the first commit due to option" \
2248 " --prepare-p4-only"
2249 break
7e5dd9f2
PW
2250 if i < last:
2251 quit = False
2252 while True:
6bbfd137
PW
2253 # prompt for what to do, or use the option/variable
2254 if self.conflict_behavior == "ask":
2255 print "What do you want to do?"
2256 response = raw_input("[s]kip this commit but apply"
2257 " the rest, or [q]uit? ")
2258 if not response:
2259 continue
2260 elif self.conflict_behavior == "skip":
2261 response = "s"
2262 elif self.conflict_behavior == "quit":
2263 response = "q"
2264 else:
2265 die("Unknown conflict_behavior '%s'" %
2266 self.conflict_behavior)
2267
7e5dd9f2
PW
2268 if response[0] == "s":
2269 print "Skipping this commit, but applying the rest"
2270 break
2271 if response[0] == "q":
2272 print "Quitting"
2273 quit = True
2274 break
2275 if quit:
2276 break
4f5cf76a 2277
67b0fe2e 2278 chdir(self.oldWorkingDirectory)
b34fa577 2279 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2280 if self.dry_run:
2281 pass
728b7ad8
PW
2282 elif self.prepare_p4_only:
2283 pass
ef739f08 2284 elif len(commits) == len(applied):
b34fa577 2285 print ("All commits {0}!".format(shelved_applied))
14594f4b 2286
4c750c0d 2287 sync = P4Sync()
44e8d26c
PW
2288 if self.branch:
2289 sync.branch = self.branch
b9d34db9
LD
2290 if self.disable_p4sync:
2291 sync.sync_origin_only()
2292 else:
2293 sync.run([])
14594f4b 2294
b9d34db9
LD
2295 if not self.disable_rebase:
2296 rebase = P4Rebase()
2297 rebase.rebase()
4f5cf76a 2298
67b0fe2e
PW
2299 else:
2300 if len(applied) == 0:
b34fa577 2301 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2302 else:
b34fa577 2303 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2304 for c in commits:
2305 if c in applied:
2306 star = "*"
2307 else:
2308 star = " "
2309 print star, read_pipe(["git", "show", "-s",
2310 "--format=format:%h %s", c])
2311 print "You will have to do 'git p4 sync' and rebase."
2312
0d609032 2313 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2314 self.exportLabels = True
06804c76
LD
2315
2316 if self.exportLabels:
2317 p4Labels = getP4Labels(self.depotPath)
2318 gitTags = getGitTags()
2319
2320 missingGitTags = gitTags - p4Labels
2321 self.exportGitTags(missingGitTags)
2322
98e023de 2323 # exit with error unless everything applied perfectly
67b0fe2e
PW
2324 if len(commits) != len(applied):
2325 sys.exit(1)
2326
b984733c
SH
2327 return True
2328
ecb7cf98
PW
2329class View(object):
2330 """Represent a p4 view ("p4 help views"), and map files in a
2331 repo according to the view."""
2332
9d57c4a6 2333 def __init__(self, client_name):
ecb7cf98 2334 self.mappings = []
9d57c4a6
KS
2335 self.client_prefix = "//%s/" % client_name
2336 # cache results of "p4 where" to lookup client file locations
2337 self.client_spec_path_cache = {}
ecb7cf98
PW
2338
2339 def append(self, view_line):
2340 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2341 sides. Append to self.mappings, preserving order. This
2342 is only needed for tag creation."""
ecb7cf98
PW
2343
2344 # Split the view line into exactly two words. P4 enforces
2345 # structure on these lines that simplifies this quite a bit.
2346 #
2347 # Either or both words may be double-quoted.
2348 # Single quotes do not matter.
2349 # Double-quote marks cannot occur inside the words.
2350 # A + or - prefix is also inside the quotes.
2351 # There are no quotes unless they contain a space.
2352 # The line is already white-space stripped.
2353 # The two words are separated by a single space.
2354 #
2355 if view_line[0] == '"':
2356 # First word is double quoted. Find its end.
2357 close_quote_index = view_line.find('"', 1)
2358 if close_quote_index <= 0:
2359 die("No first-word closing quote found: %s" % view_line)
2360 depot_side = view_line[1:close_quote_index]
2361 # skip closing quote and space
2362 rhs_index = close_quote_index + 1 + 1
2363 else:
2364 space_index = view_line.find(" ")
2365 if space_index <= 0:
2366 die("No word-splitting space found: %s" % view_line)
2367 depot_side = view_line[0:space_index]
2368 rhs_index = space_index + 1
2369
ecb7cf98 2370 # prefix + means overlay on previous mapping
ecb7cf98 2371 if depot_side.startswith("+"):
ecb7cf98
PW
2372 depot_side = depot_side[1:]
2373
9d57c4a6 2374 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2375 exclude = False
2376 if depot_side.startswith("-"):
2377 exclude = True
2378 depot_side = depot_side[1:]
2379
9d57c4a6
KS
2380 if not exclude:
2381 self.mappings.append(depot_side)
ecb7cf98 2382
9d57c4a6
KS
2383 def convert_client_path(self, clientFile):
2384 # chop off //client/ part to make it relative
2385 if not clientFile.startswith(self.client_prefix):
2386 die("No prefix '%s' on clientFile '%s'" %
2387 (self.client_prefix, clientFile))
2388 return clientFile[len(self.client_prefix):]
ecb7cf98 2389
9d57c4a6
KS
2390 def update_client_spec_path_cache(self, files):
2391 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2392
9d57c4a6
KS
2393 # List depot file paths exclude that already cached
2394 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2395
9d57c4a6
KS
2396 if len(fileArgs) == 0:
2397 return # All files in cache
ecb7cf98 2398
9d57c4a6
KS
2399 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2400 for res in where_result:
2401 if "code" in res and res["code"] == "error":
2402 # assume error is "... file(s) not in client view"
2403 continue
2404 if "clientFile" not in res:
20005443 2405 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2406 if "unmap" in res:
2407 # it will list all of them, but only one not unmap-ped
2408 continue
a0a50d87
LS
2409 if gitConfigBool("core.ignorecase"):
2410 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2411 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2412
9d57c4a6
KS
2413 # not found files or unmap files set to ""
2414 for depotFile in fileArgs:
a0a50d87
LS
2415 if gitConfigBool("core.ignorecase"):
2416 depotFile = depotFile.lower()
9d57c4a6
KS
2417 if depotFile not in self.client_spec_path_cache:
2418 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2419
9d57c4a6
KS
2420 def map_in_client(self, depot_path):
2421 """Return the relative location in the client where this
2422 depot file should live. Returns "" if the file should
2423 not be mapped in the client."""
ecb7cf98 2424
a0a50d87
LS
2425 if gitConfigBool("core.ignorecase"):
2426 depot_path = depot_path.lower()
2427
9d57c4a6
KS
2428 if depot_path in self.client_spec_path_cache:
2429 return self.client_spec_path_cache[depot_path]
2430
2431 die( "Error: %s is not found in client spec path" % depot_path )
2432 return ""
ecb7cf98 2433
3ea2cfd4 2434class P4Sync(Command, P4UserMap):
56c09345
PW
2435 delete_actions = ( "delete", "move/delete", "purge" )
2436
b984733c
SH
2437 def __init__(self):
2438 Command.__init__(self)
3ea2cfd4 2439 P4UserMap.__init__(self)
b984733c
SH
2440 self.options = [
2441 optparse.make_option("--branch", dest="branch"),
2442 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2443 optparse.make_option("--changesfile", dest="changesFile"),
2444 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2445 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2446 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2447 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2448 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2449 optparse.make_option("--max-changes", dest="maxChanges",
2450 help="Maximum number of changes to import"),
2451 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2452 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2453 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2454 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2455 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2456 help="Only sync files that are included in the Perforce Client Spec"),
2457 optparse.make_option("-/", dest="cloneExclude",
2458 action="append", type="string",
2459 help="exclude depot path"),
b984733c
SH
2460 ]
2461 self.description = """Imports from Perforce into a git repository.\n
2462 example:
2463 //depot/my/project/ -- to import the current head
2464 //depot/my/project/@all -- to import everything
2465 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2466
2467 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2468
2469 self.usage += " //depot/path[@revRange]"
b984733c 2470 self.silent = False
1d7367dc
RG
2471 self.createdBranches = set()
2472 self.committedChanges = set()
569d1bd4 2473 self.branch = ""
b984733c 2474 self.detectBranches = False
cb53e1f8 2475 self.detectLabels = False
06804c76 2476 self.importLabels = False
b984733c 2477 self.changesFile = ""
01265103 2478 self.syncWithOrigin = True
a028a98e 2479 self.importIntoRemotes = True
01a9c9c5 2480 self.maxChanges = ""
1051ef00 2481 self.changes_block_size = None
8b41a97f 2482 self.keepRepoPath = False
6326aa58 2483 self.depotPaths = None
3c699645 2484 self.p4BranchesInGit = []
354081d5 2485 self.cloneExclude = []
3a70cdfa 2486 self.useClientSpec = False
a93d33ee 2487 self.useClientSpec_from_options = False
ecb7cf98 2488 self.clientSpecDirs = None
fed23693 2489 self.tempBranches = []
d604176d 2490 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12
LS
2491 self.largeFileSystem = None
2492
2493 if gitConfig('git-p4.largeFileSystem'):
2494 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2495 self.largeFileSystem = largeFileSystemConstructor(
2496 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2497 )
b984733c 2498
01265103
SH
2499 if gitConfig("git-p4.syncFromOrigin") == "false":
2500 self.syncWithOrigin = False
2501
fed23693
VA
2502 # Force a checkpoint in fast-import and wait for it to finish
2503 def checkpoint(self):
2504 self.gitStream.write("checkpoint\n\n")
2505 self.gitStream.write("progress checkpoint\n\n")
2506 out = self.gitOutput.readline()
2507 if self.verbose:
2508 print "checkpoint finished: " + out
2509
b984733c 2510 def extractFilesFromCommit(self, commit):
354081d5
TT
2511 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2512 for path in self.cloneExclude]
b984733c
SH
2513 files = []
2514 fnum = 0
2515 while commit.has_key("depotFile%s" % fnum):
2516 path = commit["depotFile%s" % fnum]
6326aa58 2517
354081d5 2518 if [p for p in self.cloneExclude
d53de8b9 2519 if p4PathStartsWith(path, p)]:
354081d5
TT
2520 found = False
2521 else:
2522 found = [p for p in self.depotPaths
d53de8b9 2523 if p4PathStartsWith(path, p)]
6326aa58 2524 if not found:
b984733c
SH
2525 fnum = fnum + 1
2526 continue
2527
2528 file = {}
2529 file["path"] = path
2530 file["rev"] = commit["rev%s" % fnum]
2531 file["action"] = commit["action%s" % fnum]
2532 file["type"] = commit["type%s" % fnum]
2533 files.append(file)
2534 fnum = fnum + 1
2535 return files
2536
26e6a27d
JD
2537 def extractJobsFromCommit(self, commit):
2538 jobs = []
2539 jnum = 0
2540 while commit.has_key("job%s" % jnum):
2541 job = commit["job%s" % jnum]
2542 jobs.append(job)
2543 jnum = jnum + 1
2544 return jobs
2545
6326aa58 2546 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2547 """When streaming files, this is called to map a p4 depot path
2548 to where it should go in git. The prefixes are either
2549 self.depotPaths, or self.branchPrefixes in the case of
2550 branch detection."""
2551
3952710b 2552 if self.useClientSpec:
21ef5df4
PW
2553 # branch detection moves files up a level (the branch name)
2554 # from what client spec interpretation gives
0d1696ef 2555 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2556 if self.detectBranches:
2557 for b in self.knownBranches:
2558 if path.startswith(b + "/"):
2559 path = path[len(b)+1:]
2560
2561 elif self.keepRepoPath:
2562 # Preserve everything in relative path name except leading
2563 # //depot/; just look at first prefix as they all should
2564 # be in the same depot.
2565 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2566 if p4PathStartsWith(path, depot):
2567 path = path[len(depot):]
3952710b 2568
0d1696ef 2569 else:
0d1696ef
PW
2570 for p in prefixes:
2571 if p4PathStartsWith(path, p):
2572 path = path[len(p):]
21ef5df4 2573 break
8b41a97f 2574
0d1696ef 2575 path = wildcard_decode(path)
6326aa58 2576 return path
6754a299 2577
71b112d4 2578 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2579 """Look at each depotFile in the commit to figure out to what
2580 branch it belongs."""
2581
9d57c4a6
KS
2582 if self.clientSpecDirs:
2583 files = self.extractFilesFromCommit(commit)
2584 self.clientSpecDirs.update_client_spec_path_cache(files)
2585
d5904674 2586 branches = {}
71b112d4
SH
2587 fnum = 0
2588 while commit.has_key("depotFile%s" % fnum):
2589 path = commit["depotFile%s" % fnum]
6326aa58 2590 found = [p for p in self.depotPaths
d53de8b9 2591 if p4PathStartsWith(path, p)]
6326aa58 2592 if not found:
71b112d4
SH
2593 fnum = fnum + 1
2594 continue
2595
2596 file = {}
2597 file["path"] = path
2598 file["rev"] = commit["rev%s" % fnum]
2599 file["action"] = commit["action%s" % fnum]
2600 file["type"] = commit["type%s" % fnum]
2601 fnum = fnum + 1
2602
21ef5df4
PW
2603 # start with the full relative path where this file would
2604 # go in a p4 client
2605 if self.useClientSpec:
2606 relPath = self.clientSpecDirs.map_in_client(path)
2607 else:
2608 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2609
4b97ffb1 2610 for branch in self.knownBranches.keys():
21ef5df4
PW
2611 # add a trailing slash so that a commit into qt/4.2foo
2612 # doesn't end up in qt/4.2, e.g.
6754a299 2613 if relPath.startswith(branch + "/"):
d5904674
SH
2614 if branch not in branches:
2615 branches[branch] = []
71b112d4 2616 branches[branch].append(file)
6555b2cc 2617 break
b984733c
SH
2618
2619 return branches
2620
a5db4b12
LS
2621 def writeToGitStream(self, gitMode, relPath, contents):
2622 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2623 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2624 for d in contents:
2625 self.gitStream.write(d)
2626 self.gitStream.write('\n')
2627
a8b05162
LS
2628 def encodeWithUTF8(self, path):
2629 try:
2630 path.decode('ascii')
2631 except:
2632 encoding = 'utf8'
2633 if gitConfig('git-p4.pathEncoding'):
2634 encoding = gitConfig('git-p4.pathEncoding')
2635 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2636 if self.verbose:
2637 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2638 return path
2639
b932705b
LD
2640 # output one file from the P4 stream
2641 # - helper for streamP4Files
2642
2643 def streamOneP4File(self, file, contents):
b932705b 2644 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2645 relPath = self.encodeWithUTF8(relPath)
b932705b 2646 if verbose:
d2176a50
LS
2647 size = int(self.stream_file['fileSize'])
2648 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2649 sys.stdout.flush()
b932705b 2650
9cffb8c8
PW
2651 (type_base, type_mods) = split_p4_type(file["type"])
2652
2653 git_mode = "100644"
2654 if "x" in type_mods:
2655 git_mode = "100755"
2656 if type_base == "symlink":
2657 git_mode = "120000"
1292df11
AJ
2658 # p4 print on a symlink sometimes contains "target\n";
2659 # if it does, remove the newline
b39c3612 2660 data = ''.join(contents)
40f846c3
PW
2661 if not data:
2662 # Some version of p4 allowed creating a symlink that pointed
2663 # to nothing. This causes p4 errors when checking out such
2664 # a change, and errors here too. Work around it by ignoring
2665 # the bad symlink; hopefully a future change fixes it.
2666 print "\nIgnoring empty symlink in %s" % file['depotFile']
2667 return
2668 elif data[-1] == '\n':
1292df11
AJ
2669 contents = [data[:-1]]
2670 else:
2671 contents = [data]
b932705b 2672
9cffb8c8 2673 if type_base == "utf16":
55aa5714
PW
2674 # p4 delivers different text in the python output to -G
2675 # than it does when using "print -o", or normal p4 client
2676 # operations. utf16 is converted to ascii or utf8, perhaps.
2677 # But ascii text saved as -t utf16 is completely mangled.
2678 # Invoke print -o to get the real contents.
7f0e5962
PW
2679 #
2680 # On windows, the newlines will always be mangled by print, so put
2681 # them back too. This is not needed to the cygwin windows version,
2682 # just the native "NT" type.
2683 #
1f5f3907
LS
2684 try:
2685 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2686 except Exception as e:
2687 if 'Translation of file content failed' in str(e):
2688 type_base = 'binary'
2689 else:
2690 raise e
2691 else:
2692 if p4_version_string().find('/NT') >= 0:
2693 text = text.replace('\r\n', '\n')
2694 contents = [ text ]
55aa5714 2695
9f7ef0ea
PW
2696 if type_base == "apple":
2697 # Apple filetype files will be streamed as a concatenation of
2698 # its appledouble header and the contents. This is useless
2699 # on both macs and non-macs. If using "print -q -o xx", it
2700 # will create "xx" with the data, and "%xx" with the header.
2701 # This is also not very useful.
2702 #
2703 # Ideally, someday, this script can learn how to generate
2704 # appledouble files directly and import those to git, but
2705 # non-mac machines can never find a use for apple filetype.
2706 print "\nIgnoring apple filetype file %s" % file['depotFile']
2707 return
2708
55aa5714
PW
2709 # Note that we do not try to de-mangle keywords on utf16 files,
2710 # even though in theory somebody may want that.
60df071c
LD
2711 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2712 if pattern:
2713 regexp = re.compile(pattern, re.VERBOSE)
2714 text = ''.join(contents)
2715 text = regexp.sub(r'$\1$', text)
2716 contents = [ text ]
b932705b 2717
a5db4b12
LS
2718 if self.largeFileSystem:
2719 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2720
a5db4b12 2721 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2722
2723 def streamOneP4Deletion(self, file):
2724 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2725 relPath = self.encodeWithUTF8(relPath)
b932705b 2726 if verbose:
d2176a50
LS
2727 sys.stdout.write("delete %s\n" % relPath)
2728 sys.stdout.flush()
b932705b
LD
2729 self.gitStream.write("D %s\n" % relPath)
2730
a5db4b12
LS
2731 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2732 self.largeFileSystem.removeLargeFile(relPath)
2733
b932705b
LD
2734 # handle another chunk of streaming data
2735 def streamP4FilesCb(self, marshalled):
2736
78189bea
PW
2737 # catch p4 errors and complain
2738 err = None
2739 if "code" in marshalled:
2740 if marshalled["code"] == "error":
2741 if "data" in marshalled:
2742 err = marshalled["data"].rstrip()
4d25dc44
LS
2743
2744 if not err and 'fileSize' in self.stream_file:
2745 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2746 if required_bytes > 0:
2747 err = 'Not enough space left on %s! Free at least %i MB.' % (
2748 os.getcwd(), required_bytes/1024/1024
2749 )
2750
78189bea
PW
2751 if err:
2752 f = None
2753 if self.stream_have_file_info:
2754 if "depotFile" in self.stream_file:
2755 f = self.stream_file["depotFile"]
2756 # force a failure in fast-import, else an empty
2757 # commit will be made
2758 self.gitStream.write("\n")
2759 self.gitStream.write("die-now\n")
2760 self.gitStream.close()
2761 # ignore errors, but make sure it exits first
2762 self.importProcess.wait()
2763 if f:
2764 die("Error from p4 print for %s: %s" % (f, err))
2765 else:
2766 die("Error from p4 print: %s" % err)
2767
c3f6163b
AG
2768 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2769 # start of a new file - output the old one first
2770 self.streamOneP4File(self.stream_file, self.stream_contents)
2771 self.stream_file = {}
2772 self.stream_contents = []
2773 self.stream_have_file_info = False
b932705b 2774
c3f6163b
AG
2775 # pick up the new file information... for the
2776 # 'data' field we need to append to our array
2777 for k in marshalled.keys():
2778 if k == 'data':
d2176a50
LS
2779 if 'streamContentSize' not in self.stream_file:
2780 self.stream_file['streamContentSize'] = 0
2781 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2782 self.stream_contents.append(marshalled['data'])
2783 else:
2784 self.stream_file[k] = marshalled[k]
b932705b 2785
d2176a50
LS
2786 if (verbose and
2787 'streamContentSize' in self.stream_file and
2788 'fileSize' in self.stream_file and
2789 'depotFile' in self.stream_file):
2790 size = int(self.stream_file["fileSize"])
2791 if size > 0:
2792 progress = 100*self.stream_file['streamContentSize']/size
2793 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2794 sys.stdout.flush()
2795
c3f6163b 2796 self.stream_have_file_info = True
b932705b
LD
2797
2798 # Stream directly from "p4 files" into "git fast-import"
2799 def streamP4Files(self, files):
30b5940b
SH
2800 filesForCommit = []
2801 filesToRead = []
b932705b 2802 filesToDelete = []
30b5940b 2803
3a70cdfa 2804 for f in files:
ecb7cf98
PW
2805 filesForCommit.append(f)
2806 if f['action'] in self.delete_actions:
2807 filesToDelete.append(f)
2808 else:
2809 filesToRead.append(f)
6a49f8e2 2810
b932705b
LD
2811 # deleted files...
2812 for f in filesToDelete:
2813 self.streamOneP4Deletion(f)
1b9a4684 2814
b932705b
LD
2815 if len(filesToRead) > 0:
2816 self.stream_file = {}
2817 self.stream_contents = []
2818 self.stream_have_file_info = False
8ff45f2a 2819
c3f6163b
AG
2820 # curry self argument
2821 def streamP4FilesCbSelf(entry):
2822 self.streamP4FilesCb(entry)
6a49f8e2 2823
6de040df
LD
2824 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2825
2826 p4CmdList(["-x", "-", "print"],
2827 stdin=fileArgs,
2828 cb=streamP4FilesCbSelf)
30b5940b 2829
b932705b
LD
2830 # do the last chunk
2831 if self.stream_file.has_key('depotFile'):
2832 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2833
affb474f
LD
2834 def make_email(self, userid):
2835 if userid in self.users:
2836 return self.users[userid]
2837 else:
2838 return "%s <a@b>" % userid
2839
06804c76 2840 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2841 """ Stream a p4 tag.
2842 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2843 """
2844
06804c76
LD
2845 if verbose:
2846 print "writing tag %s for commit %s" % (labelName, commit)
2847 gitStream.write("tag %s\n" % labelName)
2848 gitStream.write("from %s\n" % commit)
2849
2850 if labelDetails.has_key('Owner'):
2851 owner = labelDetails["Owner"]
2852 else:
2853 owner = None
2854
2855 # Try to use the owner of the p4 label, or failing that,
2856 # the current p4 user id.
2857 if owner:
2858 email = self.make_email(owner)
2859 else:
2860 email = self.make_email(self.p4UserId())
2861 tagger = "%s %s %s" % (email, epoch, self.tz)
2862
2863 gitStream.write("tagger %s\n" % tagger)
2864
2865 print "labelDetails=",labelDetails
2866 if labelDetails.has_key('Description'):
2867 description = labelDetails['Description']
2868 else:
2869 description = 'Label from git p4'
2870
2871 gitStream.write("data %d\n" % len(description))
2872 gitStream.write(description)
2873 gitStream.write("\n")
2874
4ae048e6
LS
2875 def inClientSpec(self, path):
2876 if not self.clientSpecDirs:
2877 return True
2878 inClientSpec = self.clientSpecDirs.map_in_client(path)
2879 if not inClientSpec and self.verbose:
2880 print('Ignoring file outside of client spec: {0}'.format(path))
2881 return inClientSpec
2882
2883 def hasBranchPrefix(self, path):
2884 if not self.branchPrefixes:
2885 return True
2886 hasPrefix = [p for p in self.branchPrefixes
2887 if p4PathStartsWith(path, p)]
09667d01 2888 if not hasPrefix and self.verbose:
4ae048e6
LS
2889 print('Ignoring file outside of prefix: {0}'.format(path))
2890 return hasPrefix
2891
e63231e5 2892 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2893 epoch = details["time"]
2894 author = details["user"]
26e6a27d 2895 jobs = self.extractJobsFromCommit(details)
b984733c 2896
4b97ffb1 2897 if self.verbose:
4ae048e6 2898 print('commit into {0}'.format(branch))
96e07dd2 2899
9d57c4a6
KS
2900 if self.clientSpecDirs:
2901 self.clientSpecDirs.update_client_spec_path_cache(files)
2902
4ae048e6
LS
2903 files = [f for f in files
2904 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2905
2906 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2907 print('Ignoring revision {0} as it would produce an empty commit.'
2908 .format(details['change']))
2909 return
2910
b984733c 2911 self.gitStream.write("commit %s\n" % branch)
b43702ac 2912 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2913 self.committedChanges.add(int(details["change"]))
2914 committer = ""
b607e71e
SH
2915 if author not in self.users:
2916 self.getUserMapFromPerforceServer()
affb474f 2917 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2918
2919 self.gitStream.write("committer %s\n" % committer)
2920
2921 self.gitStream.write("data <<EOT\n")
2922 self.gitStream.write(details["desc"])
26e6a27d
JD
2923 if len(jobs) > 0:
2924 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
e63231e5
PW
2925 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2926 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2927 if len(details['options']) > 0:
2928 self.gitStream.write(": options = %s" % details['options'])
2929 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2930
2931 if len(parent) > 0:
4b97ffb1
SH
2932 if self.verbose:
2933 print "parent %s" % parent
b984733c
SH
2934 self.gitStream.write("from %s\n" % parent)
2935
4ae048e6 2936 self.streamP4Files(files)
b984733c
SH
2937 self.gitStream.write("\n")
2938
1f4ba1cb
SH
2939 change = int(details["change"])
2940
9bda3a85 2941 if self.labels.has_key(change):
1f4ba1cb
SH
2942 label = self.labels[change]
2943 labelDetails = label[0]
2944 labelRevisions = label[1]
71b112d4
SH
2945 if self.verbose:
2946 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2947
6de040df 2948 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2949 for p in self.branchPrefixes])
1f4ba1cb
SH
2950
2951 if len(files) == len(labelRevisions):
2952
2953 cleanedFiles = {}
2954 for info in files:
56c09345 2955 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2956 continue
2957 cleanedFiles[info["depotFile"]] = info["rev"]
2958
2959 if cleanedFiles == labelRevisions:
06804c76 2960 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2961
2962 else:
a46668fa 2963 if not self.silent:
cebdf5af
HWN
2964 print ("Tag %s does not match with change %s: files do not match."
2965 % (labelDetails["label"], change))
1f4ba1cb
SH
2966
2967 else:
a46668fa 2968 if not self.silent:
cebdf5af
HWN
2969 print ("Tag %s does not match with change %s: file count is different."
2970 % (labelDetails["label"], change))
b984733c 2971
06804c76 2972 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2973 def getLabels(self):
2974 self.labels = {}
2975
52a4880b 2976 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2977 if len(l) > 0 and not self.silent:
183f8436 2978 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2979
2980 for output in l:
1f4ba1cb
SH
2981 label = output["label"]
2982 revisions = {}
2983 newestChange = 0
71b112d4
SH
2984 if self.verbose:
2985 print "Querying files for label %s" % label
6de040df
LD
2986 for file in p4CmdList(["files"] +
2987 ["%s...@%s" % (p, label)
2988 for p in self.depotPaths]):
1f4ba1cb
SH
2989 revisions[file["depotFile"]] = file["rev"]
2990 change = int(file["change"])
2991 if change > newestChange:
2992 newestChange = change
2993
9bda3a85
SH
2994 self.labels[newestChange] = [output, revisions]
2995
2996 if self.verbose:
2997 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2998
06804c76
LD
2999 # Import p4 labels as git tags. A direct mapping does not
3000 # exist, so assume that if all the files are at the same revision
3001 # then we can use that, or it's something more complicated we should
3002 # just ignore.
3003 def importP4Labels(self, stream, p4Labels):
3004 if verbose:
3005 print "import p4 labels: " + ' '.join(p4Labels)
3006
3007 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 3008 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
3009 if len(validLabelRegexp) == 0:
3010 validLabelRegexp = defaultLabelRegexp
3011 m = re.compile(validLabelRegexp)
3012
3013 for name in p4Labels:
3014 commitFound = False
3015
3016 if not m.match(name):
3017 if verbose:
3018 print "label %s does not match regexp %s" % (name,validLabelRegexp)
3019 continue
3020
3021 if name in ignoredP4Labels:
3022 continue
3023
3024 labelDetails = p4CmdList(['label', "-o", name])[0]
3025
3026 # get the most recent changelist for each file in this label
3027 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3028 for p in self.depotPaths])
3029
3030 if change.has_key('change'):
3031 # find the corresponding git commit; take the oldest commit
3032 changelist = int(change['change'])
b43702ac
LD
3033 if changelist in self.committedChanges:
3034 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 3035 commitFound = True
b43702ac
LD
3036 else:
3037 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3038 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3039 if len(gitCommit) == 0:
3040 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3041 else:
3042 commitFound = True
3043 gitCommit = gitCommit.strip()
3044
3045 if commitFound:
06804c76
LD
3046 # Convert from p4 time format
3047 try:
3048 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3049 except ValueError:
a4e9054c 3050 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
3051 tmwhen = 1
3052
3053 when = int(time.mktime(tmwhen))
3054 self.streamTag(stream, name, labelDetails, gitCommit, when)
3055 if verbose:
3056 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3057 else:
3058 if verbose:
3059 print "Label %s has no changelists - possibly deleted?" % name
3060
3061 if not commitFound:
3062 # We can't import this label; don't try again as it will get very
3063 # expensive repeatedly fetching all the files for labels that will
3064 # never be imported. If the label is moved in the future, the
3065 # ignore will need to be removed manually.
3066 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3067
86dff6b6
HWN
3068 def guessProjectName(self):
3069 for p in self.depotPaths:
6e5295c4
SH
3070 if p.endswith("/"):
3071 p = p[:-1]
3072 p = p[p.strip().rfind("/") + 1:]
3073 if not p.endswith("/"):
3074 p += "/"
3075 return p
86dff6b6 3076
4b97ffb1 3077 def getBranchMapping(self):
6555b2cc
SH
3078 lostAndFoundBranches = set()
3079
8ace74c0
VA
3080 user = gitConfig("git-p4.branchUser")
3081 if len(user) > 0:
3082 command = "branches -u %s" % user
3083 else:
3084 command = "branches"
3085
3086 for info in p4CmdList(command):
52a4880b 3087 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
3088 viewIdx = 0
3089 while details.has_key("View%s" % viewIdx):
3090 paths = details["View%s" % viewIdx].split(" ")
3091 viewIdx = viewIdx + 1
3092 # require standard //depot/foo/... //depot/bar/... mapping
3093 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3094 continue
3095 source = paths[0]
3096 destination = paths[1]
6509e19c 3097 ## HACK
d53de8b9 3098 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3099 source = source[len(self.depotPaths[0]):-4]
3100 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3101
1a2edf4e
SH
3102 if destination in self.knownBranches:
3103 if not self.silent:
3104 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3105 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3106 continue
3107
6555b2cc
SH
3108 self.knownBranches[destination] = source
3109
3110 lostAndFoundBranches.discard(destination)
3111
29bdbac1 3112 if source not in self.knownBranches:
6555b2cc
SH
3113 lostAndFoundBranches.add(source)
3114
7199cf13
VA
3115 # Perforce does not strictly require branches to be defined, so we also
3116 # check git config for a branch list.
3117 #
3118 # Example of branch definition in git config file:
3119 # [git-p4]
3120 # branchList=main:branchA
3121 # branchList=main:branchB
3122 # branchList=branchA:branchC
3123 configBranches = gitConfigList("git-p4.branchList")
3124 for branch in configBranches:
3125 if branch:
3126 (source, destination) = branch.split(":")
3127 self.knownBranches[destination] = source
3128
3129 lostAndFoundBranches.discard(destination)
3130
3131 if source not in self.knownBranches:
3132 lostAndFoundBranches.add(source)
3133
6555b2cc
SH
3134
3135 for branch in lostAndFoundBranches:
3136 self.knownBranches[branch] = branch
29bdbac1 3137
38f9f5ec
SH
3138 def getBranchMappingFromGitBranches(self):
3139 branches = p4BranchesInGit(self.importIntoRemotes)
3140 for branch in branches.keys():
3141 if branch == "master":
3142 branch = "main"
3143 else:
3144 branch = branch[len(self.projectName):]
3145 self.knownBranches[branch] = branch
3146
bb6e09b2
HWN
3147 def updateOptionDict(self, d):
3148 option_keys = {}
3149 if self.keepRepoPath:
3150 option_keys['keepRepoPath'] = 1
3151
3152 d["options"] = ' '.join(sorted(option_keys.keys()))
3153
3154 def readOptions(self, d):
3155 self.keepRepoPath = (d.has_key('options')
3156 and ('keepRepoPath' in d['options']))
6326aa58 3157
8134f69c
SH
3158 def gitRefForBranch(self, branch):
3159 if branch == "main":
3160 return self.refPrefix + "master"
3161
3162 if len(branch) <= 0:
3163 return branch
3164
3165 return self.refPrefix + self.projectName + branch
3166
1ca3d710
SH
3167 def gitCommitByP4Change(self, ref, change):
3168 if self.verbose:
3169 print "looking in ref " + ref + " for change %s using bisect..." % change
3170
3171 earliestCommit = ""
3172 latestCommit = parseRevision(ref)
3173
3174 while True:
3175 if self.verbose:
3176 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3177 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3178 if len(next) == 0:
3179 if self.verbose:
3180 print "argh"
3181 return ""
3182 log = extractLogMessageFromGitCommit(next)
3183 settings = extractSettingsGitLog(log)
3184 currentChange = int(settings['change'])
3185 if self.verbose:
3186 print "current change %s" % currentChange
3187
3188 if currentChange == change:
3189 if self.verbose:
3190 print "found %s" % next
3191 return next
3192
3193 if currentChange < change:
3194 earliestCommit = "^%s" % next
3195 else:
3196 latestCommit = "%s" % next
3197
3198 return ""
3199
3200 def importNewBranch(self, branch, maxChange):
3201 # make fast-import flush all changes to disk and update the refs using the checkpoint
3202 # command so that we can try to find the branch parent in the git history
3203 self.gitStream.write("checkpoint\n\n");
3204 self.gitStream.flush();
3205 branchPrefix = self.depotPaths[0] + branch + "/"
3206 range = "@1,%s" % maxChange
3207 #print "prefix" + branchPrefix
96b2d54a 3208 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3209 if len(changes) <= 0:
3210 return False
3211 firstChange = changes[0]
3212 #print "first change in branch: %s" % firstChange
3213 sourceBranch = self.knownBranches[branch]
3214 sourceDepotPath = self.depotPaths[0] + sourceBranch
3215 sourceRef = self.gitRefForBranch(sourceBranch)
3216 #print "source " + sourceBranch
3217
52a4880b 3218 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3219 #print "branch parent: %s" % branchParentChange
3220 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3221 if len(gitParent) > 0:
3222 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3223 #print "parent git commit: %s" % gitParent
3224
3225 self.importChanges(changes)
3226 return True
3227
fed23693
VA
3228 def searchParent(self, parent, branch, target):
3229 parentFound = False
c7d34884
PW
3230 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3231 "--no-merges", parent]):
fed23693
VA
3232 blob = blob.strip()
3233 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3234 parentFound = True
3235 if self.verbose:
3236 print "Found parent of %s in commit %s" % (branch, blob)
3237 break
3238 if parentFound:
3239 return blob
3240 else:
3241 return None
3242
e87f37ae
SH
3243 def importChanges(self, changes):
3244 cnt = 1
3245 for change in changes:
18fa13d0 3246 description = p4_describe(change)
e87f37ae
SH
3247 self.updateOptionDict(description)
3248
3249 if not self.silent:
3250 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3251 sys.stdout.flush()
3252 cnt = cnt + 1
3253
3254 try:
3255 if self.detectBranches:
3256 branches = self.splitFilesIntoBranches(description)
3257 for branch in branches.keys():
3258 ## HACK --hwn
3259 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3260 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3261
3262 parent = ""
3263
3264 filesForCommit = branches[branch]
3265
3266 if self.verbose:
3267 print "branch is %s" % branch
3268
3269 self.updatedBranches.add(branch)
3270
3271 if branch not in self.createdBranches:
3272 self.createdBranches.add(branch)
3273 parent = self.knownBranches[branch]
3274 if parent == branch:
3275 parent = ""
1ca3d710
SH
3276 else:
3277 fullBranch = self.projectName + branch
3278 if fullBranch not in self.p4BranchesInGit:
3279 if not self.silent:
3280 print("\n Importing new branch %s" % fullBranch);
3281 if self.importNewBranch(branch, change - 1):
3282 parent = ""
3283 self.p4BranchesInGit.append(fullBranch)
3284 if not self.silent:
3285 print("\n Resuming with change %s" % change);
3286
3287 if self.verbose:
3288 print "parent determined through known branches: %s" % parent
e87f37ae 3289
8134f69c
SH
3290 branch = self.gitRefForBranch(branch)
3291 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3292
3293 if self.verbose:
3294 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3295
3296 if len(parent) == 0 and branch in self.initialParents:
3297 parent = self.initialParents[branch]
3298 del self.initialParents[branch]
3299
fed23693
VA
3300 blob = None
3301 if len(parent) > 0:
4f9273d2 3302 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3303 if self.verbose:
3304 print "Creating temporary branch: " + tempBranch
e63231e5 3305 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3306 self.tempBranches.append(tempBranch)
3307 self.checkpoint()
3308 blob = self.searchParent(parent, branch, tempBranch)
3309 if blob:
e63231e5 3310 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3311 else:
3312 if self.verbose:
3313 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3314 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3315 else:
3316 files = self.extractFilesFromCommit(description)
e63231e5 3317 self.commit(description, files, self.branch,
e87f37ae 3318 self.initialParent)
47497844 3319 # only needed once, to connect to the previous commit
e87f37ae
SH
3320 self.initialParent = ""
3321 except IOError:
3322 print self.gitError.read()
3323 sys.exit(1)
3324
b9d34db9
LD
3325 def sync_origin_only(self):
3326 if self.syncWithOrigin:
3327 self.hasOrigin = originP4BranchesExist()
3328 if self.hasOrigin:
3329 if not self.silent:
3330 print 'Syncing with origin first, using "git fetch origin"'
3331 system("git fetch origin")
3332
c208a243
SH
3333 def importHeadRevision(self, revision):
3334 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3335
4e2e6ce4
PW
3336 details = {}
3337 details["user"] = "git perforce import user"
1494fcbb 3338 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3339 % (' '.join(self.depotPaths), revision))
3340 details["change"] = revision
3341 newestRevision = 0
3342
3343 fileCnt = 0
6de040df
LD
3344 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3345
3346 for info in p4CmdList(["files"] + fileArgs):
c208a243 3347
68b28593 3348 if 'code' in info and info['code'] == 'error':
c208a243
SH
3349 sys.stderr.write("p4 returned an error: %s\n"
3350 % info['data'])
d88e707f
PW
3351 if info['data'].find("must refer to client") >= 0:
3352 sys.stderr.write("This particular p4 error is misleading.\n")
3353 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3354 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3355 sys.exit(1)
68b28593
PW
3356 if 'p4ExitCode' in info:
3357 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3358 sys.exit(1)
3359
3360
3361 change = int(info["change"])
3362 if change > newestRevision:
3363 newestRevision = change
3364
56c09345 3365 if info["action"] in self.delete_actions:
c208a243
SH
3366 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3367 #fileCnt = fileCnt + 1
3368 continue
3369
3370 for prop in ["depotFile", "rev", "action", "type" ]:
3371 details["%s%s" % (prop, fileCnt)] = info[prop]
3372
3373 fileCnt = fileCnt + 1
3374
3375 details["change"] = newestRevision
4e2e6ce4 3376
9dcb9f24 3377 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3378 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3379 res = p4_describe(newestRevision)
3380 details["time"] = res["time"]
4e2e6ce4 3381
c208a243
SH
3382 self.updateOptionDict(details)
3383 try:
e63231e5 3384 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3385 except IOError:
3386 print "IO error with git fast-import. Is your git version recent enough?"
3387 print self.gitError.read()
3388
3389
b984733c 3390 def run(self, args):
6326aa58 3391 self.depotPaths = []
179caebf 3392 self.changeRange = ""
6326aa58 3393 self.previousDepotPaths = []
991a2de4 3394 self.hasOrigin = False
ce6f33c8 3395
29bdbac1
SH
3396 # map from branch depot path to parent branch
3397 self.knownBranches = {}
3398 self.initialParents = {}
3399
a028a98e
SH
3400 if self.importIntoRemotes:
3401 self.refPrefix = "refs/remotes/p4/"
3402 else:
db775559 3403 self.refPrefix = "refs/heads/p4/"
a028a98e 3404
b9d34db9 3405 self.sync_origin_only()
10f880f8 3406
5a8e84cd 3407 branch_arg_given = bool(self.branch)
569d1bd4 3408 if len(self.branch) == 0:
db775559 3409 self.branch = self.refPrefix + "master"
a028a98e 3410 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3411 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3412 system("git branch -D p4")
967f72e2 3413
a93d33ee
PW
3414 # accept either the command-line option, or the configuration variable
3415 if self.useClientSpec:
3416 # will use this after clone to set the variable
3417 self.useClientSpec_from_options = True
3418 else:
0d609032 3419 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3420 self.useClientSpec = True
3421 if self.useClientSpec:
543987bd 3422 self.clientSpecDirs = getClientSpec()
3a70cdfa 3423
6a49f8e2
HWN
3424 # TODO: should always look at previous commits,
3425 # merge with previous imports, if possible.
3426 if args == []:
d414c74a 3427 if self.hasOrigin:
5ca44617 3428 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3429
3430 # branches holds mapping from branch name to sha1
3431 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3432
3433 # restrict to just this one, disabling detect-branches
3434 if branch_arg_given:
3435 short = self.branch.split("/")[-1]
3436 if short in branches:
3437 self.p4BranchesInGit = [ short ]
3438 else:
3439 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3440
3441 if len(self.p4BranchesInGit) > 1:
3442 if not self.silent:
3443 print "Importing from/into multiple branches"
3444 self.detectBranches = True
8c9e8b6e
PW
3445 for branch in branches.keys():
3446 self.initialParents[self.refPrefix + branch] = \
3447 branches[branch]
967f72e2 3448
29bdbac1
SH
3449 if self.verbose:
3450 print "branches: %s" % self.p4BranchesInGit
3451
3452 p4Change = 0
3453 for branch in self.p4BranchesInGit:
cebdf5af 3454 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3455
3456 settings = extractSettingsGitLog(logMsg)
29bdbac1 3457
bb6e09b2
HWN
3458 self.readOptions(settings)
3459 if (settings.has_key('depot-paths')
3460 and settings.has_key ('change')):
3461 change = int(settings['change']) + 1
29bdbac1
SH
3462 p4Change = max(p4Change, change)
3463
bb6e09b2
HWN
3464 depotPaths = sorted(settings['depot-paths'])
3465 if self.previousDepotPaths == []:
6326aa58 3466 self.previousDepotPaths = depotPaths
29bdbac1 3467 else:
6326aa58
HWN
3468 paths = []
3469 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3470 prev_list = prev.split("/")
3471 cur_list = cur.split("/")
3472 for i in range(0, min(len(cur_list), len(prev_list))):
3473