git-p4: git-p4 tests with p4 triggers
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
a235e85c
BC
30try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
4addad22 45verbose = False
86949eef 46
06804c76 47# Only labels/tags matching this will be imported/exported
c8942a22 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 49
1051ef00
LD
50# Grab changes in blocks of this many revisions, unless otherwise requested
51defaultBlockSize = 512
52
21a50753
AK
53def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
6de040df 60 real_cmd = ["p4"]
abcaf073
AK
61
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
6de040df 64 real_cmd += ["-u",user]
abcaf073
AK
65
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
6de040df 68 real_cmd += ["-P", password]
abcaf073
AK
69
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
6de040df 72 real_cmd += ["-p", port]
abcaf073
AK
73
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
41799aa2 76 real_cmd += ["-H", host]
abcaf073
AK
77
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
6de040df 80 real_cmd += ["-c", client]
abcaf073 81
89a6ecc5
LS
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
bc233524
IK
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
6de040df
LD
89
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
21a50753
AK
94 return real_cmd
95
378f7be1
LD
96def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
bbd84863
MF
106def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
053fd0c1 127
4d25dc44
LS
128def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
86dff6b6
HWN
138def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
6de040df 145def write_pipe(c, stdin):
4addad22 146 if verbose:
6de040df 147 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 148
6de040df
LD
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
b016d397
HWN
156
157 return val
158
6de040df 159def p4_write_pipe(c, stdin):
d9429194 160 real_cmd = p4_build_cmd(c)
6de040df 161 return write_pipe(real_cmd, stdin)
d9429194 162
78871bf4
LD
163def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
167 """
4addad22 168 if verbose:
6de040df 169 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 170
6de040df 171 expand = isinstance(c,basestring)
1f5f3907
LS
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
78871bf4
LD
174 return (p.returncode, out, err)
175
176def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
180 """
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 187 return out
b016d397 188
78871bf4
LD
189def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
192 """
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
198
d9429194
AK
199def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
b016d397 202
bce4c5fc 203def read_pipe_lines(c):
4addad22 204 if verbose:
6de040df
LD
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
b016d397 210 val = pipe.readlines()
6de040df
LD
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
b016d397
HWN
213
214 return val
caace111 215
2318121b
AK
216def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
155af834 218 real_cmd = p4_build_cmd(c)
2318121b
AK
219 return read_pipe_lines(real_cmd)
220
8e9497c2
GG
221def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
229
249da4c0
PW
230def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
235
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
248
cbff4b25 249def system(cmd, ignore_error=False):
6de040df 250 expand = isinstance(cmd,basestring)
4addad22 251 if verbose:
6de040df 252 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 253 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 254 if retcode and not ignore_error:
a235e85c 255 raise CalledProcessError(retcode, cmd)
6754a299 256
cbff4b25
LD
257 return retcode
258
bf9320f1
AK
259def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
155af834 261 real_cmd = p4_build_cmd(cmd)
6de040df 262 expand = isinstance(real_cmd, basestring)
a235e85c
BC
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
6de040df 266
7f0e5962
PW
267_p4_version_string = None
268def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
271
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276 """
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
282
6de040df 283def p4_integrate(src, dest):
9d7d446a 284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 285
8d7ec362 286def p4_sync(f, *options):
9d7d446a 287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
288
289def p4_add(f):
9d7d446a
PW
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
6de040df
LD
295
296def p4_delete(f):
9d7d446a 297 p4_system(["delete", wildcard_encode(f)])
6de040df 298
a02b8bc4
RP
299def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
301
302def p4_revert(f):
9d7d446a 303 p4_system(["revert", wildcard_encode(f)])
6de040df 304
9d7d446a
PW
305def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 307
46c609e9
LD
308def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
311
8e9497c2
GG
312def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
1051ef00
LD
315def p4_last_change():
316 results = p4CmdList(["changes", "-m", "1"])
317 return int(results[0]['change'])
318
18fa13d0
PW
319def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
323
324 ds = p4CmdList(["describe", "-s", str(change)])
325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328 d = ds[0]
329
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340 return d
341
9cffb8c8
PW
342#
343# Canonicalize the p4 type and return a tuple of the
344# base type, plus any modifiers. See "p4 help filetypes"
345# for a list and explanation.
346#
347def split_p4_type(p4type):
348
349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
366 }
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
b9fc6ea9 376
60df071c
LD
377#
378# return the raw p4 type of a file (text, text+ko, etc)
379#
79467e61
PW
380def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
382 return results[0]['headType']
383
384#
385# Given a type base and modifier, return a regexp matching
386# the keywords that can be expanded in the file
387#
388def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
6b2bf41e 400 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
406
407#
408# Given a file, return a regexp matching the possible
409# RCS keywords that will be expanded, or None for files
410# with kw expansion turned off.
411#
412def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 418
c65b670e
CP
419def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
422
423 p4Type = "+x"
424
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
431
6de040df 432 p4_reopen(p4Type, file)
c65b670e
CP
433
434def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
436
9d7d446a 437 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
439 if match:
440 return match.group(1)
441 else:
f3e5ae4f 442 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 443
06804c76
LD
444# Return the set of all p4 labels
445def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
449
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
453
454 return labels
455
456# Return the set of all git tags
457def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
463
b43b0a3c
CP
464def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
470
471def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
473
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
476
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
487
488 If the pattern is not matched, None is returned."""
489
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
501 }
502 return None
503
c65b670e
CP
504def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
508
509def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
511
b932705b 512def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
513
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
520
521 cmd = p4_build_cmd(cmd)
6a49f8e2 522 if verbose:
6de040df 523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
524
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
9f90c733
SL
536 stdin_file.flush()
537 stdin_file.seek(0)
538
6de040df
LD
539 p4 = subprocess.Popen(cmd,
540 shell=expand,
9f90c733
SL
541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
86949eef
SH
543
544 result = []
545 try:
546 while True:
9f90c733 547 entry = marshal.load(p4.stdout)
c3f6163b
AG
548 if cb is not None:
549 cb(entry)
550 else:
551 result.append(entry)
86949eef
SH
552 except EOFError:
553 pass
9f90c733
SL
554 exitCode = p4.wait()
555 if exitCode != 0:
ac3e0d79
SH
556 entry = {}
557 entry["p4ExitCode"] = exitCode
558 result.append(entry)
86949eef
SH
559
560 return result
561
562def p4Cmd(cmd):
563 list = p4CmdList(cmd)
564 result = {}
565 for entry in list:
566 result.update(entry)
567 return result;
568
cb2c9db5
SH
569def p4Where(depotPath):
570 if not depotPath.endswith("/"):
571 depotPath += "/"
cd884106
VA
572 depotPathLong = depotPath + "..."
573 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
574 output = None
575 for entry in outputList:
75bc9573 576 if "depotFile" in entry:
cd884106
VA
577 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
578 # The base path always ends with "/...".
579 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
580 output = entry
581 break
582 elif "data" in entry:
583 data = entry.get("data")
584 space = data.find(" ")
585 if data[:space] == depotPath:
586 output = entry
587 break
7f705dc3
TAL
588 if output == None:
589 return ""
dc524036
SH
590 if output["code"] == "error":
591 return ""
cb2c9db5
SH
592 clientPath = ""
593 if "path" in output:
594 clientPath = output.get("path")
595 elif "data" in output:
596 data = output.get("data")
597 lastSpace = data.rfind(" ")
598 clientPath = data[lastSpace + 1:]
599
600 if clientPath.endswith("..."):
601 clientPath = clientPath[:-3]
602 return clientPath
603
86949eef 604def currentGitBranch():
eff45110 605 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 606
4f5cf76a 607def isValidGitDir(path):
378f7be1 608 return git_dir(path) != None
4f5cf76a 609
463e8af6 610def parseRevision(ref):
b25b2065 611 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 612
28755dba
PW
613def branchExists(ref):
614 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
615 ignore_error=True)
616 return len(rev) > 0
617
6ae8de88
SH
618def extractLogMessageFromGitCommit(commit):
619 logMessage = ""
b016d397
HWN
620
621 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 622 foundTitle = False
b016d397 623 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
624 if not foundTitle:
625 if len(log) == 1:
1c094184 626 foundTitle = True
6ae8de88
SH
627 continue
628
629 logMessage += log
630 return logMessage
631
bb6e09b2 632def extractSettingsGitLog(log):
6ae8de88
SH
633 values = {}
634 for line in log.split("\n"):
635 line = line.strip()
6326aa58
HWN
636 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
637 if not m:
638 continue
639
640 assignments = m.group(1).split (':')
641 for a in assignments:
642 vals = a.split ('=')
643 key = vals[0].strip()
644 val = ('='.join (vals[1:])).strip()
645 if val.endswith ('\"') and val.startswith('"'):
646 val = val[1:-1]
647
648 values[key] = val
649
845b42cb
SH
650 paths = values.get("depot-paths")
651 if not paths:
652 paths = values.get("depot-path")
a3fdd579
SH
653 if paths:
654 values['depot-paths'] = paths.split(',')
bb6e09b2 655 return values
6ae8de88 656
8136a639 657def gitBranchExists(branch):
bb6e09b2
HWN
658 proc = subprocess.Popen(["git", "rev-parse", branch],
659 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 660 return proc.wait() == 0;
8136a639 661
36bd8446 662_gitConfig = {}
b345d6c3 663
692e1796 664def gitConfig(key, typeSpecifier=None):
36bd8446 665 if not _gitConfig.has_key(key):
692e1796
LS
666 cmd = [ "git", "config" ]
667 if typeSpecifier:
668 cmd += [ typeSpecifier ]
669 cmd += [ key ]
b345d6c3
PW
670 s = read_pipe(cmd, ignore_error=True)
671 _gitConfig[key] = s.strip()
36bd8446 672 return _gitConfig[key]
01265103 673
0d609032
PW
674def gitConfigBool(key):
675 """Return a bool, using git config --bool. It is True only if the
676 variable is set to true, and False if set to false or not present
677 in the config."""
678
36bd8446 679 if not _gitConfig.has_key(key):
692e1796 680 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 681 return _gitConfig[key]
01265103 682
cb1dafdf
LS
683def gitConfigInt(key):
684 if not _gitConfig.has_key(key):
685 cmd = [ "git", "config", "--int", key ]
0d609032
PW
686 s = read_pipe(cmd, ignore_error=True)
687 v = s.strip()
cb1dafdf
LS
688 try:
689 _gitConfig[key] = int(gitConfig(key, '--int'))
690 except ValueError:
691 _gitConfig[key] = None
36bd8446 692 return _gitConfig[key]
01265103 693
7199cf13
VA
694def gitConfigList(key):
695 if not _gitConfig.has_key(key):
2abba301 696 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 697 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
698 if _gitConfig[key] == ['']:
699 _gitConfig[key] = []
7199cf13
VA
700 return _gitConfig[key]
701
2c8037ed
PW
702def p4BranchesInGit(branchesAreInRemotes=True):
703 """Find all the branches whose names start with "p4/", looking
704 in remotes or heads as specified by the argument. Return
705 a dictionary of { branch: revision } for each one found.
706 The branch names are the short names, without any
707 "p4/" prefix."""
708
062410bb
SH
709 branches = {}
710
711 cmdline = "git rev-parse --symbolic "
712 if branchesAreInRemotes:
2c8037ed 713 cmdline += "--remotes"
062410bb 714 else:
2c8037ed 715 cmdline += "--branches"
062410bb
SH
716
717 for line in read_pipe_lines(cmdline):
718 line = line.strip()
719
2c8037ed
PW
720 # only import to p4/
721 if not line.startswith('p4/'):
722 continue
723 # special symbolic ref to p4/master
724 if line == "p4/HEAD":
062410bb 725 continue
062410bb 726
2c8037ed
PW
727 # strip off p4/ prefix
728 branch = line[len("p4/"):]
062410bb
SH
729
730 branches[branch] = parseRevision(line)
2c8037ed 731
062410bb
SH
732 return branches
733
5a8e84cd
PW
734def branch_exists(branch):
735 """Make sure that the given ref name really exists."""
736
737 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
738 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
739 out, _ = p.communicate()
740 if p.returncode:
741 return False
742 # expect exactly one line of output: the branch name
743 return out.rstrip() == branch
744
9ceab363 745def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
746 branches = p4BranchesInGit()
747 # map from depot-path to branch name
748 branchByDepotPath = {}
749 for branch in branches.keys():
750 tip = branches[branch]
751 log = extractLogMessageFromGitCommit(tip)
752 settings = extractSettingsGitLog(log)
753 if settings.has_key("depot-paths"):
754 paths = ",".join(settings["depot-paths"])
755 branchByDepotPath[paths] = "remotes/p4/" + branch
756
27d2d811 757 settings = None
27d2d811
SH
758 parent = 0
759 while parent < 65535:
9ceab363 760 commit = head + "~%s" % parent
27d2d811
SH
761 log = extractLogMessageFromGitCommit(commit)
762 settings = extractSettingsGitLog(log)
86506fe5
SH
763 if settings.has_key("depot-paths"):
764 paths = ",".join(settings["depot-paths"])
765 if branchByDepotPath.has_key(paths):
766 return [branchByDepotPath[paths], settings]
27d2d811 767
86506fe5 768 parent = parent + 1
27d2d811 769
86506fe5 770 return ["", settings]
27d2d811 771
5ca44617
SH
772def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
773 if not silent:
774 print ("Creating/updating branch(es) in %s based on origin branch(es)"
775 % localRefPrefix)
776
777 originPrefix = "origin/p4/"
778
779 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
780 line = line.strip()
781 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
782 continue
783
784 headName = line[len(originPrefix):]
785 remoteHead = localRefPrefix + headName
786 originHead = line
787
788 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
789 if (not original.has_key('depot-paths')
790 or not original.has_key('change')):
791 continue
792
793 update = False
794 if not gitBranchExists(remoteHead):
795 if verbose:
796 print "creating %s" % remoteHead
797 update = True
798 else:
799 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
800 if settings.has_key('change') > 0:
801 if settings['depot-paths'] == original['depot-paths']:
802 originP4Change = int(original['change'])
803 p4Change = int(settings['change'])
804 if originP4Change > p4Change:
805 print ("%s (%s) is newer than %s (%s). "
806 "Updating p4 branch from origin."
807 % (originHead, originP4Change,
808 remoteHead, p4Change))
809 update = True
810 else:
811 print ("Ignoring: %s was imported from %s while "
812 "%s was imported from %s"
813 % (originHead, ','.join(original['depot-paths']),
814 remoteHead, ','.join(settings['depot-paths'])))
815
816 if update:
817 system("git update-ref %s %s" % (remoteHead, originHead))
818
819def originP4BranchesExist():
820 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
821
1051ef00
LD
822
823def p4ParseNumericChangeRange(parts):
824 changeStart = int(parts[0][1:])
825 if parts[1] == '#head':
826 changeEnd = p4_last_change()
827 else:
828 changeEnd = int(parts[1])
829
830 return (changeStart, changeEnd)
831
832def chooseBlockSize(blockSize):
833 if blockSize:
834 return blockSize
835 else:
836 return defaultBlockSize
837
838def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 839 assert depotPaths
96b2d54a 840
1051ef00
LD
841 # Parse the change range into start and end. Try to find integer
842 # revision ranges as these can be broken up into blocks to avoid
843 # hitting server-side limits (maxrows, maxscanresults). But if
844 # that doesn't work, fall back to using the raw revision specifier
845 # strings, without using block mode.
846
96b2d54a 847 if changeRange is None or changeRange == '':
1051ef00
LD
848 changeStart = 1
849 changeEnd = p4_last_change()
850 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
851 else:
852 parts = changeRange.split(',')
853 assert len(parts) == 2
1051ef00
LD
854 try:
855 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
856 block_size = chooseBlockSize(requestedBlockSize)
857 except:
858 changeStart = parts[0][1:]
859 changeEnd = parts[1]
860 if requestedBlockSize:
861 die("cannot use --changes-block-size with non-numeric revisions")
862 block_size = None
4f6432d8 863
9943e5b9 864 changes = set()
96b2d54a 865
1f90a648
SH
866 # Retrieve changes a block at a time, to prevent running
867 # into a MaxResults/MaxScanRows error from the server.
1051ef00 868
1f90a648
SH
869 while True:
870 cmd = ['changes']
1051ef00 871
1f90a648
SH
872 if block_size:
873 end = min(changeEnd, changeStart + block_size)
874 revisionRange = "%d,%d" % (changeStart, end)
875 else:
876 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 877
1f90a648 878 for p in depotPaths:
1051ef00
LD
879 cmd += ["%s...@%s" % (p, revisionRange)]
880
1f90a648
SH
881 # Insert changes in chronological order
882 for line in reversed(p4_read_pipe_lines(cmd)):
9943e5b9 883 changes.add(int(line.split(" ")[1]))
1051ef00 884
1f90a648
SH
885 if not block_size:
886 break
1051ef00 887
1f90a648
SH
888 if end >= changeEnd:
889 break
1051ef00 890
1f90a648 891 changeStart = end + 1
4f6432d8 892
1f90a648
SH
893 changes = sorted(changes)
894 return changes
4f6432d8 895
d53de8b9
TAL
896def p4PathStartsWith(path, prefix):
897 # This method tries to remedy a potential mixed-case issue:
898 #
899 # If UserA adds //depot/DirA/file1
900 # and UserB adds //depot/dira/file2
901 #
902 # we may or may not have a problem. If you have core.ignorecase=true,
903 # we treat DirA and dira as the same directory
0d609032 904 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
905 return path.lower().startswith(prefix.lower())
906 return path.startswith(prefix)
907
543987bd
PW
908def getClientSpec():
909 """Look at the p4 client spec, create a View() object that contains
910 all the mappings, and return it."""
911
912 specList = p4CmdList("client -o")
913 if len(specList) != 1:
914 die('Output from "client -o" is %d lines, expecting 1' %
915 len(specList))
916
917 # dictionary of all client parameters
918 entry = specList[0]
919
9d57c4a6
KS
920 # the //client/ name
921 client_name = entry["Client"]
922
543987bd
PW
923 # just the keys that start with "View"
924 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
925
926 # hold this new View
9d57c4a6 927 view = View(client_name)
543987bd
PW
928
929 # append the lines, in order, to the view
930 for view_num in range(len(view_keys)):
931 k = "View%d" % view_num
932 if k not in view_keys:
933 die("Expected view key %s missing" % k)
934 view.append(entry[k])
935
936 return view
937
938def getClientRoot():
939 """Grab the client directory."""
940
941 output = p4CmdList("client -o")
942 if len(output) != 1:
943 die('Output from "client -o" is %d lines, expecting 1' % len(output))
944
945 entry = output[0]
946 if "Root" not in entry:
947 die('Client has no "Root"')
948
949 return entry["Root"]
950
9d7d446a
PW
951#
952# P4 wildcards are not allowed in filenames. P4 complains
953# if you simply add them, but you can force it with "-f", in
954# which case it translates them into %xx encoding internally.
955#
956def wildcard_decode(path):
957 # Search for and fix just these four characters. Do % last so
958 # that fixing it does not inadvertently create new %-escapes.
959 # Cannot have * in a filename in windows; untested as to
960 # what p4 would do in such a case.
961 if not platform.system() == "Windows":
962 path = path.replace("%2A", "*")
963 path = path.replace("%23", "#") \
964 .replace("%40", "@") \
965 .replace("%25", "%")
966 return path
967
968def wildcard_encode(path):
969 # do % first to avoid double-encoding the %s introduced here
970 path = path.replace("%", "%25") \
971 .replace("*", "%2A") \
972 .replace("#", "%23") \
973 .replace("@", "%40")
974 return path
975
976def wildcard_present(path):
598354c0
BC
977 m = re.search("[*#@%]", path)
978 return m is not None
9d7d446a 979
a5db4b12
LS
980class LargeFileSystem(object):
981 """Base class for large file system support."""
982
983 def __init__(self, writeToGitStream):
984 self.largeFiles = set()
985 self.writeToGitStream = writeToGitStream
986
987 def generatePointer(self, cloneDestination, contentFile):
988 """Return the content of a pointer file that is stored in Git instead of
989 the actual content."""
990 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
991
992 def pushFile(self, localLargeFile):
993 """Push the actual content which is not stored in the Git repository to
994 a server."""
995 assert False, "Method 'pushFile' required in " + self.__class__.__name__
996
997 def hasLargeFileExtension(self, relPath):
998 return reduce(
999 lambda a, b: a or b,
1000 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1001 False
1002 )
1003
1004 def generateTempFile(self, contents):
1005 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1006 for d in contents:
1007 contentFile.write(d)
1008 contentFile.close()
1009 return contentFile.name
1010
1011 def exceedsLargeFileThreshold(self, relPath, contents):
1012 if gitConfigInt('git-p4.largeFileThreshold'):
1013 contentsSize = sum(len(d) for d in contents)
1014 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1015 return True
1016 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1017 contentsSize = sum(len(d) for d in contents)
1018 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1019 return False
1020 contentTempFile = self.generateTempFile(contents)
1021 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1022 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1023 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1024 zf.close()
1025 compressedContentsSize = zf.infolist()[0].compress_size
1026 os.remove(contentTempFile)
1027 os.remove(compressedContentFile.name)
1028 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1029 return True
1030 return False
1031
1032 def addLargeFile(self, relPath):
1033 self.largeFiles.add(relPath)
1034
1035 def removeLargeFile(self, relPath):
1036 self.largeFiles.remove(relPath)
1037
1038 def isLargeFile(self, relPath):
1039 return relPath in self.largeFiles
1040
1041 def processContent(self, git_mode, relPath, contents):
1042 """Processes the content of git fast import. This method decides if a
1043 file is stored in the large file system and handles all necessary
1044 steps."""
1045 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1046 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1047 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1048 if pointer_git_mode:
1049 git_mode = pointer_git_mode
1050 if localLargeFile:
1051 # Move temp file to final location in large file system
1052 largeFileDir = os.path.dirname(localLargeFile)
1053 if not os.path.isdir(largeFileDir):
1054 os.makedirs(largeFileDir)
1055 shutil.move(contentTempFile, localLargeFile)
1056 self.addLargeFile(relPath)
1057 if gitConfigBool('git-p4.largeFilePush'):
1058 self.pushFile(localLargeFile)
1059 if verbose:
1060 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1061 return (git_mode, contents)
1062
1063class MockLFS(LargeFileSystem):
1064 """Mock large file system for testing."""
1065
1066 def generatePointer(self, contentFile):
1067 """The pointer content is the original content prefixed with "pointer-".
1068 The local filename of the large file storage is derived from the file content.
1069 """
1070 with open(contentFile, 'r') as f:
1071 content = next(f)
1072 gitMode = '100644'
1073 pointerContents = 'pointer-' + content
1074 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1075 return (gitMode, pointerContents, localLargeFile)
1076
1077 def pushFile(self, localLargeFile):
1078 """The remote filename of the large file storage is the same as the local
1079 one but in a different directory.
1080 """
1081 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1082 if not os.path.exists(remotePath):
1083 os.makedirs(remotePath)
1084 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1085
b47d807d
LS
1086class GitLFS(LargeFileSystem):
1087 """Git LFS as backend for the git-p4 large file system.
1088 See https://git-lfs.github.com/ for details."""
1089
1090 def __init__(self, *args):
1091 LargeFileSystem.__init__(self, *args)
1092 self.baseGitAttributes = []
1093
1094 def generatePointer(self, contentFile):
1095 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1096 mode and content which is stored in the Git repository instead of
1097 the actual content. Return also the new location of the actual
1098 content.
1099 """
d5eb3cf5
LS
1100 if os.path.getsize(contentFile) == 0:
1101 return (None, '', None)
1102
b47d807d
LS
1103 pointerProcess = subprocess.Popen(
1104 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1105 stdout=subprocess.PIPE
1106 )
1107 pointerFile = pointerProcess.stdout.read()
1108 if pointerProcess.wait():
1109 os.remove(contentFile)
1110 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1111
1112 # Git LFS removed the preamble in the output of the 'pointer' command
1113 # starting from version 1.2.0. Check for the preamble here to support
1114 # earlier versions.
1115 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1116 if pointerFile.startswith('Git LFS pointer for'):
1117 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1118
1119 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1120 localLargeFile = os.path.join(
1121 os.getcwd(),
1122 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1123 oid,
1124 )
1125 # LFS Spec states that pointer files should not have the executable bit set.
1126 gitMode = '100644'
82f2567e 1127 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1128
1129 def pushFile(self, localLargeFile):
1130 uploadProcess = subprocess.Popen(
1131 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1132 )
1133 if uploadProcess.wait():
1134 die('git-lfs push command failed. Did you define a remote?')
1135
1136 def generateGitAttributes(self):
1137 return (
1138 self.baseGitAttributes +
1139 [
1140 '\n',
1141 '#\n',
1142 '# Git LFS (see https://git-lfs.github.com/)\n',
1143 '#\n',
1144 ] +
862f9312 1145 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1146 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1147 ] +
862f9312 1148 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1149 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1150 ]
1151 )
1152
1153 def addLargeFile(self, relPath):
1154 LargeFileSystem.addLargeFile(self, relPath)
1155 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1156
1157 def removeLargeFile(self, relPath):
1158 LargeFileSystem.removeLargeFile(self, relPath)
1159 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1160
1161 def processContent(self, git_mode, relPath, contents):
1162 if relPath == '.gitattributes':
1163 self.baseGitAttributes = contents
1164 return (git_mode, self.generateGitAttributes())
1165 else:
1166 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1167
b984733c
SH
1168class Command:
1169 def __init__(self):
1170 self.usage = "usage: %prog [options]"
8910ac0e 1171 self.needsGit = True
6a10b6aa 1172 self.verbose = False
b984733c 1173
3ea2cfd4
LD
1174class P4UserMap:
1175 def __init__(self):
1176 self.userMapFromPerforceServer = False
affb474f
LD
1177 self.myP4UserId = None
1178
1179 def p4UserId(self):
1180 if self.myP4UserId:
1181 return self.myP4UserId
1182
1183 results = p4CmdList("user -o")
1184 for r in results:
1185 if r.has_key('User'):
1186 self.myP4UserId = r['User']
1187 return r['User']
1188 die("Could not find your p4 user id")
1189
1190 def p4UserIsMe(self, p4User):
1191 # return True if the given p4 user is actually me
1192 me = self.p4UserId()
1193 if not p4User or p4User != me:
1194 return False
1195 else:
1196 return True
3ea2cfd4
LD
1197
1198 def getUserCacheFilename(self):
1199 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1200 return home + "/.gitp4-usercache.txt"
1201
1202 def getUserMapFromPerforceServer(self):
1203 if self.userMapFromPerforceServer:
1204 return
1205 self.users = {}
1206 self.emails = {}
1207
1208 for output in p4CmdList("users"):
1209 if not output.has_key("User"):
1210 continue
1211 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1212 self.emails[output["Email"]] = output["User"]
1213
10d08a14
LS
1214 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1215 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1216 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1217 if mapUser and len(mapUser[0]) == 3:
1218 user = mapUser[0][0]
1219 fullname = mapUser[0][1]
1220 email = mapUser[0][2]
1221 self.users[user] = fullname + " <" + email + ">"
1222 self.emails[email] = user
3ea2cfd4
LD
1223
1224 s = ''
1225 for (key, val) in self.users.items():
1226 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1227
1228 open(self.getUserCacheFilename(), "wb").write(s)
1229 self.userMapFromPerforceServer = True
1230
1231 def loadUserMapFromCache(self):
1232 self.users = {}
1233 self.userMapFromPerforceServer = False
1234 try:
1235 cache = open(self.getUserCacheFilename(), "rb")
1236 lines = cache.readlines()
1237 cache.close()
1238 for line in lines:
1239 entry = line.strip().split("\t")
1240 self.users[entry[0]] = entry[1]
1241 except IOError:
1242 self.getUserMapFromPerforceServer()
1243
b984733c 1244class P4Debug(Command):
86949eef 1245 def __init__(self):
6ae8de88 1246 Command.__init__(self)
6a10b6aa 1247 self.options = []
c8c39116 1248 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1249 self.needsGit = False
86949eef
SH
1250
1251 def run(self, args):
b1ce9447 1252 j = 0
6de040df 1253 for output in p4CmdList(args):
b1ce9447
HWN
1254 print 'Element: %d' % j
1255 j += 1
86949eef 1256 print output
b984733c 1257 return True
86949eef 1258
5834684d
SH
1259class P4RollBack(Command):
1260 def __init__(self):
1261 Command.__init__(self)
1262 self.options = [
0c66a783 1263 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1264 ]
1265 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1266 self.rollbackLocalBranches = False
5834684d
SH
1267
1268 def run(self, args):
1269 if len(args) != 1:
1270 return False
1271 maxChange = int(args[0])
0c66a783 1272
ad192f28 1273 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1274 die("Problems executing p4");
1275
0c66a783
SH
1276 if self.rollbackLocalBranches:
1277 refPrefix = "refs/heads/"
b016d397 1278 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1279 else:
1280 refPrefix = "refs/remotes/"
b016d397 1281 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1282
1283 for line in lines:
1284 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1285 line = line.strip()
1286 ref = refPrefix + line
5834684d 1287 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1288 settings = extractSettingsGitLog(log)
1289
1290 depotPaths = settings['depot-paths']
1291 change = settings['change']
1292
5834684d 1293 changed = False
52102d47 1294
6326aa58
HWN
1295 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1296 for p in depotPaths]))) == 0:
52102d47
SH
1297 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1298 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1299 continue
1300
bb6e09b2 1301 while change and int(change) > maxChange:
5834684d 1302 changed = True
52102d47
SH
1303 if self.verbose:
1304 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1305 system("git update-ref %s \"%s^\"" % (ref, ref))
1306 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1307 settings = extractSettingsGitLog(log)
1308
1309
1310 depotPaths = settings['depot-paths']
1311 change = settings['change']
5834684d
SH
1312
1313 if changed:
52102d47 1314 print "%s rewound to %s" % (ref, change)
5834684d
SH
1315
1316 return True
1317
3ea2cfd4 1318class P4Submit(Command, P4UserMap):
6bbfd137
PW
1319
1320 conflict_behavior_choices = ("ask", "skip", "quit")
1321
4f5cf76a 1322 def __init__(self):
b984733c 1323 Command.__init__(self)
3ea2cfd4 1324 P4UserMap.__init__(self)
4f5cf76a 1325 self.options = [
4f5cf76a 1326 optparse.make_option("--origin", dest="origin"),
ae901090 1327 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1328 # preserve the user, requires relevant p4 permissions
1329 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1330 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1331 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1332 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1333 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1334 choices=self.conflict_behavior_choices),
1335 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1336 optparse.make_option("--shelve", dest="shelve", action="store_true",
1337 help="Shelve instead of submit. Shelved files are reverted, "
1338 "restoring the workspace to the state before the shelve"),
46c609e9
LD
1339 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1340 metavar="CHANGELIST",
1341 help="update an existing shelved changelist, implies --shelve")
4f5cf76a
SH
1342 ]
1343 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1344 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1345 self.origin = ""
ae901090 1346 self.detectRenames = False
0d609032 1347 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1348 self.dry_run = False
b34fa577 1349 self.shelve = False
46c609e9 1350 self.update_shelve = None
728b7ad8 1351 self.prepare_p4_only = False
6bbfd137 1352 self.conflict_behavior = None
f7baba8b 1353 self.isWindows = (platform.system() == "Windows")
06804c76 1354 self.exportLabels = False
249da4c0 1355 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1356 self.branch = None
4f5cf76a 1357
a5db4b12
LS
1358 if gitConfig('git-p4.largeFileSystem'):
1359 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1360
4f5cf76a
SH
1361 def check(self):
1362 if len(p4CmdList("opened ...")) > 0:
1363 die("You have files opened with perforce! Close them before starting the sync.")
1364
f19cb0a0
PW
1365 def separate_jobs_from_description(self, message):
1366 """Extract and return a possible Jobs field in the commit
1367 message. It goes into a separate section in the p4 change
1368 specification.
1369
1370 A jobs line starts with "Jobs:" and looks like a new field
1371 in a form. Values are white-space separated on the same
1372 line or on following lines that start with a tab.
1373
1374 This does not parse and extract the full git commit message
1375 like a p4 form. It just sees the Jobs: line as a marker
1376 to pass everything from then on directly into the p4 form,
1377 but outside the description section.
1378
1379 Return a tuple (stripped log message, jobs string)."""
1380
1381 m = re.search(r'^Jobs:', message, re.MULTILINE)
1382 if m is None:
1383 return (message, None)
1384
1385 jobtext = message[m.start():]
1386 stripped_message = message[:m.start()].rstrip()
1387 return (stripped_message, jobtext)
1388
1389 def prepareLogMessage(self, template, message, jobs):
1390 """Edits the template returned from "p4 change -o" to insert
1391 the message in the Description field, and the jobs text in
1392 the Jobs field."""
4f5cf76a
SH
1393 result = ""
1394
edae1e2f
SH
1395 inDescriptionSection = False
1396
4f5cf76a
SH
1397 for line in template.split("\n"):
1398 if line.startswith("#"):
1399 result += line + "\n"
1400 continue
1401
edae1e2f 1402 if inDescriptionSection:
c9dbab04 1403 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1404 inDescriptionSection = False
f19cb0a0
PW
1405 # insert Jobs section
1406 if jobs:
1407 result += jobs + "\n"
edae1e2f
SH
1408 else:
1409 continue
1410 else:
1411 if line.startswith("Description:"):
1412 inDescriptionSection = True
1413 line += "\n"
1414 for messageLine in message.split("\n"):
1415 line += "\t" + messageLine + "\n"
1416
1417 result += line + "\n"
4f5cf76a
SH
1418
1419 return result
1420
60df071c
LD
1421 def patchRCSKeywords(self, file, pattern):
1422 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1423 (handle, outFileName) = tempfile.mkstemp(dir='.')
1424 try:
1425 outFile = os.fdopen(handle, "w+")
1426 inFile = open(file, "r")
1427 regexp = re.compile(pattern, re.VERBOSE)
1428 for line in inFile.readlines():
1429 line = regexp.sub(r'$\1$', line)
1430 outFile.write(line)
1431 inFile.close()
1432 outFile.close()
1433 # Forcibly overwrite the original file
1434 os.unlink(file)
1435 shutil.move(outFileName, file)
1436 except:
1437 # cleanup our temporary file
1438 os.unlink(outFileName)
1439 print "Failed to strip RCS keywords in %s" % file
1440 raise
1441
1442 print "Patched up RCS keywords in %s" % file
1443
3ea2cfd4
LD
1444 def p4UserForCommit(self,id):
1445 # Return the tuple (perforce user,git email) for a given git commit id
1446 self.getUserMapFromPerforceServer()
9bf28855
PW
1447 gitEmail = read_pipe(["git", "log", "--max-count=1",
1448 "--format=%ae", id])
3ea2cfd4
LD
1449 gitEmail = gitEmail.strip()
1450 if not self.emails.has_key(gitEmail):
1451 return (None,gitEmail)
1452 else:
1453 return (self.emails[gitEmail],gitEmail)
1454
1455 def checkValidP4Users(self,commits):
1456 # check if any git authors cannot be mapped to p4 users
1457 for id in commits:
1458 (user,email) = self.p4UserForCommit(id)
1459 if not user:
1460 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1461 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1462 print "%s" % msg
1463 else:
1464 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1465
1466 def lastP4Changelist(self):
1467 # Get back the last changelist number submitted in this client spec. This
1468 # then gets used to patch up the username in the change. If the same
1469 # client spec is being used by multiple processes then this might go
1470 # wrong.
1471 results = p4CmdList("client -o") # find the current client
1472 client = None
1473 for r in results:
1474 if r.has_key('Client'):
1475 client = r['Client']
1476 break
1477 if not client:
1478 die("could not get client spec")
6de040df 1479 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1480 for r in results:
1481 if r.has_key('change'):
1482 return r['change']
1483 die("Could not get changelist number for last submit - cannot patch up user details")
1484
1485 def modifyChangelistUser(self, changelist, newUser):
1486 # fixup the user field of a changelist after it has been submitted.
1487 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1488 if len(changes) != 1:
1489 die("Bad output from p4 change modifying %s to user %s" %
1490 (changelist, newUser))
1491
1492 c = changes[0]
1493 if c['User'] == newUser: return # nothing to do
1494 c['User'] = newUser
1495 input = marshal.dumps(c)
1496
3ea2cfd4
LD
1497 result = p4CmdList("change -f -i", stdin=input)
1498 for r in result:
1499 if r.has_key('code'):
1500 if r['code'] == 'error':
1501 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1502 if r.has_key('data'):
1503 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1504 return
1505 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1506
1507 def canChangeChangelists(self):
1508 # check to see if we have p4 admin or super-user permissions, either of
1509 # which are required to modify changelists.
52a4880b 1510 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1511 for r in results:
1512 if r.has_key('perm'):
1513 if r['perm'] == 'admin':
1514 return 1
1515 if r['perm'] == 'super':
1516 return 1
1517 return 0
1518
46c609e9 1519 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1520 """Run "p4 change -o" to grab a change specification template.
1521 This does not use "p4 -G", as it is nice to keep the submission
1522 template in original order, since a human might edit it.
1523
1524 Remove lines in the Files section that show changes to files
1525 outside the depot path we're committing into."""
1526
cbc69242
SH
1527 [upstream, settings] = findUpstreamBranchPoint()
1528
ea99c3ae
SH
1529 template = ""
1530 inFilesSection = False
46c609e9
LD
1531 args = ['change', '-o']
1532 if changelist:
1533 args.append(str(changelist))
1534
1535 for line in p4_read_pipe_lines(args):
f3e5ae4f
MSO
1536 if line.endswith("\r\n"):
1537 line = line[:-2] + "\n"
ea99c3ae
SH
1538 if inFilesSection:
1539 if line.startswith("\t"):
1540 # path starts and ends with a tab
1541 path = line[1:]
1542 lastTab = path.rfind("\t")
1543 if lastTab != -1:
1544 path = path[:lastTab]
cbc69242
SH
1545 if settings.has_key('depot-paths'):
1546 if not [p for p in settings['depot-paths']
1547 if p4PathStartsWith(path, p)]:
1548 continue
1549 else:
1550 if not p4PathStartsWith(path, self.depotPath):
1551 continue
ea99c3ae
SH
1552 else:
1553 inFilesSection = False
1554 else:
1555 if line.startswith("Files:"):
1556 inFilesSection = True
1557
1558 template += line
1559
1560 return template
1561
7c766e57
PW
1562 def edit_template(self, template_file):
1563 """Invoke the editor to let the user change the submission
1564 message. Return true if okay to continue with the submit."""
1565
1566 # if configured to skip the editing part, just submit
0d609032 1567 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1568 return True
1569
1570 # look at the modification time, to check later if the user saved
1571 # the file
1572 mtime = os.stat(template_file).st_mtime
1573
1574 # invoke the editor
f95ceaf0 1575 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1576 editor = os.environ.get("P4EDITOR")
1577 else:
1578 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1579 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1580
1581 # If the file was not saved, prompt to see if this patch should
1582 # be skipped. But skip this verification step if configured so.
0d609032 1583 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1584 return True
1585
d1652049
PW
1586 # modification time updated means user saved the file
1587 if os.stat(template_file).st_mtime > mtime:
1588 return True
1589
1590 while True:
1591 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1592 if response == 'y':
1593 return True
1594 if response == 'n':
1595 return False
7c766e57 1596
df8a9e86 1597 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3
MC
1598 # diff
1599 if os.environ.has_key("P4DIFF"):
1600 del(os.environ["P4DIFF"])
1601 diff = ""
1602 for editedFile in editedFiles:
1603 diff += p4_read_pipe(['diff', '-du',
1604 wildcard_encode(editedFile)])
1605
1606 # new file diff
1607 newdiff = ""
1608 for newFile in filesToAdd:
1609 newdiff += "==== new file ====\n"
1610 newdiff += "--- /dev/null\n"
1611 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1612
1613 is_link = os.path.islink(newFile)
1614 expect_link = newFile in symlinks
1615
1616 if is_link and expect_link:
1617 newdiff += "+%s\n" % os.readlink(newFile)
1618 else:
1619 f = open(newFile, "r")
1620 for line in f.readlines():
1621 newdiff += "+" + line
1622 f.close()
b4073bb3 1623
e2a892ee 1624 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1625
7cb5cbef 1626 def applyCommit(self, id):
67b0fe2e
PW
1627 """Apply one commit, return True if it succeeded."""
1628
1629 print "Applying", read_pipe(["git", "show", "-s",
1630 "--format=format:%h %s", id])
ae901090 1631
848de9c3 1632 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1633
84cb0003 1634 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1635 filesToAdd = set()
a02b8bc4 1636 filesToChangeType = set()
4f5cf76a 1637 filesToDelete = set()
d336c158 1638 editedFiles = set()
b6ad6dcc 1639 pureRenameCopy = set()
df8a9e86 1640 symlinks = set()
c65b670e 1641 filesToChangeExecBit = {}
46c609e9 1642 all_files = list()
60df071c 1643
4f5cf76a 1644 for line in diff:
b43b0a3c
CP
1645 diff = parseDiffTreeEntry(line)
1646 modifier = diff['status']
1647 path = diff['src']
46c609e9
LD
1648 all_files.append(path)
1649
4f5cf76a 1650 if modifier == "M":
6de040df 1651 p4_edit(path)
c65b670e
CP
1652 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1653 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1654 editedFiles.add(path)
4f5cf76a
SH
1655 elif modifier == "A":
1656 filesToAdd.add(path)
c65b670e 1657 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1658 if path in filesToDelete:
1659 filesToDelete.remove(path)
df8a9e86
LD
1660
1661 dst_mode = int(diff['dst_mode'], 8)
1662 if dst_mode == 0120000:
1663 symlinks.add(path)
1664
4f5cf76a
SH
1665 elif modifier == "D":
1666 filesToDelete.add(path)
1667 if path in filesToAdd:
1668 filesToAdd.remove(path)
4fddb41b
VA
1669 elif modifier == "C":
1670 src, dest = diff['src'], diff['dst']
6de040df 1671 p4_integrate(src, dest)
b6ad6dcc 1672 pureRenameCopy.add(dest)
4fddb41b 1673 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1674 p4_edit(dest)
b6ad6dcc 1675 pureRenameCopy.discard(dest)
4fddb41b 1676 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1677 p4_edit(dest)
b6ad6dcc 1678 pureRenameCopy.discard(dest)
4fddb41b 1679 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1680 if self.isWindows:
1681 # turn off read-only attribute
1682 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1683 os.unlink(dest)
1684 editedFiles.add(dest)
d9a5f25b 1685 elif modifier == "R":
b43b0a3c 1686 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1687 if self.p4HasMoveCommand:
1688 p4_edit(src) # src must be open before move
1689 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1690 else:
8e9497c2
GG
1691 p4_integrate(src, dest)
1692 if diff['src_sha1'] != diff['dst_sha1']:
1693 p4_edit(dest)
1694 else:
1695 pureRenameCopy.add(dest)
c65b670e 1696 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1697 if not self.p4HasMoveCommand:
1698 p4_edit(dest) # with move: already open, writable
c65b670e 1699 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1700 if not self.p4HasMoveCommand:
d20f0f8e
PW
1701 if self.isWindows:
1702 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1703 os.unlink(dest)
1704 filesToDelete.add(src)
d9a5f25b 1705 editedFiles.add(dest)
a02b8bc4
RP
1706 elif modifier == "T":
1707 filesToChangeType.add(path)
4f5cf76a
SH
1708 else:
1709 die("unknown modifier %s for %s" % (modifier, path))
1710
749b668c 1711 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1712 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1713 tryPatchCmd = patchcmd + "--check -"
1714 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1715 patch_succeeded = True
51a2640a 1716
47a130b7 1717 if os.system(tryPatchCmd) != 0:
60df071c
LD
1718 fixed_rcs_keywords = False
1719 patch_succeeded = False
51a2640a 1720 print "Unfortunately applying the change failed!"
60df071c
LD
1721
1722 # Patch failed, maybe it's just RCS keyword woes. Look through
1723 # the patch to see if that's possible.
0d609032 1724 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1725 file = None
1726 pattern = None
1727 kwfiles = {}
1728 for file in editedFiles | filesToDelete:
1729 # did this file's delta contain RCS keywords?
1730 pattern = p4_keywords_regexp_for_file(file)
1731
1732 if pattern:
1733 # this file is a possibility...look for RCS keywords.
1734 regexp = re.compile(pattern, re.VERBOSE)
1735 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1736 if regexp.search(line):
1737 if verbose:
1738 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1739 kwfiles[file] = pattern
1740 break
1741
1742 for file in kwfiles:
1743 if verbose:
1744 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1745 # File is being deleted, so not open in p4. Must
1746 # disable the read-only bit on windows.
1747 if self.isWindows and file not in editedFiles:
1748 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1749 self.patchRCSKeywords(file, kwfiles[file])
1750 fixed_rcs_keywords = True
1751
1752 if fixed_rcs_keywords:
1753 print "Retrying the patch with RCS keywords cleaned up"
1754 if os.system(tryPatchCmd) == 0:
1755 patch_succeeded = True
1756
1757 if not patch_succeeded:
7e5dd9f2
PW
1758 for f in editedFiles:
1759 p4_revert(f)
7e5dd9f2 1760 return False
51a2640a 1761
55ac2ed6
PW
1762 #
1763 # Apply the patch for real, and do add/delete/+x handling.
1764 #
47a130b7 1765 system(applyPatchCmd)
4f5cf76a 1766
a02b8bc4
RP
1767 for f in filesToChangeType:
1768 p4_edit(f, "-t", "auto")
4f5cf76a 1769 for f in filesToAdd:
6de040df 1770 p4_add(f)
4f5cf76a 1771 for f in filesToDelete:
6de040df
LD
1772 p4_revert(f)
1773 p4_delete(f)
4f5cf76a 1774
c65b670e
CP
1775 # Set/clear executable bits
1776 for f in filesToChangeExecBit.keys():
1777 mode = filesToChangeExecBit[f]
1778 setP4ExecBit(f, mode)
1779
46c609e9
LD
1780 if self.update_shelve:
1781 print("all_files = %s" % str(all_files))
1782 p4_reopen_in_change(self.update_shelve, all_files)
1783
55ac2ed6
PW
1784 #
1785 # Build p4 change description, starting with the contents
1786 # of the git commit message.
1787 #
0e36f2d7 1788 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1789 logMessage = logMessage.strip()
f19cb0a0 1790 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1791
46c609e9 1792 template = self.prepareSubmitTemplate(self.update_shelve)
f19cb0a0 1793 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1794
c47178d4 1795 if self.preserveUser:
55ac2ed6 1796 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1797
55ac2ed6
PW
1798 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1799 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1800 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1801 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1802
55ac2ed6 1803 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1804 if not self.prepare_p4_only:
1805 submitTemplate += separatorLine
df8a9e86 1806 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1807
c47178d4 1808 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1809 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1810 if self.isWindows:
1811 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1812 tmpFile.write(submitTemplate)
c47178d4
PW
1813 tmpFile.close()
1814
728b7ad8
PW
1815 if self.prepare_p4_only:
1816 #
1817 # Leave the p4 tree prepared, and the submit template around
1818 # and let the user decide what to do next
1819 #
1820 print
1821 print "P4 workspace prepared for submission."
1822 print "To submit or revert, go to client workspace"
1823 print " " + self.clientPath
1824 print
1825 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1826 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1827 " \"git p4\"." % fileName
1828 print "You can delete the file \"%s\" when finished." % fileName
1829
1830 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1831 print "To preserve change ownership by user %s, you must\n" \
1832 "do \"p4 change -f <change>\" after submitting and\n" \
1833 "edit the User field."
1834 if pureRenameCopy:
1835 print "After submitting, renamed files must be re-synced."
1836 print "Invoke \"p4 sync -f\" on each of these files:"
1837 for f in pureRenameCopy:
1838 print " " + f
1839
1840 print
1841 print "To revert the changes, use \"p4 revert ...\", and delete"
1842 print "the submit template file \"%s\"" % fileName
1843 if filesToAdd:
1844 print "Since the commit adds new files, they must be deleted:"
1845 for f in filesToAdd:
1846 print " " + f
1847 print
1848 return True
1849
55ac2ed6
PW
1850 #
1851 # Let the user edit the change description, then submit it.
1852 #
b7638fed 1853 submitted = False
cdc7e388 1854
b7638fed
GE
1855 try:
1856 if self.edit_template(fileName):
1857 # read the edited message and submit
1858 tmpFile = open(fileName, "rb")
1859 message = tmpFile.read()
1860 tmpFile.close()
1861 if self.isWindows:
1862 message = message.replace("\r\n", "\n")
1863 submitTemplate = message[:message.index(separatorLine)]
46c609e9
LD
1864
1865 if self.update_shelve:
1866 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1867 elif self.shelve:
b34fa577
VK
1868 p4_write_pipe(['shelve', '-i'], submitTemplate)
1869 else:
1870 p4_write_pipe(['submit', '-i'], submitTemplate)
1871 # The rename/copy happened by applying a patch that created a
1872 # new file. This leaves it writable, which confuses p4.
1873 for f in pureRenameCopy:
1874 p4_sync(f, "-f")
b7638fed
GE
1875
1876 if self.preserveUser:
1877 if p4User:
1878 # Get last changelist number. Cannot easily get it from
1879 # the submit command output as the output is
1880 # unmarshalled.
1881 changelist = self.lastP4Changelist()
1882 self.modifyChangelistUser(changelist, p4User)
1883
b7638fed
GE
1884 submitted = True
1885
1886 finally:
c47178d4 1887 # skip this patch
b34fa577
VK
1888 if not submitted or self.shelve:
1889 if self.shelve:
1890 print ("Reverting shelved files.")
1891 else:
1892 print ("Submission cancelled, undoing p4 changes.")
1893 for f in editedFiles | filesToDelete:
b7638fed
GE
1894 p4_revert(f)
1895 for f in filesToAdd:
1896 p4_revert(f)
1897 os.remove(f)
c47178d4
PW
1898
1899 os.remove(fileName)
b7638fed 1900 return submitted
4f5cf76a 1901
06804c76
LD
1902 # Export git tags as p4 labels. Create a p4 label and then tag
1903 # with that.
1904 def exportGitTags(self, gitTags):
c8942a22
LD
1905 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1906 if len(validLabelRegexp) == 0:
1907 validLabelRegexp = defaultLabelRegexp
1908 m = re.compile(validLabelRegexp)
06804c76
LD
1909
1910 for name in gitTags:
1911
1912 if not m.match(name):
1913 if verbose:
05a3cec5 1914 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1915 continue
1916
1917 # Get the p4 commit this corresponds to
c8942a22
LD
1918 logMessage = extractLogMessageFromGitCommit(name)
1919 values = extractSettingsGitLog(logMessage)
06804c76 1920
c8942a22 1921 if not values.has_key('change'):
06804c76
LD
1922 # a tag pointing to something not sent to p4; ignore
1923 if verbose:
1924 print "git tag %s does not give a p4 commit" % name
1925 continue
c8942a22
LD
1926 else:
1927 changelist = values['change']
06804c76
LD
1928
1929 # Get the tag details.
1930 inHeader = True
1931 isAnnotated = False
1932 body = []
1933 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1934 l = l.strip()
1935 if inHeader:
1936 if re.match(r'tag\s+', l):
1937 isAnnotated = True
1938 elif re.match(r'\s*$', l):
1939 inHeader = False
1940 continue
1941 else:
1942 body.append(l)
1943
1944 if not isAnnotated:
1945 body = ["lightweight tag imported by git p4\n"]
1946
1947 # Create the label - use the same view as the client spec we are using
1948 clientSpec = getClientSpec()
1949
1950 labelTemplate = "Label: %s\n" % name
1951 labelTemplate += "Description:\n"
1952 for b in body:
1953 labelTemplate += "\t" + b + "\n"
1954 labelTemplate += "View:\n"
9d57c4a6
KS
1955 for depot_side in clientSpec.mappings:
1956 labelTemplate += "\t%s\n" % depot_side
06804c76 1957
ef739f08
PW
1958 if self.dry_run:
1959 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1960 elif self.prepare_p4_only:
1961 print "Not creating p4 label %s for tag due to option" \
1962 " --prepare-p4-only" % name
ef739f08
PW
1963 else:
1964 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1965
ef739f08
PW
1966 # Use the label
1967 p4_system(["tag", "-l", name] +
9d57c4a6 1968 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1969
ef739f08
PW
1970 if verbose:
1971 print "created p4 label for tag %s" % name
06804c76 1972
4f5cf76a 1973 def run(self, args):
c9b50e63
SH
1974 if len(args) == 0:
1975 self.master = currentGitBranch()
c9b50e63
SH
1976 elif len(args) == 1:
1977 self.master = args[0]
28755dba
PW
1978 if not branchExists(self.master):
1979 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1980 else:
1981 return False
1982
00ad6e31
LD
1983 if self.master:
1984 allowSubmit = gitConfig("git-p4.allowSubmit")
1985 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1986 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 1987
27d2d811 1988 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1989 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1990 if len(self.origin) == 0:
1991 self.origin = upstream
a3fdd579 1992
46c609e9
LD
1993 if self.update_shelve:
1994 self.shelve = True
1995
3ea2cfd4
LD
1996 if self.preserveUser:
1997 if not self.canChangeChangelists():
1998 die("Cannot preserve user names without p4 super-user or admin permissions")
1999
6bbfd137
PW
2000 # if not set from the command line, try the config file
2001 if self.conflict_behavior is None:
2002 val = gitConfig("git-p4.conflict")
2003 if val:
2004 if val not in self.conflict_behavior_choices:
2005 die("Invalid value '%s' for config git-p4.conflict" % val)
2006 else:
2007 val = "ask"
2008 self.conflict_behavior = val
2009
a3fdd579
SH
2010 if self.verbose:
2011 print "Origin branch is " + self.origin
9512497b 2012
ea99c3ae 2013 if len(self.depotPath) == 0:
9512497b
SH
2014 print "Internal error: cannot locate perforce depot path from existing branches"
2015 sys.exit(128)
2016
543987bd 2017 self.useClientSpec = False
0d609032 2018 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2019 self.useClientSpec = True
2020 if self.useClientSpec:
2021 self.clientSpecDirs = getClientSpec()
9512497b 2022
2e3a16b2 2023 # Check for the existence of P4 branches
cd884106
VA
2024 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2025
2026 if self.useClientSpec and not branchesDetected:
543987bd
PW
2027 # all files are relative to the client spec
2028 self.clientPath = getClientRoot()
2029 else:
2030 self.clientPath = p4Where(self.depotPath)
9512497b 2031
543987bd
PW
2032 if self.clientPath == "":
2033 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2034
ea99c3ae 2035 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2036 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2037
0591cfa8 2038 # ensure the clientPath exists
8d7ec362 2039 new_client_dir = False
0591cfa8 2040 if not os.path.exists(self.clientPath):
8d7ec362 2041 new_client_dir = True
0591cfa8
GG
2042 os.makedirs(self.clientPath)
2043
bbd84863 2044 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2045 if self.dry_run:
2046 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2047 else:
ef739f08
PW
2048 print "Synchronizing p4 checkout..."
2049 if new_client_dir:
2050 # old one was destroyed, and maybe nobody told p4
2051 p4_sync("...", "-f")
2052 else:
2053 p4_sync("...")
4f5cf76a 2054 self.check()
4f5cf76a 2055
4c750c0d 2056 commits = []
00ad6e31
LD
2057 if self.master:
2058 commitish = self.master
2059 else:
2060 commitish = 'HEAD'
2061
2062 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
4c750c0d
SH
2063 commits.append(line.strip())
2064 commits.reverse()
4f5cf76a 2065
0d609032 2066 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2067 self.checkAuthorship = False
2068 else:
2069 self.checkAuthorship = True
2070
3ea2cfd4
LD
2071 if self.preserveUser:
2072 self.checkValidP4Users(commits)
2073
84cb0003
GG
2074 #
2075 # Build up a set of options to be passed to diff when
2076 # submitting each commit to p4.
2077 #
2078 if self.detectRenames:
2079 # command-line -M arg
2080 self.diffOpts = "-M"
2081 else:
2082 # If not explicitly set check the config variable
2083 detectRenames = gitConfig("git-p4.detectRenames")
2084
2085 if detectRenames.lower() == "false" or detectRenames == "":
2086 self.diffOpts = ""
2087 elif detectRenames.lower() == "true":
2088 self.diffOpts = "-M"
2089 else:
2090 self.diffOpts = "-M%s" % detectRenames
2091
2092 # no command-line arg for -C or --find-copies-harder, just
2093 # config variables
2094 detectCopies = gitConfig("git-p4.detectCopies")
2095 if detectCopies.lower() == "false" or detectCopies == "":
2096 pass
2097 elif detectCopies.lower() == "true":
2098 self.diffOpts += " -C"
2099 else:
2100 self.diffOpts += " -C%s" % detectCopies
2101
0d609032 2102 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2103 self.diffOpts += " --find-copies-harder"
2104
7e5dd9f2
PW
2105 #
2106 # Apply the commits, one at a time. On failure, ask if should
2107 # continue to try the rest of the patches, or quit.
2108 #
ef739f08
PW
2109 if self.dry_run:
2110 print "Would apply"
67b0fe2e 2111 applied = []
7e5dd9f2
PW
2112 last = len(commits) - 1
2113 for i, commit in enumerate(commits):
ef739f08
PW
2114 if self.dry_run:
2115 print " ", read_pipe(["git", "show", "-s",
2116 "--format=format:%h %s", commit])
2117 ok = True
2118 else:
2119 ok = self.applyCommit(commit)
67b0fe2e
PW
2120 if ok:
2121 applied.append(commit)
7e5dd9f2 2122 else:
728b7ad8
PW
2123 if self.prepare_p4_only and i < last:
2124 print "Processing only the first commit due to option" \
2125 " --prepare-p4-only"
2126 break
7e5dd9f2
PW
2127 if i < last:
2128 quit = False
2129 while True:
6bbfd137
PW
2130 # prompt for what to do, or use the option/variable
2131 if self.conflict_behavior == "ask":
2132 print "What do you want to do?"
2133 response = raw_input("[s]kip this commit but apply"
2134 " the rest, or [q]uit? ")
2135 if not response:
2136 continue
2137 elif self.conflict_behavior == "skip":
2138 response = "s"
2139 elif self.conflict_behavior == "quit":
2140 response = "q"
2141 else:
2142 die("Unknown conflict_behavior '%s'" %
2143 self.conflict_behavior)
2144
7e5dd9f2
PW
2145 if response[0] == "s":
2146 print "Skipping this commit, but applying the rest"
2147 break
2148 if response[0] == "q":
2149 print "Quitting"
2150 quit = True
2151 break
2152 if quit:
2153 break
4f5cf76a 2154
67b0fe2e 2155 chdir(self.oldWorkingDirectory)
b34fa577 2156 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2157 if self.dry_run:
2158 pass
728b7ad8
PW
2159 elif self.prepare_p4_only:
2160 pass
ef739f08 2161 elif len(commits) == len(applied):
b34fa577 2162 print ("All commits {0}!".format(shelved_applied))
14594f4b 2163
4c750c0d 2164 sync = P4Sync()
44e8d26c
PW
2165 if self.branch:
2166 sync.branch = self.branch
4c750c0d 2167 sync.run([])
14594f4b 2168
4c750c0d
SH
2169 rebase = P4Rebase()
2170 rebase.rebase()
4f5cf76a 2171
67b0fe2e
PW
2172 else:
2173 if len(applied) == 0:
b34fa577 2174 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2175 else:
b34fa577 2176 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2177 for c in commits:
2178 if c in applied:
2179 star = "*"
2180 else:
2181 star = " "
2182 print star, read_pipe(["git", "show", "-s",
2183 "--format=format:%h %s", c])
2184 print "You will have to do 'git p4 sync' and rebase."
2185
0d609032 2186 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2187 self.exportLabels = True
06804c76
LD
2188
2189 if self.exportLabels:
2190 p4Labels = getP4Labels(self.depotPath)
2191 gitTags = getGitTags()
2192
2193 missingGitTags = gitTags - p4Labels
2194 self.exportGitTags(missingGitTags)
2195
98e023de 2196 # exit with error unless everything applied perfectly
67b0fe2e
PW
2197 if len(commits) != len(applied):
2198 sys.exit(1)
2199
b984733c
SH
2200 return True
2201
ecb7cf98
PW
2202class View(object):
2203 """Represent a p4 view ("p4 help views"), and map files in a
2204 repo according to the view."""
2205
9d57c4a6 2206 def __init__(self, client_name):
ecb7cf98 2207 self.mappings = []
9d57c4a6
KS
2208 self.client_prefix = "//%s/" % client_name
2209 # cache results of "p4 where" to lookup client file locations
2210 self.client_spec_path_cache = {}
ecb7cf98
PW
2211
2212 def append(self, view_line):
2213 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2214 sides. Append to self.mappings, preserving order. This
2215 is only needed for tag creation."""
ecb7cf98
PW
2216
2217 # Split the view line into exactly two words. P4 enforces
2218 # structure on these lines that simplifies this quite a bit.
2219 #
2220 # Either or both words may be double-quoted.
2221 # Single quotes do not matter.
2222 # Double-quote marks cannot occur inside the words.
2223 # A + or - prefix is also inside the quotes.
2224 # There are no quotes unless they contain a space.
2225 # The line is already white-space stripped.
2226 # The two words are separated by a single space.
2227 #
2228 if view_line[0] == '"':
2229 # First word is double quoted. Find its end.
2230 close_quote_index = view_line.find('"', 1)
2231 if close_quote_index <= 0:
2232 die("No first-word closing quote found: %s" % view_line)
2233 depot_side = view_line[1:close_quote_index]
2234 # skip closing quote and space
2235 rhs_index = close_quote_index + 1 + 1
2236 else:
2237 space_index = view_line.find(" ")
2238 if space_index <= 0:
2239 die("No word-splitting space found: %s" % view_line)
2240 depot_side = view_line[0:space_index]
2241 rhs_index = space_index + 1
2242
ecb7cf98 2243 # prefix + means overlay on previous mapping
ecb7cf98 2244 if depot_side.startswith("+"):
ecb7cf98
PW
2245 depot_side = depot_side[1:]
2246
9d57c4a6 2247 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2248 exclude = False
2249 if depot_side.startswith("-"):
2250 exclude = True
2251 depot_side = depot_side[1:]
2252
9d57c4a6
KS
2253 if not exclude:
2254 self.mappings.append(depot_side)
ecb7cf98 2255
9d57c4a6
KS
2256 def convert_client_path(self, clientFile):
2257 # chop off //client/ part to make it relative
2258 if not clientFile.startswith(self.client_prefix):
2259 die("No prefix '%s' on clientFile '%s'" %
2260 (self.client_prefix, clientFile))
2261 return clientFile[len(self.client_prefix):]
ecb7cf98 2262
9d57c4a6
KS
2263 def update_client_spec_path_cache(self, files):
2264 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2265
9d57c4a6
KS
2266 # List depot file paths exclude that already cached
2267 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2268
9d57c4a6
KS
2269 if len(fileArgs) == 0:
2270 return # All files in cache
ecb7cf98 2271
9d57c4a6
KS
2272 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2273 for res in where_result:
2274 if "code" in res and res["code"] == "error":
2275 # assume error is "... file(s) not in client view"
2276 continue
2277 if "clientFile" not in res:
20005443 2278 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2279 if "unmap" in res:
2280 # it will list all of them, but only one not unmap-ped
2281 continue
a0a50d87
LS
2282 if gitConfigBool("core.ignorecase"):
2283 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2284 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2285
9d57c4a6
KS
2286 # not found files or unmap files set to ""
2287 for depotFile in fileArgs:
a0a50d87
LS
2288 if gitConfigBool("core.ignorecase"):
2289 depotFile = depotFile.lower()
9d57c4a6
KS
2290 if depotFile not in self.client_spec_path_cache:
2291 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2292
9d57c4a6
KS
2293 def map_in_client(self, depot_path):
2294 """Return the relative location in the client where this
2295 depot file should live. Returns "" if the file should
2296 not be mapped in the client."""
ecb7cf98 2297
a0a50d87
LS
2298 if gitConfigBool("core.ignorecase"):
2299 depot_path = depot_path.lower()
2300
9d57c4a6
KS
2301 if depot_path in self.client_spec_path_cache:
2302 return self.client_spec_path_cache[depot_path]
2303
2304 die( "Error: %s is not found in client spec path" % depot_path )
2305 return ""
ecb7cf98 2306
3ea2cfd4 2307class P4Sync(Command, P4UserMap):
56c09345
PW
2308 delete_actions = ( "delete", "move/delete", "purge" )
2309
b984733c
SH
2310 def __init__(self):
2311 Command.__init__(self)
3ea2cfd4 2312 P4UserMap.__init__(self)
b984733c
SH
2313 self.options = [
2314 optparse.make_option("--branch", dest="branch"),
2315 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2316 optparse.make_option("--changesfile", dest="changesFile"),
2317 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2318 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2319 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2320 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2321 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2322 optparse.make_option("--max-changes", dest="maxChanges",
2323 help="Maximum number of changes to import"),
2324 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2325 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2326 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2327 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2328 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2329 help="Only sync files that are included in the Perforce Client Spec"),
2330 optparse.make_option("-/", dest="cloneExclude",
2331 action="append", type="string",
2332 help="exclude depot path"),
b984733c
SH
2333 ]
2334 self.description = """Imports from Perforce into a git repository.\n
2335 example:
2336 //depot/my/project/ -- to import the current head
2337 //depot/my/project/@all -- to import everything
2338 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2339
2340 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2341
2342 self.usage += " //depot/path[@revRange]"
b984733c 2343 self.silent = False
1d7367dc
RG
2344 self.createdBranches = set()
2345 self.committedChanges = set()
569d1bd4 2346 self.branch = ""
b984733c 2347 self.detectBranches = False
cb53e1f8 2348 self.detectLabels = False
06804c76 2349 self.importLabels = False
b984733c 2350 self.changesFile = ""
01265103 2351 self.syncWithOrigin = True
a028a98e 2352 self.importIntoRemotes = True
01a9c9c5 2353 self.maxChanges = ""
1051ef00 2354 self.changes_block_size = None
8b41a97f 2355 self.keepRepoPath = False
6326aa58 2356 self.depotPaths = None
3c699645 2357 self.p4BranchesInGit = []
354081d5 2358 self.cloneExclude = []
3a70cdfa 2359 self.useClientSpec = False
a93d33ee 2360 self.useClientSpec_from_options = False
ecb7cf98 2361 self.clientSpecDirs = None
fed23693 2362 self.tempBranches = []
d604176d 2363 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12
LS
2364 self.largeFileSystem = None
2365
2366 if gitConfig('git-p4.largeFileSystem'):
2367 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2368 self.largeFileSystem = largeFileSystemConstructor(
2369 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2370 )
b984733c 2371
01265103
SH
2372 if gitConfig("git-p4.syncFromOrigin") == "false":
2373 self.syncWithOrigin = False
2374
51334bb0
LD
2375 # This is required for the "append" cloneExclude action
2376 def ensure_value(self, attr, value):
2377 if not hasattr(self, attr) or getattr(self, attr) is None:
2378 setattr(self, attr, value)
2379 return getattr(self, attr)
2380
fed23693
VA
2381 # Force a checkpoint in fast-import and wait for it to finish
2382 def checkpoint(self):
2383 self.gitStream.write("checkpoint\n\n")
2384 self.gitStream.write("progress checkpoint\n\n")
2385 out = self.gitOutput.readline()
2386 if self.verbose:
2387 print "checkpoint finished: " + out
2388
b984733c 2389 def extractFilesFromCommit(self, commit):
354081d5
TT
2390 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2391 for path in self.cloneExclude]
b984733c
SH
2392 files = []
2393 fnum = 0
2394 while commit.has_key("depotFile%s" % fnum):
2395 path = commit["depotFile%s" % fnum]
6326aa58 2396
354081d5 2397 if [p for p in self.cloneExclude
d53de8b9 2398 if p4PathStartsWith(path, p)]:
354081d5
TT
2399 found = False
2400 else:
2401 found = [p for p in self.depotPaths
d53de8b9 2402 if p4PathStartsWith(path, p)]
6326aa58 2403 if not found:
b984733c
SH
2404 fnum = fnum + 1
2405 continue
2406
2407 file = {}
2408 file["path"] = path
2409 file["rev"] = commit["rev%s" % fnum]
2410 file["action"] = commit["action%s" % fnum]
2411 file["type"] = commit["type%s" % fnum]
2412 files.append(file)
2413 fnum = fnum + 1
2414 return files
2415
26e6a27d
JD
2416 def extractJobsFromCommit(self, commit):
2417 jobs = []
2418 jnum = 0
2419 while commit.has_key("job%s" % jnum):
2420 job = commit["job%s" % jnum]
2421 jobs.append(job)
2422 jnum = jnum + 1
2423 return jobs
2424
6326aa58 2425 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2426 """When streaming files, this is called to map a p4 depot path
2427 to where it should go in git. The prefixes are either
2428 self.depotPaths, or self.branchPrefixes in the case of
2429 branch detection."""
2430
3952710b 2431 if self.useClientSpec:
21ef5df4
PW
2432 # branch detection moves files up a level (the branch name)
2433 # from what client spec interpretation gives
0d1696ef 2434 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2435 if self.detectBranches:
2436 for b in self.knownBranches:
2437 if path.startswith(b + "/"):
2438 path = path[len(b)+1:]
2439
2440 elif self.keepRepoPath:
2441 # Preserve everything in relative path name except leading
2442 # //depot/; just look at first prefix as they all should
2443 # be in the same depot.
2444 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2445 if p4PathStartsWith(path, depot):
2446 path = path[len(depot):]
3952710b 2447
0d1696ef 2448 else:
0d1696ef
PW
2449 for p in prefixes:
2450 if p4PathStartsWith(path, p):
2451 path = path[len(p):]
21ef5df4 2452 break
8b41a97f 2453
0d1696ef 2454 path = wildcard_decode(path)
6326aa58 2455 return path
6754a299 2456
71b112d4 2457 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2458 """Look at each depotFile in the commit to figure out to what
2459 branch it belongs."""
2460
9d57c4a6
KS
2461 if self.clientSpecDirs:
2462 files = self.extractFilesFromCommit(commit)
2463 self.clientSpecDirs.update_client_spec_path_cache(files)
2464
d5904674 2465 branches = {}
71b112d4
SH
2466 fnum = 0
2467 while commit.has_key("depotFile%s" % fnum):
2468 path = commit["depotFile%s" % fnum]
6326aa58 2469 found = [p for p in self.depotPaths
d53de8b9 2470 if p4PathStartsWith(path, p)]
6326aa58 2471 if not found:
71b112d4
SH
2472 fnum = fnum + 1
2473 continue
2474
2475 file = {}
2476 file["path"] = path
2477 file["rev"] = commit["rev%s" % fnum]
2478 file["action"] = commit["action%s" % fnum]
2479 file["type"] = commit["type%s" % fnum]
2480 fnum = fnum + 1
2481
21ef5df4
PW
2482 # start with the full relative path where this file would
2483 # go in a p4 client
2484 if self.useClientSpec:
2485 relPath = self.clientSpecDirs.map_in_client(path)
2486 else:
2487 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2488
4b97ffb1 2489 for branch in self.knownBranches.keys():
21ef5df4
PW
2490 # add a trailing slash so that a commit into qt/4.2foo
2491 # doesn't end up in qt/4.2, e.g.
6754a299 2492 if relPath.startswith(branch + "/"):
d5904674
SH
2493 if branch not in branches:
2494 branches[branch] = []
71b112d4 2495 branches[branch].append(file)
6555b2cc 2496 break
b984733c
SH
2497
2498 return branches
2499
a5db4b12
LS
2500 def writeToGitStream(self, gitMode, relPath, contents):
2501 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2502 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2503 for d in contents:
2504 self.gitStream.write(d)
2505 self.gitStream.write('\n')
2506
a8b05162
LS
2507 def encodeWithUTF8(self, path):
2508 try:
2509 path.decode('ascii')
2510 except:
2511 encoding = 'utf8'
2512 if gitConfig('git-p4.pathEncoding'):
2513 encoding = gitConfig('git-p4.pathEncoding')
2514 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2515 if self.verbose:
2516 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2517 return path
2518
b932705b
LD
2519 # output one file from the P4 stream
2520 # - helper for streamP4Files
2521
2522 def streamOneP4File(self, file, contents):
b932705b 2523 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2524 relPath = self.encodeWithUTF8(relPath)
b932705b 2525 if verbose:
d2176a50
LS
2526 size = int(self.stream_file['fileSize'])
2527 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2528 sys.stdout.flush()
b932705b 2529
9cffb8c8
PW
2530 (type_base, type_mods) = split_p4_type(file["type"])
2531
2532 git_mode = "100644"
2533 if "x" in type_mods:
2534 git_mode = "100755"
2535 if type_base == "symlink":
2536 git_mode = "120000"
1292df11
AJ
2537 # p4 print on a symlink sometimes contains "target\n";
2538 # if it does, remove the newline
b39c3612 2539 data = ''.join(contents)
40f846c3
PW
2540 if not data:
2541 # Some version of p4 allowed creating a symlink that pointed
2542 # to nothing. This causes p4 errors when checking out such
2543 # a change, and errors here too. Work around it by ignoring
2544 # the bad symlink; hopefully a future change fixes it.
2545 print "\nIgnoring empty symlink in %s" % file['depotFile']
2546 return
2547 elif data[-1] == '\n':
1292df11
AJ
2548 contents = [data[:-1]]
2549 else:
2550 contents = [data]
b932705b 2551
9cffb8c8 2552 if type_base == "utf16":
55aa5714
PW
2553 # p4 delivers different text in the python output to -G
2554 # than it does when using "print -o", or normal p4 client
2555 # operations. utf16 is converted to ascii or utf8, perhaps.
2556 # But ascii text saved as -t utf16 is completely mangled.
2557 # Invoke print -o to get the real contents.
7f0e5962
PW
2558 #
2559 # On windows, the newlines will always be mangled by print, so put
2560 # them back too. This is not needed to the cygwin windows version,
2561 # just the native "NT" type.
2562 #
1f5f3907
LS
2563 try:
2564 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2565 except Exception as e:
2566 if 'Translation of file content failed' in str(e):
2567 type_base = 'binary'
2568 else:
2569 raise e
2570 else:
2571 if p4_version_string().find('/NT') >= 0:
2572 text = text.replace('\r\n', '\n')
2573 contents = [ text ]
55aa5714 2574
9f7ef0ea
PW
2575 if type_base == "apple":
2576 # Apple filetype files will be streamed as a concatenation of
2577 # its appledouble header and the contents. This is useless
2578 # on both macs and non-macs. If using "print -q -o xx", it
2579 # will create "xx" with the data, and "%xx" with the header.
2580 # This is also not very useful.
2581 #
2582 # Ideally, someday, this script can learn how to generate
2583 # appledouble files directly and import those to git, but
2584 # non-mac machines can never find a use for apple filetype.
2585 print "\nIgnoring apple filetype file %s" % file['depotFile']
2586 return
2587
55aa5714
PW
2588 # Note that we do not try to de-mangle keywords on utf16 files,
2589 # even though in theory somebody may want that.
60df071c
LD
2590 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2591 if pattern:
2592 regexp = re.compile(pattern, re.VERBOSE)
2593 text = ''.join(contents)
2594 text = regexp.sub(r'$\1$', text)
2595 contents = [ text ]
b932705b 2596
a5db4b12
LS
2597 if self.largeFileSystem:
2598 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2599
a5db4b12 2600 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2601
2602 def streamOneP4Deletion(self, file):
2603 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2604 relPath = self.encodeWithUTF8(relPath)
b932705b 2605 if verbose:
d2176a50
LS
2606 sys.stdout.write("delete %s\n" % relPath)
2607 sys.stdout.flush()
b932705b
LD
2608 self.gitStream.write("D %s\n" % relPath)
2609
a5db4b12
LS
2610 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2611 self.largeFileSystem.removeLargeFile(relPath)
2612
b932705b
LD
2613 # handle another chunk of streaming data
2614 def streamP4FilesCb(self, marshalled):
2615
78189bea
PW
2616 # catch p4 errors and complain
2617 err = None
2618 if "code" in marshalled:
2619 if marshalled["code"] == "error":
2620 if "data" in marshalled:
2621 err = marshalled["data"].rstrip()
4d25dc44
LS
2622
2623 if not err and 'fileSize' in self.stream_file:
2624 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2625 if required_bytes > 0:
2626 err = 'Not enough space left on %s! Free at least %i MB.' % (
2627 os.getcwd(), required_bytes/1024/1024
2628 )
2629
78189bea
PW
2630 if err:
2631 f = None
2632 if self.stream_have_file_info:
2633 if "depotFile" in self.stream_file:
2634 f = self.stream_file["depotFile"]
2635 # force a failure in fast-import, else an empty
2636 # commit will be made
2637 self.gitStream.write("\n")
2638 self.gitStream.write("die-now\n")
2639 self.gitStream.close()
2640 # ignore errors, but make sure it exits first
2641 self.importProcess.wait()
2642 if f:
2643 die("Error from p4 print for %s: %s" % (f, err))
2644 else:
2645 die("Error from p4 print: %s" % err)
2646
c3f6163b
AG
2647 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2648 # start of a new file - output the old one first
2649 self.streamOneP4File(self.stream_file, self.stream_contents)
2650 self.stream_file = {}
2651 self.stream_contents = []
2652 self.stream_have_file_info = False
b932705b 2653
c3f6163b
AG
2654 # pick up the new file information... for the
2655 # 'data' field we need to append to our array
2656 for k in marshalled.keys():
2657 if k == 'data':
d2176a50
LS
2658 if 'streamContentSize' not in self.stream_file:
2659 self.stream_file['streamContentSize'] = 0
2660 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2661 self.stream_contents.append(marshalled['data'])
2662 else:
2663 self.stream_file[k] = marshalled[k]
b932705b 2664
d2176a50
LS
2665 if (verbose and
2666 'streamContentSize' in self.stream_file and
2667 'fileSize' in self.stream_file and
2668 'depotFile' in self.stream_file):
2669 size = int(self.stream_file["fileSize"])
2670 if size > 0:
2671 progress = 100*self.stream_file['streamContentSize']/size
2672 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2673 sys.stdout.flush()
2674
c3f6163b 2675 self.stream_have_file_info = True
b932705b
LD
2676
2677 # Stream directly from "p4 files" into "git fast-import"
2678 def streamP4Files(self, files):
30b5940b
SH
2679 filesForCommit = []
2680 filesToRead = []
b932705b 2681 filesToDelete = []
30b5940b 2682
3a70cdfa 2683 for f in files:
ecb7cf98
PW
2684 filesForCommit.append(f)
2685 if f['action'] in self.delete_actions:
2686 filesToDelete.append(f)
2687 else:
2688 filesToRead.append(f)
6a49f8e2 2689
b932705b
LD
2690 # deleted files...
2691 for f in filesToDelete:
2692 self.streamOneP4Deletion(f)
1b9a4684 2693
b932705b
LD
2694 if len(filesToRead) > 0:
2695 self.stream_file = {}
2696 self.stream_contents = []
2697 self.stream_have_file_info = False
8ff45f2a 2698
c3f6163b
AG
2699 # curry self argument
2700 def streamP4FilesCbSelf(entry):
2701 self.streamP4FilesCb(entry)
6a49f8e2 2702
6de040df
LD
2703 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2704
2705 p4CmdList(["-x", "-", "print"],
2706 stdin=fileArgs,
2707 cb=streamP4FilesCbSelf)
30b5940b 2708
b932705b
LD
2709 # do the last chunk
2710 if self.stream_file.has_key('depotFile'):
2711 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2712
affb474f
LD
2713 def make_email(self, userid):
2714 if userid in self.users:
2715 return self.users[userid]
2716 else:
2717 return "%s <a@b>" % userid
2718
06804c76 2719 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2720 """ Stream a p4 tag.
2721 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2722 """
2723
06804c76
LD
2724 if verbose:
2725 print "writing tag %s for commit %s" % (labelName, commit)
2726 gitStream.write("tag %s\n" % labelName)
2727 gitStream.write("from %s\n" % commit)
2728
2729 if labelDetails.has_key('Owner'):
2730 owner = labelDetails["Owner"]
2731 else:
2732 owner = None
2733
2734 # Try to use the owner of the p4 label, or failing that,
2735 # the current p4 user id.
2736 if owner:
2737 email = self.make_email(owner)
2738 else:
2739 email = self.make_email(self.p4UserId())
2740 tagger = "%s %s %s" % (email, epoch, self.tz)
2741
2742 gitStream.write("tagger %s\n" % tagger)
2743
2744 print "labelDetails=",labelDetails
2745 if labelDetails.has_key('Description'):
2746 description = labelDetails['Description']
2747 else:
2748 description = 'Label from git p4'
2749
2750 gitStream.write("data %d\n" % len(description))
2751 gitStream.write(description)
2752 gitStream.write("\n")
2753
4ae048e6
LS
2754 def inClientSpec(self, path):
2755 if not self.clientSpecDirs:
2756 return True
2757 inClientSpec = self.clientSpecDirs.map_in_client(path)
2758 if not inClientSpec and self.verbose:
2759 print('Ignoring file outside of client spec: {0}'.format(path))
2760 return inClientSpec
2761
2762 def hasBranchPrefix(self, path):
2763 if not self.branchPrefixes:
2764 return True
2765 hasPrefix = [p for p in self.branchPrefixes
2766 if p4PathStartsWith(path, p)]
09667d01 2767 if not hasPrefix and self.verbose:
4ae048e6
LS
2768 print('Ignoring file outside of prefix: {0}'.format(path))
2769 return hasPrefix
2770
e63231e5 2771 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2772 epoch = details["time"]
2773 author = details["user"]
26e6a27d 2774 jobs = self.extractJobsFromCommit(details)
b984733c 2775
4b97ffb1 2776 if self.verbose:
4ae048e6 2777 print('commit into {0}'.format(branch))
96e07dd2 2778
9d57c4a6
KS
2779 if self.clientSpecDirs:
2780 self.clientSpecDirs.update_client_spec_path_cache(files)
2781
4ae048e6
LS
2782 files = [f for f in files
2783 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2784
2785 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2786 print('Ignoring revision {0} as it would produce an empty commit.'
2787 .format(details['change']))
2788 return
2789
b984733c 2790 self.gitStream.write("commit %s\n" % branch)
b43702ac 2791 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2792 self.committedChanges.add(int(details["change"]))
2793 committer = ""
b607e71e
SH
2794 if author not in self.users:
2795 self.getUserMapFromPerforceServer()
affb474f 2796 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2797
2798 self.gitStream.write("committer %s\n" % committer)
2799
2800 self.gitStream.write("data <<EOT\n")
2801 self.gitStream.write(details["desc"])
26e6a27d
JD
2802 if len(jobs) > 0:
2803 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
e63231e5
PW
2804 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2805 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2806 if len(details['options']) > 0:
2807 self.gitStream.write(": options = %s" % details['options'])
2808 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2809
2810 if len(parent) > 0:
4b97ffb1
SH
2811 if self.verbose:
2812 print "parent %s" % parent
b984733c
SH
2813 self.gitStream.write("from %s\n" % parent)
2814
4ae048e6 2815 self.streamP4Files(files)
b984733c
SH
2816 self.gitStream.write("\n")
2817
1f4ba1cb
SH
2818 change = int(details["change"])
2819
9bda3a85 2820 if self.labels.has_key(change):
1f4ba1cb
SH
2821 label = self.labels[change]
2822 labelDetails = label[0]
2823 labelRevisions = label[1]
71b112d4
SH
2824 if self.verbose:
2825 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2826
6de040df 2827 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2828 for p in self.branchPrefixes])
1f4ba1cb
SH
2829
2830 if len(files) == len(labelRevisions):
2831
2832 cleanedFiles = {}
2833 for info in files:
56c09345 2834 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2835 continue
2836 cleanedFiles[info["depotFile"]] = info["rev"]
2837
2838 if cleanedFiles == labelRevisions:
06804c76 2839 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2840
2841 else:
a46668fa 2842 if not self.silent:
cebdf5af
HWN
2843 print ("Tag %s does not match with change %s: files do not match."
2844 % (labelDetails["label"], change))
1f4ba1cb
SH
2845
2846 else:
a46668fa 2847 if not self.silent:
cebdf5af
HWN
2848 print ("Tag %s does not match with change %s: file count is different."
2849 % (labelDetails["label"], change))
b984733c 2850
06804c76 2851 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2852 def getLabels(self):
2853 self.labels = {}
2854
52a4880b 2855 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2856 if len(l) > 0 and not self.silent:
183f8436 2857 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2858
2859 for output in l:
1f4ba1cb
SH
2860 label = output["label"]
2861 revisions = {}
2862 newestChange = 0
71b112d4
SH
2863 if self.verbose:
2864 print "Querying files for label %s" % label
6de040df
LD
2865 for file in p4CmdList(["files"] +
2866 ["%s...@%s" % (p, label)
2867 for p in self.depotPaths]):
1f4ba1cb
SH
2868 revisions[file["depotFile"]] = file["rev"]
2869 change = int(file["change"])
2870 if change > newestChange:
2871 newestChange = change
2872
9bda3a85
SH
2873 self.labels[newestChange] = [output, revisions]
2874
2875 if self.verbose:
2876 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2877
06804c76
LD
2878 # Import p4 labels as git tags. A direct mapping does not
2879 # exist, so assume that if all the files are at the same revision
2880 # then we can use that, or it's something more complicated we should
2881 # just ignore.
2882 def importP4Labels(self, stream, p4Labels):
2883 if verbose:
2884 print "import p4 labels: " + ' '.join(p4Labels)
2885
2886 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2887 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2888 if len(validLabelRegexp) == 0:
2889 validLabelRegexp = defaultLabelRegexp
2890 m = re.compile(validLabelRegexp)
2891
2892 for name in p4Labels:
2893 commitFound = False
2894
2895 if not m.match(name):
2896 if verbose:
2897 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2898 continue
2899
2900 if name in ignoredP4Labels:
2901 continue
2902
2903 labelDetails = p4CmdList(['label', "-o", name])[0]
2904
2905 # get the most recent changelist for each file in this label
2906 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2907 for p in self.depotPaths])
2908
2909 if change.has_key('change'):
2910 # find the corresponding git commit; take the oldest commit
2911 changelist = int(change['change'])
b43702ac
LD
2912 if changelist in self.committedChanges:
2913 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2914 commitFound = True
b43702ac
LD
2915 else:
2916 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2917 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2918 if len(gitCommit) == 0:
2919 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2920 else:
2921 commitFound = True
2922 gitCommit = gitCommit.strip()
2923
2924 if commitFound:
06804c76
LD
2925 # Convert from p4 time format
2926 try:
2927 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2928 except ValueError:
a4e9054c 2929 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2930 tmwhen = 1
2931
2932 when = int(time.mktime(tmwhen))
2933 self.streamTag(stream, name, labelDetails, gitCommit, when)
2934 if verbose:
2935 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2936 else:
2937 if verbose:
2938 print "Label %s has no changelists - possibly deleted?" % name
2939
2940 if not commitFound:
2941 # We can't import this label; don't try again as it will get very
2942 # expensive repeatedly fetching all the files for labels that will
2943 # never be imported. If the label is moved in the future, the
2944 # ignore will need to be removed manually.
2945 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2946
86dff6b6
HWN
2947 def guessProjectName(self):
2948 for p in self.depotPaths:
6e5295c4
SH
2949 if p.endswith("/"):
2950 p = p[:-1]
2951 p = p[p.strip().rfind("/") + 1:]
2952 if not p.endswith("/"):
2953 p += "/"
2954 return p
86dff6b6 2955
4b97ffb1 2956 def getBranchMapping(self):
6555b2cc
SH
2957 lostAndFoundBranches = set()
2958
8ace74c0
VA
2959 user = gitConfig("git-p4.branchUser")
2960 if len(user) > 0:
2961 command = "branches -u %s" % user
2962 else:
2963 command = "branches"
2964
2965 for info in p4CmdList(command):
52a4880b 2966 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2967 viewIdx = 0
2968 while details.has_key("View%s" % viewIdx):
2969 paths = details["View%s" % viewIdx].split(" ")
2970 viewIdx = viewIdx + 1
2971 # require standard //depot/foo/... //depot/bar/... mapping
2972 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2973 continue
2974 source = paths[0]
2975 destination = paths[1]
6509e19c 2976 ## HACK
d53de8b9 2977 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2978 source = source[len(self.depotPaths[0]):-4]
2979 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2980
1a2edf4e
SH
2981 if destination in self.knownBranches:
2982 if not self.silent:
2983 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2984 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2985 continue
2986
6555b2cc
SH
2987 self.knownBranches[destination] = source
2988
2989 lostAndFoundBranches.discard(destination)
2990
29bdbac1 2991 if source not in self.knownBranches:
6555b2cc
SH
2992 lostAndFoundBranches.add(source)
2993
7199cf13
VA
2994 # Perforce does not strictly require branches to be defined, so we also
2995 # check git config for a branch list.
2996 #
2997 # Example of branch definition in git config file:
2998 # [git-p4]
2999 # branchList=main:branchA
3000 # branchList=main:branchB
3001 # branchList=branchA:branchC
3002 configBranches = gitConfigList("git-p4.branchList")
3003 for branch in configBranches:
3004 if branch:
3005 (source, destination) = branch.split(":")
3006 self.knownBranches[destination] = source
3007
3008 lostAndFoundBranches.discard(destination)
3009
3010 if source not in self.knownBranches:
3011 lostAndFoundBranches.add(source)
3012
6555b2cc
SH
3013
3014 for branch in lostAndFoundBranches:
3015 self.knownBranches[branch] = branch
29bdbac1 3016
38f9f5ec
SH
3017 def getBranchMappingFromGitBranches(self):
3018 branches = p4BranchesInGit(self.importIntoRemotes)
3019 for branch in branches.keys():
3020 if branch == "master":
3021 branch = "main"
3022 else:
3023 branch = branch[len(self.projectName):]
3024 self.knownBranches[branch] = branch
3025
bb6e09b2
HWN
3026 def updateOptionDict(self, d):
3027 option_keys = {}
3028 if self.keepRepoPath:
3029 option_keys['keepRepoPath'] = 1
3030
3031 d["options"] = ' '.join(sorted(option_keys.keys()))
3032
3033 def readOptions(self, d):
3034 self.keepRepoPath = (d.has_key('options')
3035 and ('keepRepoPath' in d['options']))
6326aa58 3036
8134f69c
SH
3037 def gitRefForBranch(self, branch):
3038 if branch == "main":
3039 return self.refPrefix + "master"
3040
3041 if len(branch) <= 0:
3042 return branch
3043
3044 return self.refPrefix + self.projectName + branch
3045
1ca3d710
SH
3046 def gitCommitByP4Change(self, ref, change):
3047 if self.verbose:
3048 print "looking in ref " + ref + " for change %s using bisect..." % change
3049
3050 earliestCommit = ""
3051 latestCommit = parseRevision(ref)
3052
3053 while True:
3054 if self.verbose:
3055 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3056 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3057 if len(next) == 0:
3058 if self.verbose:
3059 print "argh"
3060 return ""
3061 log = extractLogMessageFromGitCommit(next)
3062 settings = extractSettingsGitLog(log)
3063 currentChange = int(settings['change'])
3064 if self.verbose:
3065 print "current change %s" % currentChange
3066
3067 if currentChange == change:
3068 if self.verbose:
3069 print "found %s" % next
3070 return next
3071
3072 if currentChange < change:
3073 earliestCommit = "^%s" % next
3074 else:
3075 latestCommit = "%s" % next
3076
3077 return ""
3078
3079 def importNewBranch(self, branch, maxChange):
3080 # make fast-import flush all changes to disk and update the refs using the checkpoint
3081 # command so that we can try to find the branch parent in the git history
3082 self.gitStream.write("checkpoint\n\n");
3083 self.gitStream.flush();
3084 branchPrefix = self.depotPaths[0] + branch + "/"
3085 range = "@1,%s" % maxChange
3086 #print "prefix" + branchPrefix
96b2d54a 3087 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3088 if len(changes) <= 0:
3089 return False
3090 firstChange = changes[0]
3091 #print "first change in branch: %s" % firstChange
3092 sourceBranch = self.knownBranches[branch]
3093 sourceDepotPath = self.depotPaths[0] + sourceBranch
3094 sourceRef = self.gitRefForBranch(sourceBranch)
3095 #print "source " + sourceBranch
3096
52a4880b 3097 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3098 #print "branch parent: %s" % branchParentChange
3099 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3100 if len(gitParent) > 0:
3101 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3102 #print "parent git commit: %s" % gitParent
3103
3104 self.importChanges(changes)
3105 return True
3106
fed23693
VA
3107 def searchParent(self, parent, branch, target):
3108 parentFound = False
c7d34884
PW
3109 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3110 "--no-merges", parent]):
fed23693
VA
3111 blob = blob.strip()
3112 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3113 parentFound = True
3114 if self.verbose:
3115 print "Found parent of %s in commit %s" % (branch, blob)
3116 break
3117 if parentFound:
3118 return blob
3119 else:
3120 return None
3121
e87f37ae
SH
3122 def importChanges(self, changes):
3123 cnt = 1
3124 for change in changes:
18fa13d0 3125 description = p4_describe(change)
e87f37ae
SH
3126 self.updateOptionDict(description)
3127
3128 if not self.silent:
3129 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3130 sys.stdout.flush()
3131 cnt = cnt + 1
3132
3133 try:
3134 if self.detectBranches:
3135 branches = self.splitFilesIntoBranches(description)
3136 for branch in branches.keys():
3137 ## HACK --hwn
3138 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3139 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3140
3141 parent = ""
3142
3143 filesForCommit = branches[branch]
3144
3145 if self.verbose:
3146 print "branch is %s" % branch
3147
3148 self.updatedBranches.add(branch)
3149
3150 if branch not in self.createdBranches:
3151 self.createdBranches.add(branch)
3152 parent = self.knownBranches[branch]
3153 if parent == branch:
3154 parent = ""
1ca3d710
SH
3155 else:
3156 fullBranch = self.projectName + branch
3157 if fullBranch not in self.p4BranchesInGit:
3158 if not self.silent:
3159 print("\n Importing new branch %s" % fullBranch);
3160 if self.importNewBranch(branch, change - 1):
3161 parent = ""
3162 self.p4BranchesInGit.append(fullBranch)
3163 if not self.silent:
3164 print("\n Resuming with change %s" % change);
3165
3166 if self.verbose:
3167 print "parent determined through known branches: %s" % parent
e87f37ae 3168
8134f69c
SH
3169 branch = self.gitRefForBranch(branch)
3170 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3171
3172 if self.verbose:
3173 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3174
3175 if len(parent) == 0 and branch in self.initialParents:
3176 parent = self.initialParents[branch]
3177 del self.initialParents[branch]
3178
fed23693
VA
3179 blob = None
3180 if len(parent) > 0:
4f9273d2 3181 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3182 if self.verbose:
3183 print "Creating temporary branch: " + tempBranch
e63231e5 3184 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3185 self.tempBranches.append(tempBranch)
3186 self.checkpoint()
3187 blob = self.searchParent(parent, branch, tempBranch)
3188 if blob:
e63231e5 3189 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3190 else:
3191 if self.verbose:
3192 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3193 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3194 else:
3195 files = self.extractFilesFromCommit(description)
e63231e5 3196 self.commit(description, files, self.branch,
e87f37ae 3197 self.initialParent)
47497844 3198 # only needed once, to connect to the previous commit
e87f37ae
SH
3199 self.initialParent = ""
3200 except IOError:
3201 print self.gitError.read()
3202 sys.exit(1)
3203
c208a243
SH
3204 def importHeadRevision(self, revision):
3205 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3206
4e2e6ce4
PW
3207 details = {}
3208 details["user"] = "git perforce import user"
1494fcbb 3209 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3210 % (' '.join(self.depotPaths), revision))
3211 details["change"] = revision
3212 newestRevision = 0
3213
3214 fileCnt = 0
6de040df
LD
3215 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3216
3217 for info in p4CmdList(["files"] + fileArgs):
c208a243 3218
68b28593 3219 if 'code' in info and info['code'] == 'error':
c208a243
SH
3220 sys.stderr.write("p4 returned an error: %s\n"
3221 % info['data'])
d88e707f
PW
3222 if info['data'].find("must refer to client") >= 0:
3223 sys.stderr.write("This particular p4 error is misleading.\n")
3224 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3225 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3226 sys.exit(1)
68b28593
PW
3227 if 'p4ExitCode' in info:
3228 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3229 sys.exit(1)
3230
3231
3232 change = int(info["change"])
3233 if change > newestRevision:
3234 newestRevision = change
3235
56c09345 3236 if info["action"] in self.delete_actions:
c208a243
SH
3237 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3238 #fileCnt = fileCnt + 1
3239 continue
3240
3241 for prop in ["depotFile", "rev", "action", "type" ]:
3242 details["%s%s" % (prop, fileCnt)] = info[prop]
3243
3244 fileCnt = fileCnt + 1
3245
3246 details["change"] = newestRevision
4e2e6ce4 3247
9dcb9f24 3248 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3249 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3250 res = p4_describe(newestRevision)
3251 details["time"] = res["time"]
4e2e6ce4 3252
c208a243
SH
3253 self.updateOptionDict(details)
3254 try:
e63231e5 3255 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3256 except IOError:
3257 print "IO error with git fast-import. Is your git version recent enough?"
3258 print self.gitError.read()
3259
3260
b984733c 3261 def run(self, args):
6326aa58 3262 self.depotPaths = []
179caebf 3263 self.changeRange = ""
6326aa58 3264 self.previousDepotPaths = []
991a2de4 3265 self.hasOrigin = False
ce6f33c8 3266
29bdbac1
SH
3267 # map from branch depot path to parent branch
3268 self.knownBranches = {}
3269 self.initialParents = {}
3270
a028a98e
SH
3271 if self.importIntoRemotes:
3272 self.refPrefix = "refs/remotes/p4/"
3273 else:
db775559 3274 self.refPrefix = "refs/heads/p4/"
a028a98e 3275
991a2de4
PW
3276 if self.syncWithOrigin:
3277 self.hasOrigin = originP4BranchesExist()
3278 if self.hasOrigin:
3279 if not self.silent:
3280 print 'Syncing with origin first, using "git fetch origin"'
3281 system("git fetch origin")
10f880f8 3282
5a8e84cd 3283 branch_arg_given = bool(self.branch)
569d1bd4 3284 if len(self.branch) == 0:
db775559 3285 self.branch = self.refPrefix + "master"
a028a98e 3286 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3287 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3288 system("git branch -D p4")
967f72e2 3289
a93d33ee
PW
3290 # accept either the command-line option, or the configuration variable
3291 if self.useClientSpec:
3292 # will use this after clone to set the variable
3293 self.useClientSpec_from_options = True
3294 else:
0d609032 3295 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3296 self.useClientSpec = True
3297 if self.useClientSpec:
543987bd 3298 self.clientSpecDirs = getClientSpec()
3a70cdfa 3299
6a49f8e2
HWN
3300 # TODO: should always look at previous commits,
3301 # merge with previous imports, if possible.
3302 if args == []:
d414c74a 3303 if self.hasOrigin:
5ca44617 3304 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3305
3306 # branches holds mapping from branch name to sha1
3307 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3308
3309 # restrict to just this one, disabling detect-branches
3310 if branch_arg_given:
3311 short = self.branch.split("/")[-1]
3312 if short in branches:
3313 self.p4BranchesInGit = [ short ]
3314 else:
3315 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3316
3317 if len(self.p4BranchesInGit) > 1:
3318 if not self.silent:
3319 print "Importing from/into multiple branches"
3320 self.detectBranches = True
8c9e8b6e
PW
3321 for branch in branches.keys():
3322 self.initialParents[self.refPrefix + branch] = \
3323 branches[branch]
967f72e2 3324
29bdbac1
SH
3325 if self.verbose:
3326 print "branches: %s" % self.p4BranchesInGit
3327
3328 p4Change = 0
3329 for branch in self.p4BranchesInGit:
cebdf5af 3330 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3331
3332 settings = extractSettingsGitLog(logMsg)
29bdbac1 3333
bb6e09b2
HWN
3334 self.readOptions(settings)
3335 if (settings.has_key('depot-paths')
3336 and settings.has_key ('change')):
3337 change = int(settings['change']) + 1
29bdbac1
SH
3338 p4Change = max(p4Change, change)
3339
bb6e09b2
HWN
3340 depotPaths = sorted(settings['depot-paths'])
3341 if self.previousDepotPaths == []:
6326aa58 3342 self.previousDepotPaths = depotPaths
29bdbac1 3343 else:
6326aa58
HWN
3344 paths = []
3345 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3346 prev_list = prev.split("/")
3347 cur_list = cur.split("/")
3348 for i in range(0, min(len(cur_list), len(prev_list))):
3349 if cur_list[i] <> prev_list[i]:
583e1707 3350 i = i - 1
6326aa58
HWN
3351 break
3352
04d277b3 3353 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3354
3355 self.previousDepotPaths = paths
29bdbac1
SH
3356
3357 if p4Change > 0:
bb6e09b2 3358 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3359 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3360 if not self.silent and not self.detectBranches:
967f72e2 3361 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3362
40d69ac3
PW
3363 # accept multiple ref name abbreviations:
3364 # refs/foo/bar/branch -> use it exactly
3365 # p4/branch -> prepend refs/remotes/ or refs/heads/
3366 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3367 if not self.branch.startswith("refs/"):
40d69ac3
PW
3368 if self.importIntoRemotes:
3369 prepend = "refs/remotes/"
3370 else:
3371 prepend = "refs/heads/"
3372 if not self.branch.startswith("p4/"):
3373 prepend += "p4/"
3374 self.branch = prepend + self.branch
179caebf 3375
6326aa58 3376 if len(args) == 0 and self.depotPaths:
b984733c 3377 if not self.silent:
6326aa58 3378 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3379 else:
6326aa58 3380 if self.depotPaths and self.depotPaths != args:
cebdf5af 3381 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3382 "This doesn't work!" % (' '.join (self.depotPaths),
3383 ' '.join (args)))
b984733c 3384 sys.exit(1)
6326aa58 3385
bb6e09b2 3386 self.depotPaths = sorted(args)
b984733c 3387
1c49fc19 3388 revision = ""
b984733c 3389 self.users = {}
b984733c 3390
58c8bc7c
PW
3391 # Make sure no revision specifiers are used when --changesfile
3392 # is specified.
3393 bad_changesfile = False
3394 if len(self.changesFile) > 0:
3395 for p in self.depotPaths:
3396 if p.find("@") >= 0 or p.find("#") >= 0:
3397 bad_changesfile = True
3398 break
3399 if bad_changesfile:
3400 die("Option --changesfile is incompatible with revision specifiers")
3401
6326aa58
HWN
3402 newPaths = []
3403 for p in self.depotPaths:
3404 if p.find("@") != -1:
3405 atIdx = p.index("@")
3406 self.changeRange = p[atIdx:]
3407 if self.changeRange == "@all":
3408 self.changeRange = ""
6a49f8e2 3409 elif ',' not in self.changeRange:
1c49fc19 3410 revision = self.changeRange
6326aa58 3411 self.changeRange = ""
7fcff9de 3412 p = p[:atIdx]
6326aa58
HWN
3413 elif p.find("#") != -1:
3414 hashIdx = p.index("#")
1c49fc19 3415 revision = p[hashIdx:]
7fcff9de 3416 p = p[:hashIdx]
6326aa58 3417 elif self.previousDepotPaths == []:
58c8bc7c
PW
3418 # pay attention to changesfile, if given, else import
3419 # the entire p4 tree at the head revision
3420 if len(self.changesFile) == 0:
3421 revision = "#head"
6326aa58
HWN
3422
3423 p = re.sub ("\.\.\.$", "", p)
3424 if not p.endswith("/"):
3425 p += "/"
3426
3427 newPaths.append(p)
3428
3429 self.depotPaths = newPaths
3430
e63231e5
PW
3431 # --detect-branches may change this for each branch
3432 self.branchPrefixes = self.depotPaths
3433
b607e71e 3434 self.loadUserMapFromCache()
cb53e1f8
SH
3435 self.labels = {}
3436 if self.detectLabels:
3437 self.getLabels();
b984733c 3438
4b97ffb1 3439 if self.detectBranches:
df450923
SH
3440 ## FIXME - what's a P4 projectName ?
3441 self.projectName = self.guessProjectName()
3442
38f9f5ec
SH
3443 if self.hasOrigin:
3444 self.getBranchMappingFromGitBranches()
3445 else:
3446 self.getBranchMapping()
29bdbac1
SH
3447 if self.verbose:
3448 print "p4-git branches: %s" % self.p4BranchesInGit
3449 print "initial parents: %s" % self.initialParents