Fourth batch for 2.17
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
a235e85c
BC
30try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
4addad22 45verbose = False
86949eef 46
06804c76 47# Only labels/tags matching this will be imported/exported
c8942a22 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 49
1051ef00
LD
50# Grab changes in blocks of this many revisions, unless otherwise requested
51defaultBlockSize = 512
52
21a50753
AK
53def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
6de040df 60 real_cmd = ["p4"]
abcaf073
AK
61
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
6de040df 64 real_cmd += ["-u",user]
abcaf073
AK
65
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
6de040df 68 real_cmd += ["-P", password]
abcaf073
AK
69
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
6de040df 72 real_cmd += ["-p", port]
abcaf073
AK
73
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
41799aa2 76 real_cmd += ["-H", host]
abcaf073
AK
77
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
6de040df 80 real_cmd += ["-c", client]
abcaf073 81
89a6ecc5
LS
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
bc233524
IK
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
6de040df
LD
89
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
21a50753
AK
94 return real_cmd
95
378f7be1
LD
96def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
bbd84863
MF
106def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
053fd0c1 127
4d25dc44
LS
128def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
86dff6b6
HWN
138def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
6de040df 145def write_pipe(c, stdin):
4addad22 146 if verbose:
6de040df 147 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 148
6de040df
LD
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
b016d397
HWN
156
157 return val
158
6de040df 159def p4_write_pipe(c, stdin):
d9429194 160 real_cmd = p4_build_cmd(c)
6de040df 161 return write_pipe(real_cmd, stdin)
d9429194 162
78871bf4
LD
163def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
167 """
4addad22 168 if verbose:
6de040df 169 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 170
6de040df 171 expand = isinstance(c,basestring)
1f5f3907
LS
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
78871bf4
LD
174 return (p.returncode, out, err)
175
176def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
180 """
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 187 return out
b016d397 188
78871bf4
LD
189def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
192 """
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
198
d9429194
AK
199def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
b016d397 202
bce4c5fc 203def read_pipe_lines(c):
4addad22 204 if verbose:
6de040df
LD
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
b016d397 210 val = pipe.readlines()
6de040df
LD
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
b016d397
HWN
213
214 return val
caace111 215
2318121b
AK
216def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
155af834 218 real_cmd = p4_build_cmd(c)
2318121b
AK
219 return read_pipe_lines(real_cmd)
220
8e9497c2
GG
221def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
229
249da4c0
PW
230def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
235
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
248
cbff4b25 249def system(cmd, ignore_error=False):
6de040df 250 expand = isinstance(cmd,basestring)
4addad22 251 if verbose:
6de040df 252 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 253 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 254 if retcode and not ignore_error:
a235e85c 255 raise CalledProcessError(retcode, cmd)
6754a299 256
cbff4b25
LD
257 return retcode
258
bf9320f1
AK
259def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
155af834 261 real_cmd = p4_build_cmd(cmd)
6de040df 262 expand = isinstance(real_cmd, basestring)
a235e85c
BC
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
6de040df 266
7f0e5962
PW
267_p4_version_string = None
268def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
271
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276 """
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
282
6de040df 283def p4_integrate(src, dest):
9d7d446a 284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 285
8d7ec362 286def p4_sync(f, *options):
9d7d446a 287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
288
289def p4_add(f):
9d7d446a
PW
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
6de040df
LD
295
296def p4_delete(f):
9d7d446a 297 p4_system(["delete", wildcard_encode(f)])
6de040df 298
a02b8bc4
RP
299def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
301
302def p4_revert(f):
9d7d446a 303 p4_system(["revert", wildcard_encode(f)])
6de040df 304
9d7d446a
PW
305def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 307
46c609e9
LD
308def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
311
8e9497c2
GG
312def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
1051ef00 315def p4_last_change():
1997e91f 316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
317 return int(results[0]['change'])
318
18fa13d0
PW
319def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
323
1997e91f 324 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
18fa13d0
PW
325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328 d = ds[0]
329
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340 return d
341
9cffb8c8
PW
342#
343# Canonicalize the p4 type and return a tuple of the
344# base type, plus any modifiers. See "p4 help filetypes"
345# for a list and explanation.
346#
347def split_p4_type(p4type):
348
349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
366 }
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
b9fc6ea9 376
60df071c
LD
377#
378# return the raw p4 type of a file (text, text+ko, etc)
379#
79467e61
PW
380def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
382 return results[0]['headType']
383
384#
385# Given a type base and modifier, return a regexp matching
386# the keywords that can be expanded in the file
387#
388def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
6b2bf41e 400 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
406
407#
408# Given a file, return a regexp matching the possible
409# RCS keywords that will be expanded, or None for files
410# with kw expansion turned off.
411#
412def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 418
c65b670e
CP
419def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
422
423 p4Type = "+x"
424
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
431
6de040df 432 p4_reopen(p4Type, file)
c65b670e
CP
433
434def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
436
9d7d446a 437 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
439 if match:
440 return match.group(1)
441 else:
f3e5ae4f 442 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 443
06804c76
LD
444# Return the set of all p4 labels
445def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
449
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
453
454 return labels
455
456# Return the set of all git tags
457def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
463
b43b0a3c
CP
464def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
470
471def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
473
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
476
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
487
488 If the pattern is not matched, None is returned."""
489
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
501 }
502 return None
503
c65b670e
CP
504def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
508
509def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
511
1997e91f 512def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
6de040df
LD
513
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
520
521 cmd = p4_build_cmd(cmd)
6a49f8e2 522 if verbose:
6de040df 523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
524
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
9f90c733
SL
536 stdin_file.flush()
537 stdin_file.seek(0)
538
6de040df
LD
539 p4 = subprocess.Popen(cmd,
540 shell=expand,
9f90c733
SL
541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
86949eef
SH
543
544 result = []
545 try:
546 while True:
9f90c733 547 entry = marshal.load(p4.stdout)
1997e91f
MT
548 if skip_info:
549 if 'code' in entry and entry['code'] == 'info':
550 continue
c3f6163b
AG
551 if cb is not None:
552 cb(entry)
553 else:
554 result.append(entry)
86949eef
SH
555 except EOFError:
556 pass
9f90c733
SL
557 exitCode = p4.wait()
558 if exitCode != 0:
ac3e0d79
SH
559 entry = {}
560 entry["p4ExitCode"] = exitCode
561 result.append(entry)
86949eef
SH
562
563 return result
564
565def p4Cmd(cmd):
566 list = p4CmdList(cmd)
567 result = {}
568 for entry in list:
569 result.update(entry)
570 return result;
571
cb2c9db5
SH
572def p4Where(depotPath):
573 if not depotPath.endswith("/"):
574 depotPath += "/"
cd884106
VA
575 depotPathLong = depotPath + "..."
576 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
577 output = None
578 for entry in outputList:
75bc9573 579 if "depotFile" in entry:
cd884106
VA
580 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
581 # The base path always ends with "/...".
582 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
583 output = entry
584 break
585 elif "data" in entry:
586 data = entry.get("data")
587 space = data.find(" ")
588 if data[:space] == depotPath:
589 output = entry
590 break
7f705dc3
TAL
591 if output == None:
592 return ""
dc524036
SH
593 if output["code"] == "error":
594 return ""
cb2c9db5
SH
595 clientPath = ""
596 if "path" in output:
597 clientPath = output.get("path")
598 elif "data" in output:
599 data = output.get("data")
600 lastSpace = data.rfind(" ")
601 clientPath = data[lastSpace + 1:]
602
603 if clientPath.endswith("..."):
604 clientPath = clientPath[:-3]
605 return clientPath
606
86949eef 607def currentGitBranch():
eff45110 608 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 609
4f5cf76a 610def isValidGitDir(path):
378f7be1 611 return git_dir(path) != None
4f5cf76a 612
463e8af6 613def parseRevision(ref):
b25b2065 614 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 615
28755dba
PW
616def branchExists(ref):
617 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
618 ignore_error=True)
619 return len(rev) > 0
620
6ae8de88
SH
621def extractLogMessageFromGitCommit(commit):
622 logMessage = ""
b016d397
HWN
623
624 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 625 foundTitle = False
b016d397 626 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
627 if not foundTitle:
628 if len(log) == 1:
1c094184 629 foundTitle = True
6ae8de88
SH
630 continue
631
632 logMessage += log
633 return logMessage
634
bb6e09b2 635def extractSettingsGitLog(log):
6ae8de88
SH
636 values = {}
637 for line in log.split("\n"):
638 line = line.strip()
6326aa58
HWN
639 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
640 if not m:
641 continue
642
643 assignments = m.group(1).split (':')
644 for a in assignments:
645 vals = a.split ('=')
646 key = vals[0].strip()
647 val = ('='.join (vals[1:])).strip()
648 if val.endswith ('\"') and val.startswith('"'):
649 val = val[1:-1]
650
651 values[key] = val
652
845b42cb
SH
653 paths = values.get("depot-paths")
654 if not paths:
655 paths = values.get("depot-path")
a3fdd579
SH
656 if paths:
657 values['depot-paths'] = paths.split(',')
bb6e09b2 658 return values
6ae8de88 659
8136a639 660def gitBranchExists(branch):
bb6e09b2
HWN
661 proc = subprocess.Popen(["git", "rev-parse", branch],
662 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 663 return proc.wait() == 0;
8136a639 664
36bd8446 665_gitConfig = {}
b345d6c3 666
692e1796 667def gitConfig(key, typeSpecifier=None):
36bd8446 668 if not _gitConfig.has_key(key):
692e1796
LS
669 cmd = [ "git", "config" ]
670 if typeSpecifier:
671 cmd += [ typeSpecifier ]
672 cmd += [ key ]
b345d6c3
PW
673 s = read_pipe(cmd, ignore_error=True)
674 _gitConfig[key] = s.strip()
36bd8446 675 return _gitConfig[key]
01265103 676
0d609032
PW
677def gitConfigBool(key):
678 """Return a bool, using git config --bool. It is True only if the
679 variable is set to true, and False if set to false or not present
680 in the config."""
681
36bd8446 682 if not _gitConfig.has_key(key):
692e1796 683 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 684 return _gitConfig[key]
01265103 685
cb1dafdf
LS
686def gitConfigInt(key):
687 if not _gitConfig.has_key(key):
688 cmd = [ "git", "config", "--int", key ]
0d609032
PW
689 s = read_pipe(cmd, ignore_error=True)
690 v = s.strip()
cb1dafdf
LS
691 try:
692 _gitConfig[key] = int(gitConfig(key, '--int'))
693 except ValueError:
694 _gitConfig[key] = None
36bd8446 695 return _gitConfig[key]
01265103 696
7199cf13
VA
697def gitConfigList(key):
698 if not _gitConfig.has_key(key):
2abba301 699 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 700 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
701 if _gitConfig[key] == ['']:
702 _gitConfig[key] = []
7199cf13
VA
703 return _gitConfig[key]
704
2c8037ed
PW
705def p4BranchesInGit(branchesAreInRemotes=True):
706 """Find all the branches whose names start with "p4/", looking
707 in remotes or heads as specified by the argument. Return
708 a dictionary of { branch: revision } for each one found.
709 The branch names are the short names, without any
710 "p4/" prefix."""
711
062410bb
SH
712 branches = {}
713
714 cmdline = "git rev-parse --symbolic "
715 if branchesAreInRemotes:
2c8037ed 716 cmdline += "--remotes"
062410bb 717 else:
2c8037ed 718 cmdline += "--branches"
062410bb
SH
719
720 for line in read_pipe_lines(cmdline):
721 line = line.strip()
722
2c8037ed
PW
723 # only import to p4/
724 if not line.startswith('p4/'):
725 continue
726 # special symbolic ref to p4/master
727 if line == "p4/HEAD":
062410bb 728 continue
062410bb 729
2c8037ed
PW
730 # strip off p4/ prefix
731 branch = line[len("p4/"):]
062410bb
SH
732
733 branches[branch] = parseRevision(line)
2c8037ed 734
062410bb
SH
735 return branches
736
5a8e84cd
PW
737def branch_exists(branch):
738 """Make sure that the given ref name really exists."""
739
740 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
741 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
742 out, _ = p.communicate()
743 if p.returncode:
744 return False
745 # expect exactly one line of output: the branch name
746 return out.rstrip() == branch
747
9ceab363 748def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
749 branches = p4BranchesInGit()
750 # map from depot-path to branch name
751 branchByDepotPath = {}
752 for branch in branches.keys():
753 tip = branches[branch]
754 log = extractLogMessageFromGitCommit(tip)
755 settings = extractSettingsGitLog(log)
756 if settings.has_key("depot-paths"):
757 paths = ",".join(settings["depot-paths"])
758 branchByDepotPath[paths] = "remotes/p4/" + branch
759
27d2d811 760 settings = None
27d2d811
SH
761 parent = 0
762 while parent < 65535:
9ceab363 763 commit = head + "~%s" % parent
27d2d811
SH
764 log = extractLogMessageFromGitCommit(commit)
765 settings = extractSettingsGitLog(log)
86506fe5
SH
766 if settings.has_key("depot-paths"):
767 paths = ",".join(settings["depot-paths"])
768 if branchByDepotPath.has_key(paths):
769 return [branchByDepotPath[paths], settings]
27d2d811 770
86506fe5 771 parent = parent + 1
27d2d811 772
86506fe5 773 return ["", settings]
27d2d811 774
5ca44617
SH
775def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
776 if not silent:
777 print ("Creating/updating branch(es) in %s based on origin branch(es)"
778 % localRefPrefix)
779
780 originPrefix = "origin/p4/"
781
782 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
783 line = line.strip()
784 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
785 continue
786
787 headName = line[len(originPrefix):]
788 remoteHead = localRefPrefix + headName
789 originHead = line
790
791 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
792 if (not original.has_key('depot-paths')
793 or not original.has_key('change')):
794 continue
795
796 update = False
797 if not gitBranchExists(remoteHead):
798 if verbose:
799 print "creating %s" % remoteHead
800 update = True
801 else:
802 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
803 if settings.has_key('change') > 0:
804 if settings['depot-paths'] == original['depot-paths']:
805 originP4Change = int(original['change'])
806 p4Change = int(settings['change'])
807 if originP4Change > p4Change:
808 print ("%s (%s) is newer than %s (%s). "
809 "Updating p4 branch from origin."
810 % (originHead, originP4Change,
811 remoteHead, p4Change))
812 update = True
813 else:
814 print ("Ignoring: %s was imported from %s while "
815 "%s was imported from %s"
816 % (originHead, ','.join(original['depot-paths']),
817 remoteHead, ','.join(settings['depot-paths'])))
818
819 if update:
820 system("git update-ref %s %s" % (remoteHead, originHead))
821
822def originP4BranchesExist():
823 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
824
1051ef00
LD
825
826def p4ParseNumericChangeRange(parts):
827 changeStart = int(parts[0][1:])
828 if parts[1] == '#head':
829 changeEnd = p4_last_change()
830 else:
831 changeEnd = int(parts[1])
832
833 return (changeStart, changeEnd)
834
835def chooseBlockSize(blockSize):
836 if blockSize:
837 return blockSize
838 else:
839 return defaultBlockSize
840
841def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 842 assert depotPaths
96b2d54a 843
1051ef00
LD
844 # Parse the change range into start and end. Try to find integer
845 # revision ranges as these can be broken up into blocks to avoid
846 # hitting server-side limits (maxrows, maxscanresults). But if
847 # that doesn't work, fall back to using the raw revision specifier
848 # strings, without using block mode.
849
96b2d54a 850 if changeRange is None or changeRange == '':
1051ef00
LD
851 changeStart = 1
852 changeEnd = p4_last_change()
853 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
854 else:
855 parts = changeRange.split(',')
856 assert len(parts) == 2
1051ef00
LD
857 try:
858 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
859 block_size = chooseBlockSize(requestedBlockSize)
860 except:
861 changeStart = parts[0][1:]
862 changeEnd = parts[1]
863 if requestedBlockSize:
864 die("cannot use --changes-block-size with non-numeric revisions")
865 block_size = None
4f6432d8 866
9943e5b9 867 changes = set()
96b2d54a 868
1f90a648
SH
869 # Retrieve changes a block at a time, to prevent running
870 # into a MaxResults/MaxScanRows error from the server.
1051ef00 871
1f90a648
SH
872 while True:
873 cmd = ['changes']
1051ef00 874
1f90a648
SH
875 if block_size:
876 end = min(changeEnd, changeStart + block_size)
877 revisionRange = "%d,%d" % (changeStart, end)
878 else:
879 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 880
1f90a648 881 for p in depotPaths:
1051ef00
LD
882 cmd += ["%s...@%s" % (p, revisionRange)]
883
1f90a648 884 # Insert changes in chronological order
b596b3b9
MT
885 for entry in reversed(p4CmdList(cmd)):
886 if entry.has_key('p4ExitCode'):
887 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
888 if not entry.has_key('change'):
889 continue
890 changes.add(int(entry['change']))
1051ef00 891
1f90a648
SH
892 if not block_size:
893 break
1051ef00 894
1f90a648
SH
895 if end >= changeEnd:
896 break
1051ef00 897
1f90a648 898 changeStart = end + 1
4f6432d8 899
1f90a648
SH
900 changes = sorted(changes)
901 return changes
4f6432d8 902
d53de8b9
TAL
903def p4PathStartsWith(path, prefix):
904 # This method tries to remedy a potential mixed-case issue:
905 #
906 # If UserA adds //depot/DirA/file1
907 # and UserB adds //depot/dira/file2
908 #
909 # we may or may not have a problem. If you have core.ignorecase=true,
910 # we treat DirA and dira as the same directory
0d609032 911 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
912 return path.lower().startswith(prefix.lower())
913 return path.startswith(prefix)
914
543987bd
PW
915def getClientSpec():
916 """Look at the p4 client spec, create a View() object that contains
917 all the mappings, and return it."""
918
919 specList = p4CmdList("client -o")
920 if len(specList) != 1:
921 die('Output from "client -o" is %d lines, expecting 1' %
922 len(specList))
923
924 # dictionary of all client parameters
925 entry = specList[0]
926
9d57c4a6
KS
927 # the //client/ name
928 client_name = entry["Client"]
929
543987bd
PW
930 # just the keys that start with "View"
931 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
932
933 # hold this new View
9d57c4a6 934 view = View(client_name)
543987bd
PW
935
936 # append the lines, in order, to the view
937 for view_num in range(len(view_keys)):
938 k = "View%d" % view_num
939 if k not in view_keys:
940 die("Expected view key %s missing" % k)
941 view.append(entry[k])
942
943 return view
944
945def getClientRoot():
946 """Grab the client directory."""
947
948 output = p4CmdList("client -o")
949 if len(output) != 1:
950 die('Output from "client -o" is %d lines, expecting 1' % len(output))
951
952 entry = output[0]
953 if "Root" not in entry:
954 die('Client has no "Root"')
955
956 return entry["Root"]
957
9d7d446a
PW
958#
959# P4 wildcards are not allowed in filenames. P4 complains
960# if you simply add them, but you can force it with "-f", in
961# which case it translates them into %xx encoding internally.
962#
963def wildcard_decode(path):
964 # Search for and fix just these four characters. Do % last so
965 # that fixing it does not inadvertently create new %-escapes.
966 # Cannot have * in a filename in windows; untested as to
967 # what p4 would do in such a case.
968 if not platform.system() == "Windows":
969 path = path.replace("%2A", "*")
970 path = path.replace("%23", "#") \
971 .replace("%40", "@") \
972 .replace("%25", "%")
973 return path
974
975def wildcard_encode(path):
976 # do % first to avoid double-encoding the %s introduced here
977 path = path.replace("%", "%25") \
978 .replace("*", "%2A") \
979 .replace("#", "%23") \
980 .replace("@", "%40")
981 return path
982
983def wildcard_present(path):
598354c0
BC
984 m = re.search("[*#@%]", path)
985 return m is not None
9d7d446a 986
a5db4b12
LS
987class LargeFileSystem(object):
988 """Base class for large file system support."""
989
990 def __init__(self, writeToGitStream):
991 self.largeFiles = set()
992 self.writeToGitStream = writeToGitStream
993
994 def generatePointer(self, cloneDestination, contentFile):
995 """Return the content of a pointer file that is stored in Git instead of
996 the actual content."""
997 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
998
999 def pushFile(self, localLargeFile):
1000 """Push the actual content which is not stored in the Git repository to
1001 a server."""
1002 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1003
1004 def hasLargeFileExtension(self, relPath):
1005 return reduce(
1006 lambda a, b: a or b,
1007 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1008 False
1009 )
1010
1011 def generateTempFile(self, contents):
1012 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1013 for d in contents:
1014 contentFile.write(d)
1015 contentFile.close()
1016 return contentFile.name
1017
1018 def exceedsLargeFileThreshold(self, relPath, contents):
1019 if gitConfigInt('git-p4.largeFileThreshold'):
1020 contentsSize = sum(len(d) for d in contents)
1021 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1022 return True
1023 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024 contentsSize = sum(len(d) for d in contents)
1025 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1026 return False
1027 contentTempFile = self.generateTempFile(contents)
1028 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1029 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1030 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1031 zf.close()
1032 compressedContentsSize = zf.infolist()[0].compress_size
1033 os.remove(contentTempFile)
1034 os.remove(compressedContentFile.name)
1035 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1036 return True
1037 return False
1038
1039 def addLargeFile(self, relPath):
1040 self.largeFiles.add(relPath)
1041
1042 def removeLargeFile(self, relPath):
1043 self.largeFiles.remove(relPath)
1044
1045 def isLargeFile(self, relPath):
1046 return relPath in self.largeFiles
1047
1048 def processContent(self, git_mode, relPath, contents):
1049 """Processes the content of git fast import. This method decides if a
1050 file is stored in the large file system and handles all necessary
1051 steps."""
1052 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1053 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1054 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1055 if pointer_git_mode:
1056 git_mode = pointer_git_mode
1057 if localLargeFile:
1058 # Move temp file to final location in large file system
1059 largeFileDir = os.path.dirname(localLargeFile)
1060 if not os.path.isdir(largeFileDir):
1061 os.makedirs(largeFileDir)
1062 shutil.move(contentTempFile, localLargeFile)
1063 self.addLargeFile(relPath)
1064 if gitConfigBool('git-p4.largeFilePush'):
1065 self.pushFile(localLargeFile)
1066 if verbose:
1067 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1068 return (git_mode, contents)
1069
1070class MockLFS(LargeFileSystem):
1071 """Mock large file system for testing."""
1072
1073 def generatePointer(self, contentFile):
1074 """The pointer content is the original content prefixed with "pointer-".
1075 The local filename of the large file storage is derived from the file content.
1076 """
1077 with open(contentFile, 'r') as f:
1078 content = next(f)
1079 gitMode = '100644'
1080 pointerContents = 'pointer-' + content
1081 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1082 return (gitMode, pointerContents, localLargeFile)
1083
1084 def pushFile(self, localLargeFile):
1085 """The remote filename of the large file storage is the same as the local
1086 one but in a different directory.
1087 """
1088 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1089 if not os.path.exists(remotePath):
1090 os.makedirs(remotePath)
1091 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1092
b47d807d
LS
1093class GitLFS(LargeFileSystem):
1094 """Git LFS as backend for the git-p4 large file system.
1095 See https://git-lfs.github.com/ for details."""
1096
1097 def __init__(self, *args):
1098 LargeFileSystem.__init__(self, *args)
1099 self.baseGitAttributes = []
1100
1101 def generatePointer(self, contentFile):
1102 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1103 mode and content which is stored in the Git repository instead of
1104 the actual content. Return also the new location of the actual
1105 content.
1106 """
d5eb3cf5
LS
1107 if os.path.getsize(contentFile) == 0:
1108 return (None, '', None)
1109
b47d807d
LS
1110 pointerProcess = subprocess.Popen(
1111 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1112 stdout=subprocess.PIPE
1113 )
1114 pointerFile = pointerProcess.stdout.read()
1115 if pointerProcess.wait():
1116 os.remove(contentFile)
1117 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1118
1119 # Git LFS removed the preamble in the output of the 'pointer' command
1120 # starting from version 1.2.0. Check for the preamble here to support
1121 # earlier versions.
1122 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1123 if pointerFile.startswith('Git LFS pointer for'):
1124 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1125
1126 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1127 localLargeFile = os.path.join(
1128 os.getcwd(),
1129 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1130 oid,
1131 )
1132 # LFS Spec states that pointer files should not have the executable bit set.
1133 gitMode = '100644'
82f2567e 1134 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1135
1136 def pushFile(self, localLargeFile):
1137 uploadProcess = subprocess.Popen(
1138 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1139 )
1140 if uploadProcess.wait():
1141 die('git-lfs push command failed. Did you define a remote?')
1142
1143 def generateGitAttributes(self):
1144 return (
1145 self.baseGitAttributes +
1146 [
1147 '\n',
1148 '#\n',
1149 '# Git LFS (see https://git-lfs.github.com/)\n',
1150 '#\n',
1151 ] +
862f9312 1152 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1153 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1154 ] +
862f9312 1155 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1156 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1157 ]
1158 )
1159
1160 def addLargeFile(self, relPath):
1161 LargeFileSystem.addLargeFile(self, relPath)
1162 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1163
1164 def removeLargeFile(self, relPath):
1165 LargeFileSystem.removeLargeFile(self, relPath)
1166 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1167
1168 def processContent(self, git_mode, relPath, contents):
1169 if relPath == '.gitattributes':
1170 self.baseGitAttributes = contents
1171 return (git_mode, self.generateGitAttributes())
1172 else:
1173 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1174
b984733c
SH
1175class Command:
1176 def __init__(self):
1177 self.usage = "usage: %prog [options]"
8910ac0e 1178 self.needsGit = True
6a10b6aa 1179 self.verbose = False
b984733c 1180
8cf422db
LD
1181 # This is required for the "append" cloneExclude action
1182 def ensure_value(self, attr, value):
1183 if not hasattr(self, attr) or getattr(self, attr) is None:
1184 setattr(self, attr, value)
1185 return getattr(self, attr)
1186
3ea2cfd4
LD
1187class P4UserMap:
1188 def __init__(self):
1189 self.userMapFromPerforceServer = False
affb474f
LD
1190 self.myP4UserId = None
1191
1192 def p4UserId(self):
1193 if self.myP4UserId:
1194 return self.myP4UserId
1195
1196 results = p4CmdList("user -o")
1197 for r in results:
1198 if r.has_key('User'):
1199 self.myP4UserId = r['User']
1200 return r['User']
1201 die("Could not find your p4 user id")
1202
1203 def p4UserIsMe(self, p4User):
1204 # return True if the given p4 user is actually me
1205 me = self.p4UserId()
1206 if not p4User or p4User != me:
1207 return False
1208 else:
1209 return True
3ea2cfd4
LD
1210
1211 def getUserCacheFilename(self):
1212 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1213 return home + "/.gitp4-usercache.txt"
1214
1215 def getUserMapFromPerforceServer(self):
1216 if self.userMapFromPerforceServer:
1217 return
1218 self.users = {}
1219 self.emails = {}
1220
1221 for output in p4CmdList("users"):
1222 if not output.has_key("User"):
1223 continue
1224 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1225 self.emails[output["Email"]] = output["User"]
1226
10d08a14
LS
1227 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1228 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1229 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1230 if mapUser and len(mapUser[0]) == 3:
1231 user = mapUser[0][0]
1232 fullname = mapUser[0][1]
1233 email = mapUser[0][2]
1234 self.users[user] = fullname + " <" + email + ">"
1235 self.emails[email] = user
3ea2cfd4
LD
1236
1237 s = ''
1238 for (key, val) in self.users.items():
1239 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1240
1241 open(self.getUserCacheFilename(), "wb").write(s)
1242 self.userMapFromPerforceServer = True
1243
1244 def loadUserMapFromCache(self):
1245 self.users = {}
1246 self.userMapFromPerforceServer = False
1247 try:
1248 cache = open(self.getUserCacheFilename(), "rb")
1249 lines = cache.readlines()
1250 cache.close()
1251 for line in lines:
1252 entry = line.strip().split("\t")
1253 self.users[entry[0]] = entry[1]
1254 except IOError:
1255 self.getUserMapFromPerforceServer()
1256
b984733c 1257class P4Debug(Command):
86949eef 1258 def __init__(self):
6ae8de88 1259 Command.__init__(self)
6a10b6aa 1260 self.options = []
c8c39116 1261 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1262 self.needsGit = False
86949eef
SH
1263
1264 def run(self, args):
b1ce9447 1265 j = 0
6de040df 1266 for output in p4CmdList(args):
b1ce9447
HWN
1267 print 'Element: %d' % j
1268 j += 1
86949eef 1269 print output
b984733c 1270 return True
86949eef 1271
5834684d
SH
1272class P4RollBack(Command):
1273 def __init__(self):
1274 Command.__init__(self)
1275 self.options = [
0c66a783 1276 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1277 ]
1278 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1279 self.rollbackLocalBranches = False
5834684d
SH
1280
1281 def run(self, args):
1282 if len(args) != 1:
1283 return False
1284 maxChange = int(args[0])
0c66a783 1285
ad192f28 1286 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1287 die("Problems executing p4");
1288
0c66a783
SH
1289 if self.rollbackLocalBranches:
1290 refPrefix = "refs/heads/"
b016d397 1291 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1292 else:
1293 refPrefix = "refs/remotes/"
b016d397 1294 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1295
1296 for line in lines:
1297 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1298 line = line.strip()
1299 ref = refPrefix + line
5834684d 1300 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1301 settings = extractSettingsGitLog(log)
1302
1303 depotPaths = settings['depot-paths']
1304 change = settings['change']
1305
5834684d 1306 changed = False
52102d47 1307
6326aa58
HWN
1308 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1309 for p in depotPaths]))) == 0:
52102d47
SH
1310 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1311 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1312 continue
1313
bb6e09b2 1314 while change and int(change) > maxChange:
5834684d 1315 changed = True
52102d47
SH
1316 if self.verbose:
1317 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1318 system("git update-ref %s \"%s^\"" % (ref, ref))
1319 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1320 settings = extractSettingsGitLog(log)
1321
1322
1323 depotPaths = settings['depot-paths']
1324 change = settings['change']
5834684d
SH
1325
1326 if changed:
52102d47 1327 print "%s rewound to %s" % (ref, change)
5834684d
SH
1328
1329 return True
1330
3ea2cfd4 1331class P4Submit(Command, P4UserMap):
6bbfd137
PW
1332
1333 conflict_behavior_choices = ("ask", "skip", "quit")
1334
4f5cf76a 1335 def __init__(self):
b984733c 1336 Command.__init__(self)
3ea2cfd4 1337 P4UserMap.__init__(self)
4f5cf76a 1338 self.options = [
4f5cf76a 1339 optparse.make_option("--origin", dest="origin"),
ae901090 1340 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1341 # preserve the user, requires relevant p4 permissions
1342 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1343 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1344 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1345 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1346 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1347 choices=self.conflict_behavior_choices),
1348 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1349 optparse.make_option("--shelve", dest="shelve", action="store_true",
1350 help="Shelve instead of submit. Shelved files are reverted, "
1351 "restoring the workspace to the state before the shelve"),
8cf422db 1352 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1353 metavar="CHANGELIST",
8cf422db
LD
1354 help="update an existing shelved changelist, implies --shelve, "
1355 "repeat in-order for multiple shelved changelists")
4f5cf76a
SH
1356 ]
1357 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1358 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1359 self.origin = ""
ae901090 1360 self.detectRenames = False
0d609032 1361 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1362 self.dry_run = False
b34fa577 1363 self.shelve = False
8cf422db 1364 self.update_shelve = list()
728b7ad8 1365 self.prepare_p4_only = False
6bbfd137 1366 self.conflict_behavior = None
f7baba8b 1367 self.isWindows = (platform.system() == "Windows")
06804c76 1368 self.exportLabels = False
249da4c0 1369 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1370 self.branch = None
4f5cf76a 1371
a5db4b12
LS
1372 if gitConfig('git-p4.largeFileSystem'):
1373 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1374
4f5cf76a
SH
1375 def check(self):
1376 if len(p4CmdList("opened ...")) > 0:
1377 die("You have files opened with perforce! Close them before starting the sync.")
1378
f19cb0a0
PW
1379 def separate_jobs_from_description(self, message):
1380 """Extract and return a possible Jobs field in the commit
1381 message. It goes into a separate section in the p4 change
1382 specification.
1383
1384 A jobs line starts with "Jobs:" and looks like a new field
1385 in a form. Values are white-space separated on the same
1386 line or on following lines that start with a tab.
1387
1388 This does not parse and extract the full git commit message
1389 like a p4 form. It just sees the Jobs: line as a marker
1390 to pass everything from then on directly into the p4 form,
1391 but outside the description section.
1392
1393 Return a tuple (stripped log message, jobs string)."""
1394
1395 m = re.search(r'^Jobs:', message, re.MULTILINE)
1396 if m is None:
1397 return (message, None)
1398
1399 jobtext = message[m.start():]
1400 stripped_message = message[:m.start()].rstrip()
1401 return (stripped_message, jobtext)
1402
1403 def prepareLogMessage(self, template, message, jobs):
1404 """Edits the template returned from "p4 change -o" to insert
1405 the message in the Description field, and the jobs text in
1406 the Jobs field."""
4f5cf76a
SH
1407 result = ""
1408
edae1e2f
SH
1409 inDescriptionSection = False
1410
4f5cf76a
SH
1411 for line in template.split("\n"):
1412 if line.startswith("#"):
1413 result += line + "\n"
1414 continue
1415
edae1e2f 1416 if inDescriptionSection:
c9dbab04 1417 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1418 inDescriptionSection = False
f19cb0a0
PW
1419 # insert Jobs section
1420 if jobs:
1421 result += jobs + "\n"
edae1e2f
SH
1422 else:
1423 continue
1424 else:
1425 if line.startswith("Description:"):
1426 inDescriptionSection = True
1427 line += "\n"
1428 for messageLine in message.split("\n"):
1429 line += "\t" + messageLine + "\n"
1430
1431 result += line + "\n"
4f5cf76a
SH
1432
1433 return result
1434
60df071c
LD
1435 def patchRCSKeywords(self, file, pattern):
1436 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1437 (handle, outFileName) = tempfile.mkstemp(dir='.')
1438 try:
1439 outFile = os.fdopen(handle, "w+")
1440 inFile = open(file, "r")
1441 regexp = re.compile(pattern, re.VERBOSE)
1442 for line in inFile.readlines():
1443 line = regexp.sub(r'$\1$', line)
1444 outFile.write(line)
1445 inFile.close()
1446 outFile.close()
1447 # Forcibly overwrite the original file
1448 os.unlink(file)
1449 shutil.move(outFileName, file)
1450 except:
1451 # cleanup our temporary file
1452 os.unlink(outFileName)
1453 print "Failed to strip RCS keywords in %s" % file
1454 raise
1455
1456 print "Patched up RCS keywords in %s" % file
1457
3ea2cfd4
LD
1458 def p4UserForCommit(self,id):
1459 # Return the tuple (perforce user,git email) for a given git commit id
1460 self.getUserMapFromPerforceServer()
9bf28855
PW
1461 gitEmail = read_pipe(["git", "log", "--max-count=1",
1462 "--format=%ae", id])
3ea2cfd4
LD
1463 gitEmail = gitEmail.strip()
1464 if not self.emails.has_key(gitEmail):
1465 return (None,gitEmail)
1466 else:
1467 return (self.emails[gitEmail],gitEmail)
1468
1469 def checkValidP4Users(self,commits):
1470 # check if any git authors cannot be mapped to p4 users
1471 for id in commits:
1472 (user,email) = self.p4UserForCommit(id)
1473 if not user:
1474 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1475 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1476 print "%s" % msg
1477 else:
1478 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1479
1480 def lastP4Changelist(self):
1481 # Get back the last changelist number submitted in this client spec. This
1482 # then gets used to patch up the username in the change. If the same
1483 # client spec is being used by multiple processes then this might go
1484 # wrong.
1485 results = p4CmdList("client -o") # find the current client
1486 client = None
1487 for r in results:
1488 if r.has_key('Client'):
1489 client = r['Client']
1490 break
1491 if not client:
1492 die("could not get client spec")
6de040df 1493 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1494 for r in results:
1495 if r.has_key('change'):
1496 return r['change']
1497 die("Could not get changelist number for last submit - cannot patch up user details")
1498
1499 def modifyChangelistUser(self, changelist, newUser):
1500 # fixup the user field of a changelist after it has been submitted.
1501 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1502 if len(changes) != 1:
1503 die("Bad output from p4 change modifying %s to user %s" %
1504 (changelist, newUser))
1505
1506 c = changes[0]
1507 if c['User'] == newUser: return # nothing to do
1508 c['User'] = newUser
1509 input = marshal.dumps(c)
1510
3ea2cfd4
LD
1511 result = p4CmdList("change -f -i", stdin=input)
1512 for r in result:
1513 if r.has_key('code'):
1514 if r['code'] == 'error':
1515 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1516 if r.has_key('data'):
1517 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1518 return
1519 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1520
1521 def canChangeChangelists(self):
1522 # check to see if we have p4 admin or super-user permissions, either of
1523 # which are required to modify changelists.
52a4880b 1524 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1525 for r in results:
1526 if r.has_key('perm'):
1527 if r['perm'] == 'admin':
1528 return 1
1529 if r['perm'] == 'super':
1530 return 1
1531 return 0
1532
46c609e9 1533 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1534 """Run "p4 change -o" to grab a change specification template.
1535 This does not use "p4 -G", as it is nice to keep the submission
1536 template in original order, since a human might edit it.
1537
1538 Remove lines in the Files section that show changes to files
1539 outside the depot path we're committing into."""
1540
cbc69242
SH
1541 [upstream, settings] = findUpstreamBranchPoint()
1542
b596b3b9
MT
1543 template = """\
1544# A Perforce Change Specification.
1545#
1546# Change: The change number. 'new' on a new changelist.
1547# Date: The date this specification was last modified.
1548# Client: The client on which the changelist was created. Read-only.
1549# User: The user who created the changelist.
1550# Status: Either 'pending' or 'submitted'. Read-only.
1551# Type: Either 'public' or 'restricted'. Default is 'public'.
1552# Description: Comments about the changelist. Required.
1553# Jobs: What opened jobs are to be closed by this changelist.
1554# You may delete jobs from this list. (New changelists only.)
1555# Files: What opened files from the default changelist are to be added
1556# to this changelist. You may delete files from this list.
1557# (New changelists only.)
1558"""
1559 files_list = []
ea99c3ae 1560 inFilesSection = False
b596b3b9 1561 change_entry = None
46c609e9
LD
1562 args = ['change', '-o']
1563 if changelist:
1564 args.append(str(changelist))
b596b3b9
MT
1565 for entry in p4CmdList(args):
1566 if not entry.has_key('code'):
1567 continue
1568 if entry['code'] == 'stat':
1569 change_entry = entry
1570 break
1571 if not change_entry:
1572 die('Failed to decode output of p4 change -o')
1573 for key, value in change_entry.iteritems():
1574 if key.startswith('File'):
1575 if settings.has_key('depot-paths'):
1576 if not [p for p in settings['depot-paths']
1577 if p4PathStartsWith(value, p)]:
1578 continue
ea99c3ae 1579 else:
b596b3b9
MT
1580 if not p4PathStartsWith(value, self.depotPath):
1581 continue
1582 files_list.append(value)
1583 continue
1584 # Output in the order expected by prepareLogMessage
1585 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1586 if not change_entry.has_key(key):
1587 continue
1588 template += '\n'
1589 template += key + ':'
1590 if key == 'Description':
1591 template += '\n'
1592 for field_line in change_entry[key].splitlines():
1593 template += '\t'+field_line+'\n'
1594 if len(files_list) > 0:
1595 template += '\n'
1596 template += 'Files:\n'
1597 for path in files_list:
1598 template += '\t'+path+'\n'
ea99c3ae
SH
1599 return template
1600
7c766e57
PW
1601 def edit_template(self, template_file):
1602 """Invoke the editor to let the user change the submission
1603 message. Return true if okay to continue with the submit."""
1604
1605 # if configured to skip the editing part, just submit
0d609032 1606 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1607 return True
1608
1609 # look at the modification time, to check later if the user saved
1610 # the file
1611 mtime = os.stat(template_file).st_mtime
1612
1613 # invoke the editor
f95ceaf0 1614 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1615 editor = os.environ.get("P4EDITOR")
1616 else:
1617 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1618 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1619
1620 # If the file was not saved, prompt to see if this patch should
1621 # be skipped. But skip this verification step if configured so.
0d609032 1622 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1623 return True
1624
d1652049
PW
1625 # modification time updated means user saved the file
1626 if os.stat(template_file).st_mtime > mtime:
1627 return True
1628
1629 while True:
1630 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1631 if response == 'y':
1632 return True
1633 if response == 'n':
1634 return False
7c766e57 1635
df8a9e86 1636 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3
MC
1637 # diff
1638 if os.environ.has_key("P4DIFF"):
1639 del(os.environ["P4DIFF"])
1640 diff = ""
1641 for editedFile in editedFiles:
1642 diff += p4_read_pipe(['diff', '-du',
1643 wildcard_encode(editedFile)])
1644
1645 # new file diff
1646 newdiff = ""
1647 for newFile in filesToAdd:
1648 newdiff += "==== new file ====\n"
1649 newdiff += "--- /dev/null\n"
1650 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1651
1652 is_link = os.path.islink(newFile)
1653 expect_link = newFile in symlinks
1654
1655 if is_link and expect_link:
1656 newdiff += "+%s\n" % os.readlink(newFile)
1657 else:
1658 f = open(newFile, "r")
1659 for line in f.readlines():
1660 newdiff += "+" + line
1661 f.close()
b4073bb3 1662
e2a892ee 1663 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1664
7cb5cbef 1665 def applyCommit(self, id):
67b0fe2e
PW
1666 """Apply one commit, return True if it succeeded."""
1667
1668 print "Applying", read_pipe(["git", "show", "-s",
1669 "--format=format:%h %s", id])
ae901090 1670
848de9c3 1671 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1672
84cb0003 1673 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1674 filesToAdd = set()
a02b8bc4 1675 filesToChangeType = set()
4f5cf76a 1676 filesToDelete = set()
d336c158 1677 editedFiles = set()
b6ad6dcc 1678 pureRenameCopy = set()
df8a9e86 1679 symlinks = set()
c65b670e 1680 filesToChangeExecBit = {}
46c609e9 1681 all_files = list()
60df071c 1682
4f5cf76a 1683 for line in diff:
b43b0a3c
CP
1684 diff = parseDiffTreeEntry(line)
1685 modifier = diff['status']
1686 path = diff['src']
46c609e9
LD
1687 all_files.append(path)
1688
4f5cf76a 1689 if modifier == "M":
6de040df 1690 p4_edit(path)
c65b670e
CP
1691 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1692 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1693 editedFiles.add(path)
4f5cf76a
SH
1694 elif modifier == "A":
1695 filesToAdd.add(path)
c65b670e 1696 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1697 if path in filesToDelete:
1698 filesToDelete.remove(path)
df8a9e86
LD
1699
1700 dst_mode = int(diff['dst_mode'], 8)
1701 if dst_mode == 0120000:
1702 symlinks.add(path)
1703
4f5cf76a
SH
1704 elif modifier == "D":
1705 filesToDelete.add(path)
1706 if path in filesToAdd:
1707 filesToAdd.remove(path)
4fddb41b
VA
1708 elif modifier == "C":
1709 src, dest = diff['src'], diff['dst']
6de040df 1710 p4_integrate(src, dest)
b6ad6dcc 1711 pureRenameCopy.add(dest)
4fddb41b 1712 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1713 p4_edit(dest)
b6ad6dcc 1714 pureRenameCopy.discard(dest)
4fddb41b 1715 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1716 p4_edit(dest)
b6ad6dcc 1717 pureRenameCopy.discard(dest)
4fddb41b 1718 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1719 if self.isWindows:
1720 # turn off read-only attribute
1721 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1722 os.unlink(dest)
1723 editedFiles.add(dest)
d9a5f25b 1724 elif modifier == "R":
b43b0a3c 1725 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1726 if self.p4HasMoveCommand:
1727 p4_edit(src) # src must be open before move
1728 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1729 else:
8e9497c2
GG
1730 p4_integrate(src, dest)
1731 if diff['src_sha1'] != diff['dst_sha1']:
1732 p4_edit(dest)
1733 else:
1734 pureRenameCopy.add(dest)
c65b670e 1735 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1736 if not self.p4HasMoveCommand:
1737 p4_edit(dest) # with move: already open, writable
c65b670e 1738 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1739 if not self.p4HasMoveCommand:
d20f0f8e
PW
1740 if self.isWindows:
1741 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1742 os.unlink(dest)
1743 filesToDelete.add(src)
d9a5f25b 1744 editedFiles.add(dest)
a02b8bc4
RP
1745 elif modifier == "T":
1746 filesToChangeType.add(path)
4f5cf76a
SH
1747 else:
1748 die("unknown modifier %s for %s" % (modifier, path))
1749
749b668c 1750 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1751 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1752 tryPatchCmd = patchcmd + "--check -"
1753 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1754 patch_succeeded = True
51a2640a 1755
47a130b7 1756 if os.system(tryPatchCmd) != 0:
60df071c
LD
1757 fixed_rcs_keywords = False
1758 patch_succeeded = False
51a2640a 1759 print "Unfortunately applying the change failed!"
60df071c
LD
1760
1761 # Patch failed, maybe it's just RCS keyword woes. Look through
1762 # the patch to see if that's possible.
0d609032 1763 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1764 file = None
1765 pattern = None
1766 kwfiles = {}
1767 for file in editedFiles | filesToDelete:
1768 # did this file's delta contain RCS keywords?
1769 pattern = p4_keywords_regexp_for_file(file)
1770
1771 if pattern:
1772 # this file is a possibility...look for RCS keywords.
1773 regexp = re.compile(pattern, re.VERBOSE)
1774 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1775 if regexp.search(line):
1776 if verbose:
1777 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1778 kwfiles[file] = pattern
1779 break
1780
1781 for file in kwfiles:
1782 if verbose:
1783 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1784 # File is being deleted, so not open in p4. Must
1785 # disable the read-only bit on windows.
1786 if self.isWindows and file not in editedFiles:
1787 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1788 self.patchRCSKeywords(file, kwfiles[file])
1789 fixed_rcs_keywords = True
1790
1791 if fixed_rcs_keywords:
1792 print "Retrying the patch with RCS keywords cleaned up"
1793 if os.system(tryPatchCmd) == 0:
1794 patch_succeeded = True
1795
1796 if not patch_succeeded:
7e5dd9f2
PW
1797 for f in editedFiles:
1798 p4_revert(f)
7e5dd9f2 1799 return False
51a2640a 1800
55ac2ed6
PW
1801 #
1802 # Apply the patch for real, and do add/delete/+x handling.
1803 #
47a130b7 1804 system(applyPatchCmd)
4f5cf76a 1805
a02b8bc4
RP
1806 for f in filesToChangeType:
1807 p4_edit(f, "-t", "auto")
4f5cf76a 1808 for f in filesToAdd:
6de040df 1809 p4_add(f)
4f5cf76a 1810 for f in filesToDelete:
6de040df
LD
1811 p4_revert(f)
1812 p4_delete(f)
4f5cf76a 1813
c65b670e
CP
1814 # Set/clear executable bits
1815 for f in filesToChangeExecBit.keys():
1816 mode = filesToChangeExecBit[f]
1817 setP4ExecBit(f, mode)
1818
8cf422db
LD
1819 update_shelve = 0
1820 if len(self.update_shelve) > 0:
1821 update_shelve = self.update_shelve.pop(0)
1822 p4_reopen_in_change(update_shelve, all_files)
46c609e9 1823
55ac2ed6
PW
1824 #
1825 # Build p4 change description, starting with the contents
1826 # of the git commit message.
1827 #
0e36f2d7 1828 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1829 logMessage = logMessage.strip()
f19cb0a0 1830 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1831
8cf422db 1832 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 1833 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1834
c47178d4 1835 if self.preserveUser:
55ac2ed6 1836 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1837
55ac2ed6
PW
1838 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1839 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1840 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1841 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1842
55ac2ed6 1843 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1844 if not self.prepare_p4_only:
1845 submitTemplate += separatorLine
df8a9e86 1846 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1847
c47178d4 1848 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1849 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1850 if self.isWindows:
1851 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1852 tmpFile.write(submitTemplate)
c47178d4
PW
1853 tmpFile.close()
1854
728b7ad8
PW
1855 if self.prepare_p4_only:
1856 #
1857 # Leave the p4 tree prepared, and the submit template around
1858 # and let the user decide what to do next
1859 #
1860 print
1861 print "P4 workspace prepared for submission."
1862 print "To submit or revert, go to client workspace"
1863 print " " + self.clientPath
1864 print
1865 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1866 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1867 " \"git p4\"." % fileName
1868 print "You can delete the file \"%s\" when finished." % fileName
1869
1870 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1871 print "To preserve change ownership by user %s, you must\n" \
1872 "do \"p4 change -f <change>\" after submitting and\n" \
1873 "edit the User field."
1874 if pureRenameCopy:
1875 print "After submitting, renamed files must be re-synced."
1876 print "Invoke \"p4 sync -f\" on each of these files:"
1877 for f in pureRenameCopy:
1878 print " " + f
1879
1880 print
1881 print "To revert the changes, use \"p4 revert ...\", and delete"
1882 print "the submit template file \"%s\"" % fileName
1883 if filesToAdd:
1884 print "Since the commit adds new files, they must be deleted:"
1885 for f in filesToAdd:
1886 print " " + f
1887 print
1888 return True
1889
55ac2ed6
PW
1890 #
1891 # Let the user edit the change description, then submit it.
1892 #
b7638fed 1893 submitted = False
cdc7e388 1894
b7638fed
GE
1895 try:
1896 if self.edit_template(fileName):
1897 # read the edited message and submit
1898 tmpFile = open(fileName, "rb")
1899 message = tmpFile.read()
1900 tmpFile.close()
1901 if self.isWindows:
1902 message = message.replace("\r\n", "\n")
1903 submitTemplate = message[:message.index(separatorLine)]
46c609e9 1904
8cf422db 1905 if update_shelve:
46c609e9
LD
1906 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1907 elif self.shelve:
b34fa577
VK
1908 p4_write_pipe(['shelve', '-i'], submitTemplate)
1909 else:
1910 p4_write_pipe(['submit', '-i'], submitTemplate)
1911 # The rename/copy happened by applying a patch that created a
1912 # new file. This leaves it writable, which confuses p4.
1913 for f in pureRenameCopy:
1914 p4_sync(f, "-f")
b7638fed
GE
1915
1916 if self.preserveUser:
1917 if p4User:
1918 # Get last changelist number. Cannot easily get it from
1919 # the submit command output as the output is
1920 # unmarshalled.
1921 changelist = self.lastP4Changelist()
1922 self.modifyChangelistUser(changelist, p4User)
1923
b7638fed
GE
1924 submitted = True
1925
1926 finally:
c47178d4 1927 # skip this patch
b34fa577
VK
1928 if not submitted or self.shelve:
1929 if self.shelve:
1930 print ("Reverting shelved files.")
1931 else:
1932 print ("Submission cancelled, undoing p4 changes.")
1933 for f in editedFiles | filesToDelete:
b7638fed
GE
1934 p4_revert(f)
1935 for f in filesToAdd:
1936 p4_revert(f)
1937 os.remove(f)
c47178d4
PW
1938
1939 os.remove(fileName)
b7638fed 1940 return submitted
4f5cf76a 1941
06804c76
LD
1942 # Export git tags as p4 labels. Create a p4 label and then tag
1943 # with that.
1944 def exportGitTags(self, gitTags):
c8942a22
LD
1945 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1946 if len(validLabelRegexp) == 0:
1947 validLabelRegexp = defaultLabelRegexp
1948 m = re.compile(validLabelRegexp)
06804c76
LD
1949
1950 for name in gitTags:
1951
1952 if not m.match(name):
1953 if verbose:
05a3cec5 1954 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1955 continue
1956
1957 # Get the p4 commit this corresponds to
c8942a22
LD
1958 logMessage = extractLogMessageFromGitCommit(name)
1959 values = extractSettingsGitLog(logMessage)
06804c76 1960
c8942a22 1961 if not values.has_key('change'):
06804c76
LD
1962 # a tag pointing to something not sent to p4; ignore
1963 if verbose:
1964 print "git tag %s does not give a p4 commit" % name
1965 continue
c8942a22
LD
1966 else:
1967 changelist = values['change']
06804c76
LD
1968
1969 # Get the tag details.
1970 inHeader = True
1971 isAnnotated = False
1972 body = []
1973 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1974 l = l.strip()
1975 if inHeader:
1976 if re.match(r'tag\s+', l):
1977 isAnnotated = True
1978 elif re.match(r'\s*$', l):
1979 inHeader = False
1980 continue
1981 else:
1982 body.append(l)
1983
1984 if not isAnnotated:
1985 body = ["lightweight tag imported by git p4\n"]
1986
1987 # Create the label - use the same view as the client spec we are using
1988 clientSpec = getClientSpec()
1989
1990 labelTemplate = "Label: %s\n" % name
1991 labelTemplate += "Description:\n"
1992 for b in body:
1993 labelTemplate += "\t" + b + "\n"
1994 labelTemplate += "View:\n"
9d57c4a6
KS
1995 for depot_side in clientSpec.mappings:
1996 labelTemplate += "\t%s\n" % depot_side
06804c76 1997
ef739f08
PW
1998 if self.dry_run:
1999 print "Would create p4 label %s for tag" % name
728b7ad8
PW
2000 elif self.prepare_p4_only:
2001 print "Not creating p4 label %s for tag due to option" \
2002 " --prepare-p4-only" % name
ef739f08
PW
2003 else:
2004 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2005
ef739f08
PW
2006 # Use the label
2007 p4_system(["tag", "-l", name] +
9d57c4a6 2008 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2009
ef739f08
PW
2010 if verbose:
2011 print "created p4 label for tag %s" % name
06804c76 2012
4f5cf76a 2013 def run(self, args):
c9b50e63
SH
2014 if len(args) == 0:
2015 self.master = currentGitBranch()
c9b50e63
SH
2016 elif len(args) == 1:
2017 self.master = args[0]
28755dba
PW
2018 if not branchExists(self.master):
2019 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2020 else:
2021 return False
2022
8cf422db
LD
2023 for i in self.update_shelve:
2024 if i <= 0:
2025 sys.exit("invalid changelist %d" % i)
2026
00ad6e31
LD
2027 if self.master:
2028 allowSubmit = gitConfig("git-p4.allowSubmit")
2029 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2030 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2031
27d2d811 2032 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 2033 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2034 if len(self.origin) == 0:
2035 self.origin = upstream
a3fdd579 2036
8cf422db 2037 if len(self.update_shelve) > 0:
46c609e9
LD
2038 self.shelve = True
2039
3ea2cfd4
LD
2040 if self.preserveUser:
2041 if not self.canChangeChangelists():
2042 die("Cannot preserve user names without p4 super-user or admin permissions")
2043
6bbfd137
PW
2044 # if not set from the command line, try the config file
2045 if self.conflict_behavior is None:
2046 val = gitConfig("git-p4.conflict")
2047 if val:
2048 if val not in self.conflict_behavior_choices:
2049 die("Invalid value '%s' for config git-p4.conflict" % val)
2050 else:
2051 val = "ask"
2052 self.conflict_behavior = val
2053
a3fdd579
SH
2054 if self.verbose:
2055 print "Origin branch is " + self.origin
9512497b 2056
ea99c3ae 2057 if len(self.depotPath) == 0:
9512497b
SH
2058 print "Internal error: cannot locate perforce depot path from existing branches"
2059 sys.exit(128)
2060
543987bd 2061 self.useClientSpec = False
0d609032 2062 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2063 self.useClientSpec = True
2064 if self.useClientSpec:
2065 self.clientSpecDirs = getClientSpec()
9512497b 2066
2e3a16b2 2067 # Check for the existence of P4 branches
cd884106
VA
2068 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2069
2070 if self.useClientSpec and not branchesDetected:
543987bd
PW
2071 # all files are relative to the client spec
2072 self.clientPath = getClientRoot()
2073 else:
2074 self.clientPath = p4Where(self.depotPath)
9512497b 2075
543987bd
PW
2076 if self.clientPath == "":
2077 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2078
ea99c3ae 2079 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2080 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2081
0591cfa8 2082 # ensure the clientPath exists
8d7ec362 2083 new_client_dir = False
0591cfa8 2084 if not os.path.exists(self.clientPath):
8d7ec362 2085 new_client_dir = True
0591cfa8
GG
2086 os.makedirs(self.clientPath)
2087
bbd84863 2088 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2089 if self.dry_run:
2090 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2091 else:
ef739f08
PW
2092 print "Synchronizing p4 checkout..."
2093 if new_client_dir:
2094 # old one was destroyed, and maybe nobody told p4
2095 p4_sync("...", "-f")
2096 else:
2097 p4_sync("...")
4f5cf76a 2098 self.check()
4f5cf76a 2099
4c750c0d 2100 commits = []
00ad6e31
LD
2101 if self.master:
2102 commitish = self.master
2103 else:
2104 commitish = 'HEAD'
2105
2106 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
4c750c0d
SH
2107 commits.append(line.strip())
2108 commits.reverse()
4f5cf76a 2109
0d609032 2110 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2111 self.checkAuthorship = False
2112 else:
2113 self.checkAuthorship = True
2114
3ea2cfd4
LD
2115 if self.preserveUser:
2116 self.checkValidP4Users(commits)
2117
84cb0003
GG
2118 #
2119 # Build up a set of options to be passed to diff when
2120 # submitting each commit to p4.
2121 #
2122 if self.detectRenames:
2123 # command-line -M arg
2124 self.diffOpts = "-M"
2125 else:
2126 # If not explicitly set check the config variable
2127 detectRenames = gitConfig("git-p4.detectRenames")
2128
2129 if detectRenames.lower() == "false" or detectRenames == "":
2130 self.diffOpts = ""
2131 elif detectRenames.lower() == "true":
2132 self.diffOpts = "-M"
2133 else:
2134 self.diffOpts = "-M%s" % detectRenames
2135
2136 # no command-line arg for -C or --find-copies-harder, just
2137 # config variables
2138 detectCopies = gitConfig("git-p4.detectCopies")
2139 if detectCopies.lower() == "false" or detectCopies == "":
2140 pass
2141 elif detectCopies.lower() == "true":
2142 self.diffOpts += " -C"
2143 else:
2144 self.diffOpts += " -C%s" % detectCopies
2145
0d609032 2146 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2147 self.diffOpts += " --find-copies-harder"
2148
8cf422db
LD
2149 num_shelves = len(self.update_shelve)
2150 if num_shelves > 0 and num_shelves != len(commits):
2151 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2152 (len(commits), num_shelves))
2153
7e5dd9f2
PW
2154 #
2155 # Apply the commits, one at a time. On failure, ask if should
2156 # continue to try the rest of the patches, or quit.
2157 #
ef739f08
PW
2158 if self.dry_run:
2159 print "Would apply"
67b0fe2e 2160 applied = []
7e5dd9f2
PW
2161 last = len(commits) - 1
2162 for i, commit in enumerate(commits):
ef739f08
PW
2163 if self.dry_run:
2164 print " ", read_pipe(["git", "show", "-s",
2165 "--format=format:%h %s", commit])
2166 ok = True
2167 else:
2168 ok = self.applyCommit(commit)
67b0fe2e
PW
2169 if ok:
2170 applied.append(commit)
7e5dd9f2 2171 else:
728b7ad8
PW
2172 if self.prepare_p4_only and i < last:
2173 print "Processing only the first commit due to option" \
2174 " --prepare-p4-only"
2175 break
7e5dd9f2
PW
2176 if i < last:
2177 quit = False
2178 while True:
6bbfd137
PW
2179 # prompt for what to do, or use the option/variable
2180 if self.conflict_behavior == "ask":
2181 print "What do you want to do?"
2182 response = raw_input("[s]kip this commit but apply"
2183 " the rest, or [q]uit? ")
2184 if not response:
2185 continue
2186 elif self.conflict_behavior == "skip":
2187 response = "s"
2188 elif self.conflict_behavior == "quit":
2189 response = "q"
2190 else:
2191 die("Unknown conflict_behavior '%s'" %
2192 self.conflict_behavior)
2193
7e5dd9f2
PW
2194 if response[0] == "s":
2195 print "Skipping this commit, but applying the rest"
2196 break
2197 if response[0] == "q":
2198 print "Quitting"
2199 quit = True
2200 break
2201 if quit:
2202 break
4f5cf76a 2203
67b0fe2e 2204 chdir(self.oldWorkingDirectory)
b34fa577 2205 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2206 if self.dry_run:
2207 pass
728b7ad8
PW
2208 elif self.prepare_p4_only:
2209 pass
ef739f08 2210 elif len(commits) == len(applied):
b34fa577 2211 print ("All commits {0}!".format(shelved_applied))
14594f4b 2212
4c750c0d 2213 sync = P4Sync()
44e8d26c
PW
2214 if self.branch:
2215 sync.branch = self.branch
4c750c0d 2216 sync.run([])
14594f4b 2217
4c750c0d
SH
2218 rebase = P4Rebase()
2219 rebase.rebase()
4f5cf76a 2220
67b0fe2e
PW
2221 else:
2222 if len(applied) == 0:
b34fa577 2223 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2224 else:
b34fa577 2225 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2226 for c in commits:
2227 if c in applied:
2228 star = "*"
2229 else:
2230 star = " "
2231 print star, read_pipe(["git", "show", "-s",
2232 "--format=format:%h %s", c])
2233 print "You will have to do 'git p4 sync' and rebase."
2234
0d609032 2235 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2236 self.exportLabels = True
06804c76
LD
2237
2238 if self.exportLabels:
2239 p4Labels = getP4Labels(self.depotPath)
2240 gitTags = getGitTags()
2241
2242 missingGitTags = gitTags - p4Labels
2243 self.exportGitTags(missingGitTags)
2244
98e023de 2245 # exit with error unless everything applied perfectly
67b0fe2e
PW
2246 if len(commits) != len(applied):
2247 sys.exit(1)
2248
b984733c
SH
2249 return True
2250
ecb7cf98
PW
2251class View(object):
2252 """Represent a p4 view ("p4 help views"), and map files in a
2253 repo according to the view."""
2254
9d57c4a6 2255 def __init__(self, client_name):
ecb7cf98 2256 self.mappings = []
9d57c4a6
KS
2257 self.client_prefix = "//%s/" % client_name
2258 # cache results of "p4 where" to lookup client file locations
2259 self.client_spec_path_cache = {}
ecb7cf98
PW
2260
2261 def append(self, view_line):
2262 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2263 sides. Append to self.mappings, preserving order. This
2264 is only needed for tag creation."""
ecb7cf98
PW
2265
2266 # Split the view line into exactly two words. P4 enforces
2267 # structure on these lines that simplifies this quite a bit.
2268 #
2269 # Either or both words may be double-quoted.
2270 # Single quotes do not matter.
2271 # Double-quote marks cannot occur inside the words.
2272 # A + or - prefix is also inside the quotes.
2273 # There are no quotes unless they contain a space.
2274 # The line is already white-space stripped.
2275 # The two words are separated by a single space.
2276 #
2277 if view_line[0] == '"':
2278 # First word is double quoted. Find its end.
2279 close_quote_index = view_line.find('"', 1)
2280 if close_quote_index <= 0:
2281 die("No first-word closing quote found: %s" % view_line)
2282 depot_side = view_line[1:close_quote_index]
2283 # skip closing quote and space
2284 rhs_index = close_quote_index + 1 + 1
2285 else:
2286 space_index = view_line.find(" ")
2287 if space_index <= 0:
2288 die("No word-splitting space found: %s" % view_line)
2289 depot_side = view_line[0:space_index]
2290 rhs_index = space_index + 1
2291
ecb7cf98 2292 # prefix + means overlay on previous mapping
ecb7cf98 2293 if depot_side.startswith("+"):
ecb7cf98
PW
2294 depot_side = depot_side[1:]
2295
9d57c4a6 2296 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2297 exclude = False
2298 if depot_side.startswith("-"):
2299 exclude = True
2300 depot_side = depot_side[1:]
2301
9d57c4a6
KS
2302 if not exclude:
2303 self.mappings.append(depot_side)
ecb7cf98 2304
9d57c4a6
KS
2305 def convert_client_path(self, clientFile):
2306 # chop off //client/ part to make it relative
2307 if not clientFile.startswith(self.client_prefix):
2308 die("No prefix '%s' on clientFile '%s'" %
2309 (self.client_prefix, clientFile))
2310 return clientFile[len(self.client_prefix):]
ecb7cf98 2311
9d57c4a6
KS
2312 def update_client_spec_path_cache(self, files):
2313 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2314
9d57c4a6
KS
2315 # List depot file paths exclude that already cached
2316 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2317
9d57c4a6
KS
2318 if len(fileArgs) == 0:
2319 return # All files in cache
ecb7cf98 2320
9d57c4a6
KS
2321 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2322 for res in where_result:
2323 if "code" in res and res["code"] == "error":
2324 # assume error is "... file(s) not in client view"
2325 continue
2326 if "clientFile" not in res:
20005443 2327 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2328 if "unmap" in res:
2329 # it will list all of them, but only one not unmap-ped
2330 continue
a0a50d87
LS
2331 if gitConfigBool("core.ignorecase"):
2332 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2333 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2334
9d57c4a6
KS
2335 # not found files or unmap files set to ""
2336 for depotFile in fileArgs:
a0a50d87
LS
2337 if gitConfigBool("core.ignorecase"):
2338 depotFile = depotFile.lower()
9d57c4a6
KS
2339 if depotFile not in self.client_spec_path_cache:
2340 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2341
9d57c4a6
KS
2342 def map_in_client(self, depot_path):
2343 """Return the relative location in the client where this
2344 depot file should live. Returns "" if the file should
2345 not be mapped in the client."""
ecb7cf98 2346
a0a50d87
LS
2347 if gitConfigBool("core.ignorecase"):
2348 depot_path = depot_path.lower()
2349
9d57c4a6
KS
2350 if depot_path in self.client_spec_path_cache:
2351 return self.client_spec_path_cache[depot_path]
2352
2353 die( "Error: %s is not found in client spec path" % depot_path )
2354 return ""
ecb7cf98 2355
3ea2cfd4 2356class P4Sync(Command, P4UserMap):
56c09345
PW
2357 delete_actions = ( "delete", "move/delete", "purge" )
2358
b984733c
SH
2359 def __init__(self):
2360 Command.__init__(self)
3ea2cfd4 2361 P4UserMap.__init__(self)
b984733c
SH
2362 self.options = [
2363 optparse.make_option("--branch", dest="branch"),
2364 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2365 optparse.make_option("--changesfile", dest="changesFile"),
2366 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2367 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2368 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2369 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2370 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2371 optparse.make_option("--max-changes", dest="maxChanges",
2372 help="Maximum number of changes to import"),
2373 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2374 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2375 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2376 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2377 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2378 help="Only sync files that are included in the Perforce Client Spec"),
2379 optparse.make_option("-/", dest="cloneExclude",
2380 action="append", type="string",
2381 help="exclude depot path"),
b984733c
SH
2382 ]
2383 self.description = """Imports from Perforce into a git repository.\n
2384 example:
2385 //depot/my/project/ -- to import the current head
2386 //depot/my/project/@all -- to import everything
2387 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2388
2389 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2390
2391 self.usage += " //depot/path[@revRange]"
b984733c 2392 self.silent = False
1d7367dc
RG
2393 self.createdBranches = set()
2394 self.committedChanges = set()
569d1bd4 2395 self.branch = ""
b984733c 2396 self.detectBranches = False
cb53e1f8 2397 self.detectLabels = False
06804c76 2398 self.importLabels = False
b984733c 2399 self.changesFile = ""
01265103 2400 self.syncWithOrigin = True
a028a98e 2401 self.importIntoRemotes = True
01a9c9c5 2402 self.maxChanges = ""
1051ef00 2403 self.changes_block_size = None
8b41a97f 2404 self.keepRepoPath = False
6326aa58 2405 self.depotPaths = None
3c699645 2406 self.p4BranchesInGit = []
354081d5 2407 self.cloneExclude = []
3a70cdfa 2408 self.useClientSpec = False
a93d33ee 2409 self.useClientSpec_from_options = False
ecb7cf98 2410 self.clientSpecDirs = None
fed23693 2411 self.tempBranches = []
d604176d 2412 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12
LS
2413 self.largeFileSystem = None
2414
2415 if gitConfig('git-p4.largeFileSystem'):
2416 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2417 self.largeFileSystem = largeFileSystemConstructor(
2418 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2419 )
b984733c 2420
01265103
SH
2421 if gitConfig("git-p4.syncFromOrigin") == "false":
2422 self.syncWithOrigin = False
2423
fed23693
VA
2424 # Force a checkpoint in fast-import and wait for it to finish
2425 def checkpoint(self):
2426 self.gitStream.write("checkpoint\n\n")
2427 self.gitStream.write("progress checkpoint\n\n")
2428 out = self.gitOutput.readline()
2429 if self.verbose:
2430 print "checkpoint finished: " + out
2431
b984733c 2432 def extractFilesFromCommit(self, commit):
354081d5
TT
2433 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2434 for path in self.cloneExclude]
b984733c
SH
2435 files = []
2436 fnum = 0
2437 while commit.has_key("depotFile%s" % fnum):
2438 path = commit["depotFile%s" % fnum]
6326aa58 2439
354081d5 2440 if [p for p in self.cloneExclude
d53de8b9 2441 if p4PathStartsWith(path, p)]:
354081d5
TT
2442 found = False
2443 else:
2444 found = [p for p in self.depotPaths
d53de8b9 2445 if p4PathStartsWith(path, p)]
6326aa58 2446 if not found:
b984733c
SH
2447 fnum = fnum + 1
2448 continue
2449
2450 file = {}
2451 file["path"] = path
2452 file["rev"] = commit["rev%s" % fnum]
2453 file["action"] = commit["action%s" % fnum]
2454 file["type"] = commit["type%s" % fnum]
2455 files.append(file)
2456 fnum = fnum + 1
2457 return files
2458
26e6a27d
JD
2459 def extractJobsFromCommit(self, commit):
2460 jobs = []
2461 jnum = 0
2462 while commit.has_key("job%s" % jnum):
2463 job = commit["job%s" % jnum]
2464 jobs.append(job)
2465 jnum = jnum + 1
2466 return jobs
2467
6326aa58 2468 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2469 """When streaming files, this is called to map a p4 depot path
2470 to where it should go in git. The prefixes are either
2471 self.depotPaths, or self.branchPrefixes in the case of
2472 branch detection."""
2473
3952710b 2474 if self.useClientSpec:
21ef5df4
PW
2475 # branch detection moves files up a level (the branch name)
2476 # from what client spec interpretation gives
0d1696ef 2477 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2478 if self.detectBranches:
2479 for b in self.knownBranches:
2480 if path.startswith(b + "/"):
2481 path = path[len(b)+1:]
2482
2483 elif self.keepRepoPath:
2484 # Preserve everything in relative path name except leading
2485 # //depot/; just look at first prefix as they all should
2486 # be in the same depot.
2487 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2488 if p4PathStartsWith(path, depot):
2489 path = path[len(depot):]
3952710b 2490
0d1696ef 2491 else:
0d1696ef
PW
2492 for p in prefixes:
2493 if p4PathStartsWith(path, p):
2494 path = path[len(p):]
21ef5df4 2495 break
8b41a97f 2496
0d1696ef 2497 path = wildcard_decode(path)
6326aa58 2498 return path
6754a299 2499
71b112d4 2500 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2501 """Look at each depotFile in the commit to figure out to what
2502 branch it belongs."""
2503
9d57c4a6
KS
2504 if self.clientSpecDirs:
2505 files = self.extractFilesFromCommit(commit)
2506 self.clientSpecDirs.update_client_spec_path_cache(files)
2507
d5904674 2508 branches = {}
71b112d4
SH
2509 fnum = 0
2510 while commit.has_key("depotFile%s" % fnum):
2511 path = commit["depotFile%s" % fnum]
6326aa58 2512 found = [p for p in self.depotPaths
d53de8b9 2513 if p4PathStartsWith(path, p)]
6326aa58 2514 if not found:
71b112d4
SH
2515 fnum = fnum + 1
2516 continue
2517
2518 file = {}
2519 file["path"] = path
2520 file["rev"] = commit["rev%s" % fnum]
2521 file["action"] = commit["action%s" % fnum]
2522 file["type"] = commit["type%s" % fnum]
2523 fnum = fnum + 1
2524
21ef5df4
PW
2525 # start with the full relative path where this file would
2526 # go in a p4 client
2527 if self.useClientSpec:
2528 relPath = self.clientSpecDirs.map_in_client(path)
2529 else:
2530 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2531
4b97ffb1 2532 for branch in self.knownBranches.keys():
21ef5df4
PW
2533 # add a trailing slash so that a commit into qt/4.2foo
2534 # doesn't end up in qt/4.2, e.g.
6754a299 2535 if relPath.startswith(branch + "/"):
d5904674
SH
2536 if branch not in branches:
2537 branches[branch] = []
71b112d4 2538 branches[branch].append(file)
6555b2cc 2539 break
b984733c
SH
2540
2541 return branches
2542
a5db4b12
LS
2543 def writeToGitStream(self, gitMode, relPath, contents):
2544 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2545 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2546 for d in contents:
2547 self.gitStream.write(d)
2548 self.gitStream.write('\n')
2549
a8b05162
LS
2550 def encodeWithUTF8(self, path):
2551 try:
2552 path.decode('ascii')
2553 except:
2554 encoding = 'utf8'
2555 if gitConfig('git-p4.pathEncoding'):
2556 encoding = gitConfig('git-p4.pathEncoding')
2557 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2558 if self.verbose:
2559 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2560 return path
2561
b932705b
LD
2562 # output one file from the P4 stream
2563 # - helper for streamP4Files
2564
2565 def streamOneP4File(self, file, contents):
b932705b 2566 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2567 relPath = self.encodeWithUTF8(relPath)
b932705b 2568 if verbose:
d2176a50
LS
2569 size = int(self.stream_file['fileSize'])
2570 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2571 sys.stdout.flush()
b932705b 2572
9cffb8c8
PW
2573 (type_base, type_mods) = split_p4_type(file["type"])
2574
2575 git_mode = "100644"
2576 if "x" in type_mods:
2577 git_mode = "100755"
2578 if type_base == "symlink":
2579 git_mode = "120000"
1292df11
AJ
2580 # p4 print on a symlink sometimes contains "target\n";
2581 # if it does, remove the newline
b39c3612 2582 data = ''.join(contents)
40f846c3
PW
2583 if not data:
2584 # Some version of p4 allowed creating a symlink that pointed
2585 # to nothing. This causes p4 errors when checking out such
2586 # a change, and errors here too. Work around it by ignoring
2587 # the bad symlink; hopefully a future change fixes it.
2588 print "\nIgnoring empty symlink in %s" % file['depotFile']
2589 return
2590 elif data[-1] == '\n':
1292df11
AJ
2591 contents = [data[:-1]]
2592 else:
2593 contents = [data]
b932705b 2594
9cffb8c8 2595 if type_base == "utf16":
55aa5714
PW
2596 # p4 delivers different text in the python output to -G
2597 # than it does when using "print -o", or normal p4 client
2598 # operations. utf16 is converted to ascii or utf8, perhaps.
2599 # But ascii text saved as -t utf16 is completely mangled.
2600 # Invoke print -o to get the real contents.
7f0e5962
PW
2601 #
2602 # On windows, the newlines will always be mangled by print, so put
2603 # them back too. This is not needed to the cygwin windows version,
2604 # just the native "NT" type.
2605 #
1f5f3907
LS
2606 try:
2607 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2608 except Exception as e:
2609 if 'Translation of file content failed' in str(e):
2610 type_base = 'binary'
2611 else:
2612 raise e
2613 else:
2614 if p4_version_string().find('/NT') >= 0:
2615 text = text.replace('\r\n', '\n')
2616 contents = [ text ]
55aa5714 2617
9f7ef0ea
PW
2618 if type_base == "apple":
2619 # Apple filetype files will be streamed as a concatenation of
2620 # its appledouble header and the contents. This is useless
2621 # on both macs and non-macs. If using "print -q -o xx", it
2622 # will create "xx" with the data, and "%xx" with the header.
2623 # This is also not very useful.
2624 #
2625 # Ideally, someday, this script can learn how to generate
2626 # appledouble files directly and import those to git, but
2627 # non-mac machines can never find a use for apple filetype.
2628 print "\nIgnoring apple filetype file %s" % file['depotFile']
2629 return
2630
55aa5714
PW
2631 # Note that we do not try to de-mangle keywords on utf16 files,
2632 # even though in theory somebody may want that.
60df071c
LD
2633 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2634 if pattern:
2635 regexp = re.compile(pattern, re.VERBOSE)
2636 text = ''.join(contents)
2637 text = regexp.sub(r'$\1$', text)
2638 contents = [ text ]
b932705b 2639
a5db4b12
LS
2640 if self.largeFileSystem:
2641 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2642
a5db4b12 2643 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2644
2645 def streamOneP4Deletion(self, file):
2646 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2647 relPath = self.encodeWithUTF8(relPath)
b932705b 2648 if verbose:
d2176a50
LS
2649 sys.stdout.write("delete %s\n" % relPath)
2650 sys.stdout.flush()
b932705b
LD
2651 self.gitStream.write("D %s\n" % relPath)
2652
a5db4b12
LS
2653 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2654 self.largeFileSystem.removeLargeFile(relPath)
2655
b932705b
LD
2656 # handle another chunk of streaming data
2657 def streamP4FilesCb(self, marshalled):
2658
78189bea
PW
2659 # catch p4 errors and complain
2660 err = None
2661 if "code" in marshalled:
2662 if marshalled["code"] == "error":
2663 if "data" in marshalled:
2664 err = marshalled["data"].rstrip()
4d25dc44
LS
2665
2666 if not err and 'fileSize' in self.stream_file:
2667 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2668 if required_bytes > 0:
2669 err = 'Not enough space left on %s! Free at least %i MB.' % (
2670 os.getcwd(), required_bytes/1024/1024
2671 )
2672
78189bea
PW
2673 if err:
2674 f = None
2675 if self.stream_have_file_info:
2676 if "depotFile" in self.stream_file:
2677 f = self.stream_file["depotFile"]
2678 # force a failure in fast-import, else an empty
2679 # commit will be made
2680 self.gitStream.write("\n")
2681 self.gitStream.write("die-now\n")
2682 self.gitStream.close()
2683 # ignore errors, but make sure it exits first
2684 self.importProcess.wait()
2685 if f:
2686 die("Error from p4 print for %s: %s" % (f, err))
2687 else:
2688 die("Error from p4 print: %s" % err)
2689
c3f6163b
AG
2690 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2691 # start of a new file - output the old one first
2692 self.streamOneP4File(self.stream_file, self.stream_contents)
2693 self.stream_file = {}
2694 self.stream_contents = []
2695 self.stream_have_file_info = False
b932705b 2696
c3f6163b
AG
2697 # pick up the new file information... for the
2698 # 'data' field we need to append to our array
2699 for k in marshalled.keys():
2700 if k == 'data':
d2176a50
LS
2701 if 'streamContentSize' not in self.stream_file:
2702 self.stream_file['streamContentSize'] = 0
2703 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2704 self.stream_contents.append(marshalled['data'])
2705 else:
2706 self.stream_file[k] = marshalled[k]
b932705b 2707
d2176a50
LS
2708 if (verbose and
2709 'streamContentSize' in self.stream_file and
2710 'fileSize' in self.stream_file and
2711 'depotFile' in self.stream_file):
2712 size = int(self.stream_file["fileSize"])
2713 if size > 0:
2714 progress = 100*self.stream_file['streamContentSize']/size
2715 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2716 sys.stdout.flush()
2717
c3f6163b 2718 self.stream_have_file_info = True
b932705b
LD
2719
2720 # Stream directly from "p4 files" into "git fast-import"
2721 def streamP4Files(self, files):
30b5940b
SH
2722 filesForCommit = []
2723 filesToRead = []
b932705b 2724 filesToDelete = []
30b5940b 2725
3a70cdfa 2726 for f in files:
ecb7cf98
PW
2727 filesForCommit.append(f)
2728 if f['action'] in self.delete_actions:
2729 filesToDelete.append(f)
2730 else:
2731 filesToRead.append(f)
6a49f8e2 2732
b932705b
LD
2733 # deleted files...
2734 for f in filesToDelete:
2735 self.streamOneP4Deletion(f)
1b9a4684 2736
b932705b
LD
2737 if len(filesToRead) > 0:
2738 self.stream_file = {}
2739 self.stream_contents = []
2740 self.stream_have_file_info = False
8ff45f2a 2741
c3f6163b
AG
2742 # curry self argument
2743 def streamP4FilesCbSelf(entry):
2744 self.streamP4FilesCb(entry)
6a49f8e2 2745
6de040df
LD
2746 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2747
2748 p4CmdList(["-x", "-", "print"],
2749 stdin=fileArgs,
2750 cb=streamP4FilesCbSelf)
30b5940b 2751
b932705b
LD
2752 # do the last chunk
2753 if self.stream_file.has_key('depotFile'):
2754 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2755
affb474f
LD
2756 def make_email(self, userid):
2757 if userid in self.users:
2758 return self.users[userid]
2759 else:
2760 return "%s <a@b>" % userid
2761
06804c76 2762 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2763 """ Stream a p4 tag.
2764 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2765 """
2766
06804c76
LD
2767 if verbose:
2768 print "writing tag %s for commit %s" % (labelName, commit)
2769 gitStream.write("tag %s\n" % labelName)
2770 gitStream.write("from %s\n" % commit)
2771
2772 if labelDetails.has_key('Owner'):
2773 owner = labelDetails["Owner"]
2774 else:
2775 owner = None
2776
2777 # Try to use the owner of the p4 label, or failing that,
2778 # the current p4 user id.
2779 if owner:
2780 email = self.make_email(owner)
2781 else:
2782 email = self.make_email(self.p4UserId())
2783 tagger = "%s %s %s" % (email, epoch, self.tz)
2784
2785 gitStream.write("tagger %s\n" % tagger)
2786
2787 print "labelDetails=",labelDetails
2788 if labelDetails.has_key('Description'):
2789 description = labelDetails['Description']
2790 else:
2791 description = 'Label from git p4'
2792
2793 gitStream.write("data %d\n" % len(description))
2794 gitStream.write(description)
2795 gitStream.write("\n")
2796
4ae048e6
LS
2797 def inClientSpec(self, path):
2798 if not self.clientSpecDirs:
2799 return True
2800 inClientSpec = self.clientSpecDirs.map_in_client(path)
2801 if not inClientSpec and self.verbose:
2802 print('Ignoring file outside of client spec: {0}'.format(path))
2803 return inClientSpec
2804
2805 def hasBranchPrefix(self, path):
2806 if not self.branchPrefixes:
2807 return True
2808 hasPrefix = [p for p in self.branchPrefixes
2809 if p4PathStartsWith(path, p)]
09667d01 2810 if not hasPrefix and self.verbose:
4ae048e6
LS
2811 print('Ignoring file outside of prefix: {0}'.format(path))
2812 return hasPrefix
2813
e63231e5 2814 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2815 epoch = details["time"]
2816 author = details["user"]
26e6a27d 2817 jobs = self.extractJobsFromCommit(details)
b984733c 2818
4b97ffb1 2819 if self.verbose:
4ae048e6 2820 print('commit into {0}'.format(branch))
96e07dd2 2821
9d57c4a6
KS
2822 if self.clientSpecDirs:
2823 self.clientSpecDirs.update_client_spec_path_cache(files)
2824
4ae048e6
LS
2825 files = [f for f in files
2826 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2827
2828 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2829 print('Ignoring revision {0} as it would produce an empty commit.'
2830 .format(details['change']))
2831 return
2832
b984733c 2833 self.gitStream.write("commit %s\n" % branch)
b43702ac 2834 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2835 self.committedChanges.add(int(details["change"]))
2836 committer = ""
b607e71e
SH
2837 if author not in self.users:
2838 self.getUserMapFromPerforceServer()
affb474f 2839 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2840
2841 self.gitStream.write("committer %s\n" % committer)
2842
2843 self.gitStream.write("data <<EOT\n")
2844 self.gitStream.write(details["desc"])
26e6a27d
JD
2845 if len(jobs) > 0:
2846 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
e63231e5
PW
2847 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2848 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2849 if len(details['options']) > 0:
2850 self.gitStream.write(": options = %s" % details['options'])
2851 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2852
2853 if len(parent) > 0:
4b97ffb1
SH
2854 if self.verbose:
2855 print "parent %s" % parent
b984733c
SH
2856 self.gitStream.write("from %s\n" % parent)
2857
4ae048e6 2858 self.streamP4Files(files)
b984733c
SH
2859 self.gitStream.write("\n")
2860
1f4ba1cb
SH
2861 change = int(details["change"])
2862
9bda3a85 2863 if self.labels.has_key(change):
1f4ba1cb
SH
2864 label = self.labels[change]
2865 labelDetails = label[0]
2866 labelRevisions = label[1]
71b112d4
SH
2867 if self.verbose:
2868 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2869
6de040df 2870 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2871 for p in self.branchPrefixes])
1f4ba1cb
SH
2872
2873 if len(files) == len(labelRevisions):
2874
2875 cleanedFiles = {}
2876 for info in files:
56c09345 2877 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2878 continue
2879 cleanedFiles[info["depotFile"]] = info["rev"]
2880
2881 if cleanedFiles == labelRevisions:
06804c76 2882 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2883
2884 else:
a46668fa 2885 if not self.silent:
cebdf5af
HWN
2886 print ("Tag %s does not match with change %s: files do not match."
2887 % (labelDetails["label"], change))
1f4ba1cb
SH
2888
2889 else:
a46668fa 2890 if not self.silent:
cebdf5af
HWN
2891 print ("Tag %s does not match with change %s: file count is different."
2892 % (labelDetails["label"], change))
b984733c 2893
06804c76 2894 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2895 def getLabels(self):
2896 self.labels = {}
2897
52a4880b 2898 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2899 if len(l) > 0 and not self.silent:
183f8436 2900 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2901
2902 for output in l:
1f4ba1cb
SH
2903 label = output["label"]
2904 revisions = {}
2905 newestChange = 0
71b112d4
SH
2906 if self.verbose:
2907 print "Querying files for label %s" % label
6de040df
LD
2908 for file in p4CmdList(["files"] +
2909 ["%s...@%s" % (p, label)
2910 for p in self.depotPaths]):
1f4ba1cb
SH
2911 revisions[file["depotFile"]] = file["rev"]
2912 change = int(file["change"])
2913 if change > newestChange:
2914 newestChange = change
2915
9bda3a85
SH
2916 self.labels[newestChange] = [output, revisions]
2917
2918 if self.verbose:
2919 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2920
06804c76
LD
2921 # Import p4 labels as git tags. A direct mapping does not
2922 # exist, so assume that if all the files are at the same revision
2923 # then we can use that, or it's something more complicated we should
2924 # just ignore.
2925 def importP4Labels(self, stream, p4Labels):
2926 if verbose:
2927 print "import p4 labels: " + ' '.join(p4Labels)
2928
2929 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2930 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2931 if len(validLabelRegexp) == 0:
2932 validLabelRegexp = defaultLabelRegexp
2933 m = re.compile(validLabelRegexp)
2934
2935 for name in p4Labels:
2936 commitFound = False
2937
2938 if not m.match(name):
2939 if verbose:
2940 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2941 continue
2942
2943 if name in ignoredP4Labels:
2944 continue
2945
2946 labelDetails = p4CmdList(['label', "-o", name])[0]
2947
2948 # get the most recent changelist for each file in this label
2949 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2950 for p in self.depotPaths])
2951
2952 if change.has_key('change'):
2953 # find the corresponding git commit; take the oldest commit
2954 changelist = int(change['change'])
b43702ac
LD
2955 if changelist in self.committedChanges:
2956 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2957 commitFound = True
b43702ac
LD
2958 else:
2959 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2960 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2961 if len(gitCommit) == 0:
2962 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2963 else:
2964 commitFound = True
2965 gitCommit = gitCommit.strip()
2966
2967 if commitFound:
06804c76
LD
2968 # Convert from p4 time format
2969 try:
2970 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2971 except ValueError:
a4e9054c 2972 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2973 tmwhen = 1
2974
2975 when = int(time.mktime(tmwhen))
2976 self.streamTag(stream, name, labelDetails, gitCommit, when)
2977 if verbose:
2978 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2979 else:
2980 if verbose:
2981 print "Label %s has no changelists - possibly deleted?" % name
2982
2983 if not commitFound:
2984 # We can't import this label; don't try again as it will get very
2985 # expensive repeatedly fetching all the files for labels that will
2986 # never be imported. If the label is moved in the future, the
2987 # ignore will need to be removed manually.
2988 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2989
86dff6b6
HWN
2990 def guessProjectName(self):
2991 for p in self.depotPaths:
6e5295c4
SH
2992 if p.endswith("/"):
2993 p = p[:-1]
2994 p = p[p.strip().rfind("/") + 1:]
2995 if not p.endswith("/"):
2996 p += "/"
2997 return p
86dff6b6 2998
4b97ffb1 2999 def getBranchMapping(self):
6555b2cc
SH
3000 lostAndFoundBranches = set()
3001
8ace74c0
VA
3002 user = gitConfig("git-p4.branchUser")
3003 if len(user) > 0:
3004 command = "branches -u %s" % user
3005 else:
3006 command = "branches"
3007
3008 for info in p4CmdList(command):
52a4880b 3009 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
3010 viewIdx = 0
3011 while details.has_key("View%s" % viewIdx):
3012 paths = details["View%s" % viewIdx].split(" ")
3013 viewIdx = viewIdx + 1
3014 # require standard //depot/foo/... //depot/bar/... mapping
3015 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3016 continue
3017 source = paths[0]
3018 destination = paths[1]
6509e19c 3019 ## HACK
d53de8b9 3020 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3021 source = source[len(self.depotPaths[0]):-4]
3022 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3023
1a2edf4e
SH
3024 if destination in self.knownBranches:
3025 if not self.silent:
3026 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3027 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3028 continue
3029
6555b2cc
SH
3030 self.knownBranches[destination] = source
3031
3032 lostAndFoundBranches.discard(destination)
3033
29bdbac1 3034 if source not in self.knownBranches:
6555b2cc
SH
3035 lostAndFoundBranches.add(source)
3036
7199cf13
VA
3037 # Perforce does not strictly require branches to be defined, so we also
3038 # check git config for a branch list.
3039 #
3040 # Example of branch definition in git config file:
3041 # [git-p4]
3042 # branchList=main:branchA
3043 # branchList=main:branchB
3044 # branchList=branchA:branchC
3045 configBranches = gitConfigList("git-p4.branchList")
3046 for branch in configBranches:
3047 if branch:
3048 (source, destination) = branch.split(":")
3049 self.knownBranches[destination] = source
3050
3051 lostAndFoundBranches.discard(destination)
3052
3053 if source not in self.knownBranches:
3054 lostAndFoundBranches.add(source)
3055
6555b2cc
SH
3056
3057 for branch in lostAndFoundBranches:
3058 self.knownBranches[branch] = branch
29bdbac1 3059
38f9f5ec
SH
3060 def getBranchMappingFromGitBranches(self):
3061 branches = p4BranchesInGit(self.importIntoRemotes)
3062 for branch in branches.keys():
3063 if branch == "master":
3064 branch = "main"
3065 else:
3066 branch = branch[len(self.projectName):]
3067 self.knownBranches[branch] = branch
3068
bb6e09b2
HWN
3069 def updateOptionDict(self, d):
3070 option_keys = {}
3071 if self.keepRepoPath:
3072 option_keys['keepRepoPath'] = 1
3073
3074 d["options"] = ' '.join(sorted(option_keys.keys()))
3075
3076 def readOptions(self, d):
3077 self.keepRepoPath = (d.has_key('options')
3078 and ('keepRepoPath' in d['options']))
6326aa58 3079
8134f69c
SH
3080 def gitRefForBranch(self, branch):
3081 if branch == "main":
3082 return self.refPrefix + "master"
3083
3084 if len(branch) <= 0:
3085 return branch
3086
3087 return self.refPrefix + self.projectName + branch
3088
1ca3d710
SH
3089 def gitCommitByP4Change(self, ref, change):
3090 if self.verbose:
3091 print "looking in ref " + ref + " for change %s using bisect..." % change
3092
3093 earliestCommit = ""
3094 latestCommit = parseRevision(ref)
3095
3096 while True:
3097 if self.verbose:
3098 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3099 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3100 if len(next) == 0:
3101 if self.verbose:
3102 print "argh"
3103 return ""
3104 log = extractLogMessageFromGitCommit(next)
3105 settings = extractSettingsGitLog(log)
3106 currentChange = int(settings['change'])
3107 if self.verbose:
3108 print "current change %s" % currentChange
3109
3110 if currentChange == change:
3111 if self.verbose:
3112 print "found %s" % next
3113 return next
3114
3115 if currentChange < change:
3116 earliestCommit = "^%s" % next
3117 else:
3118 latestCommit = "%s" % next
3119
3120 return ""
3121
3122 def importNewBranch(self, branch, maxChange):
3123 # make fast-import flush all changes to disk and update the refs using the checkpoint
3124 # command so that we can try to find the branch parent in the git history
3125 self.gitStream.write("checkpoint\n\n");
3126 self.gitStream.flush();
3127 branchPrefix = self.depotPaths[0] + branch + "/"
3128 range = "@1,%s" % maxChange
3129 #print "prefix" + branchPrefix
96b2d54a 3130 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3131 if len(changes) <= 0:
3132 return False
3133 firstChange = changes[0]
3134 #print "first change in branch: %s" % firstChange
3135 sourceBranch = self.knownBranches[branch]
3136 sourceDepotPath = self.depotPaths[0] + sourceBranch
3137 sourceRef = self.gitRefForBranch(sourceBranch)
3138 #print "source " + sourceBranch
3139
52a4880b 3140 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3141 #print "branch parent: %s" % branchParentChange
3142 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3143 if len(gitParent) > 0:
3144 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3145 #print "parent git commit: %s" % gitParent
3146
3147 self.importChanges(changes)
3148 return True
3149
fed23693
VA
3150 def searchParent(self, parent, branch, target):
3151 parentFound = False
c7d34884
PW
3152 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3153 "--no-merges", parent]):
fed23693
VA
3154 blob = blob.strip()
3155 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3156 parentFound = True
3157 if self.verbose:
3158 print "Found parent of %s in commit %s" % (branch, blob)
3159 break
3160 if parentFound:
3161 return blob
3162 else:
3163 return None
3164
e87f37ae
SH
3165 def importChanges(self, changes):
3166 cnt = 1
3167 for change in changes:
18fa13d0 3168 description = p4_describe(change)
e87f37ae
SH
3169 self.updateOptionDict(description)
3170
3171 if not self.silent:
3172 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3173 sys.stdout.flush()
3174 cnt = cnt + 1
3175
3176 try:
3177 if self.detectBranches:
3178 branches = self.splitFilesIntoBranches(description)
3179 for branch in branches.keys():
3180 ## HACK --hwn
3181 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3182 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3183
3184 parent = ""
3185
3186 filesForCommit = branches[branch]
3187
3188 if self.verbose:
3189 print "branch is %s" % branch
3190
3191 self.updatedBranches.add(branch)
3192
3193 if branch not in self.createdBranches:
3194 self.createdBranches.add(branch)
3195 parent = self.knownBranches[branch]
3196 if parent == branch:
3197 parent = ""
1ca3d710
SH
3198 else:
3199 fullBranch = self.projectName + branch
3200 if fullBranch not in self.p4BranchesInGit:
3201 if not self.silent:
3202 print("\n Importing new branch %s" % fullBranch);
3203 if self.importNewBranch(branch, change - 1):
3204 parent = ""
3205 self.p4BranchesInGit.append(fullBranch)
3206 if not self.silent:
3207 print("\n Resuming with change %s" % change);
3208
3209 if self.verbose:
3210 print "parent determined through known branches: %s" % parent
e87f37ae 3211
8134f69c
SH
3212 branch = self.gitRefForBranch(branch)
3213 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3214
3215 if self.verbose:
3216 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3217
3218 if len(parent) == 0 and branch in self.initialParents:
3219 parent = self.initialParents[branch]
3220 del self.initialParents[branch]
3221
fed23693
VA
3222 blob = None
3223 if len(parent) > 0:
4f9273d2 3224 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3225 if self.verbose:
3226 print "Creating temporary branch: " + tempBranch
e63231e5 3227 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3228 self.tempBranches.append(tempBranch)
3229 self.checkpoint()
3230 blob = self.searchParent(parent, branch, tempBranch)
3231 if blob:
e63231e5 3232 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3233 else:
3234 if self.verbose:
3235 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3236 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3237 else:
3238 files = self.extractFilesFromCommit(description)
e63231e5 3239 self.commit(description, files, self.branch,
e87f37ae 3240 self.initialParent)
47497844 3241 # only needed once, to connect to the previous commit
e87f37ae
SH
3242 self.initialParent = ""
3243 except IOError:
3244 print self.gitError.read()
3245 sys.exit(1)
3246
c208a243
SH
3247 def importHeadRevision(self, revision):
3248 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3249
4e2e6ce4
PW
3250 details = {}
3251 details["user"] = "git perforce import user"
1494fcbb 3252 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3253 % (' '.join(self.depotPaths), revision))
3254 details["change"] = revision
3255 newestRevision = 0
3256
3257 fileCnt = 0
6de040df
LD
3258 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3259
3260 for info in p4CmdList(["files"] + fileArgs):
c208a243 3261
68b28593 3262 if 'code' in info and info['code'] == 'error':
c208a243
SH
3263 sys.stderr.write("p4 returned an error: %s\n"
3264 % info['data'])
d88e707f
PW
3265 if info['data'].find("must refer to client") >= 0:
3266 sys.stderr.write("This particular p4 error is misleading.\n")
3267 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3268 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3269 sys.exit(1)
68b28593
PW
3270 if 'p4ExitCode' in info:
3271 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3272 sys.exit(1)
3273
3274
3275 change = int(info["change"])
3276 if change > newestRevision:
3277 newestRevision = change
3278
56c09345 3279 if info["action"] in self.delete_actions:
c208a243
SH
3280 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3281 #fileCnt = fileCnt + 1
3282 continue
3283
3284 for prop in ["depotFile", "rev", "action", "type" ]:
3285 details["%s%s" % (prop, fileCnt)] = info[prop]
3286
3287 fileCnt = fileCnt + 1
3288
3289 details["change"] = newestRevision
4e2e6ce4 3290
9dcb9f24 3291 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3292 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3293 res = p4_describe(newestRevision)
3294 details["time"] = res["time"]
4e2e6ce4 3295
c208a243
SH
3296 self.updateOptionDict(details)
3297 try:
e63231e5 3298 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3299 except IOError:
3300 print "IO error with git fast-import. Is your git version recent enough?"
3301 print self.gitError.read()
3302
3303
b984733c 3304 def run(self, args):
6326aa58 3305 self.depotPaths = []
179caebf 3306 self.changeRange = ""
6326aa58 3307 self.previousDepotPaths = []
991a2de4 3308 self.hasOrigin = False
ce6f33c8 3309
29bdbac1
SH
3310 # map from branch depot path to parent branch
3311 self.knownBranches = {}
3312 self.initialParents = {}
3313
a028a98e
SH
3314 if self.importIntoRemotes:
3315 self.refPrefix = "refs/remotes/p4/"
3316 else:
db775559 3317 self.refPrefix = "refs/heads/p4/"
a028a98e 3318
991a2de4
PW
3319 if self.syncWithOrigin:
3320 self.hasOrigin = originP4BranchesExist()
3321 if self.hasOrigin:
3322 if not self.silent:
3323 print 'Syncing with origin first, using "git fetch origin"'
3324 system("git fetch origin")
10f880f8 3325
5a8e84cd 3326 branch_arg_given = bool(self.branch)
569d1bd4 3327 if len(self.branch) == 0:
db775559 3328 self.branch = self.refPrefix + "master"
a028a98e 3329 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3330 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3331 system("git branch -D p4")
967f72e2 3332
a93d33ee
PW
3333 # accept either the command-line option, or the configuration variable
3334 if self.useClientSpec:
3335 # will use this after clone to set the variable
3336 self.useClientSpec_from_options = True
3337 else:
0d609032 3338 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3339 self.useClientSpec = True
3340 if self.useClientSpec:
543987bd 3341 self.clientSpecDirs = getClientSpec()
3a70cdfa 3342
6a49f8e2
HWN
3343 # TODO: should always look at previous commits,
3344 # merge with previous imports, if possible.
3345 if args == []:
d414c74a 3346 if self.hasOrigin:
5ca44617 3347 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3348
3349 # branches holds mapping from branch name to sha1
3350 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3351
3352 # restrict to just this one, disabling detect-branches
3353 if branch_arg_given:
3354 short = self.branch.split("/")[-1]
3355 if short in branches:
3356 self.p4BranchesInGit = [ short ]
3357 else:
3358 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3359
3360 if len(self.p4BranchesInGit) > 1:
3361 if not self.silent:
3362 print "Importing from/into multiple branches"
3363 self.detectBranches = True
8c9e8b6e
PW
3364 for branch in branches.keys():
3365 self.initialParents[self.refPrefix + branch] = \
3366 branches[branch]
967f72e2 3367
29bdbac1
SH
3368 if self.verbose:
3369 print "branches: %s" % self.p4BranchesInGit
3370
3371 p4Change = 0
3372 for branch in self.p4BranchesInGit:
cebdf5af 3373 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3374
3375 settings = extractSettingsGitLog(logMsg)
29bdbac1 3376
bb6e09b2
HWN
3377 self.readOptions(settings)
3378 if (settings.has_key('depot-paths')
3379 and settings.has_key ('change')):
3380 change = int(settings['change']) + 1
29bdbac1
SH
3381 p4Change = max(p4Change, change)
3382
bb6e09b2
HWN
3383 depotPaths = sorted(settings['depot-paths'])
3384 if self.previousDepotPaths == []:
6326aa58 3385 self.previousDepotPaths = depotPaths
29bdbac1 3386 else:
6326aa58
HWN
3387 paths = []
3388 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3389 prev_list = prev.split("/")
3390 cur_list = cur.split("/")
3391 for i in range(0, min(len(cur_list), len(prev_list))):
3392 if cur_list[i] <> prev_list[i]:
583e1707 3393 i = i - 1
6326aa58
HWN
3394 break
3395
04d277b3 3396 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3397
3398 self.previousDepotPaths = paths
29bdbac1
SH
3399
3400 if p4Change > 0:
bb6e09b2 3401 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3402 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3403 if not self.silent and not self.detectBranches:
967f72e2 3404 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3405
40d69ac3
PW
3406 # accept multiple ref name abbreviations:
3407 # refs/foo/bar/branch -> use it exactly
3408 # p4/branch -> prepend refs/remotes/ or refs/heads/
3409 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3410 if not self.branch.startswith("refs/"):
40d69ac3
PW
3411 if self.importIntoRemotes:
3412 prepend = "refs/remotes/"
3413 else:
3414 prepend = "refs/heads/"
3415 if not self.branch.startswith("p4/"):
3416 prepend += "p4/"
3417 self.branch = prepend + self.branch
179caebf 3418
6326aa58 3419 if len(args) == 0 and self.depotPaths:
b984733c 3420 if not self.silent:
6326aa58 3421 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3422 else:
6326aa58 3423 if self.depotPaths and self.depotPaths != args:
cebdf5af 3424 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3425 "This doesn't work!" % (' '.join (self.depotPaths),
3426 ' '.join (args)))
b984733c 3427 sys.exit(1)
6326aa58 3428
bb6e09b2 3429 self.depotPaths = sorted(args)
b984733c 3430
1c49fc19 3431 revision = ""
b984733c 3432 self.users = {}
b984733c 3433
58c8bc7c
PW
3434 # Make sure no revision specifiers are used when --changesfile
3435 # is specified.
3436 bad_changesfile = False
3437 if len(self.changesFile) > 0:
3438 for p in self.depotPaths:
3439 if p.find("@") >= 0 or p.find("#") >= 0:
3440 bad_changesfile = True
3441 break
3442 if bad_changesfile:
3443 die("Option --changesfile is incompatible with revision specifiers")
3444
6326aa58
HWN
3445 newPaths = []
3446 for p in self.depotPaths:
3447 if p.find("@") != -1:
3448 atIdx = p.index("@")
3449 self.changeRange = p[atIdx:]
3450 if self.changeRange == "@all":
3451 self.changeRange = ""
6a49f8e2 3452 elif ',' not in self.changeRange: