git-p4: add option to disable syncing of p4/master with p4
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
df8a9e86 28import errno
8b41a97f 29
a235e85c
BC
30try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
4addad22 45verbose = False
86949eef 46
06804c76 47# Only labels/tags matching this will be imported/exported
c8942a22 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 49
1051ef00
LD
50# Grab changes in blocks of this many revisions, unless otherwise requested
51defaultBlockSize = 512
52
21a50753
AK
53def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
6de040df 60 real_cmd = ["p4"]
abcaf073
AK
61
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
6de040df 64 real_cmd += ["-u",user]
abcaf073
AK
65
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
6de040df 68 real_cmd += ["-P", password]
abcaf073
AK
69
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
6de040df 72 real_cmd += ["-p", port]
abcaf073
AK
73
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
41799aa2 76 real_cmd += ["-H", host]
abcaf073
AK
77
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
6de040df 80 real_cmd += ["-c", client]
abcaf073 81
89a6ecc5
LS
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
bc233524
IK
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
6de040df
LD
89
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
21a50753
AK
94 return real_cmd
95
378f7be1
LD
96def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
bbd84863
MF
106def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
053fd0c1 127
4d25dc44
LS
128def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
86dff6b6
HWN
138def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
6de040df 145def write_pipe(c, stdin):
4addad22 146 if verbose:
6de040df 147 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 148
6de040df
LD
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
b016d397
HWN
156
157 return val
158
6de040df 159def p4_write_pipe(c, stdin):
d9429194 160 real_cmd = p4_build_cmd(c)
6de040df 161 return write_pipe(real_cmd, stdin)
d9429194 162
78871bf4
LD
163def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
167 """
4addad22 168 if verbose:
6de040df 169 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 170
6de040df 171 expand = isinstance(c,basestring)
1f5f3907
LS
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
78871bf4
LD
174 return (p.returncode, out, err)
175
176def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
180 """
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
1f5f3907 187 return out
b016d397 188
78871bf4
LD
189def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
192 """
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
198
d9429194
AK
199def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
b016d397 202
bce4c5fc 203def read_pipe_lines(c):
4addad22 204 if verbose:
6de040df
LD
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
b016d397 210 val = pipe.readlines()
6de040df
LD
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
b016d397
HWN
213
214 return val
caace111 215
2318121b
AK
216def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
155af834 218 real_cmd = p4_build_cmd(c)
2318121b
AK
219 return read_pipe_lines(real_cmd)
220
8e9497c2
GG
221def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
229
249da4c0
PW
230def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
235
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
248
cbff4b25 249def system(cmd, ignore_error=False):
6de040df 250 expand = isinstance(cmd,basestring)
4addad22 251 if verbose:
6de040df 252 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 253 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 254 if retcode and not ignore_error:
a235e85c 255 raise CalledProcessError(retcode, cmd)
6754a299 256
cbff4b25
LD
257 return retcode
258
bf9320f1
AK
259def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
155af834 261 real_cmd = p4_build_cmd(cmd)
6de040df 262 expand = isinstance(real_cmd, basestring)
a235e85c
BC
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
6de040df 266
7f0e5962
PW
267_p4_version_string = None
268def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
271
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276 """
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
282
6de040df 283def p4_integrate(src, dest):
9d7d446a 284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 285
8d7ec362 286def p4_sync(f, *options):
9d7d446a 287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
288
289def p4_add(f):
9d7d446a
PW
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
6de040df
LD
295
296def p4_delete(f):
9d7d446a 297 p4_system(["delete", wildcard_encode(f)])
6de040df 298
a02b8bc4
RP
299def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
6de040df
LD
301
302def p4_revert(f):
9d7d446a 303 p4_system(["revert", wildcard_encode(f)])
6de040df 304
9d7d446a
PW
305def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 307
46c609e9
LD
308def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
311
8e9497c2
GG
312def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
1051ef00 315def p4_last_change():
1997e91f 316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
1051ef00
LD
317 return int(results[0]['change'])
318
18fa13d0
PW
319def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
323
1997e91f 324 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
18fa13d0
PW
325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328 d = ds[0]
329
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340 return d
341
9cffb8c8
PW
342#
343# Canonicalize the p4 type and return a tuple of the
344# base type, plus any modifiers. See "p4 help filetypes"
345# for a list and explanation.
346#
347def split_p4_type(p4type):
348
349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
366 }
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
b9fc6ea9 376
60df071c
LD
377#
378# return the raw p4 type of a file (text, text+ko, etc)
379#
79467e61
PW
380def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
382 return results[0]['headType']
383
384#
385# Given a type base and modifier, return a regexp matching
386# the keywords that can be expanded in the file
387#
388def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
6b2bf41e 400 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
406
407#
408# Given a file, return a regexp matching the possible
409# RCS keywords that will be expanded, or None for files
410# with kw expansion turned off.
411#
412def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 418
c65b670e
CP
419def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
422
423 p4Type = "+x"
424
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
431
6de040df 432 p4_reopen(p4Type, file)
c65b670e
CP
433
434def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
436
9d7d446a 437 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
439 if match:
440 return match.group(1)
441 else:
f3e5ae4f 442 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 443
06804c76
LD
444# Return the set of all p4 labels
445def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
449
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
453
454 return labels
455
456# Return the set of all git tags
457def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
463
b43b0a3c
CP
464def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
470
471def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
473
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
476
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
487
488 If the pattern is not matched, None is returned."""
489
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
501 }
502 return None
503
c65b670e
CP
504def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
508
509def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
511
1997e91f 512def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
6de040df
LD
513
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
520
521 cmd = p4_build_cmd(cmd)
6a49f8e2 522 if verbose:
6de040df 523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
524
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
9f90c733
SL
536 stdin_file.flush()
537 stdin_file.seek(0)
538
6de040df
LD
539 p4 = subprocess.Popen(cmd,
540 shell=expand,
9f90c733
SL
541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
86949eef
SH
543
544 result = []
545 try:
546 while True:
9f90c733 547 entry = marshal.load(p4.stdout)
1997e91f
MT
548 if skip_info:
549 if 'code' in entry and entry['code'] == 'info':
550 continue
c3f6163b
AG
551 if cb is not None:
552 cb(entry)
553 else:
554 result.append(entry)
86949eef
SH
555 except EOFError:
556 pass
9f90c733
SL
557 exitCode = p4.wait()
558 if exitCode != 0:
ac3e0d79
SH
559 entry = {}
560 entry["p4ExitCode"] = exitCode
561 result.append(entry)
86949eef
SH
562
563 return result
564
565def p4Cmd(cmd):
566 list = p4CmdList(cmd)
567 result = {}
568 for entry in list:
569 result.update(entry)
570 return result;
571
cb2c9db5
SH
572def p4Where(depotPath):
573 if not depotPath.endswith("/"):
574 depotPath += "/"
cd884106
VA
575 depotPathLong = depotPath + "..."
576 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
577 output = None
578 for entry in outputList:
75bc9573 579 if "depotFile" in entry:
cd884106
VA
580 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
581 # The base path always ends with "/...".
582 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
583 output = entry
584 break
585 elif "data" in entry:
586 data = entry.get("data")
587 space = data.find(" ")
588 if data[:space] == depotPath:
589 output = entry
590 break
7f705dc3
TAL
591 if output == None:
592 return ""
dc524036
SH
593 if output["code"] == "error":
594 return ""
cb2c9db5
SH
595 clientPath = ""
596 if "path" in output:
597 clientPath = output.get("path")
598 elif "data" in output:
599 data = output.get("data")
600 lastSpace = data.rfind(" ")
601 clientPath = data[lastSpace + 1:]
602
603 if clientPath.endswith("..."):
604 clientPath = clientPath[:-3]
605 return clientPath
606
86949eef 607def currentGitBranch():
eff45110 608 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
86949eef 609
4f5cf76a 610def isValidGitDir(path):
378f7be1 611 return git_dir(path) != None
4f5cf76a 612
463e8af6 613def parseRevision(ref):
b25b2065 614 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 615
28755dba
PW
616def branchExists(ref):
617 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
618 ignore_error=True)
619 return len(rev) > 0
620
6ae8de88
SH
621def extractLogMessageFromGitCommit(commit):
622 logMessage = ""
b016d397
HWN
623
624 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 625 foundTitle = False
b016d397 626 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
627 if not foundTitle:
628 if len(log) == 1:
1c094184 629 foundTitle = True
6ae8de88
SH
630 continue
631
632 logMessage += log
633 return logMessage
634
bb6e09b2 635def extractSettingsGitLog(log):
6ae8de88
SH
636 values = {}
637 for line in log.split("\n"):
638 line = line.strip()
6326aa58
HWN
639 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
640 if not m:
641 continue
642
643 assignments = m.group(1).split (':')
644 for a in assignments:
645 vals = a.split ('=')
646 key = vals[0].strip()
647 val = ('='.join (vals[1:])).strip()
648 if val.endswith ('\"') and val.startswith('"'):
649 val = val[1:-1]
650
651 values[key] = val
652
845b42cb
SH
653 paths = values.get("depot-paths")
654 if not paths:
655 paths = values.get("depot-path")
a3fdd579
SH
656 if paths:
657 values['depot-paths'] = paths.split(',')
bb6e09b2 658 return values
6ae8de88 659
8136a639 660def gitBranchExists(branch):
bb6e09b2
HWN
661 proc = subprocess.Popen(["git", "rev-parse", branch],
662 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 663 return proc.wait() == 0;
8136a639 664
36bd8446 665_gitConfig = {}
b345d6c3 666
692e1796 667def gitConfig(key, typeSpecifier=None):
36bd8446 668 if not _gitConfig.has_key(key):
692e1796
LS
669 cmd = [ "git", "config" ]
670 if typeSpecifier:
671 cmd += [ typeSpecifier ]
672 cmd += [ key ]
b345d6c3
PW
673 s = read_pipe(cmd, ignore_error=True)
674 _gitConfig[key] = s.strip()
36bd8446 675 return _gitConfig[key]
01265103 676
0d609032
PW
677def gitConfigBool(key):
678 """Return a bool, using git config --bool. It is True only if the
679 variable is set to true, and False if set to false or not present
680 in the config."""
681
36bd8446 682 if not _gitConfig.has_key(key):
692e1796 683 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 684 return _gitConfig[key]
01265103 685
cb1dafdf
LS
686def gitConfigInt(key):
687 if not _gitConfig.has_key(key):
688 cmd = [ "git", "config", "--int", key ]
0d609032
PW
689 s = read_pipe(cmd, ignore_error=True)
690 v = s.strip()
cb1dafdf
LS
691 try:
692 _gitConfig[key] = int(gitConfig(key, '--int'))
693 except ValueError:
694 _gitConfig[key] = None
36bd8446 695 return _gitConfig[key]
01265103 696
7199cf13
VA
697def gitConfigList(key):
698 if not _gitConfig.has_key(key):
2abba301 699 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
c3c2b057 700 _gitConfig[key] = s.strip().splitlines()
7960e707
LS
701 if _gitConfig[key] == ['']:
702 _gitConfig[key] = []
7199cf13
VA
703 return _gitConfig[key]
704
2c8037ed
PW
705def p4BranchesInGit(branchesAreInRemotes=True):
706 """Find all the branches whose names start with "p4/", looking
707 in remotes or heads as specified by the argument. Return
708 a dictionary of { branch: revision } for each one found.
709 The branch names are the short names, without any
710 "p4/" prefix."""
711
062410bb
SH
712 branches = {}
713
714 cmdline = "git rev-parse --symbolic "
715 if branchesAreInRemotes:
2c8037ed 716 cmdline += "--remotes"
062410bb 717 else:
2c8037ed 718 cmdline += "--branches"
062410bb
SH
719
720 for line in read_pipe_lines(cmdline):
721 line = line.strip()
722
2c8037ed
PW
723 # only import to p4/
724 if not line.startswith('p4/'):
725 continue
726 # special symbolic ref to p4/master
727 if line == "p4/HEAD":
062410bb 728 continue
062410bb 729
2c8037ed
PW
730 # strip off p4/ prefix
731 branch = line[len("p4/"):]
062410bb
SH
732
733 branches[branch] = parseRevision(line)
2c8037ed 734
062410bb
SH
735 return branches
736
5a8e84cd
PW
737def branch_exists(branch):
738 """Make sure that the given ref name really exists."""
739
740 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
741 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
742 out, _ = p.communicate()
743 if p.returncode:
744 return False
745 # expect exactly one line of output: the branch name
746 return out.rstrip() == branch
747
9ceab363 748def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
749 branches = p4BranchesInGit()
750 # map from depot-path to branch name
751 branchByDepotPath = {}
752 for branch in branches.keys():
753 tip = branches[branch]
754 log = extractLogMessageFromGitCommit(tip)
755 settings = extractSettingsGitLog(log)
756 if settings.has_key("depot-paths"):
757 paths = ",".join(settings["depot-paths"])
758 branchByDepotPath[paths] = "remotes/p4/" + branch
759
27d2d811 760 settings = None
27d2d811
SH
761 parent = 0
762 while parent < 65535:
9ceab363 763 commit = head + "~%s" % parent
27d2d811
SH
764 log = extractLogMessageFromGitCommit(commit)
765 settings = extractSettingsGitLog(log)
86506fe5
SH
766 if settings.has_key("depot-paths"):
767 paths = ",".join(settings["depot-paths"])
768 if branchByDepotPath.has_key(paths):
769 return [branchByDepotPath[paths], settings]
27d2d811 770
86506fe5 771 parent = parent + 1
27d2d811 772
86506fe5 773 return ["", settings]
27d2d811 774
5ca44617
SH
775def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
776 if not silent:
777 print ("Creating/updating branch(es) in %s based on origin branch(es)"
778 % localRefPrefix)
779
780 originPrefix = "origin/p4/"
781
782 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
783 line = line.strip()
784 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
785 continue
786
787 headName = line[len(originPrefix):]
788 remoteHead = localRefPrefix + headName
789 originHead = line
790
791 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
792 if (not original.has_key('depot-paths')
793 or not original.has_key('change')):
794 continue
795
796 update = False
797 if not gitBranchExists(remoteHead):
798 if verbose:
799 print "creating %s" % remoteHead
800 update = True
801 else:
802 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
803 if settings.has_key('change') > 0:
804 if settings['depot-paths'] == original['depot-paths']:
805 originP4Change = int(original['change'])
806 p4Change = int(settings['change'])
807 if originP4Change > p4Change:
808 print ("%s (%s) is newer than %s (%s). "
809 "Updating p4 branch from origin."
810 % (originHead, originP4Change,
811 remoteHead, p4Change))
812 update = True
813 else:
814 print ("Ignoring: %s was imported from %s while "
815 "%s was imported from %s"
816 % (originHead, ','.join(original['depot-paths']),
817 remoteHead, ','.join(settings['depot-paths'])))
818
819 if update:
820 system("git update-ref %s %s" % (remoteHead, originHead))
821
822def originP4BranchesExist():
823 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
824
1051ef00
LD
825
826def p4ParseNumericChangeRange(parts):
827 changeStart = int(parts[0][1:])
828 if parts[1] == '#head':
829 changeEnd = p4_last_change()
830 else:
831 changeEnd = int(parts[1])
832
833 return (changeStart, changeEnd)
834
835def chooseBlockSize(blockSize):
836 if blockSize:
837 return blockSize
838 else:
839 return defaultBlockSize
840
841def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 842 assert depotPaths
96b2d54a 843
1051ef00
LD
844 # Parse the change range into start and end. Try to find integer
845 # revision ranges as these can be broken up into blocks to avoid
846 # hitting server-side limits (maxrows, maxscanresults). But if
847 # that doesn't work, fall back to using the raw revision specifier
848 # strings, without using block mode.
849
96b2d54a 850 if changeRange is None or changeRange == '':
1051ef00
LD
851 changeStart = 1
852 changeEnd = p4_last_change()
853 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
854 else:
855 parts = changeRange.split(',')
856 assert len(parts) == 2
1051ef00
LD
857 try:
858 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
859 block_size = chooseBlockSize(requestedBlockSize)
860 except:
861 changeStart = parts[0][1:]
862 changeEnd = parts[1]
863 if requestedBlockSize:
864 die("cannot use --changes-block-size with non-numeric revisions")
865 block_size = None
4f6432d8 866
9943e5b9 867 changes = set()
96b2d54a 868
1f90a648
SH
869 # Retrieve changes a block at a time, to prevent running
870 # into a MaxResults/MaxScanRows error from the server.
1051ef00 871
1f90a648
SH
872 while True:
873 cmd = ['changes']
1051ef00 874
1f90a648
SH
875 if block_size:
876 end = min(changeEnd, changeStart + block_size)
877 revisionRange = "%d,%d" % (changeStart, end)
878 else:
879 revisionRange = "%s,%s" % (changeStart, changeEnd)
1051ef00 880
1f90a648 881 for p in depotPaths:
1051ef00
LD
882 cmd += ["%s...@%s" % (p, revisionRange)]
883
1f90a648 884 # Insert changes in chronological order
b596b3b9
MT
885 for entry in reversed(p4CmdList(cmd)):
886 if entry.has_key('p4ExitCode'):
887 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
888 if not entry.has_key('change'):
889 continue
890 changes.add(int(entry['change']))
1051ef00 891
1f90a648
SH
892 if not block_size:
893 break
1051ef00 894
1f90a648
SH
895 if end >= changeEnd:
896 break
1051ef00 897
1f90a648 898 changeStart = end + 1
4f6432d8 899
1f90a648
SH
900 changes = sorted(changes)
901 return changes
4f6432d8 902
d53de8b9
TAL
903def p4PathStartsWith(path, prefix):
904 # This method tries to remedy a potential mixed-case issue:
905 #
906 # If UserA adds //depot/DirA/file1
907 # and UserB adds //depot/dira/file2
908 #
909 # we may or may not have a problem. If you have core.ignorecase=true,
910 # we treat DirA and dira as the same directory
0d609032 911 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
912 return path.lower().startswith(prefix.lower())
913 return path.startswith(prefix)
914
543987bd
PW
915def getClientSpec():
916 """Look at the p4 client spec, create a View() object that contains
917 all the mappings, and return it."""
918
919 specList = p4CmdList("client -o")
920 if len(specList) != 1:
921 die('Output from "client -o" is %d lines, expecting 1' %
922 len(specList))
923
924 # dictionary of all client parameters
925 entry = specList[0]
926
9d57c4a6
KS
927 # the //client/ name
928 client_name = entry["Client"]
929
543987bd
PW
930 # just the keys that start with "View"
931 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
932
933 # hold this new View
9d57c4a6 934 view = View(client_name)
543987bd
PW
935
936 # append the lines, in order, to the view
937 for view_num in range(len(view_keys)):
938 k = "View%d" % view_num
939 if k not in view_keys:
940 die("Expected view key %s missing" % k)
941 view.append(entry[k])
942
943 return view
944
945def getClientRoot():
946 """Grab the client directory."""
947
948 output = p4CmdList("client -o")
949 if len(output) != 1:
950 die('Output from "client -o" is %d lines, expecting 1' % len(output))
951
952 entry = output[0]
953 if "Root" not in entry:
954 die('Client has no "Root"')
955
956 return entry["Root"]
957
9d7d446a
PW
958#
959# P4 wildcards are not allowed in filenames. P4 complains
960# if you simply add them, but you can force it with "-f", in
961# which case it translates them into %xx encoding internally.
962#
963def wildcard_decode(path):
964 # Search for and fix just these four characters. Do % last so
965 # that fixing it does not inadvertently create new %-escapes.
966 # Cannot have * in a filename in windows; untested as to
967 # what p4 would do in such a case.
968 if not platform.system() == "Windows":
969 path = path.replace("%2A", "*")
970 path = path.replace("%23", "#") \
971 .replace("%40", "@") \
972 .replace("%25", "%")
973 return path
974
975def wildcard_encode(path):
976 # do % first to avoid double-encoding the %s introduced here
977 path = path.replace("%", "%25") \
978 .replace("*", "%2A") \
979 .replace("#", "%23") \
980 .replace("@", "%40")
981 return path
982
983def wildcard_present(path):
598354c0
BC
984 m = re.search("[*#@%]", path)
985 return m is not None
9d7d446a 986
a5db4b12
LS
987class LargeFileSystem(object):
988 """Base class for large file system support."""
989
990 def __init__(self, writeToGitStream):
991 self.largeFiles = set()
992 self.writeToGitStream = writeToGitStream
993
994 def generatePointer(self, cloneDestination, contentFile):
995 """Return the content of a pointer file that is stored in Git instead of
996 the actual content."""
997 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
998
999 def pushFile(self, localLargeFile):
1000 """Push the actual content which is not stored in the Git repository to
1001 a server."""
1002 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1003
1004 def hasLargeFileExtension(self, relPath):
1005 return reduce(
1006 lambda a, b: a or b,
1007 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1008 False
1009 )
1010
1011 def generateTempFile(self, contents):
1012 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1013 for d in contents:
1014 contentFile.write(d)
1015 contentFile.close()
1016 return contentFile.name
1017
1018 def exceedsLargeFileThreshold(self, relPath, contents):
1019 if gitConfigInt('git-p4.largeFileThreshold'):
1020 contentsSize = sum(len(d) for d in contents)
1021 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1022 return True
1023 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024 contentsSize = sum(len(d) for d in contents)
1025 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1026 return False
1027 contentTempFile = self.generateTempFile(contents)
1028 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1029 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1030 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1031 zf.close()
1032 compressedContentsSize = zf.infolist()[0].compress_size
1033 os.remove(contentTempFile)
1034 os.remove(compressedContentFile.name)
1035 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1036 return True
1037 return False
1038
1039 def addLargeFile(self, relPath):
1040 self.largeFiles.add(relPath)
1041
1042 def removeLargeFile(self, relPath):
1043 self.largeFiles.remove(relPath)
1044
1045 def isLargeFile(self, relPath):
1046 return relPath in self.largeFiles
1047
1048 def processContent(self, git_mode, relPath, contents):
1049 """Processes the content of git fast import. This method decides if a
1050 file is stored in the large file system and handles all necessary
1051 steps."""
1052 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1053 contentTempFile = self.generateTempFile(contents)
d5eb3cf5
LS
1054 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1055 if pointer_git_mode:
1056 git_mode = pointer_git_mode
1057 if localLargeFile:
1058 # Move temp file to final location in large file system
1059 largeFileDir = os.path.dirname(localLargeFile)
1060 if not os.path.isdir(largeFileDir):
1061 os.makedirs(largeFileDir)
1062 shutil.move(contentTempFile, localLargeFile)
1063 self.addLargeFile(relPath)
1064 if gitConfigBool('git-p4.largeFilePush'):
1065 self.pushFile(localLargeFile)
1066 if verbose:
1067 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
a5db4b12
LS
1068 return (git_mode, contents)
1069
1070class MockLFS(LargeFileSystem):
1071 """Mock large file system for testing."""
1072
1073 def generatePointer(self, contentFile):
1074 """The pointer content is the original content prefixed with "pointer-".
1075 The local filename of the large file storage is derived from the file content.
1076 """
1077 with open(contentFile, 'r') as f:
1078 content = next(f)
1079 gitMode = '100644'
1080 pointerContents = 'pointer-' + content
1081 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1082 return (gitMode, pointerContents, localLargeFile)
1083
1084 def pushFile(self, localLargeFile):
1085 """The remote filename of the large file storage is the same as the local
1086 one but in a different directory.
1087 """
1088 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1089 if not os.path.exists(remotePath):
1090 os.makedirs(remotePath)
1091 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1092
b47d807d
LS
1093class GitLFS(LargeFileSystem):
1094 """Git LFS as backend for the git-p4 large file system.
1095 See https://git-lfs.github.com/ for details."""
1096
1097 def __init__(self, *args):
1098 LargeFileSystem.__init__(self, *args)
1099 self.baseGitAttributes = []
1100
1101 def generatePointer(self, contentFile):
1102 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1103 mode and content which is stored in the Git repository instead of
1104 the actual content. Return also the new location of the actual
1105 content.
1106 """
d5eb3cf5
LS
1107 if os.path.getsize(contentFile) == 0:
1108 return (None, '', None)
1109
b47d807d
LS
1110 pointerProcess = subprocess.Popen(
1111 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1112 stdout=subprocess.PIPE
1113 )
1114 pointerFile = pointerProcess.stdout.read()
1115 if pointerProcess.wait():
1116 os.remove(contentFile)
1117 die('git-lfs pointer command failed. Did you install the extension?')
82f2567e
LS
1118
1119 # Git LFS removed the preamble in the output of the 'pointer' command
1120 # starting from version 1.2.0. Check for the preamble here to support
1121 # earlier versions.
1122 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1123 if pointerFile.startswith('Git LFS pointer for'):
1124 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1125
1126 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
b47d807d
LS
1127 localLargeFile = os.path.join(
1128 os.getcwd(),
1129 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1130 oid,
1131 )
1132 # LFS Spec states that pointer files should not have the executable bit set.
1133 gitMode = '100644'
82f2567e 1134 return (gitMode, pointerFile, localLargeFile)
b47d807d
LS
1135
1136 def pushFile(self, localLargeFile):
1137 uploadProcess = subprocess.Popen(
1138 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1139 )
1140 if uploadProcess.wait():
1141 die('git-lfs push command failed. Did you define a remote?')
1142
1143 def generateGitAttributes(self):
1144 return (
1145 self.baseGitAttributes +
1146 [
1147 '\n',
1148 '#\n',
1149 '# Git LFS (see https://git-lfs.github.com/)\n',
1150 '#\n',
1151 ] +
862f9312 1152 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1153 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1154 ] +
862f9312 1155 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
b47d807d
LS
1156 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1157 ]
1158 )
1159
1160 def addLargeFile(self, relPath):
1161 LargeFileSystem.addLargeFile(self, relPath)
1162 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1163
1164 def removeLargeFile(self, relPath):
1165 LargeFileSystem.removeLargeFile(self, relPath)
1166 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1167
1168 def processContent(self, git_mode, relPath, contents):
1169 if relPath == '.gitattributes':
1170 self.baseGitAttributes = contents
1171 return (git_mode, self.generateGitAttributes())
1172 else:
1173 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1174
b984733c
SH
1175class Command:
1176 def __init__(self):
1177 self.usage = "usage: %prog [options]"
8910ac0e 1178 self.needsGit = True
6a10b6aa 1179 self.verbose = False
b984733c 1180
8cf422db
LD
1181 # This is required for the "append" cloneExclude action
1182 def ensure_value(self, attr, value):
1183 if not hasattr(self, attr) or getattr(self, attr) is None:
1184 setattr(self, attr, value)
1185 return getattr(self, attr)
1186
3ea2cfd4
LD
1187class P4UserMap:
1188 def __init__(self):
1189 self.userMapFromPerforceServer = False
affb474f
LD
1190 self.myP4UserId = None
1191
1192 def p4UserId(self):
1193 if self.myP4UserId:
1194 return self.myP4UserId
1195
1196 results = p4CmdList("user -o")
1197 for r in results:
1198 if r.has_key('User'):
1199 self.myP4UserId = r['User']
1200 return r['User']
1201 die("Could not find your p4 user id")
1202
1203 def p4UserIsMe(self, p4User):
1204 # return True if the given p4 user is actually me
1205 me = self.p4UserId()
1206 if not p4User or p4User != me:
1207 return False
1208 else:
1209 return True
3ea2cfd4
LD
1210
1211 def getUserCacheFilename(self):
1212 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1213 return home + "/.gitp4-usercache.txt"
1214
1215 def getUserMapFromPerforceServer(self):
1216 if self.userMapFromPerforceServer:
1217 return
1218 self.users = {}
1219 self.emails = {}
1220
1221 for output in p4CmdList("users"):
1222 if not output.has_key("User"):
1223 continue
1224 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1225 self.emails[output["Email"]] = output["User"]
1226
10d08a14
LS
1227 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1228 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1229 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1230 if mapUser and len(mapUser[0]) == 3:
1231 user = mapUser[0][0]
1232 fullname = mapUser[0][1]
1233 email = mapUser[0][2]
1234 self.users[user] = fullname + " <" + email + ">"
1235 self.emails[email] = user
3ea2cfd4
LD
1236
1237 s = ''
1238 for (key, val) in self.users.items():
1239 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1240
1241 open(self.getUserCacheFilename(), "wb").write(s)
1242 self.userMapFromPerforceServer = True
1243
1244 def loadUserMapFromCache(self):
1245 self.users = {}
1246 self.userMapFromPerforceServer = False
1247 try:
1248 cache = open(self.getUserCacheFilename(), "rb")
1249 lines = cache.readlines()
1250 cache.close()
1251 for line in lines:
1252 entry = line.strip().split("\t")
1253 self.users[entry[0]] = entry[1]
1254 except IOError:
1255 self.getUserMapFromPerforceServer()
1256
b984733c 1257class P4Debug(Command):
86949eef 1258 def __init__(self):
6ae8de88 1259 Command.__init__(self)
6a10b6aa 1260 self.options = []
c8c39116 1261 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1262 self.needsGit = False
86949eef
SH
1263
1264 def run(self, args):
b1ce9447 1265 j = 0
6de040df 1266 for output in p4CmdList(args):
b1ce9447
HWN
1267 print 'Element: %d' % j
1268 j += 1
86949eef 1269 print output
b984733c 1270 return True
86949eef 1271
5834684d
SH
1272class P4RollBack(Command):
1273 def __init__(self):
1274 Command.__init__(self)
1275 self.options = [
0c66a783 1276 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1277 ]
1278 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1279 self.rollbackLocalBranches = False
5834684d
SH
1280
1281 def run(self, args):
1282 if len(args) != 1:
1283 return False
1284 maxChange = int(args[0])
0c66a783 1285
ad192f28 1286 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1287 die("Problems executing p4");
1288
0c66a783
SH
1289 if self.rollbackLocalBranches:
1290 refPrefix = "refs/heads/"
b016d397 1291 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1292 else:
1293 refPrefix = "refs/remotes/"
b016d397 1294 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1295
1296 for line in lines:
1297 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1298 line = line.strip()
1299 ref = refPrefix + line
5834684d 1300 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1301 settings = extractSettingsGitLog(log)
1302
1303 depotPaths = settings['depot-paths']
1304 change = settings['change']
1305
5834684d 1306 changed = False
52102d47 1307
6326aa58
HWN
1308 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1309 for p in depotPaths]))) == 0:
52102d47
SH
1310 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1311 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1312 continue
1313
bb6e09b2 1314 while change and int(change) > maxChange:
5834684d 1315 changed = True
52102d47
SH
1316 if self.verbose:
1317 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1318 system("git update-ref %s \"%s^\"" % (ref, ref))
1319 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1320 settings = extractSettingsGitLog(log)
1321
1322
1323 depotPaths = settings['depot-paths']
1324 change = settings['change']
5834684d
SH
1325
1326 if changed:
52102d47 1327 print "%s rewound to %s" % (ref, change)
5834684d
SH
1328
1329 return True
1330
3ea2cfd4 1331class P4Submit(Command, P4UserMap):
6bbfd137
PW
1332
1333 conflict_behavior_choices = ("ask", "skip", "quit")
1334
4f5cf76a 1335 def __init__(self):
b984733c 1336 Command.__init__(self)
3ea2cfd4 1337 P4UserMap.__init__(self)
4f5cf76a 1338 self.options = [
4f5cf76a 1339 optparse.make_option("--origin", dest="origin"),
ae901090 1340 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1341 # preserve the user, requires relevant p4 permissions
1342 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1343 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1344 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1345 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1346 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1347 choices=self.conflict_behavior_choices),
1348 optparse.make_option("--branch", dest="branch"),
b34fa577
VK
1349 optparse.make_option("--shelve", dest="shelve", action="store_true",
1350 help="Shelve instead of submit. Shelved files are reverted, "
1351 "restoring the workspace to the state before the shelve"),
8cf422db 1352 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
46c609e9 1353 metavar="CHANGELIST",
8cf422db 1354 help="update an existing shelved changelist, implies --shelve, "
f55b87c1
RM
1355 "repeat in-order for multiple shelved changelists"),
1356 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1357 help="submit only the specified commit(s), one commit or xxx..xxx"),
1358 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1359 help="Disable rebase after submit is completed. Can be useful if you "
b9d34db9
LD
1360 "work from a local git branch that is not master"),
1361 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1362 help="Skip Perforce sync of p4/master after submit or shelve"),
4f5cf76a
SH
1363 ]
1364 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1365 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1366 self.origin = ""
ae901090 1367 self.detectRenames = False
0d609032 1368 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1369 self.dry_run = False
b34fa577 1370 self.shelve = False
8cf422db 1371 self.update_shelve = list()
f55b87c1 1372 self.commit = ""
3b3477ea 1373 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
b9d34db9 1374 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
728b7ad8 1375 self.prepare_p4_only = False
6bbfd137 1376 self.conflict_behavior = None
f7baba8b 1377 self.isWindows = (platform.system() == "Windows")
06804c76 1378 self.exportLabels = False
249da4c0 1379 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1380 self.branch = None
4f5cf76a 1381
a5db4b12
LS
1382 if gitConfig('git-p4.largeFileSystem'):
1383 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1384
4f5cf76a
SH
1385 def check(self):
1386 if len(p4CmdList("opened ...")) > 0:
1387 die("You have files opened with perforce! Close them before starting the sync.")
1388
f19cb0a0
PW
1389 def separate_jobs_from_description(self, message):
1390 """Extract and return a possible Jobs field in the commit
1391 message. It goes into a separate section in the p4 change
1392 specification.
1393
1394 A jobs line starts with "Jobs:" and looks like a new field
1395 in a form. Values are white-space separated on the same
1396 line or on following lines that start with a tab.
1397
1398 This does not parse and extract the full git commit message
1399 like a p4 form. It just sees the Jobs: line as a marker
1400 to pass everything from then on directly into the p4 form,
1401 but outside the description section.
1402
1403 Return a tuple (stripped log message, jobs string)."""
1404
1405 m = re.search(r'^Jobs:', message, re.MULTILINE)
1406 if m is None:
1407 return (message, None)
1408
1409 jobtext = message[m.start():]
1410 stripped_message = message[:m.start()].rstrip()
1411 return (stripped_message, jobtext)
1412
1413 def prepareLogMessage(self, template, message, jobs):
1414 """Edits the template returned from "p4 change -o" to insert
1415 the message in the Description field, and the jobs text in
1416 the Jobs field."""
4f5cf76a
SH
1417 result = ""
1418
edae1e2f
SH
1419 inDescriptionSection = False
1420
4f5cf76a
SH
1421 for line in template.split("\n"):
1422 if line.startswith("#"):
1423 result += line + "\n"
1424 continue
1425
edae1e2f 1426 if inDescriptionSection:
c9dbab04 1427 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1428 inDescriptionSection = False
f19cb0a0
PW
1429 # insert Jobs section
1430 if jobs:
1431 result += jobs + "\n"
edae1e2f
SH
1432 else:
1433 continue
1434 else:
1435 if line.startswith("Description:"):
1436 inDescriptionSection = True
1437 line += "\n"
1438 for messageLine in message.split("\n"):
1439 line += "\t" + messageLine + "\n"
1440
1441 result += line + "\n"
4f5cf76a
SH
1442
1443 return result
1444
60df071c
LD
1445 def patchRCSKeywords(self, file, pattern):
1446 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1447 (handle, outFileName) = tempfile.mkstemp(dir='.')
1448 try:
1449 outFile = os.fdopen(handle, "w+")
1450 inFile = open(file, "r")
1451 regexp = re.compile(pattern, re.VERBOSE)
1452 for line in inFile.readlines():
1453 line = regexp.sub(r'$\1$', line)
1454 outFile.write(line)
1455 inFile.close()
1456 outFile.close()
1457 # Forcibly overwrite the original file
1458 os.unlink(file)
1459 shutil.move(outFileName, file)
1460 except:
1461 # cleanup our temporary file
1462 os.unlink(outFileName)
1463 print "Failed to strip RCS keywords in %s" % file
1464 raise
1465
1466 print "Patched up RCS keywords in %s" % file
1467
3ea2cfd4
LD
1468 def p4UserForCommit(self,id):
1469 # Return the tuple (perforce user,git email) for a given git commit id
1470 self.getUserMapFromPerforceServer()
9bf28855
PW
1471 gitEmail = read_pipe(["git", "log", "--max-count=1",
1472 "--format=%ae", id])
3ea2cfd4
LD
1473 gitEmail = gitEmail.strip()
1474 if not self.emails.has_key(gitEmail):
1475 return (None,gitEmail)
1476 else:
1477 return (self.emails[gitEmail],gitEmail)
1478
1479 def checkValidP4Users(self,commits):
1480 # check if any git authors cannot be mapped to p4 users
1481 for id in commits:
1482 (user,email) = self.p4UserForCommit(id)
1483 if not user:
1484 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1485 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1486 print "%s" % msg
1487 else:
1488 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1489
1490 def lastP4Changelist(self):
1491 # Get back the last changelist number submitted in this client spec. This
1492 # then gets used to patch up the username in the change. If the same
1493 # client spec is being used by multiple processes then this might go
1494 # wrong.
1495 results = p4CmdList("client -o") # find the current client
1496 client = None
1497 for r in results:
1498 if r.has_key('Client'):
1499 client = r['Client']
1500 break
1501 if not client:
1502 die("could not get client spec")
6de040df 1503 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1504 for r in results:
1505 if r.has_key('change'):
1506 return r['change']
1507 die("Could not get changelist number for last submit - cannot patch up user details")
1508
1509 def modifyChangelistUser(self, changelist, newUser):
1510 # fixup the user field of a changelist after it has been submitted.
1511 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1512 if len(changes) != 1:
1513 die("Bad output from p4 change modifying %s to user %s" %
1514 (changelist, newUser))
1515
1516 c = changes[0]
1517 if c['User'] == newUser: return # nothing to do
1518 c['User'] = newUser
1519 input = marshal.dumps(c)
1520
3ea2cfd4
LD
1521 result = p4CmdList("change -f -i", stdin=input)
1522 for r in result:
1523 if r.has_key('code'):
1524 if r['code'] == 'error':
1525 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1526 if r.has_key('data'):
1527 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1528 return
1529 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1530
1531 def canChangeChangelists(self):
1532 # check to see if we have p4 admin or super-user permissions, either of
1533 # which are required to modify changelists.
52a4880b 1534 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1535 for r in results:
1536 if r.has_key('perm'):
1537 if r['perm'] == 'admin':
1538 return 1
1539 if r['perm'] == 'super':
1540 return 1
1541 return 0
1542
46c609e9 1543 def prepareSubmitTemplate(self, changelist=None):
f19cb0a0
PW
1544 """Run "p4 change -o" to grab a change specification template.
1545 This does not use "p4 -G", as it is nice to keep the submission
1546 template in original order, since a human might edit it.
1547
1548 Remove lines in the Files section that show changes to files
1549 outside the depot path we're committing into."""
1550
cbc69242
SH
1551 [upstream, settings] = findUpstreamBranchPoint()
1552
b596b3b9
MT
1553 template = """\
1554# A Perforce Change Specification.
1555#
1556# Change: The change number. 'new' on a new changelist.
1557# Date: The date this specification was last modified.
1558# Client: The client on which the changelist was created. Read-only.
1559# User: The user who created the changelist.
1560# Status: Either 'pending' or 'submitted'. Read-only.
1561# Type: Either 'public' or 'restricted'. Default is 'public'.
1562# Description: Comments about the changelist. Required.
1563# Jobs: What opened jobs are to be closed by this changelist.
1564# You may delete jobs from this list. (New changelists only.)
1565# Files: What opened files from the default changelist are to be added
1566# to this changelist. You may delete files from this list.
1567# (New changelists only.)
1568"""
1569 files_list = []
ea99c3ae 1570 inFilesSection = False
b596b3b9 1571 change_entry = None
46c609e9
LD
1572 args = ['change', '-o']
1573 if changelist:
1574 args.append(str(changelist))
b596b3b9
MT
1575 for entry in p4CmdList(args):
1576 if not entry.has_key('code'):
1577 continue
1578 if entry['code'] == 'stat':
1579 change_entry = entry
1580 break
1581 if not change_entry:
1582 die('Failed to decode output of p4 change -o')
1583 for key, value in change_entry.iteritems():
1584 if key.startswith('File'):
1585 if settings.has_key('depot-paths'):
1586 if not [p for p in settings['depot-paths']
1587 if p4PathStartsWith(value, p)]:
1588 continue
ea99c3ae 1589 else:
b596b3b9
MT
1590 if not p4PathStartsWith(value, self.depotPath):
1591 continue
1592 files_list.append(value)
1593 continue
1594 # Output in the order expected by prepareLogMessage
1595 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1596 if not change_entry.has_key(key):
1597 continue
1598 template += '\n'
1599 template += key + ':'
1600 if key == 'Description':
1601 template += '\n'
1602 for field_line in change_entry[key].splitlines():
1603 template += '\t'+field_line+'\n'
1604 if len(files_list) > 0:
1605 template += '\n'
1606 template += 'Files:\n'
1607 for path in files_list:
1608 template += '\t'+path+'\n'
ea99c3ae
SH
1609 return template
1610
7c766e57
PW
1611 def edit_template(self, template_file):
1612 """Invoke the editor to let the user change the submission
1613 message. Return true if okay to continue with the submit."""
1614
1615 # if configured to skip the editing part, just submit
0d609032 1616 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1617 return True
1618
1619 # look at the modification time, to check later if the user saved
1620 # the file
1621 mtime = os.stat(template_file).st_mtime
1622
1623 # invoke the editor
f95ceaf0 1624 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1625 editor = os.environ.get("P4EDITOR")
1626 else:
1627 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1628 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1629
1630 # If the file was not saved, prompt to see if this patch should
1631 # be skipped. But skip this verification step if configured so.
0d609032 1632 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1633 return True
1634
d1652049
PW
1635 # modification time updated means user saved the file
1636 if os.stat(template_file).st_mtime > mtime:
1637 return True
1638
1639 while True:
1640 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1641 if response == 'y':
1642 return True
1643 if response == 'n':
1644 return False
7c766e57 1645
df8a9e86 1646 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
b4073bb3
MC
1647 # diff
1648 if os.environ.has_key("P4DIFF"):
1649 del(os.environ["P4DIFF"])
1650 diff = ""
1651 for editedFile in editedFiles:
1652 diff += p4_read_pipe(['diff', '-du',
1653 wildcard_encode(editedFile)])
1654
1655 # new file diff
1656 newdiff = ""
1657 for newFile in filesToAdd:
1658 newdiff += "==== new file ====\n"
1659 newdiff += "--- /dev/null\n"
1660 newdiff += "+++ %s\n" % newFile
df8a9e86
LD
1661
1662 is_link = os.path.islink(newFile)
1663 expect_link = newFile in symlinks
1664
1665 if is_link and expect_link:
1666 newdiff += "+%s\n" % os.readlink(newFile)
1667 else:
1668 f = open(newFile, "r")
1669 for line in f.readlines():
1670 newdiff += "+" + line
1671 f.close()
b4073bb3 1672
e2a892ee 1673 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1674
7cb5cbef 1675 def applyCommit(self, id):
67b0fe2e
PW
1676 """Apply one commit, return True if it succeeded."""
1677
1678 print "Applying", read_pipe(["git", "show", "-s",
1679 "--format=format:%h %s", id])
ae901090 1680
848de9c3 1681 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1682
84cb0003 1683 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a 1684 filesToAdd = set()
a02b8bc4 1685 filesToChangeType = set()
4f5cf76a 1686 filesToDelete = set()
d336c158 1687 editedFiles = set()
b6ad6dcc 1688 pureRenameCopy = set()
df8a9e86 1689 symlinks = set()
c65b670e 1690 filesToChangeExecBit = {}
46c609e9 1691 all_files = list()
60df071c 1692
4f5cf76a 1693 for line in diff:
b43b0a3c
CP
1694 diff = parseDiffTreeEntry(line)
1695 modifier = diff['status']
1696 path = diff['src']
46c609e9
LD
1697 all_files.append(path)
1698
4f5cf76a 1699 if modifier == "M":
6de040df 1700 p4_edit(path)
c65b670e
CP
1701 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1702 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1703 editedFiles.add(path)
4f5cf76a
SH
1704 elif modifier == "A":
1705 filesToAdd.add(path)
c65b670e 1706 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1707 if path in filesToDelete:
1708 filesToDelete.remove(path)
df8a9e86
LD
1709
1710 dst_mode = int(diff['dst_mode'], 8)
1711 if dst_mode == 0120000:
1712 symlinks.add(path)
1713
4f5cf76a
SH
1714 elif modifier == "D":
1715 filesToDelete.add(path)
1716 if path in filesToAdd:
1717 filesToAdd.remove(path)
4fddb41b
VA
1718 elif modifier == "C":
1719 src, dest = diff['src'], diff['dst']
6de040df 1720 p4_integrate(src, dest)
b6ad6dcc 1721 pureRenameCopy.add(dest)
4fddb41b 1722 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1723 p4_edit(dest)
b6ad6dcc 1724 pureRenameCopy.discard(dest)
4fddb41b 1725 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1726 p4_edit(dest)
b6ad6dcc 1727 pureRenameCopy.discard(dest)
4fddb41b 1728 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1729 if self.isWindows:
1730 # turn off read-only attribute
1731 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1732 os.unlink(dest)
1733 editedFiles.add(dest)
d9a5f25b 1734 elif modifier == "R":
b43b0a3c 1735 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1736 if self.p4HasMoveCommand:
1737 p4_edit(src) # src must be open before move
1738 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1739 else:
8e9497c2
GG
1740 p4_integrate(src, dest)
1741 if diff['src_sha1'] != diff['dst_sha1']:
1742 p4_edit(dest)
1743 else:
1744 pureRenameCopy.add(dest)
c65b670e 1745 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1746 if not self.p4HasMoveCommand:
1747 p4_edit(dest) # with move: already open, writable
c65b670e 1748 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1749 if not self.p4HasMoveCommand:
d20f0f8e
PW
1750 if self.isWindows:
1751 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1752 os.unlink(dest)
1753 filesToDelete.add(src)
d9a5f25b 1754 editedFiles.add(dest)
a02b8bc4
RP
1755 elif modifier == "T":
1756 filesToChangeType.add(path)
4f5cf76a
SH
1757 else:
1758 die("unknown modifier %s for %s" % (modifier, path))
1759
749b668c 1760 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1761 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1762 tryPatchCmd = patchcmd + "--check -"
1763 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1764 patch_succeeded = True
51a2640a 1765
47a130b7 1766 if os.system(tryPatchCmd) != 0:
60df071c
LD
1767 fixed_rcs_keywords = False
1768 patch_succeeded = False
51a2640a 1769 print "Unfortunately applying the change failed!"
60df071c
LD
1770
1771 # Patch failed, maybe it's just RCS keyword woes. Look through
1772 # the patch to see if that's possible.
0d609032 1773 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1774 file = None
1775 pattern = None
1776 kwfiles = {}
1777 for file in editedFiles | filesToDelete:
1778 # did this file's delta contain RCS keywords?
1779 pattern = p4_keywords_regexp_for_file(file)
1780
1781 if pattern:
1782 # this file is a possibility...look for RCS keywords.
1783 regexp = re.compile(pattern, re.VERBOSE)
1784 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1785 if regexp.search(line):
1786 if verbose:
1787 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1788 kwfiles[file] = pattern
1789 break
1790
1791 for file in kwfiles:
1792 if verbose:
1793 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1794 # File is being deleted, so not open in p4. Must
1795 # disable the read-only bit on windows.
1796 if self.isWindows and file not in editedFiles:
1797 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1798 self.patchRCSKeywords(file, kwfiles[file])
1799 fixed_rcs_keywords = True
1800
1801 if fixed_rcs_keywords:
1802 print "Retrying the patch with RCS keywords cleaned up"
1803 if os.system(tryPatchCmd) == 0:
1804 patch_succeeded = True
1805
1806 if not patch_succeeded:
7e5dd9f2
PW
1807 for f in editedFiles:
1808 p4_revert(f)
7e5dd9f2 1809 return False
51a2640a 1810
55ac2ed6
PW
1811 #
1812 # Apply the patch for real, and do add/delete/+x handling.
1813 #
47a130b7 1814 system(applyPatchCmd)
4f5cf76a 1815
a02b8bc4
RP
1816 for f in filesToChangeType:
1817 p4_edit(f, "-t", "auto")
4f5cf76a 1818 for f in filesToAdd:
6de040df 1819 p4_add(f)
4f5cf76a 1820 for f in filesToDelete:
6de040df
LD
1821 p4_revert(f)
1822 p4_delete(f)
4f5cf76a 1823
c65b670e
CP
1824 # Set/clear executable bits
1825 for f in filesToChangeExecBit.keys():
1826 mode = filesToChangeExecBit[f]
1827 setP4ExecBit(f, mode)
1828
8cf422db
LD
1829 update_shelve = 0
1830 if len(self.update_shelve) > 0:
1831 update_shelve = self.update_shelve.pop(0)
1832 p4_reopen_in_change(update_shelve, all_files)
46c609e9 1833
55ac2ed6
PW
1834 #
1835 # Build p4 change description, starting with the contents
1836 # of the git commit message.
1837 #
0e36f2d7 1838 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1839 logMessage = logMessage.strip()
f19cb0a0 1840 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1841
8cf422db 1842 template = self.prepareSubmitTemplate(update_shelve)
f19cb0a0 1843 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1844
c47178d4 1845 if self.preserveUser:
55ac2ed6 1846 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1847
55ac2ed6
PW
1848 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1849 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1850 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1851 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1852
55ac2ed6 1853 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1854 if not self.prepare_p4_only:
1855 submitTemplate += separatorLine
df8a9e86 1856 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
55ac2ed6 1857
c47178d4 1858 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1859 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1860 if self.isWindows:
1861 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1862 tmpFile.write(submitTemplate)
c47178d4
PW
1863 tmpFile.close()
1864
728b7ad8
PW
1865 if self.prepare_p4_only:
1866 #
1867 # Leave the p4 tree prepared, and the submit template around
1868 # and let the user decide what to do next
1869 #
1870 print
1871 print "P4 workspace prepared for submission."
1872 print "To submit or revert, go to client workspace"
1873 print " " + self.clientPath
1874 print
1875 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1876 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1877 " \"git p4\"." % fileName
1878 print "You can delete the file \"%s\" when finished." % fileName
1879
1880 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1881 print "To preserve change ownership by user %s, you must\n" \
1882 "do \"p4 change -f <change>\" after submitting and\n" \
1883 "edit the User field."
1884 if pureRenameCopy:
1885 print "After submitting, renamed files must be re-synced."
1886 print "Invoke \"p4 sync -f\" on each of these files:"
1887 for f in pureRenameCopy:
1888 print " " + f
1889
1890 print
1891 print "To revert the changes, use \"p4 revert ...\", and delete"
1892 print "the submit template file \"%s\"" % fileName
1893 if filesToAdd:
1894 print "Since the commit adds new files, they must be deleted:"
1895 for f in filesToAdd:
1896 print " " + f
1897 print
1898 return True
1899
55ac2ed6
PW
1900 #
1901 # Let the user edit the change description, then submit it.
1902 #
b7638fed 1903 submitted = False
cdc7e388 1904
b7638fed
GE
1905 try:
1906 if self.edit_template(fileName):
1907 # read the edited message and submit
1908 tmpFile = open(fileName, "rb")
1909 message = tmpFile.read()
1910 tmpFile.close()
1911 if self.isWindows:
1912 message = message.replace("\r\n", "\n")
1913 submitTemplate = message[:message.index(separatorLine)]
46c609e9 1914
8cf422db 1915 if update_shelve:
46c609e9
LD
1916 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1917 elif self.shelve:
b34fa577
VK
1918 p4_write_pipe(['shelve', '-i'], submitTemplate)
1919 else:
1920 p4_write_pipe(['submit', '-i'], submitTemplate)
1921 # The rename/copy happened by applying a patch that created a
1922 # new file. This leaves it writable, which confuses p4.
1923 for f in pureRenameCopy:
1924 p4_sync(f, "-f")
b7638fed
GE
1925
1926 if self.preserveUser:
1927 if p4User:
1928 # Get last changelist number. Cannot easily get it from
1929 # the submit command output as the output is
1930 # unmarshalled.
1931 changelist = self.lastP4Changelist()
1932 self.modifyChangelistUser(changelist, p4User)
1933
b7638fed
GE
1934 submitted = True
1935
1936 finally:
c47178d4 1937 # skip this patch
b34fa577
VK
1938 if not submitted or self.shelve:
1939 if self.shelve:
1940 print ("Reverting shelved files.")
1941 else:
1942 print ("Submission cancelled, undoing p4 changes.")
1943 for f in editedFiles | filesToDelete:
b7638fed
GE
1944 p4_revert(f)
1945 for f in filesToAdd:
1946 p4_revert(f)
1947 os.remove(f)
c47178d4
PW
1948
1949 os.remove(fileName)
b7638fed 1950 return submitted
4f5cf76a 1951
06804c76
LD
1952 # Export git tags as p4 labels. Create a p4 label and then tag
1953 # with that.
1954 def exportGitTags(self, gitTags):
c8942a22
LD
1955 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1956 if len(validLabelRegexp) == 0:
1957 validLabelRegexp = defaultLabelRegexp
1958 m = re.compile(validLabelRegexp)
06804c76
LD
1959
1960 for name in gitTags:
1961
1962 if not m.match(name):
1963 if verbose:
05a3cec5 1964 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1965 continue
1966
1967 # Get the p4 commit this corresponds to
c8942a22
LD
1968 logMessage = extractLogMessageFromGitCommit(name)
1969 values = extractSettingsGitLog(logMessage)
06804c76 1970
c8942a22 1971 if not values.has_key('change'):
06804c76
LD
1972 # a tag pointing to something not sent to p4; ignore
1973 if verbose:
1974 print "git tag %s does not give a p4 commit" % name
1975 continue
c8942a22
LD
1976 else:
1977 changelist = values['change']
06804c76
LD
1978
1979 # Get the tag details.
1980 inHeader = True
1981 isAnnotated = False
1982 body = []
1983 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1984 l = l.strip()
1985 if inHeader:
1986 if re.match(r'tag\s+', l):
1987 isAnnotated = True
1988 elif re.match(r'\s*$', l):
1989 inHeader = False
1990 continue
1991 else:
1992 body.append(l)
1993
1994 if not isAnnotated:
1995 body = ["lightweight tag imported by git p4\n"]
1996
1997 # Create the label - use the same view as the client spec we are using
1998 clientSpec = getClientSpec()
1999
2000 labelTemplate = "Label: %s\n" % name
2001 labelTemplate += "Description:\n"
2002 for b in body:
2003 labelTemplate += "\t" + b + "\n"
2004 labelTemplate += "View:\n"
9d57c4a6
KS
2005 for depot_side in clientSpec.mappings:
2006 labelTemplate += "\t%s\n" % depot_side
06804c76 2007
ef739f08
PW
2008 if self.dry_run:
2009 print "Would create p4 label %s for tag" % name
728b7ad8
PW
2010 elif self.prepare_p4_only:
2011 print "Not creating p4 label %s for tag due to option" \
2012 " --prepare-p4-only" % name
ef739f08
PW
2013 else:
2014 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 2015
ef739f08
PW
2016 # Use the label
2017 p4_system(["tag", "-l", name] +
9d57c4a6 2018 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 2019
ef739f08
PW
2020 if verbose:
2021 print "created p4 label for tag %s" % name
06804c76 2022
4f5cf76a 2023 def run(self, args):
c9b50e63
SH
2024 if len(args) == 0:
2025 self.master = currentGitBranch()
c9b50e63
SH
2026 elif len(args) == 1:
2027 self.master = args[0]
28755dba
PW
2028 if not branchExists(self.master):
2029 die("Branch %s does not exist" % self.master)
c9b50e63
SH
2030 else:
2031 return False
2032
8cf422db
LD
2033 for i in self.update_shelve:
2034 if i <= 0:
2035 sys.exit("invalid changelist %d" % i)
2036
00ad6e31
LD
2037 if self.master:
2038 allowSubmit = gitConfig("git-p4.allowSubmit")
2039 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2040 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 2041
27d2d811 2042 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 2043 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
2044 if len(self.origin) == 0:
2045 self.origin = upstream
a3fdd579 2046
8cf422db 2047 if len(self.update_shelve) > 0:
46c609e9
LD
2048 self.shelve = True
2049
3ea2cfd4
LD
2050 if self.preserveUser:
2051 if not self.canChangeChangelists():
2052 die("Cannot preserve user names without p4 super-user or admin permissions")
2053
6bbfd137
PW
2054 # if not set from the command line, try the config file
2055 if self.conflict_behavior is None:
2056 val = gitConfig("git-p4.conflict")
2057 if val:
2058 if val not in self.conflict_behavior_choices:
2059 die("Invalid value '%s' for config git-p4.conflict" % val)
2060 else:
2061 val = "ask"
2062 self.conflict_behavior = val
2063
a3fdd579
SH
2064 if self.verbose:
2065 print "Origin branch is " + self.origin
9512497b 2066
ea99c3ae 2067 if len(self.depotPath) == 0:
9512497b
SH
2068 print "Internal error: cannot locate perforce depot path from existing branches"
2069 sys.exit(128)
2070
543987bd 2071 self.useClientSpec = False
0d609032 2072 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
2073 self.useClientSpec = True
2074 if self.useClientSpec:
2075 self.clientSpecDirs = getClientSpec()
9512497b 2076
2e3a16b2 2077 # Check for the existence of P4 branches
cd884106
VA
2078 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2079
2080 if self.useClientSpec and not branchesDetected:
543987bd
PW
2081 # all files are relative to the client spec
2082 self.clientPath = getClientRoot()
2083 else:
2084 self.clientPath = p4Where(self.depotPath)
9512497b 2085
543987bd
PW
2086 if self.clientPath == "":
2087 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 2088
ea99c3ae 2089 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 2090 self.oldWorkingDirectory = os.getcwd()
c1b296b9 2091
0591cfa8 2092 # ensure the clientPath exists
8d7ec362 2093 new_client_dir = False
0591cfa8 2094 if not os.path.exists(self.clientPath):
8d7ec362 2095 new_client_dir = True
0591cfa8
GG
2096 os.makedirs(self.clientPath)
2097
bbd84863 2098 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
2099 if self.dry_run:
2100 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 2101 else:
ef739f08
PW
2102 print "Synchronizing p4 checkout..."
2103 if new_client_dir:
2104 # old one was destroyed, and maybe nobody told p4
2105 p4_sync("...", "-f")
2106 else:
2107 p4_sync("...")
4f5cf76a 2108 self.check()
4f5cf76a 2109
4c750c0d 2110 commits = []
00ad6e31
LD
2111 if self.master:
2112 commitish = self.master
2113 else:
2114 commitish = 'HEAD'
2115
f55b87c1
RM
2116 if self.commit != "":
2117 if self.commit.find("..") != -1:
2118 limits_ish = self.commit.split("..")
2119 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2120 commits.append(line.strip())
2121 commits.reverse()
2122 else:
2123 commits.append(self.commit)
2124 else:
2125 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2126 commits.append(line.strip())
2127 commits.reverse()
4f5cf76a 2128
0d609032 2129 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
2130 self.checkAuthorship = False
2131 else:
2132 self.checkAuthorship = True
2133
3ea2cfd4
LD
2134 if self.preserveUser:
2135 self.checkValidP4Users(commits)
2136
84cb0003
GG
2137 #
2138 # Build up a set of options to be passed to diff when
2139 # submitting each commit to p4.
2140 #
2141 if self.detectRenames:
2142 # command-line -M arg
2143 self.diffOpts = "-M"
2144 else:
2145 # If not explicitly set check the config variable
2146 detectRenames = gitConfig("git-p4.detectRenames")
2147
2148 if detectRenames.lower() == "false" or detectRenames == "":
2149 self.diffOpts = ""
2150 elif detectRenames.lower() == "true":
2151 self.diffOpts = "-M"
2152 else:
2153 self.diffOpts = "-M%s" % detectRenames
2154
2155 # no command-line arg for -C or --find-copies-harder, just
2156 # config variables
2157 detectCopies = gitConfig("git-p4.detectCopies")
2158 if detectCopies.lower() == "false" or detectCopies == "":
2159 pass
2160 elif detectCopies.lower() == "true":
2161 self.diffOpts += " -C"
2162 else:
2163 self.diffOpts += " -C%s" % detectCopies
2164
0d609032 2165 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
2166 self.diffOpts += " --find-copies-harder"
2167
8cf422db
LD
2168 num_shelves = len(self.update_shelve)
2169 if num_shelves > 0 and num_shelves != len(commits):
2170 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2171 (len(commits), num_shelves))
2172
7e5dd9f2
PW
2173 #
2174 # Apply the commits, one at a time. On failure, ask if should
2175 # continue to try the rest of the patches, or quit.
2176 #
ef739f08
PW
2177 if self.dry_run:
2178 print "Would apply"
67b0fe2e 2179 applied = []
7e5dd9f2
PW
2180 last = len(commits) - 1
2181 for i, commit in enumerate(commits):
ef739f08
PW
2182 if self.dry_run:
2183 print " ", read_pipe(["git", "show", "-s",
2184 "--format=format:%h %s", commit])
2185 ok = True
2186 else:
2187 ok = self.applyCommit(commit)
67b0fe2e
PW
2188 if ok:
2189 applied.append(commit)
7e5dd9f2 2190 else:
728b7ad8
PW
2191 if self.prepare_p4_only and i < last:
2192 print "Processing only the first commit due to option" \
2193 " --prepare-p4-only"
2194 break
7e5dd9f2
PW
2195 if i < last:
2196 quit = False
2197 while True:
6bbfd137
PW
2198 # prompt for what to do, or use the option/variable
2199 if self.conflict_behavior == "ask":
2200 print "What do you want to do?"
2201 response = raw_input("[s]kip this commit but apply"
2202 " the rest, or [q]uit? ")
2203 if not response:
2204 continue
2205 elif self.conflict_behavior == "skip":
2206 response = "s"
2207 elif self.conflict_behavior == "quit":
2208 response = "q"
2209 else:
2210 die("Unknown conflict_behavior '%s'" %
2211 self.conflict_behavior)
2212
7e5dd9f2
PW
2213 if response[0] == "s":
2214 print "Skipping this commit, but applying the rest"
2215 break
2216 if response[0] == "q":
2217 print "Quitting"
2218 quit = True
2219 break
2220 if quit:
2221 break
4f5cf76a 2222
67b0fe2e 2223 chdir(self.oldWorkingDirectory)
b34fa577 2224 shelved_applied = "shelved" if self.shelve else "applied"
ef739f08
PW
2225 if self.dry_run:
2226 pass
728b7ad8
PW
2227 elif self.prepare_p4_only:
2228 pass
ef739f08 2229 elif len(commits) == len(applied):
b34fa577 2230 print ("All commits {0}!".format(shelved_applied))
14594f4b 2231
4c750c0d 2232 sync = P4Sync()
44e8d26c
PW
2233 if self.branch:
2234 sync.branch = self.branch
b9d34db9
LD
2235 if self.disable_p4sync:
2236 sync.sync_origin_only()
2237 else:
2238 sync.run([])
14594f4b 2239
b9d34db9
LD
2240 if not self.disable_rebase:
2241 rebase = P4Rebase()
2242 rebase.rebase()
4f5cf76a 2243
67b0fe2e
PW
2244 else:
2245 if len(applied) == 0:
b34fa577 2246 print ("No commits {0}.".format(shelved_applied))
67b0fe2e 2247 else:
b34fa577 2248 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
67b0fe2e
PW
2249 for c in commits:
2250 if c in applied:
2251 star = "*"
2252 else:
2253 star = " "
2254 print star, read_pipe(["git", "show", "-s",
2255 "--format=format:%h %s", c])
2256 print "You will have to do 'git p4 sync' and rebase."
2257
0d609032 2258 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2259 self.exportLabels = True
06804c76
LD
2260
2261 if self.exportLabels:
2262 p4Labels = getP4Labels(self.depotPath)
2263 gitTags = getGitTags()
2264
2265 missingGitTags = gitTags - p4Labels
2266 self.exportGitTags(missingGitTags)
2267
98e023de 2268 # exit with error unless everything applied perfectly
67b0fe2e
PW
2269 if len(commits) != len(applied):
2270 sys.exit(1)
2271
b984733c
SH
2272 return True
2273
ecb7cf98
PW
2274class View(object):
2275 """Represent a p4 view ("p4 help views"), and map files in a
2276 repo according to the view."""
2277
9d57c4a6 2278 def __init__(self, client_name):
ecb7cf98 2279 self.mappings = []
9d57c4a6
KS
2280 self.client_prefix = "//%s/" % client_name
2281 # cache results of "p4 where" to lookup client file locations
2282 self.client_spec_path_cache = {}
ecb7cf98
PW
2283
2284 def append(self, view_line):
2285 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2286 sides. Append to self.mappings, preserving order. This
2287 is only needed for tag creation."""
ecb7cf98
PW
2288
2289 # Split the view line into exactly two words. P4 enforces
2290 # structure on these lines that simplifies this quite a bit.
2291 #
2292 # Either or both words may be double-quoted.
2293 # Single quotes do not matter.
2294 # Double-quote marks cannot occur inside the words.
2295 # A + or - prefix is also inside the quotes.
2296 # There are no quotes unless they contain a space.
2297 # The line is already white-space stripped.
2298 # The two words are separated by a single space.
2299 #
2300 if view_line[0] == '"':
2301 # First word is double quoted. Find its end.
2302 close_quote_index = view_line.find('"', 1)
2303 if close_quote_index <= 0:
2304 die("No first-word closing quote found: %s" % view_line)
2305 depot_side = view_line[1:close_quote_index]
2306 # skip closing quote and space
2307 rhs_index = close_quote_index + 1 + 1
2308 else:
2309 space_index = view_line.find(" ")
2310 if space_index <= 0:
2311 die("No word-splitting space found: %s" % view_line)
2312 depot_side = view_line[0:space_index]
2313 rhs_index = space_index + 1
2314
ecb7cf98 2315 # prefix + means overlay on previous mapping
ecb7cf98 2316 if depot_side.startswith("+"):
ecb7cf98
PW
2317 depot_side = depot_side[1:]
2318
9d57c4a6 2319 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2320 exclude = False
2321 if depot_side.startswith("-"):
2322 exclude = True
2323 depot_side = depot_side[1:]
2324
9d57c4a6
KS
2325 if not exclude:
2326 self.mappings.append(depot_side)
ecb7cf98 2327
9d57c4a6
KS
2328 def convert_client_path(self, clientFile):
2329 # chop off //client/ part to make it relative
2330 if not clientFile.startswith(self.client_prefix):
2331 die("No prefix '%s' on clientFile '%s'" %
2332 (self.client_prefix, clientFile))
2333 return clientFile[len(self.client_prefix):]
ecb7cf98 2334
9d57c4a6
KS
2335 def update_client_spec_path_cache(self, files):
2336 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2337
9d57c4a6
KS
2338 # List depot file paths exclude that already cached
2339 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2340
9d57c4a6
KS
2341 if len(fileArgs) == 0:
2342 return # All files in cache
ecb7cf98 2343
9d57c4a6
KS
2344 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2345 for res in where_result:
2346 if "code" in res and res["code"] == "error":
2347 # assume error is "... file(s) not in client view"
2348 continue
2349 if "clientFile" not in res:
20005443 2350 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2351 if "unmap" in res:
2352 # it will list all of them, but only one not unmap-ped
2353 continue
a0a50d87
LS
2354 if gitConfigBool("core.ignorecase"):
2355 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2356 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2357
9d57c4a6
KS
2358 # not found files or unmap files set to ""
2359 for depotFile in fileArgs:
a0a50d87
LS
2360 if gitConfigBool("core.ignorecase"):
2361 depotFile = depotFile.lower()
9d57c4a6
KS
2362 if depotFile not in self.client_spec_path_cache:
2363 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2364
9d57c4a6
KS
2365 def map_in_client(self, depot_path):
2366 """Return the relative location in the client where this
2367 depot file should live. Returns "" if the file should
2368 not be mapped in the client."""
ecb7cf98 2369
a0a50d87
LS
2370 if gitConfigBool("core.ignorecase"):
2371 depot_path = depot_path.lower()
2372
9d57c4a6
KS
2373 if depot_path in self.client_spec_path_cache:
2374 return self.client_spec_path_cache[depot_path]
2375
2376 die( "Error: %s is not found in client spec path" % depot_path )
2377 return ""
ecb7cf98 2378
3ea2cfd4 2379class P4Sync(Command, P4UserMap):
56c09345
PW
2380 delete_actions = ( "delete", "move/delete", "purge" )
2381
b984733c
SH
2382 def __init__(self):
2383 Command.__init__(self)
3ea2cfd4 2384 P4UserMap.__init__(self)
b984733c
SH
2385 self.options = [
2386 optparse.make_option("--branch", dest="branch"),
2387 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2388 optparse.make_option("--changesfile", dest="changesFile"),
2389 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2390 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2391 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2392 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2393 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2394 optparse.make_option("--max-changes", dest="maxChanges",
2395 help="Maximum number of changes to import"),
2396 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2397 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2398 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2399 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2400 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2401 help="Only sync files that are included in the Perforce Client Spec"),
2402 optparse.make_option("-/", dest="cloneExclude",
2403 action="append", type="string",
2404 help="exclude depot path"),
b984733c
SH
2405 ]
2406 self.description = """Imports from Perforce into a git repository.\n
2407 example:
2408 //depot/my/project/ -- to import the current head
2409 //depot/my/project/@all -- to import everything
2410 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2411
2412 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2413
2414 self.usage += " //depot/path[@revRange]"
b984733c 2415 self.silent = False
1d7367dc
RG
2416 self.createdBranches = set()
2417 self.committedChanges = set()
569d1bd4 2418 self.branch = ""
b984733c 2419 self.detectBranches = False
cb53e1f8 2420 self.detectLabels = False
06804c76 2421 self.importLabels = False
b984733c 2422 self.changesFile = ""
01265103 2423 self.syncWithOrigin = True
a028a98e 2424 self.importIntoRemotes = True
01a9c9c5 2425 self.maxChanges = ""
1051ef00 2426 self.changes_block_size = None
8b41a97f 2427 self.keepRepoPath = False
6326aa58 2428 self.depotPaths = None
3c699645 2429 self.p4BranchesInGit = []
354081d5 2430 self.cloneExclude = []
3a70cdfa 2431 self.useClientSpec = False
a93d33ee 2432 self.useClientSpec_from_options = False
ecb7cf98 2433 self.clientSpecDirs = None
fed23693 2434 self.tempBranches = []
d604176d 2435 self.tempBranchLocation = "refs/git-p4-tmp"
a5db4b12
LS
2436 self.largeFileSystem = None
2437
2438 if gitConfig('git-p4.largeFileSystem'):
2439 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2440 self.largeFileSystem = largeFileSystemConstructor(
2441 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2442 )
b984733c 2443
01265103
SH
2444 if gitConfig("git-p4.syncFromOrigin") == "false":
2445 self.syncWithOrigin = False
2446
fed23693
VA
2447 # Force a checkpoint in fast-import and wait for it to finish
2448 def checkpoint(self):
2449 self.gitStream.write("checkpoint\n\n")
2450 self.gitStream.write("progress checkpoint\n\n")
2451 out = self.gitOutput.readline()
2452 if self.verbose:
2453 print "checkpoint finished: " + out
2454
b984733c 2455 def extractFilesFromCommit(self, commit):
354081d5
TT
2456 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2457 for path in self.cloneExclude]
b984733c
SH
2458 files = []
2459 fnum = 0
2460 while commit.has_key("depotFile%s" % fnum):
2461 path = commit["depotFile%s" % fnum]
6326aa58 2462
354081d5 2463 if [p for p in self.cloneExclude
d53de8b9 2464 if p4PathStartsWith(path, p)]:
354081d5
TT
2465 found = False
2466 else:
2467 found = [p for p in self.depotPaths
d53de8b9 2468 if p4PathStartsWith(path, p)]
6326aa58 2469 if not found:
b984733c
SH
2470 fnum = fnum + 1
2471 continue
2472
2473 file = {}
2474 file["path"] = path
2475 file["rev"] = commit["rev%s" % fnum]
2476 file["action"] = commit["action%s" % fnum]
2477 file["type"] = commit["type%s" % fnum]
2478 files.append(file)
2479 fnum = fnum + 1
2480 return files
2481
26e6a27d
JD
2482 def extractJobsFromCommit(self, commit):
2483 jobs = []
2484 jnum = 0
2485 while commit.has_key("job%s" % jnum):
2486 job = commit["job%s" % jnum]
2487 jobs.append(job)
2488 jnum = jnum + 1
2489 return jobs
2490
6326aa58 2491 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2492 """When streaming files, this is called to map a p4 depot path
2493 to where it should go in git. The prefixes are either
2494 self.depotPaths, or self.branchPrefixes in the case of
2495 branch detection."""
2496
3952710b 2497 if self.useClientSpec:
21ef5df4
PW
2498 # branch detection moves files up a level (the branch name)
2499 # from what client spec interpretation gives
0d1696ef 2500 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2501 if self.detectBranches:
2502 for b in self.knownBranches:
2503 if path.startswith(b + "/"):
2504 path = path[len(b)+1:]
2505
2506 elif self.keepRepoPath:
2507 # Preserve everything in relative path name except leading
2508 # //depot/; just look at first prefix as they all should
2509 # be in the same depot.
2510 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2511 if p4PathStartsWith(path, depot):
2512 path = path[len(depot):]
3952710b 2513
0d1696ef 2514 else:
0d1696ef
PW
2515 for p in prefixes:
2516 if p4PathStartsWith(path, p):
2517 path = path[len(p):]
21ef5df4 2518 break
8b41a97f 2519
0d1696ef 2520 path = wildcard_decode(path)
6326aa58 2521 return path
6754a299 2522
71b112d4 2523 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2524 """Look at each depotFile in the commit to figure out to what
2525 branch it belongs."""
2526
9d57c4a6
KS
2527 if self.clientSpecDirs:
2528 files = self.extractFilesFromCommit(commit)
2529 self.clientSpecDirs.update_client_spec_path_cache(files)
2530
d5904674 2531 branches = {}
71b112d4
SH
2532 fnum = 0
2533 while commit.has_key("depotFile%s" % fnum):
2534 path = commit["depotFile%s" % fnum]
6326aa58 2535 found = [p for p in self.depotPaths
d53de8b9 2536 if p4PathStartsWith(path, p)]
6326aa58 2537 if not found:
71b112d4
SH
2538 fnum = fnum + 1
2539 continue
2540
2541 file = {}
2542 file["path"] = path
2543 file["rev"] = commit["rev%s" % fnum]
2544 file["action"] = commit["action%s" % fnum]
2545 file["type"] = commit["type%s" % fnum]
2546 fnum = fnum + 1
2547
21ef5df4
PW
2548 # start with the full relative path where this file would
2549 # go in a p4 client
2550 if self.useClientSpec:
2551 relPath = self.clientSpecDirs.map_in_client(path)
2552 else:
2553 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2554
4b97ffb1 2555 for branch in self.knownBranches.keys():
21ef5df4
PW
2556 # add a trailing slash so that a commit into qt/4.2foo
2557 # doesn't end up in qt/4.2, e.g.
6754a299 2558 if relPath.startswith(branch + "/"):
d5904674
SH
2559 if branch not in branches:
2560 branches[branch] = []
71b112d4 2561 branches[branch].append(file)
6555b2cc 2562 break
b984733c
SH
2563
2564 return branches
2565
a5db4b12
LS
2566 def writeToGitStream(self, gitMode, relPath, contents):
2567 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2568 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2569 for d in contents:
2570 self.gitStream.write(d)
2571 self.gitStream.write('\n')
2572
a8b05162
LS
2573 def encodeWithUTF8(self, path):
2574 try:
2575 path.decode('ascii')
2576 except:
2577 encoding = 'utf8'
2578 if gitConfig('git-p4.pathEncoding'):
2579 encoding = gitConfig('git-p4.pathEncoding')
2580 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2581 if self.verbose:
2582 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2583 return path
2584
b932705b
LD
2585 # output one file from the P4 stream
2586 # - helper for streamP4Files
2587
2588 def streamOneP4File(self, file, contents):
b932705b 2589 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
a8b05162 2590 relPath = self.encodeWithUTF8(relPath)
b932705b 2591 if verbose:
d2176a50
LS
2592 size = int(self.stream_file['fileSize'])
2593 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2594 sys.stdout.flush()
b932705b 2595
9cffb8c8
PW
2596 (type_base, type_mods) = split_p4_type(file["type"])
2597
2598 git_mode = "100644"
2599 if "x" in type_mods:
2600 git_mode = "100755"
2601 if type_base == "symlink":
2602 git_mode = "120000"
1292df11
AJ
2603 # p4 print on a symlink sometimes contains "target\n";
2604 # if it does, remove the newline
b39c3612 2605 data = ''.join(contents)
40f846c3
PW
2606 if not data:
2607 # Some version of p4 allowed creating a symlink that pointed
2608 # to nothing. This causes p4 errors when checking out such
2609 # a change, and errors here too. Work around it by ignoring
2610 # the bad symlink; hopefully a future change fixes it.
2611 print "\nIgnoring empty symlink in %s" % file['depotFile']
2612 return
2613 elif data[-1] == '\n':
1292df11
AJ
2614 contents = [data[:-1]]
2615 else:
2616 contents = [data]
b932705b 2617
9cffb8c8 2618 if type_base == "utf16":
55aa5714
PW
2619 # p4 delivers different text in the python output to -G
2620 # than it does when using "print -o", or normal p4 client
2621 # operations. utf16 is converted to ascii or utf8, perhaps.
2622 # But ascii text saved as -t utf16 is completely mangled.
2623 # Invoke print -o to get the real contents.
7f0e5962
PW
2624 #
2625 # On windows, the newlines will always be mangled by print, so put
2626 # them back too. This is not needed to the cygwin windows version,
2627 # just the native "NT" type.
2628 #
1f5f3907
LS
2629 try:
2630 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2631 except Exception as e:
2632 if 'Translation of file content failed' in str(e):
2633 type_base = 'binary'
2634 else:
2635 raise e
2636 else:
2637 if p4_version_string().find('/NT') >= 0:
2638 text = text.replace('\r\n', '\n')
2639 contents = [ text ]
55aa5714 2640
9f7ef0ea
PW
2641 if type_base == "apple":
2642 # Apple filetype files will be streamed as a concatenation of
2643 # its appledouble header and the contents. This is useless
2644 # on both macs and non-macs. If using "print -q -o xx", it
2645 # will create "xx" with the data, and "%xx" with the header.
2646 # This is also not very useful.
2647 #
2648 # Ideally, someday, this script can learn how to generate
2649 # appledouble files directly and import those to git, but
2650 # non-mac machines can never find a use for apple filetype.
2651 print "\nIgnoring apple filetype file %s" % file['depotFile']
2652 return
2653
55aa5714
PW
2654 # Note that we do not try to de-mangle keywords on utf16 files,
2655 # even though in theory somebody may want that.
60df071c
LD
2656 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2657 if pattern:
2658 regexp = re.compile(pattern, re.VERBOSE)
2659 text = ''.join(contents)
2660 text = regexp.sub(r'$\1$', text)
2661 contents = [ text ]
b932705b 2662
a5db4b12
LS
2663 if self.largeFileSystem:
2664 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2665
a5db4b12 2666 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2667
2668 def streamOneP4Deletion(self, file):
2669 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
a8b05162 2670 relPath = self.encodeWithUTF8(relPath)
b932705b 2671 if verbose:
d2176a50
LS
2672 sys.stdout.write("delete %s\n" % relPath)
2673 sys.stdout.flush()
b932705b
LD
2674 self.gitStream.write("D %s\n" % relPath)
2675
a5db4b12
LS
2676 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2677 self.largeFileSystem.removeLargeFile(relPath)
2678
b932705b
LD
2679 # handle another chunk of streaming data
2680 def streamP4FilesCb(self, marshalled):
2681
78189bea
PW
2682 # catch p4 errors and complain
2683 err = None
2684 if "code" in marshalled:
2685 if marshalled["code"] == "error":
2686 if "data" in marshalled:
2687 err = marshalled["data"].rstrip()
4d25dc44
LS
2688
2689 if not err and 'fileSize' in self.stream_file:
2690 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2691 if required_bytes > 0:
2692 err = 'Not enough space left on %s! Free at least %i MB.' % (
2693 os.getcwd(), required_bytes/1024/1024
2694 )
2695
78189bea
PW
2696 if err:
2697 f = None
2698 if self.stream_have_file_info:
2699 if "depotFile" in self.stream_file:
2700 f = self.stream_file["depotFile"]
2701 # force a failure in fast-import, else an empty
2702 # commit will be made
2703 self.gitStream.write("\n")
2704 self.gitStream.write("die-now\n")
2705 self.gitStream.close()
2706 # ignore errors, but make sure it exits first
2707 self.importProcess.wait()
2708 if f:
2709 die("Error from p4 print for %s: %s" % (f, err))
2710 else:
2711 die("Error from p4 print: %s" % err)
2712
c3f6163b
AG
2713 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2714 # start of a new file - output the old one first
2715 self.streamOneP4File(self.stream_file, self.stream_contents)
2716 self.stream_file = {}
2717 self.stream_contents = []
2718 self.stream_have_file_info = False
b932705b 2719
c3f6163b
AG
2720 # pick up the new file information... for the
2721 # 'data' field we need to append to our array
2722 for k in marshalled.keys():
2723 if k == 'data':
d2176a50
LS
2724 if 'streamContentSize' not in self.stream_file:
2725 self.stream_file['streamContentSize'] = 0
2726 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2727 self.stream_contents.append(marshalled['data'])
2728 else:
2729 self.stream_file[k] = marshalled[k]
b932705b 2730
d2176a50
LS
2731 if (verbose and
2732 'streamContentSize' in self.stream_file and
2733 'fileSize' in self.stream_file and
2734 'depotFile' in self.stream_file):
2735 size = int(self.stream_file["fileSize"])
2736 if size > 0:
2737 progress = 100*self.stream_file['streamContentSize']/size
2738 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2739 sys.stdout.flush()
2740
c3f6163b 2741 self.stream_have_file_info = True
b932705b
LD
2742
2743 # Stream directly from "p4 files" into "git fast-import"
2744 def streamP4Files(self, files):
30b5940b
SH
2745 filesForCommit = []
2746 filesToRead = []
b932705b 2747 filesToDelete = []
30b5940b 2748
3a70cdfa 2749 for f in files:
ecb7cf98
PW
2750 filesForCommit.append(f)
2751 if f['action'] in self.delete_actions:
2752 filesToDelete.append(f)
2753 else:
2754 filesToRead.append(f)
6a49f8e2 2755
b932705b
LD
2756 # deleted files...
2757 for f in filesToDelete:
2758 self.streamOneP4Deletion(f)
1b9a4684 2759
b932705b
LD
2760 if len(filesToRead) > 0:
2761 self.stream_file = {}
2762 self.stream_contents = []
2763 self.stream_have_file_info = False
8ff45f2a 2764
c3f6163b
AG
2765 # curry self argument
2766 def streamP4FilesCbSelf(entry):
2767 self.streamP4FilesCb(entry)
6a49f8e2 2768
6de040df
LD
2769 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2770
2771 p4CmdList(["-x", "-", "print"],
2772 stdin=fileArgs,
2773 cb=streamP4FilesCbSelf)
30b5940b 2774
b932705b
LD
2775 # do the last chunk
2776 if self.stream_file.has_key('depotFile'):
2777 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2778
affb474f
LD
2779 def make_email(self, userid):
2780 if userid in self.users:
2781 return self.users[userid]
2782 else:
2783 return "%s <a@b>" % userid
2784
06804c76 2785 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2786 """ Stream a p4 tag.
2787 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2788 """
2789
06804c76
LD
2790 if verbose:
2791 print "writing tag %s for commit %s" % (labelName, commit)
2792 gitStream.write("tag %s\n" % labelName)
2793 gitStream.write("from %s\n" % commit)
2794
2795 if labelDetails.has_key('Owner'):
2796 owner = labelDetails["Owner"]
2797 else:
2798 owner = None
2799
2800 # Try to use the owner of the p4 label, or failing that,
2801 # the current p4 user id.
2802 if owner:
2803 email = self.make_email(owner)
2804 else:
2805 email = self.make_email(self.p4UserId())
2806 tagger = "%s %s %s" % (email, epoch, self.tz)
2807
2808 gitStream.write("tagger %s\n" % tagger)
2809
2810 print "labelDetails=",labelDetails
2811 if labelDetails.has_key('Description'):
2812 description = labelDetails['Description']
2813 else:
2814 description = 'Label from git p4'
2815
2816 gitStream.write("data %d\n" % len(description))
2817 gitStream.write(description)
2818 gitStream.write("\n")
2819
4ae048e6
LS
2820 def inClientSpec(self, path):
2821 if not self.clientSpecDirs:
2822 return True
2823 inClientSpec = self.clientSpecDirs.map_in_client(path)
2824 if not inClientSpec and self.verbose:
2825 print('Ignoring file outside of client spec: {0}'.format(path))
2826 return inClientSpec
2827
2828 def hasBranchPrefix(self, path):
2829 if not self.branchPrefixes:
2830 return True
2831 hasPrefix = [p for p in self.branchPrefixes
2832 if p4PathStartsWith(path, p)]
09667d01 2833 if not hasPrefix and self.verbose:
4ae048e6
LS
2834 print('Ignoring file outside of prefix: {0}'.format(path))
2835 return hasPrefix
2836
e63231e5 2837 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2838 epoch = details["time"]
2839 author = details["user"]
26e6a27d 2840 jobs = self.extractJobsFromCommit(details)
b984733c 2841
4b97ffb1 2842 if self.verbose:
4ae048e6 2843 print('commit into {0}'.format(branch))
96e07dd2 2844
9d57c4a6
KS
2845 if self.clientSpecDirs:
2846 self.clientSpecDirs.update_client_spec_path_cache(files)
2847
4ae048e6
LS
2848 files = [f for f in files
2849 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2850
2851 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2852 print('Ignoring revision {0} as it would produce an empty commit.'
2853 .format(details['change']))
2854 return
2855
b984733c 2856 self.gitStream.write("commit %s\n" % branch)
b43702ac 2857 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2858 self.committedChanges.add(int(details["change"]))
2859 committer = ""
b607e71e
SH
2860 if author not in self.users:
2861 self.getUserMapFromPerforceServer()
affb474f 2862 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2863
2864 self.gitStream.write("committer %s\n" % committer)
2865
2866 self.gitStream.write("data <<EOT\n")
2867 self.gitStream.write(details["desc"])
26e6a27d
JD
2868 if len(jobs) > 0:
2869 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
e63231e5
PW
2870 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2871 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2872 if len(details['options']) > 0:
2873 self.gitStream.write(": options = %s" % details['options'])
2874 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2875
2876 if len(parent) > 0:
4b97ffb1
SH
2877 if self.verbose:
2878 print "parent %s" % parent
b984733c
SH
2879 self.gitStream.write("from %s\n" % parent)
2880
4ae048e6 2881 self.streamP4Files(files)
b984733c
SH
2882 self.gitStream.write("\n")
2883
1f4ba1cb
SH
2884 change = int(details["change"])
2885
9bda3a85 2886 if self.labels.has_key(change):
1f4ba1cb
SH
2887 label = self.labels[change]
2888 labelDetails = label[0]
2889 labelRevisions = label[1]
71b112d4
SH
2890 if self.verbose:
2891 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2892
6de040df 2893 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2894 for p in self.branchPrefixes])
1f4ba1cb
SH
2895
2896 if len(files) == len(labelRevisions):
2897
2898 cleanedFiles = {}
2899 for info in files:
56c09345 2900 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2901 continue
2902 cleanedFiles[info["depotFile"]] = info["rev"]
2903
2904 if cleanedFiles == labelRevisions:
06804c76 2905 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2906
2907 else:
a46668fa 2908 if not self.silent:
cebdf5af
HWN
2909 print ("Tag %s does not match with change %s: files do not match."
2910 % (labelDetails["label"], change))
1f4ba1cb
SH
2911
2912 else:
a46668fa 2913 if not self.silent:
cebdf5af
HWN
2914 print ("Tag %s does not match with change %s: file count is different."
2915 % (labelDetails["label"], change))
b984733c 2916
06804c76 2917 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2918 def getLabels(self):
2919 self.labels = {}
2920
52a4880b 2921 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2922 if len(l) > 0 and not self.silent:
183f8436 2923 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2924
2925 for output in l:
1f4ba1cb
SH
2926 label = output["label"]
2927 revisions = {}
2928 newestChange = 0
71b112d4
SH
2929 if self.verbose:
2930 print "Querying files for label %s" % label
6de040df
LD
2931 for file in p4CmdList(["files"] +
2932 ["%s...@%s" % (p, label)
2933 for p in self.depotPaths]):
1f4ba1cb
SH
2934 revisions[file["depotFile"]] = file["rev"]
2935 change = int(file["change"])
2936 if change > newestChange:
2937 newestChange = change
2938
9bda3a85
SH
2939 self.labels[newestChange] = [output, revisions]
2940
2941 if self.verbose:
2942 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2943
06804c76
LD
2944 # Import p4 labels as git tags. A direct mapping does not
2945 # exist, so assume that if all the files are at the same revision
2946 # then we can use that, or it's something more complicated we should
2947 # just ignore.
2948 def importP4Labels(self, stream, p4Labels):
2949 if verbose:
2950 print "import p4 labels: " + ' '.join(p4Labels)
2951
2952 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2953 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2954 if len(validLabelRegexp) == 0:
2955 validLabelRegexp = defaultLabelRegexp
2956 m = re.compile(validLabelRegexp)
2957
2958 for name in p4Labels:
2959 commitFound = False
2960
2961 if not m.match(name):
2962 if verbose:
2963 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2964 continue
2965
2966 if name in ignoredP4Labels:
2967 continue
2968
2969 labelDetails = p4CmdList(['label', "-o", name])[0]
2970
2971 # get the most recent changelist for each file in this label
2972 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2973 for p in self.depotPaths])
2974
2975 if change.has_key('change'):
2976 # find the corresponding git commit; take the oldest commit
2977 changelist = int(change['change'])
b43702ac
LD
2978 if changelist in self.committedChanges:
2979 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2980 commitFound = True
b43702ac
LD
2981 else:
2982 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2983 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2984 if len(gitCommit) == 0:
2985 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2986 else:
2987 commitFound = True
2988 gitCommit = gitCommit.strip()
2989
2990 if commitFound:
06804c76
LD
2991 # Convert from p4 time format
2992 try:
2993 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2994 except ValueError:
a4e9054c 2995 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2996 tmwhen = 1
2997
2998 when = int(time.mktime(tmwhen))
2999 self.streamTag(stream, name, labelDetails, gitCommit, when)
3000 if verbose:
3001 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3002 else:
3003 if verbose:
3004 print "Label %s has no changelists - possibly deleted?" % name
3005
3006 if not commitFound:
3007 # We can't import this label; don't try again as it will get very
3008 # expensive repeatedly fetching all the files for labels that will
3009 # never be imported. If the label is moved in the future, the
3010 # ignore will need to be removed manually.
3011 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3012
86dff6b6
HWN
3013 def guessProjectName(self):
3014 for p in self.depotPaths:
6e5295c4
SH
3015 if p.endswith("/"):
3016 p = p[:-1]
3017 p = p[p.strip().rfind("/") + 1:]
3018 if not p.endswith("/"):
3019 p += "/"
3020 return p
86dff6b6 3021
4b97ffb1 3022 def getBranchMapping(self):
6555b2cc
SH
3023 lostAndFoundBranches = set()
3024
8ace74c0
VA
3025 user = gitConfig("git-p4.branchUser")
3026 if len(user) > 0:
3027 command = "branches -u %s" % user
3028 else:
3029 command = "branches"
3030
3031 for info in p4CmdList(command):
52a4880b 3032 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
3033 viewIdx = 0
3034 while details.has_key("View%s" % viewIdx):
3035 paths = details["View%s" % viewIdx].split(" ")
3036 viewIdx = viewIdx + 1
3037 # require standard //depot/foo/... //depot/bar/... mapping
3038 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3039 continue
3040 source = paths[0]
3041 destination = paths[1]
6509e19c 3042 ## HACK
d53de8b9 3043 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
3044 source = source[len(self.depotPaths[0]):-4]
3045 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 3046
1a2edf4e
SH
3047 if destination in self.knownBranches:
3048 if not self.silent:
3049 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3050 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3051 continue
3052
6555b2cc
SH
3053 self.knownBranches[destination] = source
3054
3055 lostAndFoundBranches.discard(destination)
3056
29bdbac1 3057 if source not in self.knownBranches:
6555b2cc
SH
3058 lostAndFoundBranches.add(source)
3059
7199cf13
VA
3060 # Perforce does not strictly require branches to be defined, so we also
3061 # check git config for a branch list.
3062 #
3063 # Example of branch definition in git config file:
3064 # [git-p4]
3065 # branchList=main:branchA
3066 # branchList=main:branchB
3067 # branchList=branchA:branchC
3068 configBranches = gitConfigList("git-p4.branchList")
3069 for branch in configBranches:
3070 if branch:
3071 (source, destination) = branch.split(":")
3072 self.knownBranches[destination] = source
3073
3074 lostAndFoundBranches.discard(destination)
3075
3076 if source not in self.knownBranches:
3077 lostAndFoundBranches.add(source)
3078
6555b2cc
SH
3079
3080 for branch in lostAndFoundBranches:
3081 self.knownBranches[branch] = branch
29bdbac1 3082
38f9f5ec
SH
3083 def getBranchMappingFromGitBranches(self):
3084 branches = p4BranchesInGit(self.importIntoRemotes)
3085 for branch in branches.keys():
3086 if branch == "master":
3087 branch = "main"
3088 else:
3089 branch = branch[len(self.projectName):]
3090 self.knownBranches[branch] = branch
3091
bb6e09b2
HWN
3092 def updateOptionDict(self, d):
3093 option_keys = {}
3094 if self.keepRepoPath:
3095 option_keys['keepRepoPath'] = 1
3096
3097 d["options"] = ' '.join(sorted(option_keys.keys()))
3098
3099 def readOptions(self, d):
3100 self.keepRepoPath = (d.has_key('options')
3101 and ('keepRepoPath' in d['options']))
6326aa58 3102
8134f69c
SH
3103 def gitRefForBranch(self, branch):
3104 if branch == "main":
3105 return self.refPrefix + "master"
3106
3107 if len(branch) <= 0:
3108 return branch
3109
3110 return self.refPrefix + self.projectName + branch
3111
1ca3d710
SH
3112 def gitCommitByP4Change(self, ref, change):
3113 if self.verbose:
3114 print "looking in ref " + ref + " for change %s using bisect..." % change
3115
3116 earliestCommit = ""
3117 latestCommit = parseRevision(ref)
3118
3119 while True:
3120 if self.verbose:
3121 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3122 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3123 if len(next) == 0:
3124 if self.verbose:
3125 print "argh"
3126 return ""
3127 log = extractLogMessageFromGitCommit(next)
3128 settings = extractSettingsGitLog(log)
3129 currentChange = int(settings['change'])
3130 if self.verbose:
3131 print "current change %s" % currentChange
3132
3133 if currentChange == change:
3134 if self.verbose:
3135 print "found %s" % next
3136 return next
3137
3138 if currentChange < change:
3139 earliestCommit = "^%s" % next
3140 else:
3141 latestCommit = "%s" % next
3142
3143 return ""
3144
3145 def importNewBranch(self, branch, maxChange):
3146 # make fast-import flush all changes to disk and update the refs using the checkpoint
3147 # command so that we can try to find the branch parent in the git history
3148 self.gitStream.write("checkpoint\n\n");
3149 self.gitStream.flush();
3150 branchPrefix = self.depotPaths[0] + branch + "/"
3151 range = "@1,%s" % maxChange
3152 #print "prefix" + branchPrefix
96b2d54a 3153 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
3154 if len(changes) <= 0:
3155 return False
3156 firstChange = changes[0]
3157 #print "first change in branch: %s" % firstChange
3158 sourceBranch = self.knownBranches[branch]
3159 sourceDepotPath = self.depotPaths[0] + sourceBranch
3160 sourceRef = self.gitRefForBranch(sourceBranch)
3161 #print "source " + sourceBranch
3162
52a4880b 3163 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
3164 #print "branch parent: %s" % branchParentChange
3165 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3166 if len(gitParent) > 0:
3167 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3168 #print "parent git commit: %s" % gitParent
3169
3170 self.importChanges(changes)
3171 return True
3172
fed23693
VA
3173 def searchParent(self, parent, branch, target):
3174 parentFound = False
c7d34884
PW
3175 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3176 "--no-merges", parent]):
fed23693
VA
3177 blob = blob.strip()
3178 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3179 parentFound = True
3180 if self.verbose:
3181 print "Found parent of %s in commit %s" % (branch, blob)
3182 break
3183 if parentFound:
3184 return blob
3185 else:
3186 return None
3187
e87f37ae
SH
3188 def importChanges(self, changes):
3189 cnt = 1
3190 for change in changes:
18fa13d0 3191 description = p4_describe(change)
e87f37ae
SH
3192 self.updateOptionDict(description)
3193
3194 if not self.silent:
3195 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3196 sys.stdout.flush()
3197 cnt = cnt + 1
3198
3199 try:
3200 if self.detectBranches:
3201 branches = self.splitFilesIntoBranches(description)
3202 for branch in branches.keys():
3203 ## HACK --hwn
3204 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3205 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3206
3207 parent = ""
3208
3209 filesForCommit = branches[branch]
3210
3211 if self.verbose:
3212 print "branch is %s" % branch
3213
3214 self.updatedBranches.add(branch)
3215
3216 if branch not in self.createdBranches:
3217 self.createdBranches.add(branch)
3218 parent = self.knownBranches[branch]
3219 if parent == branch:
3220 parent = ""
1ca3d710
SH
3221 else:
3222 fullBranch = self.projectName + branch
3223 if fullBranch not in self.p4BranchesInGit:
3224 if not self.silent:
3225 print("\n Importing new branch %s" % fullBranch);
3226 if self.importNewBranch(branch, change - 1):
3227 parent = ""
3228 self.p4BranchesInGit.append(fullBranch)
3229 if not self.silent:
3230 print("\n Resuming with change %s" % change);
3231
3232 if self.verbose:
3233 print "parent determined through known branches: %s" % parent
e87f37ae 3234
8134f69c
SH
3235 branch = self.gitRefForBranch(branch)
3236 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3237
3238 if self.verbose:
3239 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3240
3241 if len(parent) == 0 and branch in self.initialParents:
3242 parent = self.initialParents[branch]
3243 del self.initialParents[branch]
3244
fed23693
VA
3245 blob = None
3246 if len(parent) > 0:
4f9273d2 3247 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3248 if self.verbose:
3249 print "Creating temporary branch: " + tempBranch
e63231e5 3250 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3251 self.tempBranches.append(tempBranch)
3252 self.checkpoint()
3253 blob = self.searchParent(parent, branch, tempBranch)
3254 if blob:
e63231e5 3255 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3256 else:
3257 if self.verbose:
3258 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3259 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3260 else:
3261 files = self.extractFilesFromCommit(description)
e63231e5 3262 self.commit(description, files, self.branch,
e87f37ae 3263 self.initialParent)
47497844 3264 # only needed once, to connect to the previous commit
e87f37ae
SH
3265 self.initialParent = ""
3266 except IOError:
3267 print self.gitError.read()
3268 sys.exit(1)
3269
b9d34db9
LD
3270 def sync_origin_only(self):
3271 if self.syncWithOrigin:
3272 self.hasOrigin = originP4BranchesExist()
3273 if self.hasOrigin:
3274 if not self.silent:
3275 print 'Syncing with origin first, using "git fetch origin"'
3276 system("git fetch origin")
3277
c208a243
SH
3278 def importHeadRevision(self, revision):
3279 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3280
4e2e6ce4
PW
3281 details = {}
3282 details["user"] = "git perforce import user"
1494fcbb 3283 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3284 % (' '.join(self.depotPaths), revision))
3285 details["change"] = revision
3286 newestRevision = 0
3287
3288 fileCnt = 0
6de040df
LD
3289 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3290
3291 for info in p4CmdList(["files"] + fileArgs):
c208a243 3292
68b28593 3293 if 'code' in info and info['code'] == 'error':
c208a243
SH
3294 sys.stderr.write("p4 returned an error: %s\n"
3295 % info['data'])
d88e707f
PW
3296 if info['data'].find("must refer to client") >= 0:
3297 sys.stderr.write("This particular p4 error is misleading.\n")
3298 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3299 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3300 sys.exit(1)
68b28593
PW
3301 if 'p4ExitCode' in info:
3302 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3303 sys.exit(1)
3304
3305
3306 change = int(info["change"])
3307 if change > newestRevision:
3308 newestRevision = change
3309
56c09345 3310 if info["action"] in self.delete_actions:
c208a243
SH
3311 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3312 #fileCnt = fileCnt + 1
3313 continue
3314
3315 for prop in ["depotFile", "rev", "action", "type" ]:
3316 details["%s%s" % (prop, fileCnt)] = info[prop]
3317
3318 fileCnt = fileCnt + 1
3319
3320 details["change"] = newestRevision
4e2e6ce4 3321
9dcb9f24 3322 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3323 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3324 res = p4_describe(newestRevision)
3325 details["time"] = res["time"]
4e2e6ce4 3326
c208a243
SH
3327 self.updateOptionDict(details)
3328 try:
e63231e5 3329 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3330 except IOError:
3331 print "IO error with git fast-import. Is your git version recent enough?"
3332 print self.gitError.read()
3333
3334
b984733c 3335 def run(self, args):
6326aa58 3336 self.depotPaths = []
179caebf 3337 self.changeRange = ""
6326aa58 3338 self.previousDepotPaths = []
991a2de4 3339 self.hasOrigin = False
ce6f33c8 3340
29bdbac1
SH
3341 # map from branch depot path to parent branch
3342 self.knownBranches = {}
3343 self.initialParents = {}
3344
a028a98e
SH
3345 if self.importIntoRemotes:
3346 self.refPrefix = "refs/remotes/p4/"
3347 else:
db775559 3348 self.refPrefix = "refs/heads/p4/"
a028a98e 3349
b9d34db9 3350 self.sync_origin_only()
10f880f8 3351
5a8e84cd 3352 branch_arg_given = bool(self.branch)
569d1bd4 3353 if len(self.branch) == 0:
db775559 3354 self.branch = self.refPrefix + "master"
a028a98e 3355 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3356 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3357 system("git branch -D p4")
967f72e2 3358
a93d33ee
PW
3359 # accept either the command-line option, or the configuration variable
3360 if self.useClientSpec:
3361 # will use this after clone to set the variable
3362 self.useClientSpec_from_options = True
3363 else:
0d609032 3364 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3365 self.useClientSpec = True
3366 if self.useClientSpec:
543987bd 3367 self.clientSpecDirs = getClientSpec()
3a70cdfa 3368
6a49f8e2
HWN
3369 # TODO: should always look at previous commits,
3370 # merge with previous imports, if possible.
3371 if args == []:
d414c74a 3372 if self.hasOrigin:
5ca44617 3373 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3374
3375 # branches holds mapping from branch name to sha1
3376 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3377
3378 # restrict to just this one, disabling detect-branches
3379 if branch_arg_given:
3380 short = self.branch.split("/")[-1]
3381 if short in branches:
3382 self.p4BranchesInGit = [ short ]
3383 else:
3384 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3385
3386 if len(self.p4BranchesInGit) > 1:
3387 if not self.silent:
3388 print "Importing from/into multiple branches"
3389 self.detectBranches = True
8c9e8b6e
PW
3390 for branch in branches.keys():
3391 self.initialParents[self.refPrefix + branch] = \
3392 branches[branch]
967f72e2 3393
29bdbac1
SH
3394 if self.verbose:
3395 print "branches: %s" % self.p4BranchesInGit
3396
3397 p4Change = 0
3398 for branch in self.p4BranchesInGit:
cebdf5af 3399 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3400
3401 settings = extractSettingsGitLog(logMsg)
29bdbac1 3402
bb6e09b2
HWN
3403 self.readOptions(settings)
3404 if (settings.has_key('depot-paths')
3405 and settings.has_key ('change')):
3406 change = int(settings['change']) + 1
29bdbac1
SH
3407 p4Change = max(p4Change, change)
3408
bb6e09b2
HWN
3409 depotPaths = sorted(settings['depot-paths'])
3410 if self.previousDepotPaths == []:
6326aa58 3411 self.previousDepotPaths = depotPaths
29bdbac1 3412 else:
6326aa58
HWN
3413 paths = []
3414 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3415 prev_list = prev.split("/")
3416 cur_list = cur.split("/")
3417 for i in range(0, min(len(cur_list), len(prev_list))):
3418 if cur_list[i] <> prev_list[i]:
583e1707 3419 i = i - 1
6326aa58
HWN
3420 break
3421
04d277b3 3422 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3423
3424 self.previousDepotPaths = paths
29bdbac1
SH
3425
3426 if p4Change > 0:
bb6e09b2 3427 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3428 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3429 if not self.silent and not self.detectBranches:
967f72e2 3430 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3431
40d69ac3
PW
3432 # accept multiple ref name abbreviations:
3433 # refs/foo/bar/branch -> use it exactly
3434 # p4/branch -> prepend refs/remotes/ or refs/heads/
3435 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3436 if not self.branch.startswith("refs/"):
40d69ac3
PW
3437 if self.importIntoRemotes:
3438 prepend = "refs/remotes/"
3439 else:
3440 prepend = "refs/heads/"
3441 if not self.branch.startswith("p4/"):
3442 prepend += "p4/"
3443 self.branch = prepend + self.branch
179caebf 3444
6326aa58 3445 if len(args) == 0 and self.depotPaths:
b984733c 3446 if not self.silent:
6326aa58 3447 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3448 else:
6326aa58 3449 if self.depotPaths and self.depotPaths != args:
cebdf5af 3450 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3451 "This doesn't work!" % (' '.join (self.depotPaths),
3452 ' '.join (args)))
b984733c