git-p4: support multiple depot paths in p4 submit
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
a5db4b12
LS
25import zipfile
26import zlib
4b07cd23 27import ctypes
8b41a97f 28
a235e85c
BC
29try:
30 from subprocess import CalledProcessError
31except ImportError:
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self, returncode, cmd):
39 self.returncode = returncode
40 self.cmd = cmd
41 def __str__(self):
42 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
43
4addad22 44verbose = False
86949eef 45
06804c76 46# Only labels/tags matching this will be imported/exported
c8942a22 47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 48
1051ef00
LD
49# Grab changes in blocks of this many revisions, unless otherwise requested
50defaultBlockSize = 512
51
21a50753
AK
52def p4_build_cmd(cmd):
53 """Build a suitable p4 command line.
54
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
58 """
6de040df 59 real_cmd = ["p4"]
abcaf073
AK
60
61 user = gitConfig("git-p4.user")
62 if len(user) > 0:
6de040df 63 real_cmd += ["-u",user]
abcaf073
AK
64
65 password = gitConfig("git-p4.password")
66 if len(password) > 0:
6de040df 67 real_cmd += ["-P", password]
abcaf073
AK
68
69 port = gitConfig("git-p4.port")
70 if len(port) > 0:
6de040df 71 real_cmd += ["-p", port]
abcaf073
AK
72
73 host = gitConfig("git-p4.host")
74 if len(host) > 0:
41799aa2 75 real_cmd += ["-H", host]
abcaf073
AK
76
77 client = gitConfig("git-p4.client")
78 if len(client) > 0:
6de040df 79 real_cmd += ["-c", client]
abcaf073 80
6de040df
LD
81
82 if isinstance(cmd,basestring):
83 real_cmd = ' '.join(real_cmd) + ' ' + cmd
84 else:
85 real_cmd += cmd
21a50753
AK
86 return real_cmd
87
bbd84863
MF
88def chdir(path, is_client_path=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
93
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
98
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
102 the client path.
103 """
104
105 os.chdir(path)
106 if not is_client_path:
107 path = os.getcwd()
108 os.environ['PWD'] = path
053fd0c1 109
4d25dc44
LS
110def calcDiskFree():
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform.system() == 'Windows':
113 free_bytes = ctypes.c_ulonglong(0)
114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
115 return free_bytes.value
116 else:
117 st = os.statvfs(os.getcwd())
118 return st.f_bavail * st.f_frsize
119
86dff6b6
HWN
120def die(msg):
121 if verbose:
122 raise Exception(msg)
123 else:
124 sys.stderr.write(msg + "\n")
125 sys.exit(1)
126
6de040df 127def write_pipe(c, stdin):
4addad22 128 if verbose:
6de040df 129 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 130
6de040df
LD
131 expand = isinstance(c,basestring)
132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
133 pipe = p.stdin
134 val = pipe.write(stdin)
135 pipe.close()
136 if p.wait():
137 die('Command failed: %s' % str(c))
b016d397
HWN
138
139 return val
140
6de040df 141def p4_write_pipe(c, stdin):
d9429194 142 real_cmd = p4_build_cmd(c)
6de040df 143 return write_pipe(real_cmd, stdin)
d9429194 144
4addad22
HWN
145def read_pipe(c, ignore_error=False):
146 if verbose:
6de040df 147 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 148
6de040df 149 expand = isinstance(c,basestring)
1f5f3907
LS
150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
151 (out, err) = p.communicate()
152 if p.returncode != 0 and not ignore_error:
153 die('Command failed: %s\nError: %s' % (str(c), err))
154 return out
b016d397 155
d9429194
AK
156def p4_read_pipe(c, ignore_error=False):
157 real_cmd = p4_build_cmd(c)
158 return read_pipe(real_cmd, ignore_error)
b016d397 159
bce4c5fc 160def read_pipe_lines(c):
4addad22 161 if verbose:
6de040df
LD
162 sys.stderr.write('Reading pipe: %s\n' % str(c))
163
164 expand = isinstance(c, basestring)
165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
166 pipe = p.stdout
b016d397 167 val = pipe.readlines()
6de040df
LD
168 if pipe.close() or p.wait():
169 die('Command failed: %s' % str(c))
b016d397
HWN
170
171 return val
caace111 172
2318121b
AK
173def p4_read_pipe_lines(c):
174 """Specifically invoke p4 on the command supplied. """
155af834 175 real_cmd = p4_build_cmd(c)
2318121b
AK
176 return read_pipe_lines(real_cmd)
177
8e9497c2
GG
178def p4_has_command(cmd):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd = p4_build_cmd(["help", cmd])
182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
183 stderr=subprocess.PIPE)
184 p.communicate()
185 return p.returncode == 0
186
249da4c0
PW
187def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
192
193 if not p4_has_command("move"):
194 return False
195 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
197 (out, err) = p.communicate()
198 # return code will be 1 in either case
199 if err.find("Invalid option") >= 0:
200 return False
201 if err.find("disabled") >= 0:
202 return False
203 # assume it failed because @... was invalid changelist
204 return True
205
cbff4b25 206def system(cmd, ignore_error=False):
6de040df 207 expand = isinstance(cmd,basestring)
4addad22 208 if verbose:
6de040df 209 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c 210 retcode = subprocess.call(cmd, shell=expand)
cbff4b25 211 if retcode and not ignore_error:
a235e85c 212 raise CalledProcessError(retcode, cmd)
6754a299 213
cbff4b25
LD
214 return retcode
215
bf9320f1
AK
216def p4_system(cmd):
217 """Specifically invoke p4 as the system command. """
155af834 218 real_cmd = p4_build_cmd(cmd)
6de040df 219 expand = isinstance(real_cmd, basestring)
a235e85c
BC
220 retcode = subprocess.call(real_cmd, shell=expand)
221 if retcode:
222 raise CalledProcessError(retcode, real_cmd)
6de040df 223
7f0e5962
PW
224_p4_version_string = None
225def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
228
229 $ p4 -V
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233 """
234 global _p4_version_string
235 if not _p4_version_string:
236 a = p4_read_pipe_lines(["-V"])
237 _p4_version_string = a[-1].rstrip()
238 return _p4_version_string
239
6de040df 240def p4_integrate(src, dest):
9d7d446a 241 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 242
8d7ec362 243def p4_sync(f, *options):
9d7d446a 244 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
245
246def p4_add(f):
9d7d446a
PW
247 # forcibly add file names with wildcards
248 if wildcard_present(f):
249 p4_system(["add", "-f", f])
250 else:
251 p4_system(["add", f])
6de040df
LD
252
253def p4_delete(f):
9d7d446a 254 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
255
256def p4_edit(f):
9d7d446a 257 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
258
259def p4_revert(f):
9d7d446a 260 p4_system(["revert", wildcard_encode(f)])
6de040df 261
9d7d446a
PW
262def p4_reopen(type, f):
263 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 264
8e9497c2
GG
265def p4_move(src, dest):
266 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
267
1051ef00
LD
268def p4_last_change():
269 results = p4CmdList(["changes", "-m", "1"])
270 return int(results[0]['change'])
271
18fa13d0
PW
272def p4_describe(change):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
275 results."""
276
277 ds = p4CmdList(["describe", "-s", str(change)])
278 if len(ds) != 1:
279 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
280
281 d = ds[0]
282
283 if "p4ExitCode" in d:
284 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
285 str(d)))
286 if "code" in d:
287 if d["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
289
290 if "time" not in d:
291 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
292
293 return d
294
9cffb8c8
PW
295#
296# Canonicalize the p4 type and return a tuple of the
297# base type, plus any modifiers. See "p4 help filetypes"
298# for a list and explanation.
299#
300def split_p4_type(p4type):
301
302 p4_filetypes_historical = {
303 "ctempobj": "binary+Sw",
304 "ctext": "text+C",
305 "cxtext": "text+Cx",
306 "ktext": "text+k",
307 "kxtext": "text+kx",
308 "ltext": "text+F",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
314 "xltext": "text+Fx",
315 "xtempobj": "binary+Swx",
316 "xtext": "text+x",
317 "xunicode": "unicode+x",
318 "xutf16": "utf16+x",
319 }
320 if p4type in p4_filetypes_historical:
321 p4type = p4_filetypes_historical[p4type]
322 mods = ""
323 s = p4type.split("+")
324 base = s[0]
325 mods = ""
326 if len(s) > 1:
327 mods = s[1]
328 return (base, mods)
b9fc6ea9 329
60df071c
LD
330#
331# return the raw p4 type of a file (text, text+ko, etc)
332#
79467e61
PW
333def p4_type(f):
334 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
335 return results[0]['headType']
336
337#
338# Given a type base and modifier, return a regexp matching
339# the keywords that can be expanded in the file
340#
341def p4_keywords_regexp_for_type(base, type_mods):
342 if base in ("text", "unicode", "binary"):
343 kwords = None
344 if "ko" in type_mods:
345 kwords = 'Id|Header'
346 elif "k" in type_mods:
347 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
348 else:
349 return None
350 pattern = r"""
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
6b2bf41e 353 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
354 \$ # another dollar
355 """ % kwords
356 return pattern
357 else:
358 return None
359
360#
361# Given a file, return a regexp matching the possible
362# RCS keywords that will be expanded, or None for files
363# with kw expansion turned off.
364#
365def p4_keywords_regexp_for_file(file):
366 if not os.path.exists(file):
367 return None
368 else:
369 (type_base, type_mods) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 371
c65b670e
CP
372def setP4ExecBit(file, mode):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
375
376 p4Type = "+x"
377
378 if not isModeExec(mode):
379 p4Type = getP4OpenedType(file)
380 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
381 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
382 if p4Type[-1] == "+":
383 p4Type = p4Type[0:-1]
384
6de040df 385 p4_reopen(p4Type, file)
c65b670e
CP
386
387def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
389
9d7d446a 390 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 391 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
392 if match:
393 return match.group(1)
394 else:
f3e5ae4f 395 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 396
06804c76
LD
397# Return the set of all p4 labels
398def getP4Labels(depotPaths):
399 labels = set()
400 if isinstance(depotPaths,basestring):
401 depotPaths = [depotPaths]
402
403 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
404 label = l['label']
405 labels.add(label)
406
407 return labels
408
409# Return the set of all git tags
410def getGitTags():
411 gitTags = set()
412 for line in read_pipe_lines(["git", "tag"]):
413 tag = line.strip()
414 gitTags.add(tag)
415 return gitTags
416
b43b0a3c
CP
417def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
421 while True:
422 yield pattern
423
424def parseDiffTreeEntry(entry):
425 """Parses a single diff tree entry into its component elements.
426
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
429
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
440
441 If the pattern is not matched, None is returned."""
442
443 match = diffTreePattern().next().match(entry)
444 if match:
445 return {
446 'src_mode': match.group(1),
447 'dst_mode': match.group(2),
448 'src_sha1': match.group(3),
449 'dst_sha1': match.group(4),
450 'status': match.group(5),
451 'status_score': match.group(6),
452 'src': match.group(7),
453 'dst': match.group(10)
454 }
455 return None
456
c65b670e
CP
457def isModeExec(mode):
458 # Returns True if the given git mode represents an executable file,
459 # otherwise False.
460 return mode[-3:] == "755"
461
462def isModeExecChanged(src_mode, dst_mode):
463 return isModeExec(src_mode) != isModeExec(dst_mode)
464
b932705b 465def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
466
467 if isinstance(cmd,basestring):
468 cmd = "-G " + cmd
469 expand = True
470 else:
471 cmd = ["-G"] + cmd
472 expand = False
473
474 cmd = p4_build_cmd(cmd)
6a49f8e2 475 if verbose:
6de040df 476 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
477
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
481 stdin_file = None
482 if stdin is not None:
483 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
484 if isinstance(stdin,basestring):
485 stdin_file.write(stdin)
486 else:
487 for i in stdin:
488 stdin_file.write(i + '\n')
9f90c733
SL
489 stdin_file.flush()
490 stdin_file.seek(0)
491
6de040df
LD
492 p4 = subprocess.Popen(cmd,
493 shell=expand,
9f90c733
SL
494 stdin=stdin_file,
495 stdout=subprocess.PIPE)
86949eef
SH
496
497 result = []
498 try:
499 while True:
9f90c733 500 entry = marshal.load(p4.stdout)
c3f6163b
AG
501 if cb is not None:
502 cb(entry)
503 else:
504 result.append(entry)
86949eef
SH
505 except EOFError:
506 pass
9f90c733
SL
507 exitCode = p4.wait()
508 if exitCode != 0:
ac3e0d79
SH
509 entry = {}
510 entry["p4ExitCode"] = exitCode
511 result.append(entry)
86949eef
SH
512
513 return result
514
515def p4Cmd(cmd):
516 list = p4CmdList(cmd)
517 result = {}
518 for entry in list:
519 result.update(entry)
520 return result;
521
cb2c9db5
SH
522def p4Where(depotPath):
523 if not depotPath.endswith("/"):
524 depotPath += "/"
cd884106
VA
525 depotPathLong = depotPath + "..."
526 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
527 output = None
528 for entry in outputList:
75bc9573 529 if "depotFile" in entry:
cd884106
VA
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
533 output = entry
534 break
535 elif "data" in entry:
536 data = entry.get("data")
537 space = data.find(" ")
538 if data[:space] == depotPath:
539 output = entry
540 break
7f705dc3
TAL
541 if output == None:
542 return ""
dc524036
SH
543 if output["code"] == "error":
544 return ""
cb2c9db5
SH
545 clientPath = ""
546 if "path" in output:
547 clientPath = output.get("path")
548 elif "data" in output:
549 data = output.get("data")
550 lastSpace = data.rfind(" ")
551 clientPath = data[lastSpace + 1:]
552
553 if clientPath.endswith("..."):
554 clientPath = clientPath[:-3]
555 return clientPath
556
86949eef 557def currentGitBranch():
00ad6e31
LD
558 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
559 if retcode != 0:
560 # on a detached head
561 return None
562 else:
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
86949eef 564
4f5cf76a 565def isValidGitDir(path):
bb6e09b2
HWN
566 if (os.path.exists(path + "/HEAD")
567 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
568 return True;
569 return False
570
463e8af6 571def parseRevision(ref):
b25b2065 572 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 573
28755dba
PW
574def branchExists(ref):
575 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
576 ignore_error=True)
577 return len(rev) > 0
578
6ae8de88
SH
579def extractLogMessageFromGitCommit(commit):
580 logMessage = ""
b016d397
HWN
581
582 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 583 foundTitle = False
b016d397 584 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
585 if not foundTitle:
586 if len(log) == 1:
1c094184 587 foundTitle = True
6ae8de88
SH
588 continue
589
590 logMessage += log
591 return logMessage
592
bb6e09b2 593def extractSettingsGitLog(log):
6ae8de88
SH
594 values = {}
595 for line in log.split("\n"):
596 line = line.strip()
6326aa58
HWN
597 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
598 if not m:
599 continue
600
601 assignments = m.group(1).split (':')
602 for a in assignments:
603 vals = a.split ('=')
604 key = vals[0].strip()
605 val = ('='.join (vals[1:])).strip()
606 if val.endswith ('\"') and val.startswith('"'):
607 val = val[1:-1]
608
609 values[key] = val
610
845b42cb
SH
611 paths = values.get("depot-paths")
612 if not paths:
613 paths = values.get("depot-path")
a3fdd579
SH
614 if paths:
615 values['depot-paths'] = paths.split(',')
bb6e09b2 616 return values
6ae8de88 617
8136a639 618def gitBranchExists(branch):
bb6e09b2
HWN
619 proc = subprocess.Popen(["git", "rev-parse", branch],
620 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 621 return proc.wait() == 0;
8136a639 622
36bd8446 623_gitConfig = {}
b345d6c3 624
692e1796 625def gitConfig(key, typeSpecifier=None):
36bd8446 626 if not _gitConfig.has_key(key):
692e1796
LS
627 cmd = [ "git", "config" ]
628 if typeSpecifier:
629 cmd += [ typeSpecifier ]
630 cmd += [ key ]
b345d6c3
PW
631 s = read_pipe(cmd, ignore_error=True)
632 _gitConfig[key] = s.strip()
36bd8446 633 return _gitConfig[key]
01265103 634
0d609032
PW
635def gitConfigBool(key):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
638 in the config."""
639
36bd8446 640 if not _gitConfig.has_key(key):
692e1796 641 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 642 return _gitConfig[key]
01265103 643
cb1dafdf
LS
644def gitConfigInt(key):
645 if not _gitConfig.has_key(key):
646 cmd = [ "git", "config", "--int", key ]
0d609032
PW
647 s = read_pipe(cmd, ignore_error=True)
648 v = s.strip()
cb1dafdf
LS
649 try:
650 _gitConfig[key] = int(gitConfig(key, '--int'))
651 except ValueError:
652 _gitConfig[key] = None
36bd8446 653 return _gitConfig[key]
01265103 654
7199cf13
VA
655def gitConfigList(key):
656 if not _gitConfig.has_key(key):
2abba301
PW
657 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
658 _gitConfig[key] = s.strip().split(os.linesep)
7960e707
LS
659 if _gitConfig[key] == ['']:
660 _gitConfig[key] = []
7199cf13
VA
661 return _gitConfig[key]
662
2c8037ed
PW
663def p4BranchesInGit(branchesAreInRemotes=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
668 "p4/" prefix."""
669
062410bb
SH
670 branches = {}
671
672 cmdline = "git rev-parse --symbolic "
673 if branchesAreInRemotes:
2c8037ed 674 cmdline += "--remotes"
062410bb 675 else:
2c8037ed 676 cmdline += "--branches"
062410bb
SH
677
678 for line in read_pipe_lines(cmdline):
679 line = line.strip()
680
2c8037ed
PW
681 # only import to p4/
682 if not line.startswith('p4/'):
683 continue
684 # special symbolic ref to p4/master
685 if line == "p4/HEAD":
062410bb 686 continue
062410bb 687
2c8037ed
PW
688 # strip off p4/ prefix
689 branch = line[len("p4/"):]
062410bb
SH
690
691 branches[branch] = parseRevision(line)
2c8037ed 692
062410bb
SH
693 return branches
694
5a8e84cd
PW
695def branch_exists(branch):
696 """Make sure that the given ref name really exists."""
697
698 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
699 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
700 out, _ = p.communicate()
701 if p.returncode:
702 return False
703 # expect exactly one line of output: the branch name
704 return out.rstrip() == branch
705
9ceab363 706def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
707 branches = p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath = {}
710 for branch in branches.keys():
711 tip = branches[branch]
712 log = extractLogMessageFromGitCommit(tip)
713 settings = extractSettingsGitLog(log)
714 if settings.has_key("depot-paths"):
715 paths = ",".join(settings["depot-paths"])
716 branchByDepotPath[paths] = "remotes/p4/" + branch
717
27d2d811 718 settings = None
27d2d811
SH
719 parent = 0
720 while parent < 65535:
9ceab363 721 commit = head + "~%s" % parent
27d2d811
SH
722 log = extractLogMessageFromGitCommit(commit)
723 settings = extractSettingsGitLog(log)
86506fe5
SH
724 if settings.has_key("depot-paths"):
725 paths = ",".join(settings["depot-paths"])
726 if branchByDepotPath.has_key(paths):
727 return [branchByDepotPath[paths], settings]
27d2d811 728
86506fe5 729 parent = parent + 1
27d2d811 730
86506fe5 731 return ["", settings]
27d2d811 732
5ca44617
SH
733def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
734 if not silent:
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
736 % localRefPrefix)
737
738 originPrefix = "origin/p4/"
739
740 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
741 line = line.strip()
742 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
743 continue
744
745 headName = line[len(originPrefix):]
746 remoteHead = localRefPrefix + headName
747 originHead = line
748
749 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
750 if (not original.has_key('depot-paths')
751 or not original.has_key('change')):
752 continue
753
754 update = False
755 if not gitBranchExists(remoteHead):
756 if verbose:
757 print "creating %s" % remoteHead
758 update = True
759 else:
760 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
761 if settings.has_key('change') > 0:
762 if settings['depot-paths'] == original['depot-paths']:
763 originP4Change = int(original['change'])
764 p4Change = int(settings['change'])
765 if originP4Change > p4Change:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead, originP4Change,
769 remoteHead, p4Change))
770 update = True
771 else:
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead, ','.join(original['depot-paths']),
775 remoteHead, ','.join(settings['depot-paths'])))
776
777 if update:
778 system("git update-ref %s %s" % (remoteHead, originHead))
779
780def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
782
1051ef00
LD
783
784def p4ParseNumericChangeRange(parts):
785 changeStart = int(parts[0][1:])
786 if parts[1] == '#head':
787 changeEnd = p4_last_change()
788 else:
789 changeEnd = int(parts[1])
790
791 return (changeStart, changeEnd)
792
793def chooseBlockSize(blockSize):
794 if blockSize:
795 return blockSize
796 else:
797 return defaultBlockSize
798
799def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 800 assert depotPaths
96b2d54a 801
1051ef00
LD
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
807
96b2d54a 808 if changeRange is None or changeRange == '':
1051ef00
LD
809 changeStart = 1
810 changeEnd = p4_last_change()
811 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
812 else:
813 parts = changeRange.split(',')
814 assert len(parts) == 2
1051ef00
LD
815 try:
816 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
817 block_size = chooseBlockSize(requestedBlockSize)
818 except:
819 changeStart = parts[0][1:]
820 changeEnd = parts[1]
821 if requestedBlockSize:
822 die("cannot use --changes-block-size with non-numeric revisions")
823 block_size = None
4f6432d8 824
96b2d54a 825 # Accumulate change numbers in a dictionary to avoid duplicates
b4b0ba06 826 changes = {}
96b2d54a
LS
827
828 for p in depotPaths:
829 # Retrieve changes a block at a time, to prevent running
1051ef00
LD
830 # into a MaxResults/MaxScanRows error from the server.
831
832 while True:
96b2d54a 833 cmd = ['changes']
1051ef00
LD
834
835 if block_size:
836 end = min(changeEnd, changeStart + block_size)
837 revisionRange = "%d,%d" % (changeStart, end)
838 else:
839 revisionRange = "%s,%s" % (changeStart, changeEnd)
840
841 cmd += ["%s...@%s" % (p, revisionRange)]
842
96b2d54a
LS
843 for line in p4_read_pipe_lines(cmd):
844 changeNum = int(line.split(" ")[1])
96b2d54a 845 changes[changeNum] = True
1051ef00
LD
846
847 if not block_size:
848 break
849
850 if end >= changeEnd:
851 break
852
853 changeStart = end + 1
4f6432d8 854
b4b0ba06
PW
855 changelist = changes.keys()
856 changelist.sort()
857 return changelist
4f6432d8 858
d53de8b9
TAL
859def p4PathStartsWith(path, prefix):
860 # This method tries to remedy a potential mixed-case issue:
861 #
862 # If UserA adds //depot/DirA/file1
863 # and UserB adds //depot/dira/file2
864 #
865 # we may or may not have a problem. If you have core.ignorecase=true,
866 # we treat DirA and dira as the same directory
0d609032 867 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
868 return path.lower().startswith(prefix.lower())
869 return path.startswith(prefix)
870
543987bd
PW
871def getClientSpec():
872 """Look at the p4 client spec, create a View() object that contains
873 all the mappings, and return it."""
874
875 specList = p4CmdList("client -o")
876 if len(specList) != 1:
877 die('Output from "client -o" is %d lines, expecting 1' %
878 len(specList))
879
880 # dictionary of all client parameters
881 entry = specList[0]
882
9d57c4a6
KS
883 # the //client/ name
884 client_name = entry["Client"]
885
543987bd
PW
886 # just the keys that start with "View"
887 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
888
889 # hold this new View
9d57c4a6 890 view = View(client_name)
543987bd
PW
891
892 # append the lines, in order, to the view
893 for view_num in range(len(view_keys)):
894 k = "View%d" % view_num
895 if k not in view_keys:
896 die("Expected view key %s missing" % k)
897 view.append(entry[k])
898
899 return view
900
901def getClientRoot():
902 """Grab the client directory."""
903
904 output = p4CmdList("client -o")
905 if len(output) != 1:
906 die('Output from "client -o" is %d lines, expecting 1' % len(output))
907
908 entry = output[0]
909 if "Root" not in entry:
910 die('Client has no "Root"')
911
912 return entry["Root"]
913
9d7d446a
PW
914#
915# P4 wildcards are not allowed in filenames. P4 complains
916# if you simply add them, but you can force it with "-f", in
917# which case it translates them into %xx encoding internally.
918#
919def wildcard_decode(path):
920 # Search for and fix just these four characters. Do % last so
921 # that fixing it does not inadvertently create new %-escapes.
922 # Cannot have * in a filename in windows; untested as to
923 # what p4 would do in such a case.
924 if not platform.system() == "Windows":
925 path = path.replace("%2A", "*")
926 path = path.replace("%23", "#") \
927 .replace("%40", "@") \
928 .replace("%25", "%")
929 return path
930
931def wildcard_encode(path):
932 # do % first to avoid double-encoding the %s introduced here
933 path = path.replace("%", "%25") \
934 .replace("*", "%2A") \
935 .replace("#", "%23") \
936 .replace("@", "%40")
937 return path
938
939def wildcard_present(path):
598354c0
BC
940 m = re.search("[*#@%]", path)
941 return m is not None
9d7d446a 942
a5db4b12
LS
943class LargeFileSystem(object):
944 """Base class for large file system support."""
945
946 def __init__(self, writeToGitStream):
947 self.largeFiles = set()
948 self.writeToGitStream = writeToGitStream
949
950 def generatePointer(self, cloneDestination, contentFile):
951 """Return the content of a pointer file that is stored in Git instead of
952 the actual content."""
953 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
954
955 def pushFile(self, localLargeFile):
956 """Push the actual content which is not stored in the Git repository to
957 a server."""
958 assert False, "Method 'pushFile' required in " + self.__class__.__name__
959
960 def hasLargeFileExtension(self, relPath):
961 return reduce(
962 lambda a, b: a or b,
963 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
964 False
965 )
966
967 def generateTempFile(self, contents):
968 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
969 for d in contents:
970 contentFile.write(d)
971 contentFile.close()
972 return contentFile.name
973
974 def exceedsLargeFileThreshold(self, relPath, contents):
975 if gitConfigInt('git-p4.largeFileThreshold'):
976 contentsSize = sum(len(d) for d in contents)
977 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
978 return True
979 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
980 contentsSize = sum(len(d) for d in contents)
981 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
982 return False
983 contentTempFile = self.generateTempFile(contents)
984 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
985 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
986 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
987 zf.close()
988 compressedContentsSize = zf.infolist()[0].compress_size
989 os.remove(contentTempFile)
990 os.remove(compressedContentFile.name)
991 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
992 return True
993 return False
994
995 def addLargeFile(self, relPath):
996 self.largeFiles.add(relPath)
997
998 def removeLargeFile(self, relPath):
999 self.largeFiles.remove(relPath)
1000
1001 def isLargeFile(self, relPath):
1002 return relPath in self.largeFiles
1003
1004 def processContent(self, git_mode, relPath, contents):
1005 """Processes the content of git fast import. This method decides if a
1006 file is stored in the large file system and handles all necessary
1007 steps."""
1008 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1009 contentTempFile = self.generateTempFile(contents)
1010 (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1011
1012 # Move temp file to final location in large file system
1013 largeFileDir = os.path.dirname(localLargeFile)
1014 if not os.path.isdir(largeFileDir):
1015 os.makedirs(largeFileDir)
1016 shutil.move(contentTempFile, localLargeFile)
1017 self.addLargeFile(relPath)
1018 if gitConfigBool('git-p4.largeFilePush'):
1019 self.pushFile(localLargeFile)
1020 if verbose:
1021 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1022 return (git_mode, contents)
1023
1024class MockLFS(LargeFileSystem):
1025 """Mock large file system for testing."""
1026
1027 def generatePointer(self, contentFile):
1028 """The pointer content is the original content prefixed with "pointer-".
1029 The local filename of the large file storage is derived from the file content.
1030 """
1031 with open(contentFile, 'r') as f:
1032 content = next(f)
1033 gitMode = '100644'
1034 pointerContents = 'pointer-' + content
1035 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1036 return (gitMode, pointerContents, localLargeFile)
1037
1038 def pushFile(self, localLargeFile):
1039 """The remote filename of the large file storage is the same as the local
1040 one but in a different directory.
1041 """
1042 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1043 if not os.path.exists(remotePath):
1044 os.makedirs(remotePath)
1045 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1046
b47d807d
LS
1047class GitLFS(LargeFileSystem):
1048 """Git LFS as backend for the git-p4 large file system.
1049 See https://git-lfs.github.com/ for details."""
1050
1051 def __init__(self, *args):
1052 LargeFileSystem.__init__(self, *args)
1053 self.baseGitAttributes = []
1054
1055 def generatePointer(self, contentFile):
1056 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1057 mode and content which is stored in the Git repository instead of
1058 the actual content. Return also the new location of the actual
1059 content.
1060 """
1061 pointerProcess = subprocess.Popen(
1062 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1063 stdout=subprocess.PIPE
1064 )
1065 pointerFile = pointerProcess.stdout.read()
1066 if pointerProcess.wait():
1067 os.remove(contentFile)
1068 die('git-lfs pointer command failed. Did you install the extension?')
1069 pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
1070 oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
1071 localLargeFile = os.path.join(
1072 os.getcwd(),
1073 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1074 oid,
1075 )
1076 # LFS Spec states that pointer files should not have the executable bit set.
1077 gitMode = '100644'
1078 return (gitMode, pointerContents, localLargeFile)
1079
1080 def pushFile(self, localLargeFile):
1081 uploadProcess = subprocess.Popen(
1082 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1083 )
1084 if uploadProcess.wait():
1085 die('git-lfs push command failed. Did you define a remote?')
1086
1087 def generateGitAttributes(self):
1088 return (
1089 self.baseGitAttributes +
1090 [
1091 '\n',
1092 '#\n',
1093 '# Git LFS (see https://git-lfs.github.com/)\n',
1094 '#\n',
1095 ] +
1096 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1097 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1098 ] +
1099 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1100 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1101 ]
1102 )
1103
1104 def addLargeFile(self, relPath):
1105 LargeFileSystem.addLargeFile(self, relPath)
1106 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1107
1108 def removeLargeFile(self, relPath):
1109 LargeFileSystem.removeLargeFile(self, relPath)
1110 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1111
1112 def processContent(self, git_mode, relPath, contents):
1113 if relPath == '.gitattributes':
1114 self.baseGitAttributes = contents
1115 return (git_mode, self.generateGitAttributes())
1116 else:
1117 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1118
b984733c
SH
1119class Command:
1120 def __init__(self):
1121 self.usage = "usage: %prog [options]"
8910ac0e 1122 self.needsGit = True
6a10b6aa 1123 self.verbose = False
b984733c 1124
3ea2cfd4
LD
1125class P4UserMap:
1126 def __init__(self):
1127 self.userMapFromPerforceServer = False
affb474f
LD
1128 self.myP4UserId = None
1129
1130 def p4UserId(self):
1131 if self.myP4UserId:
1132 return self.myP4UserId
1133
1134 results = p4CmdList("user -o")
1135 for r in results:
1136 if r.has_key('User'):
1137 self.myP4UserId = r['User']
1138 return r['User']
1139 die("Could not find your p4 user id")
1140
1141 def p4UserIsMe(self, p4User):
1142 # return True if the given p4 user is actually me
1143 me = self.p4UserId()
1144 if not p4User or p4User != me:
1145 return False
1146 else:
1147 return True
3ea2cfd4
LD
1148
1149 def getUserCacheFilename(self):
1150 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1151 return home + "/.gitp4-usercache.txt"
1152
1153 def getUserMapFromPerforceServer(self):
1154 if self.userMapFromPerforceServer:
1155 return
1156 self.users = {}
1157 self.emails = {}
1158
1159 for output in p4CmdList("users"):
1160 if not output.has_key("User"):
1161 continue
1162 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1163 self.emails[output["Email"]] = output["User"]
1164
1165
1166 s = ''
1167 for (key, val) in self.users.items():
1168 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1169
1170 open(self.getUserCacheFilename(), "wb").write(s)
1171 self.userMapFromPerforceServer = True
1172
1173 def loadUserMapFromCache(self):
1174 self.users = {}
1175 self.userMapFromPerforceServer = False
1176 try:
1177 cache = open(self.getUserCacheFilename(), "rb")
1178 lines = cache.readlines()
1179 cache.close()
1180 for line in lines:
1181 entry = line.strip().split("\t")
1182 self.users[entry[0]] = entry[1]
1183 except IOError:
1184 self.getUserMapFromPerforceServer()
1185
b984733c 1186class P4Debug(Command):
86949eef 1187 def __init__(self):
6ae8de88 1188 Command.__init__(self)
6a10b6aa 1189 self.options = []
c8c39116 1190 self.description = "A tool to debug the output of p4 -G."
8910ac0e 1191 self.needsGit = False
86949eef
SH
1192
1193 def run(self, args):
b1ce9447 1194 j = 0
6de040df 1195 for output in p4CmdList(args):
b1ce9447
HWN
1196 print 'Element: %d' % j
1197 j += 1
86949eef 1198 print output
b984733c 1199 return True
86949eef 1200
5834684d
SH
1201class P4RollBack(Command):
1202 def __init__(self):
1203 Command.__init__(self)
1204 self.options = [
0c66a783 1205 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1206 ]
1207 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1208 self.rollbackLocalBranches = False
5834684d
SH
1209
1210 def run(self, args):
1211 if len(args) != 1:
1212 return False
1213 maxChange = int(args[0])
0c66a783 1214
ad192f28 1215 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1216 die("Problems executing p4");
1217
0c66a783
SH
1218 if self.rollbackLocalBranches:
1219 refPrefix = "refs/heads/"
b016d397 1220 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1221 else:
1222 refPrefix = "refs/remotes/"
b016d397 1223 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1224
1225 for line in lines:
1226 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1227 line = line.strip()
1228 ref = refPrefix + line
5834684d 1229 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1230 settings = extractSettingsGitLog(log)
1231
1232 depotPaths = settings['depot-paths']
1233 change = settings['change']
1234
5834684d 1235 changed = False
52102d47 1236
6326aa58
HWN
1237 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1238 for p in depotPaths]))) == 0:
52102d47
SH
1239 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1240 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1241 continue
1242
bb6e09b2 1243 while change and int(change) > maxChange:
5834684d 1244 changed = True
52102d47
SH
1245 if self.verbose:
1246 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1247 system("git update-ref %s \"%s^\"" % (ref, ref))
1248 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1249 settings = extractSettingsGitLog(log)
1250
1251
1252 depotPaths = settings['depot-paths']
1253 change = settings['change']
5834684d
SH
1254
1255 if changed:
52102d47 1256 print "%s rewound to %s" % (ref, change)
5834684d
SH
1257
1258 return True
1259
3ea2cfd4 1260class P4Submit(Command, P4UserMap):
6bbfd137
PW
1261
1262 conflict_behavior_choices = ("ask", "skip", "quit")
1263
4f5cf76a 1264 def __init__(self):
b984733c 1265 Command.__init__(self)
3ea2cfd4 1266 P4UserMap.__init__(self)
4f5cf76a 1267 self.options = [
4f5cf76a 1268 optparse.make_option("--origin", dest="origin"),
ae901090 1269 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1270 # preserve the user, requires relevant p4 permissions
1271 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1272 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1273 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1274 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1275 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1276 choices=self.conflict_behavior_choices),
1277 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
1278 ]
1279 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1280 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1281 self.origin = ""
ae901090 1282 self.detectRenames = False
0d609032 1283 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1284 self.dry_run = False
728b7ad8 1285 self.prepare_p4_only = False
6bbfd137 1286 self.conflict_behavior = None
f7baba8b 1287 self.isWindows = (platform.system() == "Windows")
06804c76 1288 self.exportLabels = False
249da4c0 1289 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1290 self.branch = None
4f5cf76a 1291
a5db4b12
LS
1292 if gitConfig('git-p4.largeFileSystem'):
1293 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1294
4f5cf76a
SH
1295 def check(self):
1296 if len(p4CmdList("opened ...")) > 0:
1297 die("You have files opened with perforce! Close them before starting the sync.")
1298
f19cb0a0
PW
1299 def separate_jobs_from_description(self, message):
1300 """Extract and return a possible Jobs field in the commit
1301 message. It goes into a separate section in the p4 change
1302 specification.
1303
1304 A jobs line starts with "Jobs:" and looks like a new field
1305 in a form. Values are white-space separated on the same
1306 line or on following lines that start with a tab.
1307
1308 This does not parse and extract the full git commit message
1309 like a p4 form. It just sees the Jobs: line as a marker
1310 to pass everything from then on directly into the p4 form,
1311 but outside the description section.
1312
1313 Return a tuple (stripped log message, jobs string)."""
1314
1315 m = re.search(r'^Jobs:', message, re.MULTILINE)
1316 if m is None:
1317 return (message, None)
1318
1319 jobtext = message[m.start():]
1320 stripped_message = message[:m.start()].rstrip()
1321 return (stripped_message, jobtext)
1322
1323 def prepareLogMessage(self, template, message, jobs):
1324 """Edits the template returned from "p4 change -o" to insert
1325 the message in the Description field, and the jobs text in
1326 the Jobs field."""
4f5cf76a
SH
1327 result = ""
1328
edae1e2f
SH
1329 inDescriptionSection = False
1330
4f5cf76a
SH
1331 for line in template.split("\n"):
1332 if line.startswith("#"):
1333 result += line + "\n"
1334 continue
1335
edae1e2f 1336 if inDescriptionSection:
c9dbab04 1337 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1338 inDescriptionSection = False
f19cb0a0
PW
1339 # insert Jobs section
1340 if jobs:
1341 result += jobs + "\n"
edae1e2f
SH
1342 else:
1343 continue
1344 else:
1345 if line.startswith("Description:"):
1346 inDescriptionSection = True
1347 line += "\n"
1348 for messageLine in message.split("\n"):
1349 line += "\t" + messageLine + "\n"
1350
1351 result += line + "\n"
4f5cf76a
SH
1352
1353 return result
1354
60df071c
LD
1355 def patchRCSKeywords(self, file, pattern):
1356 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1357 (handle, outFileName) = tempfile.mkstemp(dir='.')
1358 try:
1359 outFile = os.fdopen(handle, "w+")
1360 inFile = open(file, "r")
1361 regexp = re.compile(pattern, re.VERBOSE)
1362 for line in inFile.readlines():
1363 line = regexp.sub(r'$\1$', line)
1364 outFile.write(line)
1365 inFile.close()
1366 outFile.close()
1367 # Forcibly overwrite the original file
1368 os.unlink(file)
1369 shutil.move(outFileName, file)
1370 except:
1371 # cleanup our temporary file
1372 os.unlink(outFileName)
1373 print "Failed to strip RCS keywords in %s" % file
1374 raise
1375
1376 print "Patched up RCS keywords in %s" % file
1377
3ea2cfd4
LD
1378 def p4UserForCommit(self,id):
1379 # Return the tuple (perforce user,git email) for a given git commit id
1380 self.getUserMapFromPerforceServer()
9bf28855
PW
1381 gitEmail = read_pipe(["git", "log", "--max-count=1",
1382 "--format=%ae", id])
3ea2cfd4
LD
1383 gitEmail = gitEmail.strip()
1384 if not self.emails.has_key(gitEmail):
1385 return (None,gitEmail)
1386 else:
1387 return (self.emails[gitEmail],gitEmail)
1388
1389 def checkValidP4Users(self,commits):
1390 # check if any git authors cannot be mapped to p4 users
1391 for id in commits:
1392 (user,email) = self.p4UserForCommit(id)
1393 if not user:
1394 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1395 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1396 print "%s" % msg
1397 else:
1398 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1399
1400 def lastP4Changelist(self):
1401 # Get back the last changelist number submitted in this client spec. This
1402 # then gets used to patch up the username in the change. If the same
1403 # client spec is being used by multiple processes then this might go
1404 # wrong.
1405 results = p4CmdList("client -o") # find the current client
1406 client = None
1407 for r in results:
1408 if r.has_key('Client'):
1409 client = r['Client']
1410 break
1411 if not client:
1412 die("could not get client spec")
6de040df 1413 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1414 for r in results:
1415 if r.has_key('change'):
1416 return r['change']
1417 die("Could not get changelist number for last submit - cannot patch up user details")
1418
1419 def modifyChangelistUser(self, changelist, newUser):
1420 # fixup the user field of a changelist after it has been submitted.
1421 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1422 if len(changes) != 1:
1423 die("Bad output from p4 change modifying %s to user %s" %
1424 (changelist, newUser))
1425
1426 c = changes[0]
1427 if c['User'] == newUser: return # nothing to do
1428 c['User'] = newUser
1429 input = marshal.dumps(c)
1430
3ea2cfd4
LD
1431 result = p4CmdList("change -f -i", stdin=input)
1432 for r in result:
1433 if r.has_key('code'):
1434 if r['code'] == 'error':
1435 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1436 if r.has_key('data'):
1437 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1438 return
1439 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1440
1441 def canChangeChangelists(self):
1442 # check to see if we have p4 admin or super-user permissions, either of
1443 # which are required to modify changelists.
52a4880b 1444 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1445 for r in results:
1446 if r.has_key('perm'):
1447 if r['perm'] == 'admin':
1448 return 1
1449 if r['perm'] == 'super':
1450 return 1
1451 return 0
1452
ea99c3ae 1453 def prepareSubmitTemplate(self):
f19cb0a0
PW
1454 """Run "p4 change -o" to grab a change specification template.
1455 This does not use "p4 -G", as it is nice to keep the submission
1456 template in original order, since a human might edit it.
1457
1458 Remove lines in the Files section that show changes to files
1459 outside the depot path we're committing into."""
1460
cbc69242
SH
1461 [upstream, settings] = findUpstreamBranchPoint()
1462
ea99c3ae
SH
1463 template = ""
1464 inFilesSection = False
6de040df 1465 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1466 if line.endswith("\r\n"):
1467 line = line[:-2] + "\n"
ea99c3ae
SH
1468 if inFilesSection:
1469 if line.startswith("\t"):
1470 # path starts and ends with a tab
1471 path = line[1:]
1472 lastTab = path.rfind("\t")
1473 if lastTab != -1:
1474 path = path[:lastTab]
cbc69242
SH
1475 if settings.has_key('depot-paths'):
1476 if not [p for p in settings['depot-paths']
1477 if p4PathStartsWith(path, p)]:
1478 continue
1479 else:
1480 if not p4PathStartsWith(path, self.depotPath):
1481 continue
ea99c3ae
SH
1482 else:
1483 inFilesSection = False
1484 else:
1485 if line.startswith("Files:"):
1486 inFilesSection = True
1487
1488 template += line
1489
1490 return template
1491
7c766e57
PW
1492 def edit_template(self, template_file):
1493 """Invoke the editor to let the user change the submission
1494 message. Return true if okay to continue with the submit."""
1495
1496 # if configured to skip the editing part, just submit
0d609032 1497 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1498 return True
1499
1500 # look at the modification time, to check later if the user saved
1501 # the file
1502 mtime = os.stat(template_file).st_mtime
1503
1504 # invoke the editor
f95ceaf0 1505 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1506 editor = os.environ.get("P4EDITOR")
1507 else:
1508 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1509 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1510
1511 # If the file was not saved, prompt to see if this patch should
1512 # be skipped. But skip this verification step if configured so.
0d609032 1513 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1514 return True
1515
d1652049
PW
1516 # modification time updated means user saved the file
1517 if os.stat(template_file).st_mtime > mtime:
1518 return True
1519
1520 while True:
1521 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1522 if response == 'y':
1523 return True
1524 if response == 'n':
1525 return False
7c766e57 1526
e2a892ee 1527 def get_diff_description(self, editedFiles, filesToAdd):
b4073bb3
MC
1528 # diff
1529 if os.environ.has_key("P4DIFF"):
1530 del(os.environ["P4DIFF"])
1531 diff = ""
1532 for editedFile in editedFiles:
1533 diff += p4_read_pipe(['diff', '-du',
1534 wildcard_encode(editedFile)])
1535
1536 # new file diff
1537 newdiff = ""
1538 for newFile in filesToAdd:
1539 newdiff += "==== new file ====\n"
1540 newdiff += "--- /dev/null\n"
1541 newdiff += "+++ %s\n" % newFile
1542 f = open(newFile, "r")
1543 for line in f.readlines():
1544 newdiff += "+" + line
1545 f.close()
1546
e2a892ee 1547 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1548
7cb5cbef 1549 def applyCommit(self, id):
67b0fe2e
PW
1550 """Apply one commit, return True if it succeeded."""
1551
1552 print "Applying", read_pipe(["git", "show", "-s",
1553 "--format=format:%h %s", id])
ae901090 1554
848de9c3 1555 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1556
84cb0003 1557 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1558 filesToAdd = set()
1559 filesToDelete = set()
d336c158 1560 editedFiles = set()
b6ad6dcc 1561 pureRenameCopy = set()
c65b670e 1562 filesToChangeExecBit = {}
60df071c 1563
4f5cf76a 1564 for line in diff:
b43b0a3c
CP
1565 diff = parseDiffTreeEntry(line)
1566 modifier = diff['status']
1567 path = diff['src']
4f5cf76a 1568 if modifier == "M":
6de040df 1569 p4_edit(path)
c65b670e
CP
1570 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1571 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1572 editedFiles.add(path)
4f5cf76a
SH
1573 elif modifier == "A":
1574 filesToAdd.add(path)
c65b670e 1575 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1576 if path in filesToDelete:
1577 filesToDelete.remove(path)
1578 elif modifier == "D":
1579 filesToDelete.add(path)
1580 if path in filesToAdd:
1581 filesToAdd.remove(path)
4fddb41b
VA
1582 elif modifier == "C":
1583 src, dest = diff['src'], diff['dst']
6de040df 1584 p4_integrate(src, dest)
b6ad6dcc 1585 pureRenameCopy.add(dest)
4fddb41b 1586 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1587 p4_edit(dest)
b6ad6dcc 1588 pureRenameCopy.discard(dest)
4fddb41b 1589 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1590 p4_edit(dest)
b6ad6dcc 1591 pureRenameCopy.discard(dest)
4fddb41b 1592 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1593 if self.isWindows:
1594 # turn off read-only attribute
1595 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1596 os.unlink(dest)
1597 editedFiles.add(dest)
d9a5f25b 1598 elif modifier == "R":
b43b0a3c 1599 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1600 if self.p4HasMoveCommand:
1601 p4_edit(src) # src must be open before move
1602 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1603 else:
8e9497c2
GG
1604 p4_integrate(src, dest)
1605 if diff['src_sha1'] != diff['dst_sha1']:
1606 p4_edit(dest)
1607 else:
1608 pureRenameCopy.add(dest)
c65b670e 1609 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1610 if not self.p4HasMoveCommand:
1611 p4_edit(dest) # with move: already open, writable
c65b670e 1612 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1613 if not self.p4HasMoveCommand:
d20f0f8e
PW
1614 if self.isWindows:
1615 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1616 os.unlink(dest)
1617 filesToDelete.add(src)
d9a5f25b 1618 editedFiles.add(dest)
4f5cf76a
SH
1619 else:
1620 die("unknown modifier %s for %s" % (modifier, path))
1621
749b668c 1622 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1623 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1624 tryPatchCmd = patchcmd + "--check -"
1625 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1626 patch_succeeded = True
51a2640a 1627
47a130b7 1628 if os.system(tryPatchCmd) != 0:
60df071c
LD
1629 fixed_rcs_keywords = False
1630 patch_succeeded = False
51a2640a 1631 print "Unfortunately applying the change failed!"
60df071c
LD
1632
1633 # Patch failed, maybe it's just RCS keyword woes. Look through
1634 # the patch to see if that's possible.
0d609032 1635 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1636 file = None
1637 pattern = None
1638 kwfiles = {}
1639 for file in editedFiles | filesToDelete:
1640 # did this file's delta contain RCS keywords?
1641 pattern = p4_keywords_regexp_for_file(file)
1642
1643 if pattern:
1644 # this file is a possibility...look for RCS keywords.
1645 regexp = re.compile(pattern, re.VERBOSE)
1646 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1647 if regexp.search(line):
1648 if verbose:
1649 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1650 kwfiles[file] = pattern
1651 break
1652
1653 for file in kwfiles:
1654 if verbose:
1655 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1656 # File is being deleted, so not open in p4. Must
1657 # disable the read-only bit on windows.
1658 if self.isWindows and file not in editedFiles:
1659 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1660 self.patchRCSKeywords(file, kwfiles[file])
1661 fixed_rcs_keywords = True
1662
1663 if fixed_rcs_keywords:
1664 print "Retrying the patch with RCS keywords cleaned up"
1665 if os.system(tryPatchCmd) == 0:
1666 patch_succeeded = True
1667
1668 if not patch_succeeded:
7e5dd9f2
PW
1669 for f in editedFiles:
1670 p4_revert(f)
7e5dd9f2 1671 return False
51a2640a 1672
55ac2ed6
PW
1673 #
1674 # Apply the patch for real, and do add/delete/+x handling.
1675 #
47a130b7 1676 system(applyPatchCmd)
4f5cf76a
SH
1677
1678 for f in filesToAdd:
6de040df 1679 p4_add(f)
4f5cf76a 1680 for f in filesToDelete:
6de040df
LD
1681 p4_revert(f)
1682 p4_delete(f)
4f5cf76a 1683
c65b670e
CP
1684 # Set/clear executable bits
1685 for f in filesToChangeExecBit.keys():
1686 mode = filesToChangeExecBit[f]
1687 setP4ExecBit(f, mode)
1688
55ac2ed6
PW
1689 #
1690 # Build p4 change description, starting with the contents
1691 # of the git commit message.
1692 #
0e36f2d7 1693 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1694 logMessage = logMessage.strip()
f19cb0a0 1695 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1696
ea99c3ae 1697 template = self.prepareSubmitTemplate()
f19cb0a0 1698 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1699
c47178d4 1700 if self.preserveUser:
55ac2ed6 1701 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1702
55ac2ed6
PW
1703 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1704 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1705 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1706 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1707
55ac2ed6 1708 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1709 if not self.prepare_p4_only:
1710 submitTemplate += separatorLine
e2a892ee 1711 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
55ac2ed6 1712
c47178d4 1713 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1714 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1715 if self.isWindows:
1716 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1717 tmpFile.write(submitTemplate)
c47178d4
PW
1718 tmpFile.close()
1719
728b7ad8
PW
1720 if self.prepare_p4_only:
1721 #
1722 # Leave the p4 tree prepared, and the submit template around
1723 # and let the user decide what to do next
1724 #
1725 print
1726 print "P4 workspace prepared for submission."
1727 print "To submit or revert, go to client workspace"
1728 print " " + self.clientPath
1729 print
1730 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1731 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1732 " \"git p4\"." % fileName
1733 print "You can delete the file \"%s\" when finished." % fileName
1734
1735 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1736 print "To preserve change ownership by user %s, you must\n" \
1737 "do \"p4 change -f <change>\" after submitting and\n" \
1738 "edit the User field."
1739 if pureRenameCopy:
1740 print "After submitting, renamed files must be re-synced."
1741 print "Invoke \"p4 sync -f\" on each of these files:"
1742 for f in pureRenameCopy:
1743 print " " + f
1744
1745 print
1746 print "To revert the changes, use \"p4 revert ...\", and delete"
1747 print "the submit template file \"%s\"" % fileName
1748 if filesToAdd:
1749 print "Since the commit adds new files, they must be deleted:"
1750 for f in filesToAdd:
1751 print " " + f
1752 print
1753 return True
1754
55ac2ed6
PW
1755 #
1756 # Let the user edit the change description, then submit it.
1757 #
b7638fed 1758 submitted = False
cdc7e388 1759
b7638fed
GE
1760 try:
1761 if self.edit_template(fileName):
1762 # read the edited message and submit
1763 tmpFile = open(fileName, "rb")
1764 message = tmpFile.read()
1765 tmpFile.close()
1766 if self.isWindows:
1767 message = message.replace("\r\n", "\n")
1768 submitTemplate = message[:message.index(separatorLine)]
1769 p4_write_pipe(['submit', '-i'], submitTemplate)
1770
1771 if self.preserveUser:
1772 if p4User:
1773 # Get last changelist number. Cannot easily get it from
1774 # the submit command output as the output is
1775 # unmarshalled.
1776 changelist = self.lastP4Changelist()
1777 self.modifyChangelistUser(changelist, p4User)
1778
1779 # The rename/copy happened by applying a patch that created a
1780 # new file. This leaves it writable, which confuses p4.
1781 for f in pureRenameCopy:
1782 p4_sync(f, "-f")
1783 submitted = True
1784
1785 finally:
c47178d4 1786 # skip this patch
b7638fed
GE
1787 if not submitted:
1788 print "Submission cancelled, undoing p4 changes."
1789 for f in editedFiles:
1790 p4_revert(f)
1791 for f in filesToAdd:
1792 p4_revert(f)
1793 os.remove(f)
1794 for f in filesToDelete:
1795 p4_revert(f)
c47178d4
PW
1796
1797 os.remove(fileName)
b7638fed 1798 return submitted
4f5cf76a 1799
06804c76
LD
1800 # Export git tags as p4 labels. Create a p4 label and then tag
1801 # with that.
1802 def exportGitTags(self, gitTags):
c8942a22
LD
1803 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1804 if len(validLabelRegexp) == 0:
1805 validLabelRegexp = defaultLabelRegexp
1806 m = re.compile(validLabelRegexp)
06804c76
LD
1807
1808 for name in gitTags:
1809
1810 if not m.match(name):
1811 if verbose:
05a3cec5 1812 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1813 continue
1814
1815 # Get the p4 commit this corresponds to
c8942a22
LD
1816 logMessage = extractLogMessageFromGitCommit(name)
1817 values = extractSettingsGitLog(logMessage)
06804c76 1818
c8942a22 1819 if not values.has_key('change'):
06804c76
LD
1820 # a tag pointing to something not sent to p4; ignore
1821 if verbose:
1822 print "git tag %s does not give a p4 commit" % name
1823 continue
c8942a22
LD
1824 else:
1825 changelist = values['change']
06804c76
LD
1826
1827 # Get the tag details.
1828 inHeader = True
1829 isAnnotated = False
1830 body = []
1831 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1832 l = l.strip()
1833 if inHeader:
1834 if re.match(r'tag\s+', l):
1835 isAnnotated = True
1836 elif re.match(r'\s*$', l):
1837 inHeader = False
1838 continue
1839 else:
1840 body.append(l)
1841
1842 if not isAnnotated:
1843 body = ["lightweight tag imported by git p4\n"]
1844
1845 # Create the label - use the same view as the client spec we are using
1846 clientSpec = getClientSpec()
1847
1848 labelTemplate = "Label: %s\n" % name
1849 labelTemplate += "Description:\n"
1850 for b in body:
1851 labelTemplate += "\t" + b + "\n"
1852 labelTemplate += "View:\n"
9d57c4a6
KS
1853 for depot_side in clientSpec.mappings:
1854 labelTemplate += "\t%s\n" % depot_side
06804c76 1855
ef739f08
PW
1856 if self.dry_run:
1857 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1858 elif self.prepare_p4_only:
1859 print "Not creating p4 label %s for tag due to option" \
1860 " --prepare-p4-only" % name
ef739f08
PW
1861 else:
1862 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1863
ef739f08
PW
1864 # Use the label
1865 p4_system(["tag", "-l", name] +
9d57c4a6 1866 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1867
ef739f08
PW
1868 if verbose:
1869 print "created p4 label for tag %s" % name
06804c76 1870
4f5cf76a 1871 def run(self, args):
c9b50e63
SH
1872 if len(args) == 0:
1873 self.master = currentGitBranch()
c9b50e63
SH
1874 elif len(args) == 1:
1875 self.master = args[0]
28755dba
PW
1876 if not branchExists(self.master):
1877 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1878 else:
1879 return False
1880
00ad6e31
LD
1881 if self.master:
1882 allowSubmit = gitConfig("git-p4.allowSubmit")
1883 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1884 die("%s is not in git-p4.allowSubmit" % self.master)
4c2d5d72 1885
27d2d811 1886 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1887 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1888 if len(self.origin) == 0:
1889 self.origin = upstream
a3fdd579 1890
3ea2cfd4
LD
1891 if self.preserveUser:
1892 if not self.canChangeChangelists():
1893 die("Cannot preserve user names without p4 super-user or admin permissions")
1894
6bbfd137
PW
1895 # if not set from the command line, try the config file
1896 if self.conflict_behavior is None:
1897 val = gitConfig("git-p4.conflict")
1898 if val:
1899 if val not in self.conflict_behavior_choices:
1900 die("Invalid value '%s' for config git-p4.conflict" % val)
1901 else:
1902 val = "ask"
1903 self.conflict_behavior = val
1904
a3fdd579
SH
1905 if self.verbose:
1906 print "Origin branch is " + self.origin
9512497b 1907
ea99c3ae 1908 if len(self.depotPath) == 0:
9512497b
SH
1909 print "Internal error: cannot locate perforce depot path from existing branches"
1910 sys.exit(128)
1911
543987bd 1912 self.useClientSpec = False
0d609032 1913 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1914 self.useClientSpec = True
1915 if self.useClientSpec:
1916 self.clientSpecDirs = getClientSpec()
9512497b 1917
cd884106
VA
1918 # Check for the existance of P4 branches
1919 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1920
1921 if self.useClientSpec and not branchesDetected:
543987bd
PW
1922 # all files are relative to the client spec
1923 self.clientPath = getClientRoot()
1924 else:
1925 self.clientPath = p4Where(self.depotPath)
9512497b 1926
543987bd
PW
1927 if self.clientPath == "":
1928 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1929
ea99c3ae 1930 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1931 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1932
0591cfa8 1933 # ensure the clientPath exists
8d7ec362 1934 new_client_dir = False
0591cfa8 1935 if not os.path.exists(self.clientPath):
8d7ec362 1936 new_client_dir = True
0591cfa8
GG
1937 os.makedirs(self.clientPath)
1938
bbd84863 1939 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1940 if self.dry_run:
1941 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1942 else:
ef739f08
PW
1943 print "Synchronizing p4 checkout..."
1944 if new_client_dir:
1945 # old one was destroyed, and maybe nobody told p4
1946 p4_sync("...", "-f")
1947 else:
1948 p4_sync("...")
4f5cf76a 1949 self.check()
4f5cf76a 1950
4c750c0d 1951 commits = []
00ad6e31
LD
1952 if self.master:
1953 commitish = self.master
1954 else:
1955 commitish = 'HEAD'
1956
1957 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
4c750c0d
SH
1958 commits.append(line.strip())
1959 commits.reverse()
4f5cf76a 1960
0d609032 1961 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1962 self.checkAuthorship = False
1963 else:
1964 self.checkAuthorship = True
1965
3ea2cfd4
LD
1966 if self.preserveUser:
1967 self.checkValidP4Users(commits)
1968
84cb0003
GG
1969 #
1970 # Build up a set of options to be passed to diff when
1971 # submitting each commit to p4.
1972 #
1973 if self.detectRenames:
1974 # command-line -M arg
1975 self.diffOpts = "-M"
1976 else:
1977 # If not explicitly set check the config variable
1978 detectRenames = gitConfig("git-p4.detectRenames")
1979
1980 if detectRenames.lower() == "false" or detectRenames == "":
1981 self.diffOpts = ""
1982 elif detectRenames.lower() == "true":
1983 self.diffOpts = "-M"
1984 else:
1985 self.diffOpts = "-M%s" % detectRenames
1986
1987 # no command-line arg for -C or --find-copies-harder, just
1988 # config variables
1989 detectCopies = gitConfig("git-p4.detectCopies")
1990 if detectCopies.lower() == "false" or detectCopies == "":
1991 pass
1992 elif detectCopies.lower() == "true":
1993 self.diffOpts += " -C"
1994 else:
1995 self.diffOpts += " -C%s" % detectCopies
1996
0d609032 1997 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1998 self.diffOpts += " --find-copies-harder"
1999
7e5dd9f2
PW
2000 #
2001 # Apply the commits, one at a time. On failure, ask if should
2002 # continue to try the rest of the patches, or quit.
2003 #
ef739f08
PW
2004 if self.dry_run:
2005 print "Would apply"
67b0fe2e 2006 applied = []
7e5dd9f2
PW
2007 last = len(commits) - 1
2008 for i, commit in enumerate(commits):
ef739f08
PW
2009 if self.dry_run:
2010 print " ", read_pipe(["git", "show", "-s",
2011 "--format=format:%h %s", commit])
2012 ok = True
2013 else:
2014 ok = self.applyCommit(commit)
67b0fe2e
PW
2015 if ok:
2016 applied.append(commit)
7e5dd9f2 2017 else:
728b7ad8
PW
2018 if self.prepare_p4_only and i < last:
2019 print "Processing only the first commit due to option" \
2020 " --prepare-p4-only"
2021 break
7e5dd9f2
PW
2022 if i < last:
2023 quit = False
2024 while True:
6bbfd137
PW
2025 # prompt for what to do, or use the option/variable
2026 if self.conflict_behavior == "ask":
2027 print "What do you want to do?"
2028 response = raw_input("[s]kip this commit but apply"
2029 " the rest, or [q]uit? ")
2030 if not response:
2031 continue
2032 elif self.conflict_behavior == "skip":
2033 response = "s"
2034 elif self.conflict_behavior == "quit":
2035 response = "q"
2036 else:
2037 die("Unknown conflict_behavior '%s'" %
2038 self.conflict_behavior)
2039
7e5dd9f2
PW
2040 if response[0] == "s":
2041 print "Skipping this commit, but applying the rest"
2042 break
2043 if response[0] == "q":
2044 print "Quitting"
2045 quit = True
2046 break
2047 if quit:
2048 break
4f5cf76a 2049
67b0fe2e 2050 chdir(self.oldWorkingDirectory)
4f5cf76a 2051
ef739f08
PW
2052 if self.dry_run:
2053 pass
728b7ad8
PW
2054 elif self.prepare_p4_only:
2055 pass
ef739f08 2056 elif len(commits) == len(applied):
67b0fe2e 2057 print "All commits applied!"
14594f4b 2058
4c750c0d 2059 sync = P4Sync()
44e8d26c
PW
2060 if self.branch:
2061 sync.branch = self.branch
4c750c0d 2062 sync.run([])
14594f4b 2063
4c750c0d
SH
2064 rebase = P4Rebase()
2065 rebase.rebase()
4f5cf76a 2066
67b0fe2e
PW
2067 else:
2068 if len(applied) == 0:
2069 print "No commits applied."
2070 else:
2071 print "Applied only the commits marked with '*':"
2072 for c in commits:
2073 if c in applied:
2074 star = "*"
2075 else:
2076 star = " "
2077 print star, read_pipe(["git", "show", "-s",
2078 "--format=format:%h %s", c])
2079 print "You will have to do 'git p4 sync' and rebase."
2080
0d609032 2081 if gitConfigBool("git-p4.exportLabels"):
06dcd152 2082 self.exportLabels = True
06804c76
LD
2083
2084 if self.exportLabels:
2085 p4Labels = getP4Labels(self.depotPath)
2086 gitTags = getGitTags()
2087
2088 missingGitTags = gitTags - p4Labels
2089 self.exportGitTags(missingGitTags)
2090
98e023de 2091 # exit with error unless everything applied perfectly
67b0fe2e
PW
2092 if len(commits) != len(applied):
2093 sys.exit(1)
2094
b984733c
SH
2095 return True
2096
ecb7cf98
PW
2097class View(object):
2098 """Represent a p4 view ("p4 help views"), and map files in a
2099 repo according to the view."""
2100
9d57c4a6 2101 def __init__(self, client_name):
ecb7cf98 2102 self.mappings = []
9d57c4a6
KS
2103 self.client_prefix = "//%s/" % client_name
2104 # cache results of "p4 where" to lookup client file locations
2105 self.client_spec_path_cache = {}
ecb7cf98
PW
2106
2107 def append(self, view_line):
2108 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
2109 sides. Append to self.mappings, preserving order. This
2110 is only needed for tag creation."""
ecb7cf98
PW
2111
2112 # Split the view line into exactly two words. P4 enforces
2113 # structure on these lines that simplifies this quite a bit.
2114 #
2115 # Either or both words may be double-quoted.
2116 # Single quotes do not matter.
2117 # Double-quote marks cannot occur inside the words.
2118 # A + or - prefix is also inside the quotes.
2119 # There are no quotes unless they contain a space.
2120 # The line is already white-space stripped.
2121 # The two words are separated by a single space.
2122 #
2123 if view_line[0] == '"':
2124 # First word is double quoted. Find its end.
2125 close_quote_index = view_line.find('"', 1)
2126 if close_quote_index <= 0:
2127 die("No first-word closing quote found: %s" % view_line)
2128 depot_side = view_line[1:close_quote_index]
2129 # skip closing quote and space
2130 rhs_index = close_quote_index + 1 + 1
2131 else:
2132 space_index = view_line.find(" ")
2133 if space_index <= 0:
2134 die("No word-splitting space found: %s" % view_line)
2135 depot_side = view_line[0:space_index]
2136 rhs_index = space_index + 1
2137
ecb7cf98 2138 # prefix + means overlay on previous mapping
ecb7cf98 2139 if depot_side.startswith("+"):
ecb7cf98
PW
2140 depot_side = depot_side[1:]
2141
9d57c4a6 2142 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
2143 exclude = False
2144 if depot_side.startswith("-"):
2145 exclude = True
2146 depot_side = depot_side[1:]
2147
9d57c4a6
KS
2148 if not exclude:
2149 self.mappings.append(depot_side)
ecb7cf98 2150
9d57c4a6
KS
2151 def convert_client_path(self, clientFile):
2152 # chop off //client/ part to make it relative
2153 if not clientFile.startswith(self.client_prefix):
2154 die("No prefix '%s' on clientFile '%s'" %
2155 (self.client_prefix, clientFile))
2156 return clientFile[len(self.client_prefix):]
ecb7cf98 2157
9d57c4a6
KS
2158 def update_client_spec_path_cache(self, files):
2159 """ Caching file paths by "p4 where" batch query """
ecb7cf98 2160
9d57c4a6
KS
2161 # List depot file paths exclude that already cached
2162 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 2163
9d57c4a6
KS
2164 if len(fileArgs) == 0:
2165 return # All files in cache
ecb7cf98 2166
9d57c4a6
KS
2167 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2168 for res in where_result:
2169 if "code" in res and res["code"] == "error":
2170 # assume error is "... file(s) not in client view"
2171 continue
2172 if "clientFile" not in res:
20005443 2173 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
2174 if "unmap" in res:
2175 # it will list all of them, but only one not unmap-ped
2176 continue
a0a50d87
LS
2177 if gitConfigBool("core.ignorecase"):
2178 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 2179 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 2180
9d57c4a6
KS
2181 # not found files or unmap files set to ""
2182 for depotFile in fileArgs:
a0a50d87
LS
2183 if gitConfigBool("core.ignorecase"):
2184 depotFile = depotFile.lower()
9d57c4a6
KS
2185 if depotFile not in self.client_spec_path_cache:
2186 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 2187
9d57c4a6
KS
2188 def map_in_client(self, depot_path):
2189 """Return the relative location in the client where this
2190 depot file should live. Returns "" if the file should
2191 not be mapped in the client."""
ecb7cf98 2192
a0a50d87
LS
2193 if gitConfigBool("core.ignorecase"):
2194 depot_path = depot_path.lower()
2195
9d57c4a6
KS
2196 if depot_path in self.client_spec_path_cache:
2197 return self.client_spec_path_cache[depot_path]
2198
2199 die( "Error: %s is not found in client spec path" % depot_path )
2200 return ""
ecb7cf98 2201
3ea2cfd4 2202class P4Sync(Command, P4UserMap):
56c09345
PW
2203 delete_actions = ( "delete", "move/delete", "purge" )
2204
b984733c
SH
2205 def __init__(self):
2206 Command.__init__(self)
3ea2cfd4 2207 P4UserMap.__init__(self)
b984733c
SH
2208 self.options = [
2209 optparse.make_option("--branch", dest="branch"),
2210 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2211 optparse.make_option("--changesfile", dest="changesFile"),
2212 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2213 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2214 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2215 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2216 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2217 optparse.make_option("--max-changes", dest="maxChanges",
2218 help="Maximum number of changes to import"),
2219 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2220 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2221 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2222 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2223 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2224 help="Only sync files that are included in the Perforce Client Spec"),
2225 optparse.make_option("-/", dest="cloneExclude",
2226 action="append", type="string",
2227 help="exclude depot path"),
b984733c
SH
2228 ]
2229 self.description = """Imports from Perforce into a git repository.\n
2230 example:
2231 //depot/my/project/ -- to import the current head
2232 //depot/my/project/@all -- to import everything
2233 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2234
2235 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2236
2237 self.usage += " //depot/path[@revRange]"
b984733c 2238 self.silent = False
1d7367dc
RG
2239 self.createdBranches = set()
2240 self.committedChanges = set()
569d1bd4 2241 self.branch = ""
b984733c 2242 self.detectBranches = False
cb53e1f8 2243 self.detectLabels = False
06804c76 2244 self.importLabels = False
b984733c 2245 self.changesFile = ""
01265103 2246 self.syncWithOrigin = True
a028a98e 2247 self.importIntoRemotes = True
01a9c9c5 2248 self.maxChanges = ""
1051ef00 2249 self.changes_block_size = None
8b41a97f 2250 self.keepRepoPath = False
6326aa58 2251 self.depotPaths = None
3c699645 2252 self.p4BranchesInGit = []
354081d5 2253 self.cloneExclude = []
3a70cdfa 2254 self.useClientSpec = False
a93d33ee 2255 self.useClientSpec_from_options = False
ecb7cf98 2256 self.clientSpecDirs = None
fed23693
VA
2257 self.tempBranches = []
2258 self.tempBranchLocation = "git-p4-tmp"
a5db4b12
LS
2259 self.largeFileSystem = None
2260
2261 if gitConfig('git-p4.largeFileSystem'):
2262 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2263 self.largeFileSystem = largeFileSystemConstructor(
2264 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2265 )
b984733c 2266
01265103
SH
2267 if gitConfig("git-p4.syncFromOrigin") == "false":
2268 self.syncWithOrigin = False
2269
51334bb0
LD
2270 # This is required for the "append" cloneExclude action
2271 def ensure_value(self, attr, value):
2272 if not hasattr(self, attr) or getattr(self, attr) is None:
2273 setattr(self, attr, value)
2274 return getattr(self, attr)
2275
fed23693
VA
2276 # Force a checkpoint in fast-import and wait for it to finish
2277 def checkpoint(self):
2278 self.gitStream.write("checkpoint\n\n")
2279 self.gitStream.write("progress checkpoint\n\n")
2280 out = self.gitOutput.readline()
2281 if self.verbose:
2282 print "checkpoint finished: " + out
2283
b984733c 2284 def extractFilesFromCommit(self, commit):
354081d5
TT
2285 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2286 for path in self.cloneExclude]
b984733c
SH
2287 files = []
2288 fnum = 0
2289 while commit.has_key("depotFile%s" % fnum):
2290 path = commit["depotFile%s" % fnum]
6326aa58 2291
354081d5 2292 if [p for p in self.cloneExclude
d53de8b9 2293 if p4PathStartsWith(path, p)]:
354081d5
TT
2294 found = False
2295 else:
2296 found = [p for p in self.depotPaths
d53de8b9 2297 if p4PathStartsWith(path, p)]
6326aa58 2298 if not found:
b984733c
SH
2299 fnum = fnum + 1
2300 continue
2301
2302 file = {}
2303 file["path"] = path
2304 file["rev"] = commit["rev%s" % fnum]
2305 file["action"] = commit["action%s" % fnum]
2306 file["type"] = commit["type%s" % fnum]
2307 files.append(file)
2308 fnum = fnum + 1
2309 return files
2310
6326aa58 2311 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2312 """When streaming files, this is called to map a p4 depot path
2313 to where it should go in git. The prefixes are either
2314 self.depotPaths, or self.branchPrefixes in the case of
2315 branch detection."""
2316
3952710b 2317 if self.useClientSpec:
21ef5df4
PW
2318 # branch detection moves files up a level (the branch name)
2319 # from what client spec interpretation gives
0d1696ef 2320 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2321 if self.detectBranches:
2322 for b in self.knownBranches:
2323 if path.startswith(b + "/"):
2324 path = path[len(b)+1:]
2325
2326 elif self.keepRepoPath:
2327 # Preserve everything in relative path name except leading
2328 # //depot/; just look at first prefix as they all should
2329 # be in the same depot.
2330 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2331 if p4PathStartsWith(path, depot):
2332 path = path[len(depot):]
3952710b 2333
0d1696ef 2334 else:
0d1696ef
PW
2335 for p in prefixes:
2336 if p4PathStartsWith(path, p):
2337 path = path[len(p):]
21ef5df4 2338 break
8b41a97f 2339
0d1696ef 2340 path = wildcard_decode(path)
6326aa58 2341 return path
6754a299 2342
71b112d4 2343 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2344 """Look at each depotFile in the commit to figure out to what
2345 branch it belongs."""
2346
9d57c4a6
KS
2347 if self.clientSpecDirs:
2348 files = self.extractFilesFromCommit(commit)
2349 self.clientSpecDirs.update_client_spec_path_cache(files)
2350
d5904674 2351 branches = {}
71b112d4
SH
2352 fnum = 0
2353 while commit.has_key("depotFile%s" % fnum):
2354 path = commit["depotFile%s" % fnum]
6326aa58 2355 found = [p for p in self.depotPaths
d53de8b9 2356 if p4PathStartsWith(path, p)]
6326aa58 2357 if not found:
71b112d4
SH
2358 fnum = fnum + 1
2359 continue
2360
2361 file = {}
2362 file["path"] = path
2363 file["rev"] = commit["rev%s" % fnum]
2364 file["action"] = commit["action%s" % fnum]
2365 file["type"] = commit["type%s" % fnum]
2366 fnum = fnum + 1
2367
21ef5df4
PW
2368 # start with the full relative path where this file would
2369 # go in a p4 client
2370 if self.useClientSpec:
2371 relPath = self.clientSpecDirs.map_in_client(path)
2372 else:
2373 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2374
4b97ffb1 2375 for branch in self.knownBranches.keys():
21ef5df4
PW
2376 # add a trailing slash so that a commit into qt/4.2foo
2377 # doesn't end up in qt/4.2, e.g.
6754a299 2378 if relPath.startswith(branch + "/"):
d5904674
SH
2379 if branch not in branches:
2380 branches[branch] = []
71b112d4 2381 branches[branch].append(file)
6555b2cc 2382 break
b984733c
SH
2383
2384 return branches
2385
a5db4b12
LS
2386 def writeToGitStream(self, gitMode, relPath, contents):
2387 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2388 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2389 for d in contents:
2390 self.gitStream.write(d)
2391 self.gitStream.write('\n')
2392
b932705b
LD
2393 # output one file from the P4 stream
2394 # - helper for streamP4Files
2395
2396 def streamOneP4File(self, file, contents):
b932705b
LD
2397 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2398 if verbose:
d2176a50
LS
2399 size = int(self.stream_file['fileSize'])
2400 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2401 sys.stdout.flush()
b932705b 2402
9cffb8c8
PW
2403 (type_base, type_mods) = split_p4_type(file["type"])
2404
2405 git_mode = "100644"
2406 if "x" in type_mods:
2407 git_mode = "100755"
2408 if type_base == "symlink":
2409 git_mode = "120000"
1292df11
AJ
2410 # p4 print on a symlink sometimes contains "target\n";
2411 # if it does, remove the newline
b39c3612 2412 data = ''.join(contents)
40f846c3
PW
2413 if not data:
2414 # Some version of p4 allowed creating a symlink that pointed
2415 # to nothing. This causes p4 errors when checking out such
2416 # a change, and errors here too. Work around it by ignoring
2417 # the bad symlink; hopefully a future change fixes it.
2418 print "\nIgnoring empty symlink in %s" % file['depotFile']
2419 return
2420 elif data[-1] == '\n':
1292df11
AJ
2421 contents = [data[:-1]]
2422 else:
2423 contents = [data]
b932705b 2424
9cffb8c8 2425 if type_base == "utf16":
55aa5714
PW
2426 # p4 delivers different text in the python output to -G
2427 # than it does when using "print -o", or normal p4 client
2428 # operations. utf16 is converted to ascii or utf8, perhaps.
2429 # But ascii text saved as -t utf16 is completely mangled.
2430 # Invoke print -o to get the real contents.
7f0e5962
PW
2431 #
2432 # On windows, the newlines will always be mangled by print, so put
2433 # them back too. This is not needed to the cygwin windows version,
2434 # just the native "NT" type.
2435 #
1f5f3907
LS
2436 try:
2437 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2438 except Exception as e:
2439 if 'Translation of file content failed' in str(e):
2440 type_base = 'binary'
2441 else:
2442 raise e
2443 else:
2444 if p4_version_string().find('/NT') >= 0:
2445 text = text.replace('\r\n', '\n')
2446 contents = [ text ]
55aa5714 2447
9f7ef0ea
PW
2448 if type_base == "apple":
2449 # Apple filetype files will be streamed as a concatenation of
2450 # its appledouble header and the contents. This is useless
2451 # on both macs and non-macs. If using "print -q -o xx", it
2452 # will create "xx" with the data, and "%xx" with the header.
2453 # This is also not very useful.
2454 #
2455 # Ideally, someday, this script can learn how to generate
2456 # appledouble files directly and import those to git, but
2457 # non-mac machines can never find a use for apple filetype.
2458 print "\nIgnoring apple filetype file %s" % file['depotFile']
2459 return
2460
55aa5714
PW
2461 # Note that we do not try to de-mangle keywords on utf16 files,
2462 # even though in theory somebody may want that.
60df071c
LD
2463 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2464 if pattern:
2465 regexp = re.compile(pattern, re.VERBOSE)
2466 text = ''.join(contents)
2467 text = regexp.sub(r'$\1$', text)
2468 contents = [ text ]
b932705b 2469
00a9403a
LS
2470 try:
2471 relPath.decode('ascii')
2472 except:
2473 encoding = 'utf8'
2474 if gitConfig('git-p4.pathEncoding'):
2475 encoding = gitConfig('git-p4.pathEncoding')
4cb870d8 2476 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
00a9403a
LS
2477 if self.verbose:
2478 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
a9e38359 2479
a5db4b12
LS
2480 if self.largeFileSystem:
2481 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
b932705b 2482
a5db4b12 2483 self.writeToGitStream(git_mode, relPath, contents)
b932705b
LD
2484
2485 def streamOneP4Deletion(self, file):
2486 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2487 if verbose:
d2176a50
LS
2488 sys.stdout.write("delete %s\n" % relPath)
2489 sys.stdout.flush()
b932705b
LD
2490 self.gitStream.write("D %s\n" % relPath)
2491
a5db4b12
LS
2492 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2493 self.largeFileSystem.removeLargeFile(relPath)
2494
b932705b
LD
2495 # handle another chunk of streaming data
2496 def streamP4FilesCb(self, marshalled):
2497
78189bea
PW
2498 # catch p4 errors and complain
2499 err = None
2500 if "code" in marshalled:
2501 if marshalled["code"] == "error":
2502 if "data" in marshalled:
2503 err = marshalled["data"].rstrip()
4d25dc44
LS
2504
2505 if not err and 'fileSize' in self.stream_file:
2506 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2507 if required_bytes > 0:
2508 err = 'Not enough space left on %s! Free at least %i MB.' % (
2509 os.getcwd(), required_bytes/1024/1024
2510 )
2511
78189bea
PW
2512 if err:
2513 f = None
2514 if self.stream_have_file_info:
2515 if "depotFile" in self.stream_file:
2516 f = self.stream_file["depotFile"]
2517 # force a failure in fast-import, else an empty
2518 # commit will be made
2519 self.gitStream.write("\n")
2520 self.gitStream.write("die-now\n")
2521 self.gitStream.close()
2522 # ignore errors, but make sure it exits first
2523 self.importProcess.wait()
2524 if f:
2525 die("Error from p4 print for %s: %s" % (f, err))
2526 else:
2527 die("Error from p4 print: %s" % err)
2528
c3f6163b
AG
2529 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2530 # start of a new file - output the old one first
2531 self.streamOneP4File(self.stream_file, self.stream_contents)
2532 self.stream_file = {}
2533 self.stream_contents = []
2534 self.stream_have_file_info = False
b932705b 2535
c3f6163b
AG
2536 # pick up the new file information... for the
2537 # 'data' field we need to append to our array
2538 for k in marshalled.keys():
2539 if k == 'data':
d2176a50
LS
2540 if 'streamContentSize' not in self.stream_file:
2541 self.stream_file['streamContentSize'] = 0
2542 self.stream_file['streamContentSize'] += len(marshalled['data'])
c3f6163b
AG
2543 self.stream_contents.append(marshalled['data'])
2544 else:
2545 self.stream_file[k] = marshalled[k]
b932705b 2546
d2176a50
LS
2547 if (verbose and
2548 'streamContentSize' in self.stream_file and
2549 'fileSize' in self.stream_file and
2550 'depotFile' in self.stream_file):
2551 size = int(self.stream_file["fileSize"])
2552 if size > 0:
2553 progress = 100*self.stream_file['streamContentSize']/size
2554 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2555 sys.stdout.flush()
2556
c3f6163b 2557 self.stream_have_file_info = True
b932705b
LD
2558
2559 # Stream directly from "p4 files" into "git fast-import"
2560 def streamP4Files(self, files):
30b5940b
SH
2561 filesForCommit = []
2562 filesToRead = []
b932705b 2563 filesToDelete = []
30b5940b 2564
3a70cdfa 2565 for f in files:
ecb7cf98
PW
2566 # if using a client spec, only add the files that have
2567 # a path in the client
2568 if self.clientSpecDirs:
2569 if self.clientSpecDirs.map_in_client(f['path']) == "":
2570 continue
3a70cdfa 2571
ecb7cf98
PW
2572 filesForCommit.append(f)
2573 if f['action'] in self.delete_actions:
2574 filesToDelete.append(f)
2575 else:
2576 filesToRead.append(f)
6a49f8e2 2577
b932705b
LD
2578 # deleted files...
2579 for f in filesToDelete:
2580 self.streamOneP4Deletion(f)
1b9a4684 2581
b932705b
LD
2582 if len(filesToRead) > 0:
2583 self.stream_file = {}
2584 self.stream_contents = []
2585 self.stream_have_file_info = False
8ff45f2a 2586
c3f6163b
AG
2587 # curry self argument
2588 def streamP4FilesCbSelf(entry):
2589 self.streamP4FilesCb(entry)
6a49f8e2 2590
6de040df
LD
2591 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2592
2593 p4CmdList(["-x", "-", "print"],
2594 stdin=fileArgs,
2595 cb=streamP4FilesCbSelf)
30b5940b 2596
b932705b
LD
2597 # do the last chunk
2598 if self.stream_file.has_key('depotFile'):
2599 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2600
affb474f
LD
2601 def make_email(self, userid):
2602 if userid in self.users:
2603 return self.users[userid]
2604 else:
2605 return "%s <a@b>" % userid
2606
06804c76 2607 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2608 """ Stream a p4 tag.
2609 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2610 """
2611
06804c76
LD
2612 if verbose:
2613 print "writing tag %s for commit %s" % (labelName, commit)
2614 gitStream.write("tag %s\n" % labelName)
2615 gitStream.write("from %s\n" % commit)
2616
2617 if labelDetails.has_key('Owner'):
2618 owner = labelDetails["Owner"]
2619 else:
2620 owner = None
2621
2622 # Try to use the owner of the p4 label, or failing that,
2623 # the current p4 user id.
2624 if owner:
2625 email = self.make_email(owner)
2626 else:
2627 email = self.make_email(self.p4UserId())
2628 tagger = "%s %s %s" % (email, epoch, self.tz)
2629
2630 gitStream.write("tagger %s\n" % tagger)
2631
2632 print "labelDetails=",labelDetails
2633 if labelDetails.has_key('Description'):
2634 description = labelDetails['Description']
2635 else:
2636 description = 'Label from git p4'
2637
2638 gitStream.write("data %d\n" % len(description))
2639 gitStream.write(description)
2640 gitStream.write("\n")
2641
e63231e5 2642 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2643 epoch = details["time"]
2644 author = details["user"]
2645
4b97ffb1
SH
2646 if self.verbose:
2647 print "commit into %s" % branch
2648
96e07dd2
HWN
2649 # start with reading files; if that fails, we should not
2650 # create a commit.
2651 new_files = []
2652 for f in files:
e63231e5 2653 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2654 new_files.append (f)
2655 else:
afa1dd9a 2656 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2657
9d57c4a6
KS
2658 if self.clientSpecDirs:
2659 self.clientSpecDirs.update_client_spec_path_cache(files)
2660
b984733c 2661 self.gitStream.write("commit %s\n" % branch)
b43702ac 2662 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2663 self.committedChanges.add(int(details["change"]))
2664 committer = ""
b607e71e
SH
2665 if author not in self.users:
2666 self.getUserMapFromPerforceServer()
affb474f 2667 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2668
2669 self.gitStream.write("committer %s\n" % committer)
2670
2671 self.gitStream.write("data <<EOT\n")
2672 self.gitStream.write(details["desc"])
e63231e5
PW
2673 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2674 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2675 if len(details['options']) > 0:
2676 self.gitStream.write(": options = %s" % details['options'])
2677 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2678
2679 if len(parent) > 0:
4b97ffb1
SH
2680 if self.verbose:
2681 print "parent %s" % parent
b984733c
SH
2682 self.gitStream.write("from %s\n" % parent)
2683
b932705b 2684 self.streamP4Files(new_files)
b984733c
SH
2685 self.gitStream.write("\n")
2686
1f4ba1cb
SH
2687 change = int(details["change"])
2688
9bda3a85 2689 if self.labels.has_key(change):
1f4ba1cb
SH
2690 label = self.labels[change]
2691 labelDetails = label[0]
2692 labelRevisions = label[1]
71b112d4
SH
2693 if self.verbose:
2694 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2695
6de040df 2696 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2697 for p in self.branchPrefixes])
1f4ba1cb
SH
2698
2699 if len(files) == len(labelRevisions):
2700
2701 cleanedFiles = {}
2702 for info in files:
56c09345 2703 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2704 continue
2705 cleanedFiles[info["depotFile"]] = info["rev"]
2706
2707 if cleanedFiles == labelRevisions:
06804c76 2708 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2709
2710 else:
a46668fa 2711 if not self.silent:
cebdf5af
HWN
2712 print ("Tag %s does not match with change %s: files do not match."
2713 % (labelDetails["label"], change))
1f4ba1cb
SH
2714
2715 else:
a46668fa 2716 if not self.silent:
cebdf5af
HWN
2717 print ("Tag %s does not match with change %s: file count is different."
2718 % (labelDetails["label"], change))
b984733c 2719
06804c76 2720 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2721 def getLabels(self):
2722 self.labels = {}
2723
52a4880b 2724 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2725 if len(l) > 0 and not self.silent:
183f8436 2726 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2727
2728 for output in l:
1f4ba1cb
SH
2729 label = output["label"]
2730 revisions = {}
2731 newestChange = 0
71b112d4
SH
2732 if self.verbose:
2733 print "Querying files for label %s" % label
6de040df
LD
2734 for file in p4CmdList(["files"] +
2735 ["%s...@%s" % (p, label)
2736 for p in self.depotPaths]):
1f4ba1cb
SH
2737 revisions[file["depotFile"]] = file["rev"]
2738 change = int(file["change"])
2739 if change > newestChange:
2740 newestChange = change
2741
9bda3a85
SH
2742 self.labels[newestChange] = [output, revisions]
2743
2744 if self.verbose:
2745 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2746
06804c76
LD
2747 # Import p4 labels as git tags. A direct mapping does not
2748 # exist, so assume that if all the files are at the same revision
2749 # then we can use that, or it's something more complicated we should
2750 # just ignore.
2751 def importP4Labels(self, stream, p4Labels):
2752 if verbose:
2753 print "import p4 labels: " + ' '.join(p4Labels)
2754
2755 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2756 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2757 if len(validLabelRegexp) == 0:
2758 validLabelRegexp = defaultLabelRegexp
2759 m = re.compile(validLabelRegexp)
2760
2761 for name in p4Labels:
2762 commitFound = False
2763
2764 if not m.match(name):
2765 if verbose:
2766 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2767 continue
2768
2769 if name in ignoredP4Labels:
2770 continue
2771
2772 labelDetails = p4CmdList(['label', "-o", name])[0]
2773
2774 # get the most recent changelist for each file in this label
2775 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2776 for p in self.depotPaths])
2777
2778 if change.has_key('change'):
2779 # find the corresponding git commit; take the oldest commit
2780 changelist = int(change['change'])
b43702ac
LD
2781 if changelist in self.committedChanges:
2782 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2783 commitFound = True
b43702ac
LD
2784 else:
2785 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2786 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2787 if len(gitCommit) == 0:
2788 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2789 else:
2790 commitFound = True
2791 gitCommit = gitCommit.strip()
2792
2793 if commitFound:
06804c76
LD
2794 # Convert from p4 time format
2795 try:
2796 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2797 except ValueError:
a4e9054c 2798 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2799 tmwhen = 1
2800
2801 when = int(time.mktime(tmwhen))
2802 self.streamTag(stream, name, labelDetails, gitCommit, when)
2803 if verbose:
2804 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2805 else:
2806 if verbose:
2807 print "Label %s has no changelists - possibly deleted?" % name
2808
2809 if not commitFound:
2810 # We can't import this label; don't try again as it will get very
2811 # expensive repeatedly fetching all the files for labels that will
2812 # never be imported. If the label is moved in the future, the
2813 # ignore will need to be removed manually.
2814 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2815
86dff6b6
HWN
2816 def guessProjectName(self):
2817 for p in self.depotPaths:
6e5295c4
SH
2818 if p.endswith("/"):
2819 p = p[:-1]
2820 p = p[p.strip().rfind("/") + 1:]
2821 if not p.endswith("/"):
2822 p += "/"
2823 return p
86dff6b6 2824
4b97ffb1 2825 def getBranchMapping(self):
6555b2cc
SH
2826 lostAndFoundBranches = set()
2827
8ace74c0
VA
2828 user = gitConfig("git-p4.branchUser")
2829 if len(user) > 0:
2830 command = "branches -u %s" % user
2831 else:
2832 command = "branches"
2833
2834 for info in p4CmdList(command):
52a4880b 2835 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2836 viewIdx = 0
2837 while details.has_key("View%s" % viewIdx):
2838 paths = details["View%s" % viewIdx].split(" ")
2839 viewIdx = viewIdx + 1
2840 # require standard //depot/foo/... //depot/bar/... mapping
2841 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2842 continue
2843 source = paths[0]
2844 destination = paths[1]
6509e19c 2845 ## HACK
d53de8b9 2846 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2847 source = source[len(self.depotPaths[0]):-4]
2848 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2849
1a2edf4e
SH
2850 if destination in self.knownBranches:
2851 if not self.silent:
2852 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2853 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2854 continue
2855
6555b2cc
SH
2856 self.knownBranches[destination] = source
2857
2858 lostAndFoundBranches.discard(destination)
2859
29bdbac1 2860 if source not in self.knownBranches:
6555b2cc
SH
2861 lostAndFoundBranches.add(source)
2862
7199cf13
VA
2863 # Perforce does not strictly require branches to be defined, so we also
2864 # check git config for a branch list.
2865 #
2866 # Example of branch definition in git config file:
2867 # [git-p4]
2868 # branchList=main:branchA
2869 # branchList=main:branchB
2870 # branchList=branchA:branchC
2871 configBranches = gitConfigList("git-p4.branchList")
2872 for branch in configBranches:
2873 if branch:
2874 (source, destination) = branch.split(":")
2875 self.knownBranches[destination] = source
2876
2877 lostAndFoundBranches.discard(destination)
2878
2879 if source not in self.knownBranches:
2880 lostAndFoundBranches.add(source)
2881
6555b2cc
SH
2882
2883 for branch in lostAndFoundBranches:
2884 self.knownBranches[branch] = branch
29bdbac1 2885
38f9f5ec
SH
2886 def getBranchMappingFromGitBranches(self):
2887 branches = p4BranchesInGit(self.importIntoRemotes)
2888 for branch in branches.keys():
2889 if branch == "master":
2890 branch = "main"
2891 else:
2892 branch = branch[len(self.projectName):]
2893 self.knownBranches[branch] = branch
2894
bb6e09b2
HWN
2895 def updateOptionDict(self, d):
2896 option_keys = {}
2897 if self.keepRepoPath:
2898 option_keys['keepRepoPath'] = 1
2899
2900 d["options"] = ' '.join(sorted(option_keys.keys()))
2901
2902 def readOptions(self, d):
2903 self.keepRepoPath = (d.has_key('options')
2904 and ('keepRepoPath' in d['options']))
6326aa58 2905
8134f69c
SH
2906 def gitRefForBranch(self, branch):
2907 if branch == "main":
2908 return self.refPrefix + "master"
2909
2910 if len(branch) <= 0:
2911 return branch
2912
2913 return self.refPrefix + self.projectName + branch
2914
1ca3d710
SH
2915 def gitCommitByP4Change(self, ref, change):
2916 if self.verbose:
2917 print "looking in ref " + ref + " for change %s using bisect..." % change
2918
2919 earliestCommit = ""
2920 latestCommit = parseRevision(ref)
2921
2922 while True:
2923 if self.verbose:
2924 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2925 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2926 if len(next) == 0:
2927 if self.verbose:
2928 print "argh"
2929 return ""
2930 log = extractLogMessageFromGitCommit(next)
2931 settings = extractSettingsGitLog(log)
2932 currentChange = int(settings['change'])
2933 if self.verbose:
2934 print "current change %s" % currentChange
2935
2936 if currentChange == change:
2937 if self.verbose:
2938 print "found %s" % next
2939 return next
2940
2941 if currentChange < change:
2942 earliestCommit = "^%s" % next
2943 else:
2944 latestCommit = "%s" % next
2945
2946 return ""
2947
2948 def importNewBranch(self, branch, maxChange):
2949 # make fast-import flush all changes to disk and update the refs using the checkpoint
2950 # command so that we can try to find the branch parent in the git history
2951 self.gitStream.write("checkpoint\n\n");
2952 self.gitStream.flush();
2953 branchPrefix = self.depotPaths[0] + branch + "/"
2954 range = "@1,%s" % maxChange
2955 #print "prefix" + branchPrefix
96b2d54a 2956 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
2957 if len(changes) <= 0:
2958 return False
2959 firstChange = changes[0]
2960 #print "first change in branch: %s" % firstChange
2961 sourceBranch = self.knownBranches[branch]
2962 sourceDepotPath = self.depotPaths[0] + sourceBranch
2963 sourceRef = self.gitRefForBranch(sourceBranch)
2964 #print "source " + sourceBranch
2965
52a4880b 2966 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2967 #print "branch parent: %s" % branchParentChange
2968 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2969 if len(gitParent) > 0:
2970 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2971 #print "parent git commit: %s" % gitParent
2972
2973 self.importChanges(changes)
2974 return True
2975
fed23693
VA
2976 def searchParent(self, parent, branch, target):
2977 parentFound = False
c7d34884
PW
2978 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2979 "--no-merges", parent]):
fed23693
VA
2980 blob = blob.strip()
2981 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2982 parentFound = True
2983 if self.verbose:
2984 print "Found parent of %s in commit %s" % (branch, blob)
2985 break
2986 if parentFound:
2987 return blob
2988 else:
2989 return None
2990
e87f37ae
SH
2991 def importChanges(self, changes):
2992 cnt = 1
2993 for change in changes:
18fa13d0 2994 description = p4_describe(change)
e87f37ae
SH
2995 self.updateOptionDict(description)
2996
2997 if not self.silent:
2998 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2999 sys.stdout.flush()
3000 cnt = cnt + 1
3001
3002 try:
3003 if self.detectBranches:
3004 branches = self.splitFilesIntoBranches(description)
3005 for branch in branches.keys():
3006 ## HACK --hwn
3007 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 3008 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
3009
3010 parent = ""
3011
3012 filesForCommit = branches[branch]
3013
3014 if self.verbose:
3015 print "branch is %s" % branch
3016
3017 self.updatedBranches.add(branch)
3018
3019 if branch not in self.createdBranches:
3020 self.createdBranches.add(branch)
3021 parent = self.knownBranches[branch]
3022 if parent == branch:
3023 parent = ""
1ca3d710
SH
3024 else:
3025 fullBranch = self.projectName + branch
3026 if fullBranch not in self.p4BranchesInGit:
3027 if not self.silent:
3028 print("\n Importing new branch %s" % fullBranch);
3029 if self.importNewBranch(branch, change - 1):
3030 parent = ""
3031 self.p4BranchesInGit.append(fullBranch)
3032 if not self.silent:
3033 print("\n Resuming with change %s" % change);
3034
3035 if self.verbose:
3036 print "parent determined through known branches: %s" % parent
e87f37ae 3037
8134f69c
SH
3038 branch = self.gitRefForBranch(branch)
3039 parent = self.gitRefForBranch(parent)
e87f37ae
SH
3040
3041 if self.verbose:
3042 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3043
3044 if len(parent) == 0 and branch in self.initialParents:
3045 parent = self.initialParents[branch]
3046 del self.initialParents[branch]
3047
fed23693
VA
3048 blob = None
3049 if len(parent) > 0:
4f9273d2 3050 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
3051 if self.verbose:
3052 print "Creating temporary branch: " + tempBranch
e63231e5 3053 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
3054 self.tempBranches.append(tempBranch)
3055 self.checkpoint()
3056 blob = self.searchParent(parent, branch, tempBranch)
3057 if blob:
e63231e5 3058 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
3059 else:
3060 if self.verbose:
3061 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 3062 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
3063 else:
3064 files = self.extractFilesFromCommit(description)
e63231e5 3065 self.commit(description, files, self.branch,
e87f37ae 3066 self.initialParent)
47497844 3067 # only needed once, to connect to the previous commit
e87f37ae
SH
3068 self.initialParent = ""
3069 except IOError:
3070 print self.gitError.read()
3071 sys.exit(1)
3072
c208a243
SH
3073 def importHeadRevision(self, revision):
3074 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3075
4e2e6ce4
PW
3076 details = {}
3077 details["user"] = "git perforce import user"
1494fcbb 3078 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
3079 % (' '.join(self.depotPaths), revision))
3080 details["change"] = revision
3081 newestRevision = 0
3082
3083 fileCnt = 0
6de040df
LD
3084 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3085
3086 for info in p4CmdList(["files"] + fileArgs):
c208a243 3087
68b28593 3088 if 'code' in info and info['code'] == 'error':
c208a243
SH
3089 sys.stderr.write("p4 returned an error: %s\n"
3090 % info['data'])
d88e707f
PW
3091 if info['data'].find("must refer to client") >= 0:
3092 sys.stderr.write("This particular p4 error is misleading.\n")
3093 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3094 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 3095 sys.exit(1)
68b28593
PW
3096 if 'p4ExitCode' in info:
3097 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
3098 sys.exit(1)
3099
3100
3101 change = int(info["change"])
3102 if change > newestRevision:
3103 newestRevision = change
3104
56c09345 3105 if info["action"] in self.delete_actions:
c208a243
SH
3106 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3107 #fileCnt = fileCnt + 1
3108 continue
3109
3110 for prop in ["depotFile", "rev", "action", "type" ]:
3111 details["%s%s" % (prop, fileCnt)] = info[prop]
3112
3113 fileCnt = fileCnt + 1
3114
3115 details["change"] = newestRevision
4e2e6ce4 3116
9dcb9f24 3117 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 3118 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
3119 res = p4_describe(newestRevision)
3120 details["time"] = res["time"]
4e2e6ce4 3121
c208a243
SH
3122 self.updateOptionDict(details)
3123 try:
e63231e5 3124 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
3125 except IOError:
3126 print "IO error with git fast-import. Is your git version recent enough?"
3127 print self.gitError.read()
3128
3129
b984733c 3130 def run(self, args):
6326aa58 3131 self.depotPaths = []
179caebf 3132 self.changeRange = ""
6326aa58 3133 self.previousDepotPaths = []
991a2de4 3134 self.hasOrigin = False
ce6f33c8 3135
29bdbac1
SH
3136 # map from branch depot path to parent branch
3137 self.knownBranches = {}
3138 self.initialParents = {}
3139
a028a98e
SH
3140 if self.importIntoRemotes:
3141 self.refPrefix = "refs/remotes/p4/"
3142 else:
db775559 3143 self.refPrefix = "refs/heads/p4/"
a028a98e 3144
991a2de4
PW
3145 if self.syncWithOrigin:
3146 self.hasOrigin = originP4BranchesExist()
3147 if self.hasOrigin:
3148 if not self.silent:
3149 print 'Syncing with origin first, using "git fetch origin"'
3150 system("git fetch origin")
10f880f8 3151
5a8e84cd 3152 branch_arg_given = bool(self.branch)
569d1bd4 3153 if len(self.branch) == 0:
db775559 3154 self.branch = self.refPrefix + "master"
a028a98e 3155 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 3156 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 3157 system("git branch -D p4")
967f72e2 3158
a93d33ee
PW
3159 # accept either the command-line option, or the configuration variable
3160 if self.useClientSpec:
3161 # will use this after clone to set the variable
3162 self.useClientSpec_from_options = True
3163 else:
0d609032 3164 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
3165 self.useClientSpec = True
3166 if self.useClientSpec:
543987bd 3167 self.clientSpecDirs = getClientSpec()
3a70cdfa 3168
6a49f8e2
HWN
3169 # TODO: should always look at previous commits,
3170 # merge with previous imports, if possible.
3171 if args == []:
d414c74a 3172 if self.hasOrigin:
5ca44617 3173 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
3174
3175 # branches holds mapping from branch name to sha1
3176 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
3177
3178 # restrict to just this one, disabling detect-branches
3179 if branch_arg_given:
3180 short = self.branch.split("/")[-1]
3181 if short in branches:
3182 self.p4BranchesInGit = [ short ]
3183 else:
3184 self.p4BranchesInGit = branches.keys()
abcd790f
SH
3185
3186 if len(self.p4BranchesInGit) > 1:
3187 if not self.silent:
3188 print "Importing from/into multiple branches"
3189 self.detectBranches = True
8c9e8b6e
PW
3190 for branch in branches.keys():
3191 self.initialParents[self.refPrefix + branch] = \
3192 branches[branch]
967f72e2 3193
29bdbac1
SH
3194 if self.verbose:
3195 print "branches: %s" % self.p4BranchesInGit
3196
3197 p4Change = 0
3198 for branch in self.p4BranchesInGit:
cebdf5af 3199 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
3200
3201 settings = extractSettingsGitLog(logMsg)
29bdbac1 3202
bb6e09b2
HWN
3203 self.readOptions(settings)
3204 if (settings.has_key('depot-paths')
3205 and settings.has_key ('change')):
3206 change = int(settings['change']) + 1
29bdbac1
SH
3207 p4Change = max(p4Change, change)
3208
bb6e09b2
HWN
3209 depotPaths = sorted(settings['depot-paths'])
3210 if self.previousDepotPaths == []:
6326aa58 3211 self.previousDepotPaths = depotPaths
29bdbac1 3212 else:
6326aa58
HWN
3213 paths = []
3214 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
3215 prev_list = prev.split("/")
3216 cur_list = cur.split("/")
3217 for i in range(0, min(len(cur_list), len(prev_list))):
3218 if cur_list[i] <> prev_list[i]:
583e1707 3219 i = i - 1
6326aa58
HWN
3220 break
3221
04d277b3 3222 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
3223
3224 self.previousDepotPaths = paths
29bdbac1
SH
3225
3226 if p4Change > 0:
bb6e09b2 3227 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 3228 self.changeRange = "@%s,#head" % p4Change
341dc1c1 3229 if not self.silent and not self.detectBranches:
967f72e2 3230 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 3231
40d69ac3
PW
3232 # accept multiple ref name abbreviations:
3233 # refs/foo/bar/branch -> use it exactly
3234 # p4/branch -> prepend refs/remotes/ or refs/heads/
3235 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 3236 if not self.branch.startswith("refs/"):
40d69ac3
PW
3237 if self.importIntoRemotes:
3238 prepend = "refs/remotes/"
3239 else:
3240 prepend = "refs/heads/"
3241 if not self.branch.startswith("p4/"):
3242 prepend += "p4/"
3243 self.branch = prepend + self.branch
179caebf 3244
6326aa58 3245 if len(args) == 0 and self.depotPaths:
b984733c 3246 if not self.silent:
6326aa58 3247 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 3248 else:
6326aa58 3249 if self.depotPaths and self.depotPaths != args:
cebdf5af 3250 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
3251 "This doesn't work!" % (' '.join (self.depotPaths),
3252 ' '.join (args)))
b984733c 3253 sys.exit(1)
6326aa58 3254
bb6e09b2 3255 self.depotPaths = sorted(args)
b984733c 3256
1c49fc19 3257 revision = ""
b984733c 3258 self.users = {}
b984733c 3259
58c8bc7c
PW
3260 # Make sure no revision specifiers are used when --changesfile
3261 # is specified.
3262 bad_changesfile = False
3263 if len(self.changesFile) > 0:
3264 for p in self.depotPaths:
3265 if p.find("@") >= 0 or p.find("#") >= 0:
3266 bad_changesfile = True
3267 break
3268 if bad_changesfile:
3269 die("Option --changesfile is incompatible with revision specifiers")
3270
6326aa58
HWN
3271 newPaths = []
3272 for p in self.depotPaths:
3273 if p.find("@") != -1:
3274 atIdx = p.index("@")
3275 self.changeRange = p[atIdx:]
3276 if self.changeRange == "@all":
3277 self.changeRange = ""
6a49f8e2 3278 elif ',' not in self.changeRange:
1c49fc19 3279 revision = self.changeRange
6326aa58 3280 self.changeRange = ""
7fcff9de 3281 p = p[:atIdx]
6326aa58
HWN
3282 elif p.find("#") != -1:
3283 hashIdx = p.index("#")
1c49fc19 3284 revision = p[hashIdx:]
7fcff9de 3285 p = p[:hashIdx]
6326aa58 3286 elif self.previousDepotPaths == []:
58c8bc7c
PW
3287 # pay attention to changesfile, if given, else import
3288 # the entire p4 tree at the head revision
3289 if len(self.changesFile) == 0:
3290 revision = "#head"
6326aa58
HWN
3291
3292 p = re.sub ("\.\.\.$", "", p)
3293 if not p.endswith("/"):
3294 p += "/"
3295
3296 newPaths.append(p)
3297
3298 self.depotPaths = newPaths
3299
e63231e5
PW
3300 # --detect-branches may change this for each branch
3301 self.branchPrefixes = self.depotPaths
3302
b607e71e 3303 self.loadUserMapFromCache()
cb53e1f8
SH
3304 self.labels = {}
3305 if self.detectLabels:
3306 self.getLabels();
b984733c 3307
4b97ffb1 3308 if self.detectBranches:
df450923
SH
3309 ## FIXME - what's a P4 projectName ?
3310 self.projectName = self.guessProjectName()
3311
38f9f5ec
SH
3312 if self.hasOrigin:
3313 self.getBranchMappingFromGitBranches()
3314 else:
3315 self.getBranchMapping()
29bdbac1
SH
3316 if self.verbose:
3317 print "p4-git branches: %s" % self.p4BranchesInGit
3318 print "initial parents: %s" % self.initialParents
3319 for b in self.p4BranchesInGit:
3320 if b != "master":
6326aa58
HWN
3321
3322 ## FIXME
29bdbac1
SH
3323 b = b[len(self.projectName):]
3324 self.createdBranches.add(b)
4b97ffb1 3325
f291b4e3 3326 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3327
78189bea
PW
3328 self.importProcess = subprocess.Popen(["git", "fast-import"],
3329 stdin=subprocess.PIPE,
3330 stdout=subprocess.PIPE,
3331 stderr=subprocess.PIPE);
3332 self.gitOutput = self.importProcess.stdout
3333 self.gitStream = self.importProcess.stdin
3334 self.gitError = self.importProcess.stderr
b984733c 3335
1c49fc19 3336 if revision:
c208a243 3337 self.importHeadRevision(revision)
b984733c
SH
3338 else:
3339 changes = []
3340
0828ab14 3341 if len(self.changesFile) > 0:
b984733c 3342 output = open(self.changesFile).readlines()
1d7367dc 3343 changeSet = set()
b984733c
SH
3344 for line in output:
3345 changeSet.add(int(line))
3346
3347 for change in changeSet:
3348 changes.append(change)
3349
3350 changes.sort()
3351 else:
9dcb9f24
PW
3352 # catch "git p4 sync" with no new branches, in a repo that
3353 # does not have any existing p4 branches
5a8e84cd
PW
3354 if len(args) == 0:
3355 if not self.p4BranchesInGit:
3356 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3357
3358 # The default branch is master, unless --branch is used to
3359 # specify something else. Make sure it exists, or complain
3360 # nicely about how to use --branch.
3361 if not self.detectBranches:
3362 if not branch_exists(self.branch):
3363 if branch_arg_given:
3364 die("Error: branch %s does not exist." % self.branch)
3365 else:
3366 die("Error: no branch %s; perhaps specify one with --branch." %
3367 self.branch)
3368
29bdbac1 3369 if self.verbose:
86dff6b6 3370 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3371 self.changeRange)
96b2d54a 3372 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
b984733c 3373
01a9c9c5 3374 if len(self.maxChanges) > 0:
7fcff9de 3375 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3376
b984733c 3377 if len(changes) == 0:
0828ab14 3378 if not self.silent:
341dc1c1 3379 print "No changes to import!"
06804c76
LD
3380 else:
3381 if not self.silent and not self.detectBranches:
3382 print "Import destination: %s" % self.branch
3383
3384 self.updatedBranches = set()
b984733c 3385
47497844
PW
3386 if not self.detectBranches:
3387 if args:
3388 # start a new branch
3389 self.initialParent = ""
3390 else:
3391 # build on a previous revision
3392 self.initialParent = parseRevision(self.branch)
3393
06804c76 3394 self.importChanges(changes)
a9d1a27a 3395
06804c76
LD
3396 if not self.silent:
3397 print ""
3398 if len(self.updatedBranches) > 0:
3399 sys.stdout.write("Updated branches: ")
3400 for b in self.updatedBranches:
3401 sys.stdout.write("%s " % b)
3402 sys.stdout.write("\n")
341dc1c1 3403
0d609032 3404 if gitConfigBool("git-p4.importLabels"):
06dcd152 3405 self.importLabels = True
b984733c 3406
06804c76
LD
3407 if self.importLabels:
3408 p4Labels = getP4Labels(self.depotPaths)
3409 gitTags = getGitTags()
3410
3411 missingP4Labels = p4Labels - gitTags
3412 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3413
b984733c 3414 self.gitStream.close()
78189bea 3415 if self.importProcess.wait() != 0:
29bdbac1 3416 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3417 self.gitOutput.close()
3418 self.gitError.close()
3419
fed23693
VA
3420 # Cleanup temporary branches created during import
3421 if self.tempBranches != []:
3422 for branch in self.tempBranches:
3423 read_pipe("git update-ref -d %s" % branch)
3424 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3425
55d12437
PW
3426 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3427 # a convenient shortcut refname "p4".
3428 if self.importIntoRemotes:
3429 head_ref = self.refPrefix + "HEAD"
3430 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3431 system(["git", "symbolic-ref", head_ref, self.branch])
3432
b984733c
SH
3433 return True
3434
01ce1fe9
SH
3435class P4Rebase(Command):
3436 def __init__(self):
3437 Command.__init__(self)
06804c76
LD
3438 self.options = [
3439 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3440 ]
06804c76 3441 self.importLabels = False
cebdf5af
HWN
3442 self.description = ("Fetches the latest revision from perforce and "
3443 + "rebases the current work (branch) against it")
01ce1fe9
SH
3444
3445 def run(self, args):
3446 sync = P4Sync()
06804c76 3447 sync.importLabels = self.importLabels
01ce1fe9 3448 sync.run([])
d7e3868c 3449
14594f4b
SH
3450