git-p4: fix P4 label import for unprocessed commits
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
8b41a97f 25
a235e85c
BC
26try:
27 from subprocess import CalledProcessError
28except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
4addad22 41verbose = False
86949eef 42
06804c76 43# Only labels/tags matching this will be imported/exported
c8942a22 44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 45
1051ef00
LD
46# Grab changes in blocks of this many revisions, unless otherwise requested
47defaultBlockSize = 512
48
21a50753
AK
49def p4_build_cmd(cmd):
50 """Build a suitable p4 command line.
51
52 This consolidates building and returning a p4 command line into one
53 location. It means that hooking into the environment, or other configuration
54 can be done more easily.
55 """
6de040df 56 real_cmd = ["p4"]
abcaf073
AK
57
58 user = gitConfig("git-p4.user")
59 if len(user) > 0:
6de040df 60 real_cmd += ["-u",user]
abcaf073
AK
61
62 password = gitConfig("git-p4.password")
63 if len(password) > 0:
6de040df 64 real_cmd += ["-P", password]
abcaf073
AK
65
66 port = gitConfig("git-p4.port")
67 if len(port) > 0:
6de040df 68 real_cmd += ["-p", port]
abcaf073
AK
69
70 host = gitConfig("git-p4.host")
71 if len(host) > 0:
41799aa2 72 real_cmd += ["-H", host]
abcaf073
AK
73
74 client = gitConfig("git-p4.client")
75 if len(client) > 0:
6de040df 76 real_cmd += ["-c", client]
abcaf073 77
6de040df
LD
78
79 if isinstance(cmd,basestring):
80 real_cmd = ' '.join(real_cmd) + ' ' + cmd
81 else:
82 real_cmd += cmd
21a50753
AK
83 return real_cmd
84
bbd84863
MF
85def chdir(path, is_client_path=False):
86 """Do chdir to the given path, and set the PWD environment
87 variable for use by P4. It does not look at getcwd() output.
88 Since we're not using the shell, it is necessary to set the
89 PWD environment variable explicitly.
90
91 Normally, expand the path to force it to be absolute. This
92 addresses the use of relative path names inside P4 settings,
93 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
94 as given; it looks for .p4config using PWD.
95
96 If is_client_path, the path was handed to us directly by p4,
97 and may be a symbolic link. Do not call os.getcwd() in this
98 case, because it will cause p4 to think that PWD is not inside
99 the client path.
100 """
101
102 os.chdir(path)
103 if not is_client_path:
104 path = os.getcwd()
105 os.environ['PWD'] = path
053fd0c1 106
86dff6b6
HWN
107def die(msg):
108 if verbose:
109 raise Exception(msg)
110 else:
111 sys.stderr.write(msg + "\n")
112 sys.exit(1)
113
6de040df 114def write_pipe(c, stdin):
4addad22 115 if verbose:
6de040df 116 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 117
6de040df
LD
118 expand = isinstance(c,basestring)
119 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
120 pipe = p.stdin
121 val = pipe.write(stdin)
122 pipe.close()
123 if p.wait():
124 die('Command failed: %s' % str(c))
b016d397
HWN
125
126 return val
127
6de040df 128def p4_write_pipe(c, stdin):
d9429194 129 real_cmd = p4_build_cmd(c)
6de040df 130 return write_pipe(real_cmd, stdin)
d9429194 131
4addad22
HWN
132def read_pipe(c, ignore_error=False):
133 if verbose:
6de040df 134 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 135
6de040df
LD
136 expand = isinstance(c,basestring)
137 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
138 pipe = p.stdout
b016d397 139 val = pipe.read()
6de040df
LD
140 if p.wait() and not ignore_error:
141 die('Command failed: %s' % str(c))
b016d397
HWN
142
143 return val
144
d9429194
AK
145def p4_read_pipe(c, ignore_error=False):
146 real_cmd = p4_build_cmd(c)
147 return read_pipe(real_cmd, ignore_error)
b016d397 148
bce4c5fc 149def read_pipe_lines(c):
4addad22 150 if verbose:
6de040df
LD
151 sys.stderr.write('Reading pipe: %s\n' % str(c))
152
153 expand = isinstance(c, basestring)
154 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
155 pipe = p.stdout
b016d397 156 val = pipe.readlines()
6de040df
LD
157 if pipe.close() or p.wait():
158 die('Command failed: %s' % str(c))
b016d397
HWN
159
160 return val
caace111 161
2318121b
AK
162def p4_read_pipe_lines(c):
163 """Specifically invoke p4 on the command supplied. """
155af834 164 real_cmd = p4_build_cmd(c)
2318121b
AK
165 return read_pipe_lines(real_cmd)
166
8e9497c2
GG
167def p4_has_command(cmd):
168 """Ask p4 for help on this command. If it returns an error, the
169 command does not exist in this version of p4."""
170 real_cmd = p4_build_cmd(["help", cmd])
171 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
172 stderr=subprocess.PIPE)
173 p.communicate()
174 return p.returncode == 0
175
249da4c0
PW
176def p4_has_move_command():
177 """See if the move command exists, that it supports -k, and that
178 it has not been administratively disabled. The arguments
179 must be correct, but the filenames do not have to exist. Use
180 ones with wildcards so even if they exist, it will fail."""
181
182 if not p4_has_command("move"):
183 return False
184 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
185 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
186 (out, err) = p.communicate()
187 # return code will be 1 in either case
188 if err.find("Invalid option") >= 0:
189 return False
190 if err.find("disabled") >= 0:
191 return False
192 # assume it failed because @... was invalid changelist
193 return True
194
6754a299 195def system(cmd):
6de040df 196 expand = isinstance(cmd,basestring)
4addad22 197 if verbose:
6de040df 198 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c
BC
199 retcode = subprocess.call(cmd, shell=expand)
200 if retcode:
201 raise CalledProcessError(retcode, cmd)
6754a299 202
bf9320f1
AK
203def p4_system(cmd):
204 """Specifically invoke p4 as the system command. """
155af834 205 real_cmd = p4_build_cmd(cmd)
6de040df 206 expand = isinstance(real_cmd, basestring)
a235e85c
BC
207 retcode = subprocess.call(real_cmd, shell=expand)
208 if retcode:
209 raise CalledProcessError(retcode, real_cmd)
6de040df 210
7f0e5962
PW
211_p4_version_string = None
212def p4_version_string():
213 """Read the version string, showing just the last line, which
214 hopefully is the interesting version bit.
215
216 $ p4 -V
217 Perforce - The Fast Software Configuration Management System.
218 Copyright 1995-2011 Perforce Software. All rights reserved.
219 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
220 """
221 global _p4_version_string
222 if not _p4_version_string:
223 a = p4_read_pipe_lines(["-V"])
224 _p4_version_string = a[-1].rstrip()
225 return _p4_version_string
226
6de040df 227def p4_integrate(src, dest):
9d7d446a 228 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 229
8d7ec362 230def p4_sync(f, *options):
9d7d446a 231 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
232
233def p4_add(f):
9d7d446a
PW
234 # forcibly add file names with wildcards
235 if wildcard_present(f):
236 p4_system(["add", "-f", f])
237 else:
238 p4_system(["add", f])
6de040df
LD
239
240def p4_delete(f):
9d7d446a 241 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
242
243def p4_edit(f):
9d7d446a 244 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
245
246def p4_revert(f):
9d7d446a 247 p4_system(["revert", wildcard_encode(f)])
6de040df 248
9d7d446a
PW
249def p4_reopen(type, f):
250 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 251
8e9497c2
GG
252def p4_move(src, dest):
253 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
254
1051ef00
LD
255def p4_last_change():
256 results = p4CmdList(["changes", "-m", "1"])
257 return int(results[0]['change'])
258
18fa13d0
PW
259def p4_describe(change):
260 """Make sure it returns a valid result by checking for
261 the presence of field "time". Return a dict of the
262 results."""
263
264 ds = p4CmdList(["describe", "-s", str(change)])
265 if len(ds) != 1:
266 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
267
268 d = ds[0]
269
270 if "p4ExitCode" in d:
271 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
272 str(d)))
273 if "code" in d:
274 if d["code"] == "error":
275 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
276
277 if "time" not in d:
278 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
279
280 return d
281
9cffb8c8
PW
282#
283# Canonicalize the p4 type and return a tuple of the
284# base type, plus any modifiers. See "p4 help filetypes"
285# for a list and explanation.
286#
287def split_p4_type(p4type):
288
289 p4_filetypes_historical = {
290 "ctempobj": "binary+Sw",
291 "ctext": "text+C",
292 "cxtext": "text+Cx",
293 "ktext": "text+k",
294 "kxtext": "text+kx",
295 "ltext": "text+F",
296 "tempobj": "binary+FSw",
297 "ubinary": "binary+F",
298 "uresource": "resource+F",
299 "uxbinary": "binary+Fx",
300 "xbinary": "binary+x",
301 "xltext": "text+Fx",
302 "xtempobj": "binary+Swx",
303 "xtext": "text+x",
304 "xunicode": "unicode+x",
305 "xutf16": "utf16+x",
306 }
307 if p4type in p4_filetypes_historical:
308 p4type = p4_filetypes_historical[p4type]
309 mods = ""
310 s = p4type.split("+")
311 base = s[0]
312 mods = ""
313 if len(s) > 1:
314 mods = s[1]
315 return (base, mods)
b9fc6ea9 316
60df071c
LD
317#
318# return the raw p4 type of a file (text, text+ko, etc)
319#
79467e61
PW
320def p4_type(f):
321 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
322 return results[0]['headType']
323
324#
325# Given a type base and modifier, return a regexp matching
326# the keywords that can be expanded in the file
327#
328def p4_keywords_regexp_for_type(base, type_mods):
329 if base in ("text", "unicode", "binary"):
330 kwords = None
331 if "ko" in type_mods:
332 kwords = 'Id|Header'
333 elif "k" in type_mods:
334 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
335 else:
336 return None
337 pattern = r"""
338 \$ # Starts with a dollar, followed by...
339 (%s) # one of the keywords, followed by...
6b2bf41e 340 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
341 \$ # another dollar
342 """ % kwords
343 return pattern
344 else:
345 return None
346
347#
348# Given a file, return a regexp matching the possible
349# RCS keywords that will be expanded, or None for files
350# with kw expansion turned off.
351#
352def p4_keywords_regexp_for_file(file):
353 if not os.path.exists(file):
354 return None
355 else:
356 (type_base, type_mods) = split_p4_type(p4_type(file))
357 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 358
c65b670e
CP
359def setP4ExecBit(file, mode):
360 # Reopens an already open file and changes the execute bit to match
361 # the execute bit setting in the passed in mode.
362
363 p4Type = "+x"
364
365 if not isModeExec(mode):
366 p4Type = getP4OpenedType(file)
367 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
368 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
369 if p4Type[-1] == "+":
370 p4Type = p4Type[0:-1]
371
6de040df 372 p4_reopen(p4Type, file)
c65b670e
CP
373
374def getP4OpenedType(file):
375 # Returns the perforce file type for the given file.
376
9d7d446a 377 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 378 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
379 if match:
380 return match.group(1)
381 else:
f3e5ae4f 382 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 383
06804c76
LD
384# Return the set of all p4 labels
385def getP4Labels(depotPaths):
386 labels = set()
387 if isinstance(depotPaths,basestring):
388 depotPaths = [depotPaths]
389
390 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
391 label = l['label']
392 labels.add(label)
393
394 return labels
395
396# Return the set of all git tags
397def getGitTags():
398 gitTags = set()
399 for line in read_pipe_lines(["git", "tag"]):
400 tag = line.strip()
401 gitTags.add(tag)
402 return gitTags
403
b43b0a3c
CP
404def diffTreePattern():
405 # This is a simple generator for the diff tree regex pattern. This could be
406 # a class variable if this and parseDiffTreeEntry were a part of a class.
407 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
408 while True:
409 yield pattern
410
411def parseDiffTreeEntry(entry):
412 """Parses a single diff tree entry into its component elements.
413
414 See git-diff-tree(1) manpage for details about the format of the diff
415 output. This method returns a dictionary with the following elements:
416
417 src_mode - The mode of the source file
418 dst_mode - The mode of the destination file
419 src_sha1 - The sha1 for the source file
420 dst_sha1 - The sha1 fr the destination file
421 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
422 status_score - The score for the status (applicable for 'C' and 'R'
423 statuses). This is None if there is no score.
424 src - The path for the source file.
425 dst - The path for the destination file. This is only present for
426 copy or renames. If it is not present, this is None.
427
428 If the pattern is not matched, None is returned."""
429
430 match = diffTreePattern().next().match(entry)
431 if match:
432 return {
433 'src_mode': match.group(1),
434 'dst_mode': match.group(2),
435 'src_sha1': match.group(3),
436 'dst_sha1': match.group(4),
437 'status': match.group(5),
438 'status_score': match.group(6),
439 'src': match.group(7),
440 'dst': match.group(10)
441 }
442 return None
443
c65b670e
CP
444def isModeExec(mode):
445 # Returns True if the given git mode represents an executable file,
446 # otherwise False.
447 return mode[-3:] == "755"
448
449def isModeExecChanged(src_mode, dst_mode):
450 return isModeExec(src_mode) != isModeExec(dst_mode)
451
b932705b 452def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
453
454 if isinstance(cmd,basestring):
455 cmd = "-G " + cmd
456 expand = True
457 else:
458 cmd = ["-G"] + cmd
459 expand = False
460
461 cmd = p4_build_cmd(cmd)
6a49f8e2 462 if verbose:
6de040df 463 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
464
465 # Use a temporary file to avoid deadlocks without
466 # subprocess.communicate(), which would put another copy
467 # of stdout into memory.
468 stdin_file = None
469 if stdin is not None:
470 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
471 if isinstance(stdin,basestring):
472 stdin_file.write(stdin)
473 else:
474 for i in stdin:
475 stdin_file.write(i + '\n')
9f90c733
SL
476 stdin_file.flush()
477 stdin_file.seek(0)
478
6de040df
LD
479 p4 = subprocess.Popen(cmd,
480 shell=expand,
9f90c733
SL
481 stdin=stdin_file,
482 stdout=subprocess.PIPE)
86949eef
SH
483
484 result = []
485 try:
486 while True:
9f90c733 487 entry = marshal.load(p4.stdout)
c3f6163b
AG
488 if cb is not None:
489 cb(entry)
490 else:
491 result.append(entry)
86949eef
SH
492 except EOFError:
493 pass
9f90c733
SL
494 exitCode = p4.wait()
495 if exitCode != 0:
ac3e0d79
SH
496 entry = {}
497 entry["p4ExitCode"] = exitCode
498 result.append(entry)
86949eef
SH
499
500 return result
501
502def p4Cmd(cmd):
503 list = p4CmdList(cmd)
504 result = {}
505 for entry in list:
506 result.update(entry)
507 return result;
508
cb2c9db5
SH
509def p4Where(depotPath):
510 if not depotPath.endswith("/"):
511 depotPath += "/"
cd884106
VA
512 depotPathLong = depotPath + "..."
513 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
514 output = None
515 for entry in outputList:
75bc9573 516 if "depotFile" in entry:
cd884106
VA
517 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
518 # The base path always ends with "/...".
519 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
520 output = entry
521 break
522 elif "data" in entry:
523 data = entry.get("data")
524 space = data.find(" ")
525 if data[:space] == depotPath:
526 output = entry
527 break
7f705dc3
TAL
528 if output == None:
529 return ""
dc524036
SH
530 if output["code"] == "error":
531 return ""
cb2c9db5
SH
532 clientPath = ""
533 if "path" in output:
534 clientPath = output.get("path")
535 elif "data" in output:
536 data = output.get("data")
537 lastSpace = data.rfind(" ")
538 clientPath = data[lastSpace + 1:]
539
540 if clientPath.endswith("..."):
541 clientPath = clientPath[:-3]
542 return clientPath
543
86949eef 544def currentGitBranch():
b25b2065 545 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 546
4f5cf76a 547def isValidGitDir(path):
bb6e09b2
HWN
548 if (os.path.exists(path + "/HEAD")
549 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
550 return True;
551 return False
552
463e8af6 553def parseRevision(ref):
b25b2065 554 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 555
28755dba
PW
556def branchExists(ref):
557 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
558 ignore_error=True)
559 return len(rev) > 0
560
6ae8de88
SH
561def extractLogMessageFromGitCommit(commit):
562 logMessage = ""
b016d397
HWN
563
564 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 565 foundTitle = False
b016d397 566 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
567 if not foundTitle:
568 if len(log) == 1:
1c094184 569 foundTitle = True
6ae8de88
SH
570 continue
571
572 logMessage += log
573 return logMessage
574
bb6e09b2 575def extractSettingsGitLog(log):
6ae8de88
SH
576 values = {}
577 for line in log.split("\n"):
578 line = line.strip()
6326aa58
HWN
579 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
580 if not m:
581 continue
582
583 assignments = m.group(1).split (':')
584 for a in assignments:
585 vals = a.split ('=')
586 key = vals[0].strip()
587 val = ('='.join (vals[1:])).strip()
588 if val.endswith ('\"') and val.startswith('"'):
589 val = val[1:-1]
590
591 values[key] = val
592
845b42cb
SH
593 paths = values.get("depot-paths")
594 if not paths:
595 paths = values.get("depot-path")
a3fdd579
SH
596 if paths:
597 values['depot-paths'] = paths.split(',')
bb6e09b2 598 return values
6ae8de88 599
8136a639 600def gitBranchExists(branch):
bb6e09b2
HWN
601 proc = subprocess.Popen(["git", "rev-parse", branch],
602 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 603 return proc.wait() == 0;
8136a639 604
36bd8446 605_gitConfig = {}
b345d6c3 606
0d609032 607def gitConfig(key):
36bd8446 608 if not _gitConfig.has_key(key):
0d609032 609 cmd = [ "git", "config", key ]
b345d6c3
PW
610 s = read_pipe(cmd, ignore_error=True)
611 _gitConfig[key] = s.strip()
36bd8446 612 return _gitConfig[key]
01265103 613
0d609032
PW
614def gitConfigBool(key):
615 """Return a bool, using git config --bool. It is True only if the
616 variable is set to true, and False if set to false or not present
617 in the config."""
618
36bd8446 619 if not _gitConfig.has_key(key):
0d609032
PW
620 cmd = [ "git", "config", "--bool", key ]
621 s = read_pipe(cmd, ignore_error=True)
622 v = s.strip()
623 _gitConfig[key] = v == "true"
36bd8446 624 return _gitConfig[key]
01265103 625
7199cf13
VA
626def gitConfigList(key):
627 if not _gitConfig.has_key(key):
2abba301
PW
628 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
629 _gitConfig[key] = s.strip().split(os.linesep)
7199cf13
VA
630 return _gitConfig[key]
631
2c8037ed
PW
632def p4BranchesInGit(branchesAreInRemotes=True):
633 """Find all the branches whose names start with "p4/", looking
634 in remotes or heads as specified by the argument. Return
635 a dictionary of { branch: revision } for each one found.
636 The branch names are the short names, without any
637 "p4/" prefix."""
638
062410bb
SH
639 branches = {}
640
641 cmdline = "git rev-parse --symbolic "
642 if branchesAreInRemotes:
2c8037ed 643 cmdline += "--remotes"
062410bb 644 else:
2c8037ed 645 cmdline += "--branches"
062410bb
SH
646
647 for line in read_pipe_lines(cmdline):
648 line = line.strip()
649
2c8037ed
PW
650 # only import to p4/
651 if not line.startswith('p4/'):
652 continue
653 # special symbolic ref to p4/master
654 if line == "p4/HEAD":
062410bb 655 continue
062410bb 656
2c8037ed
PW
657 # strip off p4/ prefix
658 branch = line[len("p4/"):]
062410bb
SH
659
660 branches[branch] = parseRevision(line)
2c8037ed 661
062410bb
SH
662 return branches
663
5a8e84cd
PW
664def branch_exists(branch):
665 """Make sure that the given ref name really exists."""
666
667 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
668 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
669 out, _ = p.communicate()
670 if p.returncode:
671 return False
672 # expect exactly one line of output: the branch name
673 return out.rstrip() == branch
674
9ceab363 675def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
676 branches = p4BranchesInGit()
677 # map from depot-path to branch name
678 branchByDepotPath = {}
679 for branch in branches.keys():
680 tip = branches[branch]
681 log = extractLogMessageFromGitCommit(tip)
682 settings = extractSettingsGitLog(log)
683 if settings.has_key("depot-paths"):
684 paths = ",".join(settings["depot-paths"])
685 branchByDepotPath[paths] = "remotes/p4/" + branch
686
27d2d811 687 settings = None
27d2d811
SH
688 parent = 0
689 while parent < 65535:
9ceab363 690 commit = head + "~%s" % parent
27d2d811
SH
691 log = extractLogMessageFromGitCommit(commit)
692 settings = extractSettingsGitLog(log)
86506fe5
SH
693 if settings.has_key("depot-paths"):
694 paths = ",".join(settings["depot-paths"])
695 if branchByDepotPath.has_key(paths):
696 return [branchByDepotPath[paths], settings]
27d2d811 697
86506fe5 698 parent = parent + 1
27d2d811 699
86506fe5 700 return ["", settings]
27d2d811 701
5ca44617
SH
702def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
703 if not silent:
704 print ("Creating/updating branch(es) in %s based on origin branch(es)"
705 % localRefPrefix)
706
707 originPrefix = "origin/p4/"
708
709 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
710 line = line.strip()
711 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
712 continue
713
714 headName = line[len(originPrefix):]
715 remoteHead = localRefPrefix + headName
716 originHead = line
717
718 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
719 if (not original.has_key('depot-paths')
720 or not original.has_key('change')):
721 continue
722
723 update = False
724 if not gitBranchExists(remoteHead):
725 if verbose:
726 print "creating %s" % remoteHead
727 update = True
728 else:
729 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
730 if settings.has_key('change') > 0:
731 if settings['depot-paths'] == original['depot-paths']:
732 originP4Change = int(original['change'])
733 p4Change = int(settings['change'])
734 if originP4Change > p4Change:
735 print ("%s (%s) is newer than %s (%s). "
736 "Updating p4 branch from origin."
737 % (originHead, originP4Change,
738 remoteHead, p4Change))
739 update = True
740 else:
741 print ("Ignoring: %s was imported from %s while "
742 "%s was imported from %s"
743 % (originHead, ','.join(original['depot-paths']),
744 remoteHead, ','.join(settings['depot-paths'])))
745
746 if update:
747 system("git update-ref %s %s" % (remoteHead, originHead))
748
749def originP4BranchesExist():
750 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
751
1051ef00
LD
752
753def p4ParseNumericChangeRange(parts):
754 changeStart = int(parts[0][1:])
755 if parts[1] == '#head':
756 changeEnd = p4_last_change()
757 else:
758 changeEnd = int(parts[1])
759
760 return (changeStart, changeEnd)
761
762def chooseBlockSize(blockSize):
763 if blockSize:
764 return blockSize
765 else:
766 return defaultBlockSize
767
768def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 769 assert depotPaths
96b2d54a 770
1051ef00
LD
771 # Parse the change range into start and end. Try to find integer
772 # revision ranges as these can be broken up into blocks to avoid
773 # hitting server-side limits (maxrows, maxscanresults). But if
774 # that doesn't work, fall back to using the raw revision specifier
775 # strings, without using block mode.
776
96b2d54a 777 if changeRange is None or changeRange == '':
1051ef00
LD
778 changeStart = 1
779 changeEnd = p4_last_change()
780 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
781 else:
782 parts = changeRange.split(',')
783 assert len(parts) == 2
1051ef00
LD
784 try:
785 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
786 block_size = chooseBlockSize(requestedBlockSize)
787 except:
788 changeStart = parts[0][1:]
789 changeEnd = parts[1]
790 if requestedBlockSize:
791 die("cannot use --changes-block-size with non-numeric revisions")
792 block_size = None
4f6432d8 793
96b2d54a 794 # Accumulate change numbers in a dictionary to avoid duplicates
b4b0ba06 795 changes = {}
96b2d54a
LS
796
797 for p in depotPaths:
798 # Retrieve changes a block at a time, to prevent running
1051ef00
LD
799 # into a MaxResults/MaxScanRows error from the server.
800
801 while True:
96b2d54a 802 cmd = ['changes']
1051ef00
LD
803
804 if block_size:
805 end = min(changeEnd, changeStart + block_size)
806 revisionRange = "%d,%d" % (changeStart, end)
807 else:
808 revisionRange = "%s,%s" % (changeStart, changeEnd)
809
810 cmd += ["%s...@%s" % (p, revisionRange)]
811
96b2d54a
LS
812 for line in p4_read_pipe_lines(cmd):
813 changeNum = int(line.split(" ")[1])
96b2d54a 814 changes[changeNum] = True
1051ef00
LD
815
816 if not block_size:
817 break
818
819 if end >= changeEnd:
820 break
821
822 changeStart = end + 1
4f6432d8 823
b4b0ba06
PW
824 changelist = changes.keys()
825 changelist.sort()
826 return changelist
4f6432d8 827
d53de8b9
TAL
828def p4PathStartsWith(path, prefix):
829 # This method tries to remedy a potential mixed-case issue:
830 #
831 # If UserA adds //depot/DirA/file1
832 # and UserB adds //depot/dira/file2
833 #
834 # we may or may not have a problem. If you have core.ignorecase=true,
835 # we treat DirA and dira as the same directory
0d609032 836 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
837 return path.lower().startswith(prefix.lower())
838 return path.startswith(prefix)
839
543987bd
PW
840def getClientSpec():
841 """Look at the p4 client spec, create a View() object that contains
842 all the mappings, and return it."""
843
844 specList = p4CmdList("client -o")
845 if len(specList) != 1:
846 die('Output from "client -o" is %d lines, expecting 1' %
847 len(specList))
848
849 # dictionary of all client parameters
850 entry = specList[0]
851
9d57c4a6
KS
852 # the //client/ name
853 client_name = entry["Client"]
854
543987bd
PW
855 # just the keys that start with "View"
856 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
857
858 # hold this new View
9d57c4a6 859 view = View(client_name)
543987bd
PW
860
861 # append the lines, in order, to the view
862 for view_num in range(len(view_keys)):
863 k = "View%d" % view_num
864 if k not in view_keys:
865 die("Expected view key %s missing" % k)
866 view.append(entry[k])
867
868 return view
869
870def getClientRoot():
871 """Grab the client directory."""
872
873 output = p4CmdList("client -o")
874 if len(output) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' % len(output))
876
877 entry = output[0]
878 if "Root" not in entry:
879 die('Client has no "Root"')
880
881 return entry["Root"]
882
9d7d446a
PW
883#
884# P4 wildcards are not allowed in filenames. P4 complains
885# if you simply add them, but you can force it with "-f", in
886# which case it translates them into %xx encoding internally.
887#
888def wildcard_decode(path):
889 # Search for and fix just these four characters. Do % last so
890 # that fixing it does not inadvertently create new %-escapes.
891 # Cannot have * in a filename in windows; untested as to
892 # what p4 would do in such a case.
893 if not platform.system() == "Windows":
894 path = path.replace("%2A", "*")
895 path = path.replace("%23", "#") \
896 .replace("%40", "@") \
897 .replace("%25", "%")
898 return path
899
900def wildcard_encode(path):
901 # do % first to avoid double-encoding the %s introduced here
902 path = path.replace("%", "%25") \
903 .replace("*", "%2A") \
904 .replace("#", "%23") \
905 .replace("@", "%40")
906 return path
907
908def wildcard_present(path):
598354c0
BC
909 m = re.search("[*#@%]", path)
910 return m is not None
9d7d446a 911
b984733c
SH
912class Command:
913 def __init__(self):
914 self.usage = "usage: %prog [options]"
8910ac0e 915 self.needsGit = True
6a10b6aa 916 self.verbose = False
b984733c 917
3ea2cfd4
LD
918class P4UserMap:
919 def __init__(self):
920 self.userMapFromPerforceServer = False
affb474f
LD
921 self.myP4UserId = None
922
923 def p4UserId(self):
924 if self.myP4UserId:
925 return self.myP4UserId
926
927 results = p4CmdList("user -o")
928 for r in results:
929 if r.has_key('User'):
930 self.myP4UserId = r['User']
931 return r['User']
932 die("Could not find your p4 user id")
933
934 def p4UserIsMe(self, p4User):
935 # return True if the given p4 user is actually me
936 me = self.p4UserId()
937 if not p4User or p4User != me:
938 return False
939 else:
940 return True
3ea2cfd4
LD
941
942 def getUserCacheFilename(self):
943 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
944 return home + "/.gitp4-usercache.txt"
945
946 def getUserMapFromPerforceServer(self):
947 if self.userMapFromPerforceServer:
948 return
949 self.users = {}
950 self.emails = {}
951
952 for output in p4CmdList("users"):
953 if not output.has_key("User"):
954 continue
955 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
956 self.emails[output["Email"]] = output["User"]
957
958
959 s = ''
960 for (key, val) in self.users.items():
961 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
962
963 open(self.getUserCacheFilename(), "wb").write(s)
964 self.userMapFromPerforceServer = True
965
966 def loadUserMapFromCache(self):
967 self.users = {}
968 self.userMapFromPerforceServer = False
969 try:
970 cache = open(self.getUserCacheFilename(), "rb")
971 lines = cache.readlines()
972 cache.close()
973 for line in lines:
974 entry = line.strip().split("\t")
975 self.users[entry[0]] = entry[1]
976 except IOError:
977 self.getUserMapFromPerforceServer()
978
b984733c 979class P4Debug(Command):
86949eef 980 def __init__(self):
6ae8de88 981 Command.__init__(self)
6a10b6aa 982 self.options = []
c8c39116 983 self.description = "A tool to debug the output of p4 -G."
8910ac0e 984 self.needsGit = False
86949eef
SH
985
986 def run(self, args):
b1ce9447 987 j = 0
6de040df 988 for output in p4CmdList(args):
b1ce9447
HWN
989 print 'Element: %d' % j
990 j += 1
86949eef 991 print output
b984733c 992 return True
86949eef 993
5834684d
SH
994class P4RollBack(Command):
995 def __init__(self):
996 Command.__init__(self)
997 self.options = [
0c66a783 998 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
999 ]
1000 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1001 self.rollbackLocalBranches = False
5834684d
SH
1002
1003 def run(self, args):
1004 if len(args) != 1:
1005 return False
1006 maxChange = int(args[0])
0c66a783 1007
ad192f28 1008 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1009 die("Problems executing p4");
1010
0c66a783
SH
1011 if self.rollbackLocalBranches:
1012 refPrefix = "refs/heads/"
b016d397 1013 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1014 else:
1015 refPrefix = "refs/remotes/"
b016d397 1016 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1017
1018 for line in lines:
1019 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1020 line = line.strip()
1021 ref = refPrefix + line
5834684d 1022 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1023 settings = extractSettingsGitLog(log)
1024
1025 depotPaths = settings['depot-paths']
1026 change = settings['change']
1027
5834684d 1028 changed = False
52102d47 1029
6326aa58
HWN
1030 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1031 for p in depotPaths]))) == 0:
52102d47
SH
1032 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1033 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1034 continue
1035
bb6e09b2 1036 while change and int(change) > maxChange:
5834684d 1037 changed = True
52102d47
SH
1038 if self.verbose:
1039 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1040 system("git update-ref %s \"%s^\"" % (ref, ref))
1041 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1042 settings = extractSettingsGitLog(log)
1043
1044
1045 depotPaths = settings['depot-paths']
1046 change = settings['change']
5834684d
SH
1047
1048 if changed:
52102d47 1049 print "%s rewound to %s" % (ref, change)
5834684d
SH
1050
1051 return True
1052
3ea2cfd4 1053class P4Submit(Command, P4UserMap):
6bbfd137
PW
1054
1055 conflict_behavior_choices = ("ask", "skip", "quit")
1056
4f5cf76a 1057 def __init__(self):
b984733c 1058 Command.__init__(self)
3ea2cfd4 1059 P4UserMap.__init__(self)
4f5cf76a 1060 self.options = [
4f5cf76a 1061 optparse.make_option("--origin", dest="origin"),
ae901090 1062 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1063 # preserve the user, requires relevant p4 permissions
1064 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1065 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1066 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1067 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1068 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1069 choices=self.conflict_behavior_choices),
1070 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
1071 ]
1072 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1073 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1074 self.origin = ""
ae901090 1075 self.detectRenames = False
0d609032 1076 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1077 self.dry_run = False
728b7ad8 1078 self.prepare_p4_only = False
6bbfd137 1079 self.conflict_behavior = None
f7baba8b 1080 self.isWindows = (platform.system() == "Windows")
06804c76 1081 self.exportLabels = False
249da4c0 1082 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1083 self.branch = None
4f5cf76a 1084
4f5cf76a
SH
1085 def check(self):
1086 if len(p4CmdList("opened ...")) > 0:
1087 die("You have files opened with perforce! Close them before starting the sync.")
1088
f19cb0a0
PW
1089 def separate_jobs_from_description(self, message):
1090 """Extract and return a possible Jobs field in the commit
1091 message. It goes into a separate section in the p4 change
1092 specification.
1093
1094 A jobs line starts with "Jobs:" and looks like a new field
1095 in a form. Values are white-space separated on the same
1096 line or on following lines that start with a tab.
1097
1098 This does not parse and extract the full git commit message
1099 like a p4 form. It just sees the Jobs: line as a marker
1100 to pass everything from then on directly into the p4 form,
1101 but outside the description section.
1102
1103 Return a tuple (stripped log message, jobs string)."""
1104
1105 m = re.search(r'^Jobs:', message, re.MULTILINE)
1106 if m is None:
1107 return (message, None)
1108
1109 jobtext = message[m.start():]
1110 stripped_message = message[:m.start()].rstrip()
1111 return (stripped_message, jobtext)
1112
1113 def prepareLogMessage(self, template, message, jobs):
1114 """Edits the template returned from "p4 change -o" to insert
1115 the message in the Description field, and the jobs text in
1116 the Jobs field."""
4f5cf76a
SH
1117 result = ""
1118
edae1e2f
SH
1119 inDescriptionSection = False
1120
4f5cf76a
SH
1121 for line in template.split("\n"):
1122 if line.startswith("#"):
1123 result += line + "\n"
1124 continue
1125
edae1e2f 1126 if inDescriptionSection:
c9dbab04 1127 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1128 inDescriptionSection = False
f19cb0a0
PW
1129 # insert Jobs section
1130 if jobs:
1131 result += jobs + "\n"
edae1e2f
SH
1132 else:
1133 continue
1134 else:
1135 if line.startswith("Description:"):
1136 inDescriptionSection = True
1137 line += "\n"
1138 for messageLine in message.split("\n"):
1139 line += "\t" + messageLine + "\n"
1140
1141 result += line + "\n"
4f5cf76a
SH
1142
1143 return result
1144
60df071c
LD
1145 def patchRCSKeywords(self, file, pattern):
1146 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1147 (handle, outFileName) = tempfile.mkstemp(dir='.')
1148 try:
1149 outFile = os.fdopen(handle, "w+")
1150 inFile = open(file, "r")
1151 regexp = re.compile(pattern, re.VERBOSE)
1152 for line in inFile.readlines():
1153 line = regexp.sub(r'$\1$', line)
1154 outFile.write(line)
1155 inFile.close()
1156 outFile.close()
1157 # Forcibly overwrite the original file
1158 os.unlink(file)
1159 shutil.move(outFileName, file)
1160 except:
1161 # cleanup our temporary file
1162 os.unlink(outFileName)
1163 print "Failed to strip RCS keywords in %s" % file
1164 raise
1165
1166 print "Patched up RCS keywords in %s" % file
1167
3ea2cfd4
LD
1168 def p4UserForCommit(self,id):
1169 # Return the tuple (perforce user,git email) for a given git commit id
1170 self.getUserMapFromPerforceServer()
9bf28855
PW
1171 gitEmail = read_pipe(["git", "log", "--max-count=1",
1172 "--format=%ae", id])
3ea2cfd4
LD
1173 gitEmail = gitEmail.strip()
1174 if not self.emails.has_key(gitEmail):
1175 return (None,gitEmail)
1176 else:
1177 return (self.emails[gitEmail],gitEmail)
1178
1179 def checkValidP4Users(self,commits):
1180 # check if any git authors cannot be mapped to p4 users
1181 for id in commits:
1182 (user,email) = self.p4UserForCommit(id)
1183 if not user:
1184 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1185 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1186 print "%s" % msg
1187 else:
1188 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1189
1190 def lastP4Changelist(self):
1191 # Get back the last changelist number submitted in this client spec. This
1192 # then gets used to patch up the username in the change. If the same
1193 # client spec is being used by multiple processes then this might go
1194 # wrong.
1195 results = p4CmdList("client -o") # find the current client
1196 client = None
1197 for r in results:
1198 if r.has_key('Client'):
1199 client = r['Client']
1200 break
1201 if not client:
1202 die("could not get client spec")
6de040df 1203 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1204 for r in results:
1205 if r.has_key('change'):
1206 return r['change']
1207 die("Could not get changelist number for last submit - cannot patch up user details")
1208
1209 def modifyChangelistUser(self, changelist, newUser):
1210 # fixup the user field of a changelist after it has been submitted.
1211 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1212 if len(changes) != 1:
1213 die("Bad output from p4 change modifying %s to user %s" %
1214 (changelist, newUser))
1215
1216 c = changes[0]
1217 if c['User'] == newUser: return # nothing to do
1218 c['User'] = newUser
1219 input = marshal.dumps(c)
1220
3ea2cfd4
LD
1221 result = p4CmdList("change -f -i", stdin=input)
1222 for r in result:
1223 if r.has_key('code'):
1224 if r['code'] == 'error':
1225 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1226 if r.has_key('data'):
1227 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1228 return
1229 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1230
1231 def canChangeChangelists(self):
1232 # check to see if we have p4 admin or super-user permissions, either of
1233 # which are required to modify changelists.
52a4880b 1234 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1235 for r in results:
1236 if r.has_key('perm'):
1237 if r['perm'] == 'admin':
1238 return 1
1239 if r['perm'] == 'super':
1240 return 1
1241 return 0
1242
ea99c3ae 1243 def prepareSubmitTemplate(self):
f19cb0a0
PW
1244 """Run "p4 change -o" to grab a change specification template.
1245 This does not use "p4 -G", as it is nice to keep the submission
1246 template in original order, since a human might edit it.
1247
1248 Remove lines in the Files section that show changes to files
1249 outside the depot path we're committing into."""
1250
ea99c3ae
SH
1251 template = ""
1252 inFilesSection = False
6de040df 1253 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1254 if line.endswith("\r\n"):
1255 line = line[:-2] + "\n"
ea99c3ae
SH
1256 if inFilesSection:
1257 if line.startswith("\t"):
1258 # path starts and ends with a tab
1259 path = line[1:]
1260 lastTab = path.rfind("\t")
1261 if lastTab != -1:
1262 path = path[:lastTab]
d53de8b9 1263 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
1264 continue
1265 else:
1266 inFilesSection = False
1267 else:
1268 if line.startswith("Files:"):
1269 inFilesSection = True
1270
1271 template += line
1272
1273 return template
1274
7c766e57
PW
1275 def edit_template(self, template_file):
1276 """Invoke the editor to let the user change the submission
1277 message. Return true if okay to continue with the submit."""
1278
1279 # if configured to skip the editing part, just submit
0d609032 1280 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1281 return True
1282
1283 # look at the modification time, to check later if the user saved
1284 # the file
1285 mtime = os.stat(template_file).st_mtime
1286
1287 # invoke the editor
f95ceaf0 1288 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1289 editor = os.environ.get("P4EDITOR")
1290 else:
1291 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1292 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1293
1294 # If the file was not saved, prompt to see if this patch should
1295 # be skipped. But skip this verification step if configured so.
0d609032 1296 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1297 return True
1298
d1652049
PW
1299 # modification time updated means user saved the file
1300 if os.stat(template_file).st_mtime > mtime:
1301 return True
1302
1303 while True:
1304 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1305 if response == 'y':
1306 return True
1307 if response == 'n':
1308 return False
7c766e57 1309
e2a892ee 1310 def get_diff_description(self, editedFiles, filesToAdd):
b4073bb3
MC
1311 # diff
1312 if os.environ.has_key("P4DIFF"):
1313 del(os.environ["P4DIFF"])
1314 diff = ""
1315 for editedFile in editedFiles:
1316 diff += p4_read_pipe(['diff', '-du',
1317 wildcard_encode(editedFile)])
1318
1319 # new file diff
1320 newdiff = ""
1321 for newFile in filesToAdd:
1322 newdiff += "==== new file ====\n"
1323 newdiff += "--- /dev/null\n"
1324 newdiff += "+++ %s\n" % newFile
1325 f = open(newFile, "r")
1326 for line in f.readlines():
1327 newdiff += "+" + line
1328 f.close()
1329
e2a892ee 1330 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1331
7cb5cbef 1332 def applyCommit(self, id):
67b0fe2e
PW
1333 """Apply one commit, return True if it succeeded."""
1334
1335 print "Applying", read_pipe(["git", "show", "-s",
1336 "--format=format:%h %s", id])
ae901090 1337
848de9c3 1338 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1339
84cb0003 1340 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1341 filesToAdd = set()
1342 filesToDelete = set()
d336c158 1343 editedFiles = set()
b6ad6dcc 1344 pureRenameCopy = set()
c65b670e 1345 filesToChangeExecBit = {}
60df071c 1346
4f5cf76a 1347 for line in diff:
b43b0a3c
CP
1348 diff = parseDiffTreeEntry(line)
1349 modifier = diff['status']
1350 path = diff['src']
4f5cf76a 1351 if modifier == "M":
6de040df 1352 p4_edit(path)
c65b670e
CP
1353 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1354 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1355 editedFiles.add(path)
4f5cf76a
SH
1356 elif modifier == "A":
1357 filesToAdd.add(path)
c65b670e 1358 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1359 if path in filesToDelete:
1360 filesToDelete.remove(path)
1361 elif modifier == "D":
1362 filesToDelete.add(path)
1363 if path in filesToAdd:
1364 filesToAdd.remove(path)
4fddb41b
VA
1365 elif modifier == "C":
1366 src, dest = diff['src'], diff['dst']
6de040df 1367 p4_integrate(src, dest)
b6ad6dcc 1368 pureRenameCopy.add(dest)
4fddb41b 1369 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1370 p4_edit(dest)
b6ad6dcc 1371 pureRenameCopy.discard(dest)
4fddb41b 1372 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1373 p4_edit(dest)
b6ad6dcc 1374 pureRenameCopy.discard(dest)
4fddb41b 1375 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1376 if self.isWindows:
1377 # turn off read-only attribute
1378 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1379 os.unlink(dest)
1380 editedFiles.add(dest)
d9a5f25b 1381 elif modifier == "R":
b43b0a3c 1382 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1383 if self.p4HasMoveCommand:
1384 p4_edit(src) # src must be open before move
1385 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1386 else:
8e9497c2
GG
1387 p4_integrate(src, dest)
1388 if diff['src_sha1'] != diff['dst_sha1']:
1389 p4_edit(dest)
1390 else:
1391 pureRenameCopy.add(dest)
c65b670e 1392 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1393 if not self.p4HasMoveCommand:
1394 p4_edit(dest) # with move: already open, writable
c65b670e 1395 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1396 if not self.p4HasMoveCommand:
d20f0f8e
PW
1397 if self.isWindows:
1398 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1399 os.unlink(dest)
1400 filesToDelete.add(src)
d9a5f25b 1401 editedFiles.add(dest)
4f5cf76a
SH
1402 else:
1403 die("unknown modifier %s for %s" % (modifier, path))
1404
749b668c 1405 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1406 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1407 tryPatchCmd = patchcmd + "--check -"
1408 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1409 patch_succeeded = True
51a2640a 1410
47a130b7 1411 if os.system(tryPatchCmd) != 0:
60df071c
LD
1412 fixed_rcs_keywords = False
1413 patch_succeeded = False
51a2640a 1414 print "Unfortunately applying the change failed!"
60df071c
LD
1415
1416 # Patch failed, maybe it's just RCS keyword woes. Look through
1417 # the patch to see if that's possible.
0d609032 1418 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1419 file = None
1420 pattern = None
1421 kwfiles = {}
1422 for file in editedFiles | filesToDelete:
1423 # did this file's delta contain RCS keywords?
1424 pattern = p4_keywords_regexp_for_file(file)
1425
1426 if pattern:
1427 # this file is a possibility...look for RCS keywords.
1428 regexp = re.compile(pattern, re.VERBOSE)
1429 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1430 if regexp.search(line):
1431 if verbose:
1432 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1433 kwfiles[file] = pattern
1434 break
1435
1436 for file in kwfiles:
1437 if verbose:
1438 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1439 # File is being deleted, so not open in p4. Must
1440 # disable the read-only bit on windows.
1441 if self.isWindows and file not in editedFiles:
1442 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1443 self.patchRCSKeywords(file, kwfiles[file])
1444 fixed_rcs_keywords = True
1445
1446 if fixed_rcs_keywords:
1447 print "Retrying the patch with RCS keywords cleaned up"
1448 if os.system(tryPatchCmd) == 0:
1449 patch_succeeded = True
1450
1451 if not patch_succeeded:
7e5dd9f2
PW
1452 for f in editedFiles:
1453 p4_revert(f)
7e5dd9f2 1454 return False
51a2640a 1455
55ac2ed6
PW
1456 #
1457 # Apply the patch for real, and do add/delete/+x handling.
1458 #
47a130b7 1459 system(applyPatchCmd)
4f5cf76a
SH
1460
1461 for f in filesToAdd:
6de040df 1462 p4_add(f)
4f5cf76a 1463 for f in filesToDelete:
6de040df
LD
1464 p4_revert(f)
1465 p4_delete(f)
4f5cf76a 1466
c65b670e
CP
1467 # Set/clear executable bits
1468 for f in filesToChangeExecBit.keys():
1469 mode = filesToChangeExecBit[f]
1470 setP4ExecBit(f, mode)
1471
55ac2ed6
PW
1472 #
1473 # Build p4 change description, starting with the contents
1474 # of the git commit message.
1475 #
0e36f2d7 1476 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1477 logMessage = logMessage.strip()
f19cb0a0 1478 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1479
ea99c3ae 1480 template = self.prepareSubmitTemplate()
f19cb0a0 1481 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1482
c47178d4 1483 if self.preserveUser:
55ac2ed6 1484 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1485
55ac2ed6
PW
1486 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1487 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1488 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1489 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1490
55ac2ed6 1491 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1492 if not self.prepare_p4_only:
1493 submitTemplate += separatorLine
e2a892ee 1494 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
55ac2ed6 1495
c47178d4 1496 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1497 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1498 if self.isWindows:
1499 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1500 tmpFile.write(submitTemplate)
c47178d4
PW
1501 tmpFile.close()
1502
728b7ad8
PW
1503 if self.prepare_p4_only:
1504 #
1505 # Leave the p4 tree prepared, and the submit template around
1506 # and let the user decide what to do next
1507 #
1508 print
1509 print "P4 workspace prepared for submission."
1510 print "To submit or revert, go to client workspace"
1511 print " " + self.clientPath
1512 print
1513 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1514 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1515 " \"git p4\"." % fileName
1516 print "You can delete the file \"%s\" when finished." % fileName
1517
1518 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1519 print "To preserve change ownership by user %s, you must\n" \
1520 "do \"p4 change -f <change>\" after submitting and\n" \
1521 "edit the User field."
1522 if pureRenameCopy:
1523 print "After submitting, renamed files must be re-synced."
1524 print "Invoke \"p4 sync -f\" on each of these files:"
1525 for f in pureRenameCopy:
1526 print " " + f
1527
1528 print
1529 print "To revert the changes, use \"p4 revert ...\", and delete"
1530 print "the submit template file \"%s\"" % fileName
1531 if filesToAdd:
1532 print "Since the commit adds new files, they must be deleted:"
1533 for f in filesToAdd:
1534 print " " + f
1535 print
1536 return True
1537
55ac2ed6
PW
1538 #
1539 # Let the user edit the change description, then submit it.
1540 #
c47178d4
PW
1541 if self.edit_template(fileName):
1542 # read the edited message and submit
5a41c16a 1543 ret = True
c47178d4
PW
1544 tmpFile = open(fileName, "rb")
1545 message = tmpFile.read()
e96e400f 1546 tmpFile.close()
c47178d4 1547 if self.isWindows:
e2a892ee
MC
1548 message = message.replace("\r\n", "\n")
1549 submitTemplate = message[:message.index(separatorLine)]
c47178d4 1550 p4_write_pipe(['submit', '-i'], submitTemplate)
e96e400f 1551
c47178d4
PW
1552 if self.preserveUser:
1553 if p4User:
1554 # Get last changelist number. Cannot easily get it from
1555 # the submit command output as the output is
1556 # unmarshalled.
1557 changelist = self.lastP4Changelist()
1558 self.modifyChangelistUser(changelist, p4User)
1559
1560 # The rename/copy happened by applying a patch that created a
1561 # new file. This leaves it writable, which confuses p4.
1562 for f in pureRenameCopy:
1563 p4_sync(f, "-f")
cdc7e388 1564
4f5cf76a 1565 else:
c47178d4 1566 # skip this patch
5a41c16a 1567 ret = False
c47178d4
PW
1568 print "Submission cancelled, undoing p4 changes."
1569 for f in editedFiles:
1570 p4_revert(f)
1571 for f in filesToAdd:
1572 p4_revert(f)
1573 os.remove(f)
df9c5453
PW
1574 for f in filesToDelete:
1575 p4_revert(f)
c47178d4
PW
1576
1577 os.remove(fileName)
5a41c16a 1578 return ret
4f5cf76a 1579
06804c76
LD
1580 # Export git tags as p4 labels. Create a p4 label and then tag
1581 # with that.
1582 def exportGitTags(self, gitTags):
c8942a22
LD
1583 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1584 if len(validLabelRegexp) == 0:
1585 validLabelRegexp = defaultLabelRegexp
1586 m = re.compile(validLabelRegexp)
06804c76
LD
1587
1588 for name in gitTags:
1589
1590 if not m.match(name):
1591 if verbose:
05a3cec5 1592 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1593 continue
1594
1595 # Get the p4 commit this corresponds to
c8942a22
LD
1596 logMessage = extractLogMessageFromGitCommit(name)
1597 values = extractSettingsGitLog(logMessage)
06804c76 1598
c8942a22 1599 if not values.has_key('change'):
06804c76
LD
1600 # a tag pointing to something not sent to p4; ignore
1601 if verbose:
1602 print "git tag %s does not give a p4 commit" % name
1603 continue
c8942a22
LD
1604 else:
1605 changelist = values['change']
06804c76
LD
1606
1607 # Get the tag details.
1608 inHeader = True
1609 isAnnotated = False
1610 body = []
1611 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1612 l = l.strip()
1613 if inHeader:
1614 if re.match(r'tag\s+', l):
1615 isAnnotated = True
1616 elif re.match(r'\s*$', l):
1617 inHeader = False
1618 continue
1619 else:
1620 body.append(l)
1621
1622 if not isAnnotated:
1623 body = ["lightweight tag imported by git p4\n"]
1624
1625 # Create the label - use the same view as the client spec we are using
1626 clientSpec = getClientSpec()
1627
1628 labelTemplate = "Label: %s\n" % name
1629 labelTemplate += "Description:\n"
1630 for b in body:
1631 labelTemplate += "\t" + b + "\n"
1632 labelTemplate += "View:\n"
9d57c4a6
KS
1633 for depot_side in clientSpec.mappings:
1634 labelTemplate += "\t%s\n" % depot_side
06804c76 1635
ef739f08
PW
1636 if self.dry_run:
1637 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1638 elif self.prepare_p4_only:
1639 print "Not creating p4 label %s for tag due to option" \
1640 " --prepare-p4-only" % name
ef739f08
PW
1641 else:
1642 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1643
ef739f08
PW
1644 # Use the label
1645 p4_system(["tag", "-l", name] +
9d57c4a6 1646 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1647
ef739f08
PW
1648 if verbose:
1649 print "created p4 label for tag %s" % name
06804c76 1650
4f5cf76a 1651 def run(self, args):
c9b50e63
SH
1652 if len(args) == 0:
1653 self.master = currentGitBranch()
4280e533 1654 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1655 die("Detecting current git branch failed!")
1656 elif len(args) == 1:
1657 self.master = args[0]
28755dba
PW
1658 if not branchExists(self.master):
1659 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1660 else:
1661 return False
1662
4c2d5d72
JX
1663 allowSubmit = gitConfig("git-p4.allowSubmit")
1664 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1665 die("%s is not in git-p4.allowSubmit" % self.master)
1666
27d2d811 1667 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1668 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1669 if len(self.origin) == 0:
1670 self.origin = upstream
a3fdd579 1671
3ea2cfd4
LD
1672 if self.preserveUser:
1673 if not self.canChangeChangelists():
1674 die("Cannot preserve user names without p4 super-user or admin permissions")
1675
6bbfd137
PW
1676 # if not set from the command line, try the config file
1677 if self.conflict_behavior is None:
1678 val = gitConfig("git-p4.conflict")
1679 if val:
1680 if val not in self.conflict_behavior_choices:
1681 die("Invalid value '%s' for config git-p4.conflict" % val)
1682 else:
1683 val = "ask"
1684 self.conflict_behavior = val
1685
a3fdd579
SH
1686 if self.verbose:
1687 print "Origin branch is " + self.origin
9512497b 1688
ea99c3ae 1689 if len(self.depotPath) == 0:
9512497b
SH
1690 print "Internal error: cannot locate perforce depot path from existing branches"
1691 sys.exit(128)
1692
543987bd 1693 self.useClientSpec = False
0d609032 1694 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1695 self.useClientSpec = True
1696 if self.useClientSpec:
1697 self.clientSpecDirs = getClientSpec()
9512497b 1698
cd884106
VA
1699 # Check for the existance of P4 branches
1700 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1701
1702 if self.useClientSpec and not branchesDetected:
543987bd
PW
1703 # all files are relative to the client spec
1704 self.clientPath = getClientRoot()
1705 else:
1706 self.clientPath = p4Where(self.depotPath)
9512497b 1707
543987bd
PW
1708 if self.clientPath == "":
1709 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1710
ea99c3ae 1711 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1712 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1713
0591cfa8 1714 # ensure the clientPath exists
8d7ec362 1715 new_client_dir = False
0591cfa8 1716 if not os.path.exists(self.clientPath):
8d7ec362 1717 new_client_dir = True
0591cfa8
GG
1718 os.makedirs(self.clientPath)
1719
bbd84863 1720 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1721 if self.dry_run:
1722 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1723 else:
ef739f08
PW
1724 print "Synchronizing p4 checkout..."
1725 if new_client_dir:
1726 # old one was destroyed, and maybe nobody told p4
1727 p4_sync("...", "-f")
1728 else:
1729 p4_sync("...")
4f5cf76a 1730 self.check()
4f5cf76a 1731
4c750c0d 1732 commits = []
c7d34884 1733 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
4c750c0d
SH
1734 commits.append(line.strip())
1735 commits.reverse()
4f5cf76a 1736
0d609032 1737 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1738 self.checkAuthorship = False
1739 else:
1740 self.checkAuthorship = True
1741
3ea2cfd4
LD
1742 if self.preserveUser:
1743 self.checkValidP4Users(commits)
1744
84cb0003
GG
1745 #
1746 # Build up a set of options to be passed to diff when
1747 # submitting each commit to p4.
1748 #
1749 if self.detectRenames:
1750 # command-line -M arg
1751 self.diffOpts = "-M"
1752 else:
1753 # If not explicitly set check the config variable
1754 detectRenames = gitConfig("git-p4.detectRenames")
1755
1756 if detectRenames.lower() == "false" or detectRenames == "":
1757 self.diffOpts = ""
1758 elif detectRenames.lower() == "true":
1759 self.diffOpts = "-M"
1760 else:
1761 self.diffOpts = "-M%s" % detectRenames
1762
1763 # no command-line arg for -C or --find-copies-harder, just
1764 # config variables
1765 detectCopies = gitConfig("git-p4.detectCopies")
1766 if detectCopies.lower() == "false" or detectCopies == "":
1767 pass
1768 elif detectCopies.lower() == "true":
1769 self.diffOpts += " -C"
1770 else:
1771 self.diffOpts += " -C%s" % detectCopies
1772
0d609032 1773 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1774 self.diffOpts += " --find-copies-harder"
1775
7e5dd9f2
PW
1776 #
1777 # Apply the commits, one at a time. On failure, ask if should
1778 # continue to try the rest of the patches, or quit.
1779 #
ef739f08
PW
1780 if self.dry_run:
1781 print "Would apply"
67b0fe2e 1782 applied = []
7e5dd9f2
PW
1783 last = len(commits) - 1
1784 for i, commit in enumerate(commits):
ef739f08
PW
1785 if self.dry_run:
1786 print " ", read_pipe(["git", "show", "-s",
1787 "--format=format:%h %s", commit])
1788 ok = True
1789 else:
1790 ok = self.applyCommit(commit)
67b0fe2e
PW
1791 if ok:
1792 applied.append(commit)
7e5dd9f2 1793 else:
728b7ad8
PW
1794 if self.prepare_p4_only and i < last:
1795 print "Processing only the first commit due to option" \
1796 " --prepare-p4-only"
1797 break
7e5dd9f2
PW
1798 if i < last:
1799 quit = False
1800 while True:
6bbfd137
PW
1801 # prompt for what to do, or use the option/variable
1802 if self.conflict_behavior == "ask":
1803 print "What do you want to do?"
1804 response = raw_input("[s]kip this commit but apply"
1805 " the rest, or [q]uit? ")
1806 if not response:
1807 continue
1808 elif self.conflict_behavior == "skip":
1809 response = "s"
1810 elif self.conflict_behavior == "quit":
1811 response = "q"
1812 else:
1813 die("Unknown conflict_behavior '%s'" %
1814 self.conflict_behavior)
1815
7e5dd9f2
PW
1816 if response[0] == "s":
1817 print "Skipping this commit, but applying the rest"
1818 break
1819 if response[0] == "q":
1820 print "Quitting"
1821 quit = True
1822 break
1823 if quit:
1824 break
4f5cf76a 1825
67b0fe2e 1826 chdir(self.oldWorkingDirectory)
4f5cf76a 1827
ef739f08
PW
1828 if self.dry_run:
1829 pass
728b7ad8
PW
1830 elif self.prepare_p4_only:
1831 pass
ef739f08 1832 elif len(commits) == len(applied):
67b0fe2e 1833 print "All commits applied!"
14594f4b 1834
4c750c0d 1835 sync = P4Sync()
44e8d26c
PW
1836 if self.branch:
1837 sync.branch = self.branch
4c750c0d 1838 sync.run([])
14594f4b 1839
4c750c0d
SH
1840 rebase = P4Rebase()
1841 rebase.rebase()
4f5cf76a 1842
67b0fe2e
PW
1843 else:
1844 if len(applied) == 0:
1845 print "No commits applied."
1846 else:
1847 print "Applied only the commits marked with '*':"
1848 for c in commits:
1849 if c in applied:
1850 star = "*"
1851 else:
1852 star = " "
1853 print star, read_pipe(["git", "show", "-s",
1854 "--format=format:%h %s", c])
1855 print "You will have to do 'git p4 sync' and rebase."
1856
0d609032 1857 if gitConfigBool("git-p4.exportLabels"):
06dcd152 1858 self.exportLabels = True
06804c76
LD
1859
1860 if self.exportLabels:
1861 p4Labels = getP4Labels(self.depotPath)
1862 gitTags = getGitTags()
1863
1864 missingGitTags = gitTags - p4Labels
1865 self.exportGitTags(missingGitTags)
1866
98e023de 1867 # exit with error unless everything applied perfectly
67b0fe2e
PW
1868 if len(commits) != len(applied):
1869 sys.exit(1)
1870
b984733c
SH
1871 return True
1872
ecb7cf98
PW
1873class View(object):
1874 """Represent a p4 view ("p4 help views"), and map files in a
1875 repo according to the view."""
1876
9d57c4a6 1877 def __init__(self, client_name):
ecb7cf98 1878 self.mappings = []
9d57c4a6
KS
1879 self.client_prefix = "//%s/" % client_name
1880 # cache results of "p4 where" to lookup client file locations
1881 self.client_spec_path_cache = {}
ecb7cf98
PW
1882
1883 def append(self, view_line):
1884 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
1885 sides. Append to self.mappings, preserving order. This
1886 is only needed for tag creation."""
ecb7cf98
PW
1887
1888 # Split the view line into exactly two words. P4 enforces
1889 # structure on these lines that simplifies this quite a bit.
1890 #
1891 # Either or both words may be double-quoted.
1892 # Single quotes do not matter.
1893 # Double-quote marks cannot occur inside the words.
1894 # A + or - prefix is also inside the quotes.
1895 # There are no quotes unless they contain a space.
1896 # The line is already white-space stripped.
1897 # The two words are separated by a single space.
1898 #
1899 if view_line[0] == '"':
1900 # First word is double quoted. Find its end.
1901 close_quote_index = view_line.find('"', 1)
1902 if close_quote_index <= 0:
1903 die("No first-word closing quote found: %s" % view_line)
1904 depot_side = view_line[1:close_quote_index]
1905 # skip closing quote and space
1906 rhs_index = close_quote_index + 1 + 1
1907 else:
1908 space_index = view_line.find(" ")
1909 if space_index <= 0:
1910 die("No word-splitting space found: %s" % view_line)
1911 depot_side = view_line[0:space_index]
1912 rhs_index = space_index + 1
1913
ecb7cf98 1914 # prefix + means overlay on previous mapping
ecb7cf98 1915 if depot_side.startswith("+"):
ecb7cf98
PW
1916 depot_side = depot_side[1:]
1917
9d57c4a6 1918 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
1919 exclude = False
1920 if depot_side.startswith("-"):
1921 exclude = True
1922 depot_side = depot_side[1:]
1923
9d57c4a6
KS
1924 if not exclude:
1925 self.mappings.append(depot_side)
ecb7cf98 1926
9d57c4a6
KS
1927 def convert_client_path(self, clientFile):
1928 # chop off //client/ part to make it relative
1929 if not clientFile.startswith(self.client_prefix):
1930 die("No prefix '%s' on clientFile '%s'" %
1931 (self.client_prefix, clientFile))
1932 return clientFile[len(self.client_prefix):]
ecb7cf98 1933
9d57c4a6
KS
1934 def update_client_spec_path_cache(self, files):
1935 """ Caching file paths by "p4 where" batch query """
ecb7cf98 1936
9d57c4a6
KS
1937 # List depot file paths exclude that already cached
1938 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 1939
9d57c4a6
KS
1940 if len(fileArgs) == 0:
1941 return # All files in cache
ecb7cf98 1942
9d57c4a6
KS
1943 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1944 for res in where_result:
1945 if "code" in res and res["code"] == "error":
1946 # assume error is "... file(s) not in client view"
1947 continue
1948 if "clientFile" not in res:
20005443 1949 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
1950 if "unmap" in res:
1951 # it will list all of them, but only one not unmap-ped
1952 continue
1953 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 1954
9d57c4a6
KS
1955 # not found files or unmap files set to ""
1956 for depotFile in fileArgs:
1957 if depotFile not in self.client_spec_path_cache:
1958 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 1959
9d57c4a6
KS
1960 def map_in_client(self, depot_path):
1961 """Return the relative location in the client where this
1962 depot file should live. Returns "" if the file should
1963 not be mapped in the client."""
ecb7cf98 1964
9d57c4a6
KS
1965 if depot_path in self.client_spec_path_cache:
1966 return self.client_spec_path_cache[depot_path]
1967
1968 die( "Error: %s is not found in client spec path" % depot_path )
1969 return ""
ecb7cf98 1970
3ea2cfd4 1971class P4Sync(Command, P4UserMap):
56c09345
PW
1972 delete_actions = ( "delete", "move/delete", "purge" )
1973
b984733c
SH
1974 def __init__(self):
1975 Command.__init__(self)
3ea2cfd4 1976 P4UserMap.__init__(self)
b984733c
SH
1977 self.options = [
1978 optparse.make_option("--branch", dest="branch"),
1979 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1980 optparse.make_option("--changesfile", dest="changesFile"),
1981 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1982 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 1983 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
1984 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1985 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
1986 optparse.make_option("--max-changes", dest="maxChanges",
1987 help="Maximum number of changes to import"),
1988 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1989 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 1990 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1991 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1992 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
1993 help="Only sync files that are included in the Perforce Client Spec"),
1994 optparse.make_option("-/", dest="cloneExclude",
1995 action="append", type="string",
1996 help="exclude depot path"),
b984733c
SH
1997 ]
1998 self.description = """Imports from Perforce into a git repository.\n
1999 example:
2000 //depot/my/project/ -- to import the current head
2001 //depot/my/project/@all -- to import everything
2002 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2003
2004 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2005
2006 self.usage += " //depot/path[@revRange]"
b984733c 2007 self.silent = False
1d7367dc
RG
2008 self.createdBranches = set()
2009 self.committedChanges = set()
569d1bd4 2010 self.branch = ""
b984733c 2011 self.detectBranches = False
cb53e1f8 2012 self.detectLabels = False
06804c76 2013 self.importLabels = False
b984733c 2014 self.changesFile = ""
01265103 2015 self.syncWithOrigin = True
a028a98e 2016 self.importIntoRemotes = True
01a9c9c5 2017 self.maxChanges = ""
1051ef00 2018 self.changes_block_size = None
8b41a97f 2019 self.keepRepoPath = False
6326aa58 2020 self.depotPaths = None
3c699645 2021 self.p4BranchesInGit = []
354081d5 2022 self.cloneExclude = []
3a70cdfa 2023 self.useClientSpec = False
a93d33ee 2024 self.useClientSpec_from_options = False
ecb7cf98 2025 self.clientSpecDirs = None
fed23693
VA
2026 self.tempBranches = []
2027 self.tempBranchLocation = "git-p4-tmp"
b984733c 2028
01265103
SH
2029 if gitConfig("git-p4.syncFromOrigin") == "false":
2030 self.syncWithOrigin = False
2031
51334bb0
LD
2032 # This is required for the "append" cloneExclude action
2033 def ensure_value(self, attr, value):
2034 if not hasattr(self, attr) or getattr(self, attr) is None:
2035 setattr(self, attr, value)
2036 return getattr(self, attr)
2037
fed23693
VA
2038 # Force a checkpoint in fast-import and wait for it to finish
2039 def checkpoint(self):
2040 self.gitStream.write("checkpoint\n\n")
2041 self.gitStream.write("progress checkpoint\n\n")
2042 out = self.gitOutput.readline()
2043 if self.verbose:
2044 print "checkpoint finished: " + out
2045
b984733c 2046 def extractFilesFromCommit(self, commit):
354081d5
TT
2047 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2048 for path in self.cloneExclude]
b984733c
SH
2049 files = []
2050 fnum = 0
2051 while commit.has_key("depotFile%s" % fnum):
2052 path = commit["depotFile%s" % fnum]
6326aa58 2053
354081d5 2054 if [p for p in self.cloneExclude
d53de8b9 2055 if p4PathStartsWith(path, p)]:
354081d5
TT
2056 found = False
2057 else:
2058 found = [p for p in self.depotPaths
d53de8b9 2059 if p4PathStartsWith(path, p)]
6326aa58 2060 if not found:
b984733c
SH
2061 fnum = fnum + 1
2062 continue
2063
2064 file = {}
2065 file["path"] = path
2066 file["rev"] = commit["rev%s" % fnum]
2067 file["action"] = commit["action%s" % fnum]
2068 file["type"] = commit["type%s" % fnum]
2069 files.append(file)
2070 fnum = fnum + 1
2071 return files
2072
6326aa58 2073 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2074 """When streaming files, this is called to map a p4 depot path
2075 to where it should go in git. The prefixes are either
2076 self.depotPaths, or self.branchPrefixes in the case of
2077 branch detection."""
2078
3952710b 2079 if self.useClientSpec:
21ef5df4
PW
2080 # branch detection moves files up a level (the branch name)
2081 # from what client spec interpretation gives
0d1696ef 2082 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2083 if self.detectBranches:
2084 for b in self.knownBranches:
2085 if path.startswith(b + "/"):
2086 path = path[len(b)+1:]
2087
2088 elif self.keepRepoPath:
2089 # Preserve everything in relative path name except leading
2090 # //depot/; just look at first prefix as they all should
2091 # be in the same depot.
2092 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2093 if p4PathStartsWith(path, depot):
2094 path = path[len(depot):]
3952710b 2095
0d1696ef 2096 else:
0d1696ef
PW
2097 for p in prefixes:
2098 if p4PathStartsWith(path, p):
2099 path = path[len(p):]
21ef5df4 2100 break
8b41a97f 2101
0d1696ef 2102 path = wildcard_decode(path)
6326aa58 2103 return path
6754a299 2104
71b112d4 2105 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2106 """Look at each depotFile in the commit to figure out to what
2107 branch it belongs."""
2108
9d57c4a6
KS
2109 if self.clientSpecDirs:
2110 files = self.extractFilesFromCommit(commit)
2111 self.clientSpecDirs.update_client_spec_path_cache(files)
2112
d5904674 2113 branches = {}
71b112d4
SH
2114 fnum = 0
2115 while commit.has_key("depotFile%s" % fnum):
2116 path = commit["depotFile%s" % fnum]
6326aa58 2117 found = [p for p in self.depotPaths
d53de8b9 2118 if p4PathStartsWith(path, p)]
6326aa58 2119 if not found:
71b112d4
SH
2120 fnum = fnum + 1
2121 continue
2122
2123 file = {}
2124 file["path"] = path
2125 file["rev"] = commit["rev%s" % fnum]
2126 file["action"] = commit["action%s" % fnum]
2127 file["type"] = commit["type%s" % fnum]
2128 fnum = fnum + 1
2129
21ef5df4
PW
2130 # start with the full relative path where this file would
2131 # go in a p4 client
2132 if self.useClientSpec:
2133 relPath = self.clientSpecDirs.map_in_client(path)
2134 else:
2135 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2136
4b97ffb1 2137 for branch in self.knownBranches.keys():
21ef5df4
PW
2138 # add a trailing slash so that a commit into qt/4.2foo
2139 # doesn't end up in qt/4.2, e.g.
6754a299 2140 if relPath.startswith(branch + "/"):
d5904674
SH
2141 if branch not in branches:
2142 branches[branch] = []
71b112d4 2143 branches[branch].append(file)
6555b2cc 2144 break
b984733c
SH
2145
2146 return branches
2147
b932705b
LD
2148 # output one file from the P4 stream
2149 # - helper for streamP4Files
2150
2151 def streamOneP4File(self, file, contents):
b932705b
LD
2152 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2153 if verbose:
2154 sys.stderr.write("%s\n" % relPath)
2155
9cffb8c8
PW
2156 (type_base, type_mods) = split_p4_type(file["type"])
2157
2158 git_mode = "100644"
2159 if "x" in type_mods:
2160 git_mode = "100755"
2161 if type_base == "symlink":
2162 git_mode = "120000"
1292df11
AJ
2163 # p4 print on a symlink sometimes contains "target\n";
2164 # if it does, remove the newline
b39c3612 2165 data = ''.join(contents)
40f846c3
PW
2166 if not data:
2167 # Some version of p4 allowed creating a symlink that pointed
2168 # to nothing. This causes p4 errors when checking out such
2169 # a change, and errors here too. Work around it by ignoring
2170 # the bad symlink; hopefully a future change fixes it.
2171 print "\nIgnoring empty symlink in %s" % file['depotFile']
2172 return
2173 elif data[-1] == '\n':
1292df11
AJ
2174 contents = [data[:-1]]
2175 else:
2176 contents = [data]
b932705b 2177
9cffb8c8 2178 if type_base == "utf16":
55aa5714
PW
2179 # p4 delivers different text in the python output to -G
2180 # than it does when using "print -o", or normal p4 client
2181 # operations. utf16 is converted to ascii or utf8, perhaps.
2182 # But ascii text saved as -t utf16 is completely mangled.
2183 # Invoke print -o to get the real contents.
7f0e5962
PW
2184 #
2185 # On windows, the newlines will always be mangled by print, so put
2186 # them back too. This is not needed to the cygwin windows version,
2187 # just the native "NT" type.
2188 #
f5f53f14 2189 text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
7f0e5962
PW
2190 if p4_version_string().find("/NT") >= 0:
2191 text = text.replace("\r\n", "\n")
55aa5714
PW
2192 contents = [ text ]
2193
9f7ef0ea
PW
2194 if type_base == "apple":
2195 # Apple filetype files will be streamed as a concatenation of
2196 # its appledouble header and the contents. This is useless
2197 # on both macs and non-macs. If using "print -q -o xx", it
2198 # will create "xx" with the data, and "%xx" with the header.
2199 # This is also not very useful.
2200 #
2201 # Ideally, someday, this script can learn how to generate
2202 # appledouble files directly and import those to git, but
2203 # non-mac machines can never find a use for apple filetype.
2204 print "\nIgnoring apple filetype file %s" % file['depotFile']
2205 return
2206
55aa5714
PW
2207 # Note that we do not try to de-mangle keywords on utf16 files,
2208 # even though in theory somebody may want that.
60df071c
LD
2209 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2210 if pattern:
2211 regexp = re.compile(pattern, re.VERBOSE)
2212 text = ''.join(contents)
2213 text = regexp.sub(r'$\1$', text)
2214 contents = [ text ]
b932705b 2215
9cffb8c8 2216 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
2217
2218 # total length...
2219 length = 0
2220 for d in contents:
2221 length = length + len(d)
2222
2223 self.gitStream.write("data %d\n" % length)
2224 for d in contents:
2225 self.gitStream.write(d)
2226 self.gitStream.write("\n")
2227
2228 def streamOneP4Deletion(self, file):
2229 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2230 if verbose:
2231 sys.stderr.write("delete %s\n" % relPath)
2232 self.gitStream.write("D %s\n" % relPath)
2233
2234 # handle another chunk of streaming data
2235 def streamP4FilesCb(self, marshalled):
2236
78189bea
PW
2237 # catch p4 errors and complain
2238 err = None
2239 if "code" in marshalled:
2240 if marshalled["code"] == "error":
2241 if "data" in marshalled:
2242 err = marshalled["data"].rstrip()
2243 if err:
2244 f = None
2245 if self.stream_have_file_info:
2246 if "depotFile" in self.stream_file:
2247 f = self.stream_file["depotFile"]
2248 # force a failure in fast-import, else an empty
2249 # commit will be made
2250 self.gitStream.write("\n")
2251 self.gitStream.write("die-now\n")
2252 self.gitStream.close()
2253 # ignore errors, but make sure it exits first
2254 self.importProcess.wait()
2255 if f:
2256 die("Error from p4 print for %s: %s" % (f, err))
2257 else:
2258 die("Error from p4 print: %s" % err)
2259
c3f6163b
AG
2260 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2261 # start of a new file - output the old one first
2262 self.streamOneP4File(self.stream_file, self.stream_contents)
2263 self.stream_file = {}
2264 self.stream_contents = []
2265 self.stream_have_file_info = False
b932705b 2266
c3f6163b
AG
2267 # pick up the new file information... for the
2268 # 'data' field we need to append to our array
2269 for k in marshalled.keys():
2270 if k == 'data':
2271 self.stream_contents.append(marshalled['data'])
2272 else:
2273 self.stream_file[k] = marshalled[k]
b932705b 2274
c3f6163b 2275 self.stream_have_file_info = True
b932705b
LD
2276
2277 # Stream directly from "p4 files" into "git fast-import"
2278 def streamP4Files(self, files):
30b5940b
SH
2279 filesForCommit = []
2280 filesToRead = []
b932705b 2281 filesToDelete = []
30b5940b 2282
3a70cdfa 2283 for f in files:
ecb7cf98
PW
2284 # if using a client spec, only add the files that have
2285 # a path in the client
2286 if self.clientSpecDirs:
2287 if self.clientSpecDirs.map_in_client(f['path']) == "":
2288 continue
3a70cdfa 2289
ecb7cf98
PW
2290 filesForCommit.append(f)
2291 if f['action'] in self.delete_actions:
2292 filesToDelete.append(f)
2293 else:
2294 filesToRead.append(f)
6a49f8e2 2295
b932705b
LD
2296 # deleted files...
2297 for f in filesToDelete:
2298 self.streamOneP4Deletion(f)
1b9a4684 2299
b932705b
LD
2300 if len(filesToRead) > 0:
2301 self.stream_file = {}
2302 self.stream_contents = []
2303 self.stream_have_file_info = False
8ff45f2a 2304
c3f6163b
AG
2305 # curry self argument
2306 def streamP4FilesCbSelf(entry):
2307 self.streamP4FilesCb(entry)
6a49f8e2 2308
6de040df
LD
2309 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2310
2311 p4CmdList(["-x", "-", "print"],
2312 stdin=fileArgs,
2313 cb=streamP4FilesCbSelf)
30b5940b 2314
b932705b
LD
2315 # do the last chunk
2316 if self.stream_file.has_key('depotFile'):
2317 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2318
affb474f
LD
2319 def make_email(self, userid):
2320 if userid in self.users:
2321 return self.users[userid]
2322 else:
2323 return "%s <a@b>" % userid
2324
06804c76 2325 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
b43702ac
LD
2326 """ Stream a p4 tag.
2327 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2328 """
2329
06804c76
LD
2330 if verbose:
2331 print "writing tag %s for commit %s" % (labelName, commit)
2332 gitStream.write("tag %s\n" % labelName)
2333 gitStream.write("from %s\n" % commit)
2334
2335 if labelDetails.has_key('Owner'):
2336 owner = labelDetails["Owner"]
2337 else:
2338 owner = None
2339
2340 # Try to use the owner of the p4 label, or failing that,
2341 # the current p4 user id.
2342 if owner:
2343 email = self.make_email(owner)
2344 else:
2345 email = self.make_email(self.p4UserId())
2346 tagger = "%s %s %s" % (email, epoch, self.tz)
2347
2348 gitStream.write("tagger %s\n" % tagger)
2349
2350 print "labelDetails=",labelDetails
2351 if labelDetails.has_key('Description'):
2352 description = labelDetails['Description']
2353 else:
2354 description = 'Label from git p4'
2355
2356 gitStream.write("data %d\n" % len(description))
2357 gitStream.write(description)
2358 gitStream.write("\n")
2359
e63231e5 2360 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2361 epoch = details["time"]
2362 author = details["user"]
2363
4b97ffb1
SH
2364 if self.verbose:
2365 print "commit into %s" % branch
2366
96e07dd2
HWN
2367 # start with reading files; if that fails, we should not
2368 # create a commit.
2369 new_files = []
2370 for f in files:
e63231e5 2371 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2372 new_files.append (f)
2373 else:
afa1dd9a 2374 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2375
9d57c4a6
KS
2376 if self.clientSpecDirs:
2377 self.clientSpecDirs.update_client_spec_path_cache(files)
2378
b984733c 2379 self.gitStream.write("commit %s\n" % branch)
b43702ac 2380 self.gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2381 self.committedChanges.add(int(details["change"]))
2382 committer = ""
b607e71e
SH
2383 if author not in self.users:
2384 self.getUserMapFromPerforceServer()
affb474f 2385 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2386
2387 self.gitStream.write("committer %s\n" % committer)
2388
2389 self.gitStream.write("data <<EOT\n")
2390 self.gitStream.write(details["desc"])
e63231e5
PW
2391 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2392 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2393 if len(details['options']) > 0:
2394 self.gitStream.write(": options = %s" % details['options'])
2395 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2396
2397 if len(parent) > 0:
4b97ffb1
SH
2398 if self.verbose:
2399 print "parent %s" % parent
b984733c
SH
2400 self.gitStream.write("from %s\n" % parent)
2401
b932705b 2402 self.streamP4Files(new_files)
b984733c
SH
2403 self.gitStream.write("\n")
2404
1f4ba1cb
SH
2405 change = int(details["change"])
2406
9bda3a85 2407 if self.labels.has_key(change):
1f4ba1cb
SH
2408 label = self.labels[change]
2409 labelDetails = label[0]
2410 labelRevisions = label[1]
71b112d4
SH
2411 if self.verbose:
2412 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2413
6de040df 2414 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2415 for p in self.branchPrefixes])
1f4ba1cb
SH
2416
2417 if len(files) == len(labelRevisions):
2418
2419 cleanedFiles = {}
2420 for info in files:
56c09345 2421 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2422 continue
2423 cleanedFiles[info["depotFile"]] = info["rev"]
2424
2425 if cleanedFiles == labelRevisions:
06804c76 2426 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2427
2428 else:
a46668fa 2429 if not self.silent:
cebdf5af
HWN
2430 print ("Tag %s does not match with change %s: files do not match."
2431 % (labelDetails["label"], change))
1f4ba1cb
SH
2432
2433 else:
a46668fa 2434 if not self.silent:
cebdf5af
HWN
2435 print ("Tag %s does not match with change %s: file count is different."
2436 % (labelDetails["label"], change))
b984733c 2437
06804c76 2438 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2439 def getLabels(self):
2440 self.labels = {}
2441
52a4880b 2442 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2443 if len(l) > 0 and not self.silent:
183f8436 2444 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2445
2446 for output in l:
1f4ba1cb
SH
2447 label = output["label"]
2448 revisions = {}
2449 newestChange = 0
71b112d4
SH
2450 if self.verbose:
2451 print "Querying files for label %s" % label
6de040df
LD
2452 for file in p4CmdList(["files"] +
2453 ["%s...@%s" % (p, label)
2454 for p in self.depotPaths]):
1f4ba1cb
SH
2455 revisions[file["depotFile"]] = file["rev"]
2456 change = int(file["change"])
2457 if change > newestChange:
2458 newestChange = change
2459
9bda3a85
SH
2460 self.labels[newestChange] = [output, revisions]
2461
2462 if self.verbose:
2463 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2464
06804c76
LD
2465 # Import p4 labels as git tags. A direct mapping does not
2466 # exist, so assume that if all the files are at the same revision
2467 # then we can use that, or it's something more complicated we should
2468 # just ignore.
2469 def importP4Labels(self, stream, p4Labels):
2470 if verbose:
2471 print "import p4 labels: " + ' '.join(p4Labels)
2472
2473 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2474 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2475 if len(validLabelRegexp) == 0:
2476 validLabelRegexp = defaultLabelRegexp
2477 m = re.compile(validLabelRegexp)
2478
2479 for name in p4Labels:
2480 commitFound = False
2481
2482 if not m.match(name):
2483 if verbose:
2484 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2485 continue
2486
2487 if name in ignoredP4Labels:
2488 continue
2489
2490 labelDetails = p4CmdList(['label', "-o", name])[0]
2491
2492 # get the most recent changelist for each file in this label
2493 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2494 for p in self.depotPaths])
2495
2496 if change.has_key('change'):
2497 # find the corresponding git commit; take the oldest commit
2498 changelist = int(change['change'])
b43702ac
LD
2499 if changelist in self.committedChanges:
2500 gitCommit = ":%d" % changelist # use a fast-import mark
06804c76 2501 commitFound = True
b43702ac
LD
2502 else:
2503 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2504 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2505 if len(gitCommit) == 0:
2506 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2507 else:
2508 commitFound = True
2509 gitCommit = gitCommit.strip()
2510
2511 if commitFound:
06804c76
LD
2512 # Convert from p4 time format
2513 try:
2514 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2515 except ValueError:
a4e9054c 2516 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2517 tmwhen = 1
2518
2519 when = int(time.mktime(tmwhen))
2520 self.streamTag(stream, name, labelDetails, gitCommit, when)
2521 if verbose:
2522 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2523 else:
2524 if verbose:
2525 print "Label %s has no changelists - possibly deleted?" % name
2526
2527 if not commitFound:
2528 # We can't import this label; don't try again as it will get very
2529 # expensive repeatedly fetching all the files for labels that will
2530 # never be imported. If the label is moved in the future, the
2531 # ignore will need to be removed manually.
2532 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2533
86dff6b6
HWN
2534 def guessProjectName(self):
2535 for p in self.depotPaths:
6e5295c4
SH
2536 if p.endswith("/"):
2537 p = p[:-1]
2538 p = p[p.strip().rfind("/") + 1:]
2539 if not p.endswith("/"):
2540 p += "/"
2541 return p
86dff6b6 2542
4b97ffb1 2543 def getBranchMapping(self):
6555b2cc
SH
2544 lostAndFoundBranches = set()
2545
8ace74c0
VA
2546 user = gitConfig("git-p4.branchUser")
2547 if len(user) > 0:
2548 command = "branches -u %s" % user
2549 else:
2550 command = "branches"
2551
2552 for info in p4CmdList(command):
52a4880b 2553 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2554 viewIdx = 0
2555 while details.has_key("View%s" % viewIdx):
2556 paths = details["View%s" % viewIdx].split(" ")
2557 viewIdx = viewIdx + 1
2558 # require standard //depot/foo/... //depot/bar/... mapping
2559 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2560 continue
2561 source = paths[0]
2562 destination = paths[1]
6509e19c 2563 ## HACK
d53de8b9 2564 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2565 source = source[len(self.depotPaths[0]):-4]
2566 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2567
1a2edf4e
SH
2568 if destination in self.knownBranches:
2569 if not self.silent:
2570 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2571 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2572 continue
2573
6555b2cc
SH
2574 self.knownBranches[destination] = source
2575
2576 lostAndFoundBranches.discard(destination)
2577
29bdbac1 2578 if source not in self.knownBranches:
6555b2cc
SH
2579 lostAndFoundBranches.add(source)
2580
7199cf13
VA
2581 # Perforce does not strictly require branches to be defined, so we also
2582 # check git config for a branch list.
2583 #
2584 # Example of branch definition in git config file:
2585 # [git-p4]
2586 # branchList=main:branchA
2587 # branchList=main:branchB
2588 # branchList=branchA:branchC
2589 configBranches = gitConfigList("git-p4.branchList")
2590 for branch in configBranches:
2591 if branch:
2592 (source, destination) = branch.split(":")
2593 self.knownBranches[destination] = source
2594
2595 lostAndFoundBranches.discard(destination)
2596
2597 if source not in self.knownBranches:
2598 lostAndFoundBranches.add(source)
2599
6555b2cc
SH
2600
2601 for branch in lostAndFoundBranches:
2602 self.knownBranches[branch] = branch
29bdbac1 2603
38f9f5ec
SH
2604 def getBranchMappingFromGitBranches(self):
2605 branches = p4BranchesInGit(self.importIntoRemotes)
2606 for branch in branches.keys():
2607 if branch == "master":
2608 branch = "main"
2609 else:
2610 branch = branch[len(self.projectName):]
2611 self.knownBranches[branch] = branch
2612
bb6e09b2
HWN
2613 def updateOptionDict(self, d):
2614 option_keys = {}
2615 if self.keepRepoPath:
2616 option_keys['keepRepoPath'] = 1
2617
2618 d["options"] = ' '.join(sorted(option_keys.keys()))
2619
2620 def readOptions(self, d):
2621 self.keepRepoPath = (d.has_key('options')
2622 and ('keepRepoPath' in d['options']))
6326aa58 2623
8134f69c
SH
2624 def gitRefForBranch(self, branch):
2625 if branch == "main":
2626 return self.refPrefix + "master"
2627
2628 if len(branch) <= 0:
2629 return branch
2630
2631 return self.refPrefix + self.projectName + branch
2632
1ca3d710
SH
2633 def gitCommitByP4Change(self, ref, change):
2634 if self.verbose:
2635 print "looking in ref " + ref + " for change %s using bisect..." % change
2636
2637 earliestCommit = ""
2638 latestCommit = parseRevision(ref)
2639
2640 while True:
2641 if self.verbose:
2642 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2643 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2644 if len(next) == 0:
2645 if self.verbose:
2646 print "argh"
2647 return ""
2648 log = extractLogMessageFromGitCommit(next)
2649 settings = extractSettingsGitLog(log)
2650 currentChange = int(settings['change'])
2651 if self.verbose:
2652 print "current change %s" % currentChange
2653
2654 if currentChange == change:
2655 if self.verbose:
2656 print "found %s" % next
2657 return next
2658
2659 if currentChange < change:
2660 earliestCommit = "^%s" % next
2661 else:
2662 latestCommit = "%s" % next
2663
2664 return ""
2665
2666 def importNewBranch(self, branch, maxChange):
2667 # make fast-import flush all changes to disk and update the refs using the checkpoint
2668 # command so that we can try to find the branch parent in the git history
2669 self.gitStream.write("checkpoint\n\n");
2670 self.gitStream.flush();
2671 branchPrefix = self.depotPaths[0] + branch + "/"
2672 range = "@1,%s" % maxChange
2673 #print "prefix" + branchPrefix
96b2d54a 2674 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
2675 if len(changes) <= 0:
2676 return False
2677 firstChange = changes[0]
2678 #print "first change in branch: %s" % firstChange
2679 sourceBranch = self.knownBranches[branch]
2680 sourceDepotPath = self.depotPaths[0] + sourceBranch
2681 sourceRef = self.gitRefForBranch(sourceBranch)
2682 #print "source " + sourceBranch
2683
52a4880b 2684 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2685 #print "branch parent: %s" % branchParentChange
2686 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2687 if len(gitParent) > 0:
2688 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2689 #print "parent git commit: %s" % gitParent
2690
2691 self.importChanges(changes)
2692 return True
2693
fed23693
VA
2694 def searchParent(self, parent, branch, target):
2695 parentFound = False
c7d34884
PW
2696 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2697 "--no-merges", parent]):
fed23693
VA
2698 blob = blob.strip()
2699 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2700 parentFound = True
2701 if self.verbose:
2702 print "Found parent of %s in commit %s" % (branch, blob)
2703 break
2704 if parentFound:
2705 return blob
2706 else:
2707 return None
2708
e87f37ae
SH
2709 def importChanges(self, changes):
2710 cnt = 1
2711 for change in changes:
18fa13d0 2712 description = p4_describe(change)
e87f37ae
SH
2713 self.updateOptionDict(description)
2714
2715 if not self.silent:
2716 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2717 sys.stdout.flush()
2718 cnt = cnt + 1
2719
2720 try:
2721 if self.detectBranches:
2722 branches = self.splitFilesIntoBranches(description)
2723 for branch in branches.keys():
2724 ## HACK --hwn
2725 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 2726 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
2727
2728 parent = ""
2729
2730 filesForCommit = branches[branch]
2731
2732 if self.verbose:
2733 print "branch is %s" % branch
2734
2735 self.updatedBranches.add(branch)
2736
2737 if branch not in self.createdBranches:
2738 self.createdBranches.add(branch)
2739 parent = self.knownBranches[branch]
2740 if parent == branch:
2741 parent = ""
1ca3d710
SH
2742 else:
2743 fullBranch = self.projectName + branch
2744 if fullBranch not in self.p4BranchesInGit:
2745 if not self.silent:
2746 print("\n Importing new branch %s" % fullBranch);
2747 if self.importNewBranch(branch, change - 1):
2748 parent = ""
2749 self.p4BranchesInGit.append(fullBranch)
2750 if not self.silent:
2751 print("\n Resuming with change %s" % change);
2752
2753 if self.verbose:
2754 print "parent determined through known branches: %s" % parent
e87f37ae 2755
8134f69c
SH
2756 branch = self.gitRefForBranch(branch)
2757 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2758
2759 if self.verbose:
2760 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2761
2762 if len(parent) == 0 and branch in self.initialParents:
2763 parent = self.initialParents[branch]
2764 del self.initialParents[branch]
2765
fed23693
VA
2766 blob = None
2767 if len(parent) > 0:
4f9273d2 2768 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
2769 if self.verbose:
2770 print "Creating temporary branch: " + tempBranch
e63231e5 2771 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
2772 self.tempBranches.append(tempBranch)
2773 self.checkpoint()
2774 blob = self.searchParent(parent, branch, tempBranch)
2775 if blob:
e63231e5 2776 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
2777 else:
2778 if self.verbose:
2779 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 2780 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
2781 else:
2782 files = self.extractFilesFromCommit(description)
e63231e5 2783 self.commit(description, files, self.branch,
e87f37ae 2784 self.initialParent)
47497844 2785 # only needed once, to connect to the previous commit
e87f37ae
SH
2786 self.initialParent = ""
2787 except IOError:
2788 print self.gitError.read()
2789 sys.exit(1)
2790
c208a243
SH
2791 def importHeadRevision(self, revision):
2792 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2793
4e2e6ce4
PW
2794 details = {}
2795 details["user"] = "git perforce import user"
1494fcbb 2796 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2797 % (' '.join(self.depotPaths), revision))
2798 details["change"] = revision
2799 newestRevision = 0
2800
2801 fileCnt = 0
6de040df
LD
2802 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2803
2804 for info in p4CmdList(["files"] + fileArgs):
c208a243 2805
68b28593 2806 if 'code' in info and info['code'] == 'error':
c208a243
SH
2807 sys.stderr.write("p4 returned an error: %s\n"
2808 % info['data'])
d88e707f
PW
2809 if info['data'].find("must refer to client") >= 0:
2810 sys.stderr.write("This particular p4 error is misleading.\n")
2811 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2812 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2813 sys.exit(1)
68b28593
PW
2814 if 'p4ExitCode' in info:
2815 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2816 sys.exit(1)
2817
2818
2819 change = int(info["change"])
2820 if change > newestRevision:
2821 newestRevision = change
2822
56c09345 2823 if info["action"] in self.delete_actions:
c208a243
SH
2824 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2825 #fileCnt = fileCnt + 1
2826 continue
2827
2828 for prop in ["depotFile", "rev", "action", "type" ]:
2829 details["%s%s" % (prop, fileCnt)] = info[prop]
2830
2831 fileCnt = fileCnt + 1
2832
2833 details["change"] = newestRevision
4e2e6ce4 2834
9dcb9f24 2835 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 2836 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
2837 res = p4_describe(newestRevision)
2838 details["time"] = res["time"]
4e2e6ce4 2839
c208a243
SH
2840 self.updateOptionDict(details)
2841 try:
e63231e5 2842 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
2843 except IOError:
2844 print "IO error with git fast-import. Is your git version recent enough?"
2845 print self.gitError.read()
2846
2847
b984733c 2848 def run(self, args):
6326aa58 2849 self.depotPaths = []
179caebf 2850 self.changeRange = ""
6326aa58 2851 self.previousDepotPaths = []
991a2de4 2852 self.hasOrigin = False
ce6f33c8 2853
29bdbac1
SH
2854 # map from branch depot path to parent branch
2855 self.knownBranches = {}
2856 self.initialParents = {}
2857
a028a98e
SH
2858 if self.importIntoRemotes:
2859 self.refPrefix = "refs/remotes/p4/"
2860 else:
db775559 2861 self.refPrefix = "refs/heads/p4/"
a028a98e 2862
991a2de4
PW
2863 if self.syncWithOrigin:
2864 self.hasOrigin = originP4BranchesExist()
2865 if self.hasOrigin:
2866 if not self.silent:
2867 print 'Syncing with origin first, using "git fetch origin"'
2868 system("git fetch origin")
10f880f8 2869
5a8e84cd 2870 branch_arg_given = bool(self.branch)
569d1bd4 2871 if len(self.branch) == 0:
db775559 2872 self.branch = self.refPrefix + "master"
a028a98e 2873 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2874 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 2875 system("git branch -D p4")
967f72e2 2876
a93d33ee
PW
2877 # accept either the command-line option, or the configuration variable
2878 if self.useClientSpec:
2879 # will use this after clone to set the variable
2880 self.useClientSpec_from_options = True
2881 else:
0d609032 2882 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
2883 self.useClientSpec = True
2884 if self.useClientSpec:
543987bd 2885 self.clientSpecDirs = getClientSpec()
3a70cdfa 2886
6a49f8e2
HWN
2887 # TODO: should always look at previous commits,
2888 # merge with previous imports, if possible.
2889 if args == []:
d414c74a 2890 if self.hasOrigin:
5ca44617 2891 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
2892
2893 # branches holds mapping from branch name to sha1
2894 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
2895
2896 # restrict to just this one, disabling detect-branches
2897 if branch_arg_given:
2898 short = self.branch.split("/")[-1]
2899 if short in branches:
2900 self.p4BranchesInGit = [ short ]
2901 else:
2902 self.p4BranchesInGit = branches.keys()
abcd790f
SH
2903
2904 if len(self.p4BranchesInGit) > 1:
2905 if not self.silent:
2906 print "Importing from/into multiple branches"
2907 self.detectBranches = True
8c9e8b6e
PW
2908 for branch in branches.keys():
2909 self.initialParents[self.refPrefix + branch] = \
2910 branches[branch]
967f72e2 2911
29bdbac1
SH
2912 if self.verbose:
2913 print "branches: %s" % self.p4BranchesInGit
2914
2915 p4Change = 0
2916 for branch in self.p4BranchesInGit:
cebdf5af 2917 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2918
2919 settings = extractSettingsGitLog(logMsg)
29bdbac1 2920
bb6e09b2
HWN
2921 self.readOptions(settings)
2922 if (settings.has_key('depot-paths')
2923 and settings.has_key ('change')):
2924 change = int(settings['change']) + 1
29bdbac1
SH
2925 p4Change = max(p4Change, change)
2926
bb6e09b2
HWN
2927 depotPaths = sorted(settings['depot-paths'])
2928 if self.previousDepotPaths == []:
6326aa58 2929 self.previousDepotPaths = depotPaths
29bdbac1 2930 else:
6326aa58
HWN
2931 paths = []
2932 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2933 prev_list = prev.split("/")
2934 cur_list = cur.split("/")
2935 for i in range(0, min(len(cur_list), len(prev_list))):
2936 if cur_list[i] <> prev_list[i]:
583e1707 2937 i = i - 1
6326aa58
HWN
2938 break
2939
04d277b3 2940 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2941
2942 self.previousDepotPaths = paths
29bdbac1
SH
2943
2944 if p4Change > 0:
bb6e09b2 2945 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2946 self.changeRange = "@%s,#head" % p4Change
341dc1c1 2947 if not self.silent and not self.detectBranches:
967f72e2 2948 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2949
40d69ac3
PW
2950 # accept multiple ref name abbreviations:
2951 # refs/foo/bar/branch -> use it exactly
2952 # p4/branch -> prepend refs/remotes/ or refs/heads/
2953 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 2954 if not self.branch.startswith("refs/"):
40d69ac3
PW
2955 if self.importIntoRemotes:
2956 prepend = "refs/remotes/"
2957 else:
2958 prepend = "refs/heads/"
2959 if not self.branch.startswith("p4/"):
2960 prepend += "p4/"
2961 self.branch = prepend + self.branch
179caebf 2962
6326aa58 2963 if len(args) == 0 and self.depotPaths:
b984733c 2964 if not self.silent:
6326aa58 2965 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2966 else:
6326aa58 2967 if self.depotPaths and self.depotPaths != args:
cebdf5af 2968 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2969 "This doesn't work!" % (' '.join (self.depotPaths),
2970 ' '.join (args)))
b984733c 2971 sys.exit(1)
6326aa58 2972
bb6e09b2 2973 self.depotPaths = sorted(args)
b984733c 2974
1c49fc19 2975 revision = ""
b984733c 2976 self.users = {}
b984733c 2977
58c8bc7c
PW
2978 # Make sure no revision specifiers are used when --changesfile
2979 # is specified.
2980 bad_changesfile = False
2981 if len(self.changesFile) > 0:
2982 for p in self.depotPaths:
2983 if p.find("@") >= 0 or p.find("#") >= 0:
2984 bad_changesfile = True
2985 break
2986 if bad_changesfile:
2987 die("Option --changesfile is incompatible with revision specifiers")
2988
6326aa58
HWN
2989 newPaths = []
2990 for p in self.depotPaths:
2991 if p.find("@") != -1:
2992 atIdx = p.index("@")
2993 self.changeRange = p[atIdx:]
2994 if self.changeRange == "@all":
2995 self.changeRange = ""
6a49f8e2 2996 elif ',' not in self.changeRange:
1c49fc19 2997 revision = self.changeRange
6326aa58 2998 self.changeRange = ""
7fcff9de 2999 p = p[:atIdx]
6326aa58
HWN
3000 elif p.find("#") != -1:
3001 hashIdx = p.index("#")
1c49fc19 3002 revision = p[hashIdx:]
7fcff9de 3003 p = p[:hashIdx]
6326aa58 3004 elif self.previousDepotPaths == []:
58c8bc7c
PW
3005 # pay attention to changesfile, if given, else import
3006 # the entire p4 tree at the head revision
3007 if len(self.changesFile) == 0:
3008 revision = "#head"
6326aa58
HWN
3009
3010 p = re.sub ("\.\.\.$", "", p)
3011 if not p.endswith("/"):
3012 p += "/"
3013
3014 newPaths.append(p)
3015
3016 self.depotPaths = newPaths
3017
e63231e5
PW
3018 # --detect-branches may change this for each branch
3019 self.branchPrefixes = self.depotPaths
3020
b607e71e 3021 self.loadUserMapFromCache()
cb53e1f8
SH
3022 self.labels = {}
3023 if self.detectLabels:
3024 self.getLabels();
b984733c 3025
4b97ffb1 3026 if self.detectBranches:
df450923
SH
3027 ## FIXME - what's a P4 projectName ?
3028 self.projectName = self.guessProjectName()
3029
38f9f5ec
SH
3030 if self.hasOrigin:
3031 self.getBranchMappingFromGitBranches()
3032 else:
3033 self.getBranchMapping()
29bdbac1
SH
3034 if self.verbose:
3035 print "p4-git branches: %s" % self.p4BranchesInGit
3036 print "initial parents: %s" % self.initialParents
3037 for b in self.p4BranchesInGit:
3038 if b != "master":
6326aa58
HWN
3039
3040 ## FIXME
29bdbac1
SH
3041 b = b[len(self.projectName):]
3042 self.createdBranches.add(b)
4b97ffb1 3043
f291b4e3 3044 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3045
78189bea
PW
3046 self.importProcess = subprocess.Popen(["git", "fast-import"],
3047 stdin=subprocess.PIPE,
3048 stdout=subprocess.PIPE,
3049 stderr=subprocess.PIPE);
3050 self.gitOutput = self.importProcess.stdout
3051 self.gitStream = self.importProcess.stdin
3052 self.gitError = self.importProcess.stderr
b984733c 3053
1c49fc19 3054 if revision:
c208a243 3055 self.importHeadRevision(revision)
b984733c
SH
3056 else:
3057 changes = []
3058
0828ab14 3059 if len(self.changesFile) > 0:
b984733c 3060 output = open(self.changesFile).readlines()
1d7367dc 3061 changeSet = set()
b984733c
SH
3062 for line in output:
3063 changeSet.add(int(line))
3064
3065 for change in changeSet:
3066 changes.append(change)
3067
3068 changes.sort()
3069 else:
9dcb9f24
PW
3070 # catch "git p4 sync" with no new branches, in a repo that
3071 # does not have any existing p4 branches
5a8e84cd
PW
3072 if len(args) == 0:
3073 if not self.p4BranchesInGit:
3074 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3075
3076 # The default branch is master, unless --branch is used to
3077 # specify something else. Make sure it exists, or complain
3078 # nicely about how to use --branch.
3079 if not self.detectBranches:
3080 if not branch_exists(self.branch):
3081 if branch_arg_given:
3082 die("Error: branch %s does not exist." % self.branch)
3083 else:
3084 die("Error: no branch %s; perhaps specify one with --branch." %
3085 self.branch)
3086
29bdbac1 3087 if self.verbose:
86dff6b6 3088 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3089 self.changeRange)
96b2d54a 3090 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
b984733c 3091
01a9c9c5 3092 if len(self.maxChanges) > 0:
7fcff9de 3093 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3094
b984733c 3095 if len(changes) == 0:
0828ab14 3096 if not self.silent:
341dc1c1 3097 print "No changes to import!"
06804c76
LD
3098 else:
3099 if not self.silent and not self.detectBranches:
3100 print "Import destination: %s" % self.branch
3101
3102 self.updatedBranches = set()
b984733c 3103
47497844
PW
3104 if not self.detectBranches:
3105 if args:
3106 # start a new branch
3107 self.initialParent = ""
3108 else:
3109 # build on a previous revision
3110 self.initialParent = parseRevision(self.branch)
3111
06804c76 3112 self.importChanges(changes)
a9d1a27a 3113
06804c76
LD
3114 if not self.silent:
3115 print ""
3116 if len(self.updatedBranches) > 0:
3117 sys.stdout.write("Updated branches: ")
3118 for b in self.updatedBranches:
3119 sys.stdout.write("%s " % b)
3120 sys.stdout.write("\n")
341dc1c1 3121
0d609032 3122 if gitConfigBool("git-p4.importLabels"):
06dcd152 3123 self.importLabels = True
b984733c 3124
06804c76
LD
3125 if self.importLabels:
3126 p4Labels = getP4Labels(self.depotPaths)
3127 gitTags = getGitTags()
3128
3129 missingP4Labels = p4Labels - gitTags
3130 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3131
b984733c 3132 self.gitStream.close()
78189bea 3133 if self.importProcess.wait() != 0:
29bdbac1 3134 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3135 self.gitOutput.close()
3136 self.gitError.close()
3137
fed23693
VA
3138 # Cleanup temporary branches created during import
3139 if self.tempBranches != []:
3140 for branch in self.tempBranches:
3141 read_pipe("git update-ref -d %s" % branch)
3142 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3143
55d12437
PW
3144 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3145 # a convenient shortcut refname "p4".
3146 if self.importIntoRemotes:
3147 head_ref = self.refPrefix + "HEAD"
3148 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3149 system(["git", "symbolic-ref", head_ref, self.branch])
3150
b984733c
SH
3151 return True
3152
01ce1fe9
SH
3153class P4Rebase(Command):
3154 def __init__(self):
3155 Command.__init__(self)
06804c76
LD
3156 self.options = [
3157 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3158 ]
06804c76 3159 self.importLabels = False
cebdf5af
HWN
3160 self.description = ("Fetches the latest revision from perforce and "
3161 + "rebases the current work (branch) against it")
01ce1fe9
SH
3162
3163 def run(self, args):
3164 sync = P4Sync()
06804c76 3165 sync.importLabels = self.importLabels
01ce1fe9 3166 sync.run([])
d7e3868c 3167
14594f4b
SH
3168 return self.rebase()
3169
3170 def rebase(self):
36ee4ee4
SH
3171 if os.system("git update-index --refresh") != 0:
3172 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3173 if len(read_pipe("git diff-index HEAD --")) > 0:
f7e604ed 3174 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
36ee4ee4 3175
d7e3868c
SH
3176 [upstream, settings] = findUpstreamBranchPoint()
3177 if len(upstream) == 0:
3178 die("Cannot find upstream branchpoint for rebase")
3179
3180 # the branchpoint may be p4/foo~3, so strip off the parent
3181 upstream = re.sub("~[0-9]+$", "", upstream)
3182
3183 print "Rebasing the current branch onto %s" % upstream
b25b2065 3184 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 3185 system("git rebase %s" % upstream)
4e49d95e 3186 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
01ce1fe9
SH
3187 return True
3188
f9a3a4f7
SH
3189class P4Clone(P4Sync):
3190 def __init__(self):
3191 P4Sync.__init__(self)
3192 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 3193 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 3194 self.options += [
bb6e09b2
HWN
3195 optparse.make_option("--destination", dest="cloneDestination",
3196 action='store', default=None,
354081d5 3197 help="where to leave result of the clone"),
38200076
PW
3198 optparse.make_option("--bare", dest="cloneBare",
3199 action="store_true", default=False),
354081d5 3200 ]
bb6e09b2 3201 self.cloneDestination = None
f9a3a4f7 3202 self.needsGit = False
38200076 3203 self.cloneBare = False
f9a3a4f7 3204
6a49f8e2
HWN
3205 def defaultDestination(self, args):
3206 ## TODO: use common prefix of args?
3207 depotPath = args[0]
3208 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3209 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 3210 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
3211 depotDir = re.sub(r"/$", "", depotDir)
3212 return os.path.split(depotDir)[1]
3213
f9a3a4f7
SH
3214 def run(self, args):
3215 if len(args) < 1:
3216 return False
bb6e09b2
HWN
3217
3218 if self.keepRepoPath and not self.cloneDestination:
3219 sys.stderr.write("Must specify destination for --keep-path\n")
3220 sys.exit(1)
f9a3a4f7 3221
6326aa58 3222 depotPaths = args
5e100b5c
SH
3223
3224 if not self.cloneDestination and len(depotPaths) > 1:
3225 self.cloneDestination = depotPaths[-1]
3226 depotPaths = depotPaths[:-1]
3227
354081d5 3228 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
3229 for p in depotPaths:
3230 if not p.startswith("//"):
0f487d30 3231 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
6326aa58 3232 return False
f9a3a4f7 3233
bb6e09b2 3234 if not self.cloneDestination:
98ad4faf 3235 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 3236
86dff6b6 3237 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 3238
c3bf3f13
KG
3239 if not os.path.exists(self.cloneDestination):
3240 os.makedirs(self.cloneDestination)
053fd0c1 3241 chdir(self.cloneDestination)
38200076
PW
3242
3243 init_cmd = [ "git", "init" ]
3244 if self.cloneBare:
3245 init_cmd.append("--bare")
a235e85c
BC
3246 retcode = subprocess.call(init_cmd)
3247 if retcode:
3248 raise CalledProcessError(retcode, init_cmd)
38200076 3249
6326aa58 3250 if not P4Sync.run(self, depotPaths):
f9a3a4f7 3251 return False
c595956d
PW
3252
3253 # create a master branch and check out a work tree
3254 if gitBranchExists(self.branch):
3255 system([ "git", "branch", "master", self.branch ])
3256 if not self.cloneBare:
3257 system([ "git", "checkout", "-f" ])
3258 else:
3259 print 'Not checking out any branch, use ' \
3260 '"git checkout -q -b master <branch>"'
86dff6b6 3261
a93d33ee
PW
3262 # auto-set this variable if invoked with --use-client-spec
3263 if self.useClientSpec_from_options:
3264 system("git config --bool git-p4.useclientspec true")
3265
f9a3a4f7
SH
3266 return True
3267
09d89de2
SH
3268class P4Branches(Command):
3269 def __init__(self):
3270 Command.__init__(self)
3271 self.options = [ ]
3272 self.description = ("Shows the git branches that hold imports and their "
3273 + "corresponding perforce depot paths")
3274 self.verbose = False
3275
3276 def run(self, args):
5ca44617
SH
3277 if originP4BranchesExist():
3278 createOrUpdateBranchesFromOrigin()
3279
09d89de2
SH
3280 cmdline = "git rev-parse --symbolic "
3281 cmdline += " --remotes"
3282
3283 for line in read_pipe_lines(cmdline):
3284 line = line.strip()
3285
3286 if not line.startswith('p4/') or line == "p4/HEAD":
3287 continue
3288 branch = line
3289
3290 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3291 settings = extractSettingsGitLog(log)
3292
3293 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3294 return True
3295
b984733c
SH
3296class HelpFormatter(optparse.IndentedHelpFormatter):
3297 def __init__(self):
3298 optparse.IndentedHelpFormatter.__init__(self)
3299
3300 def format_description(self, description):
3301 if description:
3302 return description + "\n"
3303 else:
3304 return ""
4f5cf76a 3305
86949eef
SH
3306def printUsage(commands):
3307 print "usage: %s <command> [options]" % sys.argv[0]
3308 print ""
3309 print "valid commands: %s" % ", ".join(commands)
3310 print ""
3311 print "Try %s <command> --help for command specific help." % sys.argv[0]
3312 print ""
3313
3314commands = {
b86f7378
HWN
3315 "debug" : P4Debug,
3316 "submit" : P4Submit,
a9834f58 3317 "commit" : P4Submit,
b86f7378
HWN
3318 "sync" : P4Sync,
3319 "rebase" : P4Rebase,
3320 "clone" : P4Clone,
09d89de2
SH
3321 "rollback" : P4RollBack,
3322 "branches" : P4Branches
86949eef
SH
3323}
3324
86949eef 3325
bb6e09b2
HWN
3326def main():
3327 if len(sys.argv[1:]) == 0:
3328 printUsage(commands.keys())
3329 sys.exit(2)
4f5cf76a 3330
bb6e09b2
HWN
3331 cmdName = sys.argv[1]
3332 try:
b86f7378
HWN
3333 klass = commands[cmdName]
3334 cmd = klass()
bb6e09b2
HWN
3335 except KeyError:
3336 print "unknown command %s" % cmdName
3337 print ""
3338 printUsage(commands.keys())
3339 sys.exit(2)
3340
3341 options = cmd.options
b86f7378 3342 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
3343
3344 args = sys.argv[2:]
3345
b0ccc80d 3346 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
6a10b6aa
LD
3347 if cmd.needsGit:
3348 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 3349
6a10b6aa
LD
3350 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3351 options,
3352 description = cmd.description,
3353 formatter = HelpFormatter())
bb6e09b2 3354
6a10b6aa 3355 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
bb6e09b2
HWN
3356 global verbose
3357 verbose = cmd.verbose
3358 if cmd.needsGit:
b86f7378
HWN
3359 if cmd.gitdir == None:
3360 cmd.gitdir = os.path.abspath(".git")
3361 if not isValidGitDir(cmd.gitdir):
3362 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3363 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
3364 cdup = read_pipe("git rev-parse --show-cdup").strip()
3365 if len(cdup) > 0:
053fd0c1 3366 chdir(cdup);
e20a9e53 3367
b86f7378
HWN
3368 if not isValidGitDir(cmd.gitdir):
3369 if isValidGitDir(cmd.gitdir + "/.git"):
3370 cmd.gitdir += "/.git"
bb6e09b2 3371 else:
b86f7378 3372 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 3373
b86f7378 3374 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 3375
bb6e09b2
HWN
3376 if not cmd.run(args):
3377 parser.print_help()
09fca77b 3378 sys.exit(2)
4f5cf76a 3379
4f5cf76a 3380
bb6e09b2
HWN
3381if __name__ == '__main__':
3382 main()