git-p4: return an empty list if a list config has no values
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
8b41a97f 25
a235e85c
BC
26try:
27 from subprocess import CalledProcessError
28except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
4addad22 41verbose = False
86949eef 42
06804c76 43# Only labels/tags matching this will be imported/exported
c8942a22 44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753 45
1051ef00
LD
46# Grab changes in blocks of this many revisions, unless otherwise requested
47defaultBlockSize = 512
48
21a50753
AK
49def p4_build_cmd(cmd):
50 """Build a suitable p4 command line.
51
52 This consolidates building and returning a p4 command line into one
53 location. It means that hooking into the environment, or other configuration
54 can be done more easily.
55 """
6de040df 56 real_cmd = ["p4"]
abcaf073
AK
57
58 user = gitConfig("git-p4.user")
59 if len(user) > 0:
6de040df 60 real_cmd += ["-u",user]
abcaf073
AK
61
62 password = gitConfig("git-p4.password")
63 if len(password) > 0:
6de040df 64 real_cmd += ["-P", password]
abcaf073
AK
65
66 port = gitConfig("git-p4.port")
67 if len(port) > 0:
6de040df 68 real_cmd += ["-p", port]
abcaf073
AK
69
70 host = gitConfig("git-p4.host")
71 if len(host) > 0:
41799aa2 72 real_cmd += ["-H", host]
abcaf073
AK
73
74 client = gitConfig("git-p4.client")
75 if len(client) > 0:
6de040df 76 real_cmd += ["-c", client]
abcaf073 77
6de040df
LD
78
79 if isinstance(cmd,basestring):
80 real_cmd = ' '.join(real_cmd) + ' ' + cmd
81 else:
82 real_cmd += cmd
21a50753
AK
83 return real_cmd
84
bbd84863
MF
85def chdir(path, is_client_path=False):
86 """Do chdir to the given path, and set the PWD environment
87 variable for use by P4. It does not look at getcwd() output.
88 Since we're not using the shell, it is necessary to set the
89 PWD environment variable explicitly.
90
91 Normally, expand the path to force it to be absolute. This
92 addresses the use of relative path names inside P4 settings,
93 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
94 as given; it looks for .p4config using PWD.
95
96 If is_client_path, the path was handed to us directly by p4,
97 and may be a symbolic link. Do not call os.getcwd() in this
98 case, because it will cause p4 to think that PWD is not inside
99 the client path.
100 """
101
102 os.chdir(path)
103 if not is_client_path:
104 path = os.getcwd()
105 os.environ['PWD'] = path
053fd0c1 106
86dff6b6
HWN
107def die(msg):
108 if verbose:
109 raise Exception(msg)
110 else:
111 sys.stderr.write(msg + "\n")
112 sys.exit(1)
113
6de040df 114def write_pipe(c, stdin):
4addad22 115 if verbose:
6de040df 116 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 117
6de040df
LD
118 expand = isinstance(c,basestring)
119 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
120 pipe = p.stdin
121 val = pipe.write(stdin)
122 pipe.close()
123 if p.wait():
124 die('Command failed: %s' % str(c))
b016d397
HWN
125
126 return val
127
6de040df 128def p4_write_pipe(c, stdin):
d9429194 129 real_cmd = p4_build_cmd(c)
6de040df 130 return write_pipe(real_cmd, stdin)
d9429194 131
4addad22
HWN
132def read_pipe(c, ignore_error=False):
133 if verbose:
6de040df 134 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 135
6de040df
LD
136 expand = isinstance(c,basestring)
137 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
138 pipe = p.stdout
b016d397 139 val = pipe.read()
6de040df
LD
140 if p.wait() and not ignore_error:
141 die('Command failed: %s' % str(c))
b016d397
HWN
142
143 return val
144
d9429194
AK
145def p4_read_pipe(c, ignore_error=False):
146 real_cmd = p4_build_cmd(c)
147 return read_pipe(real_cmd, ignore_error)
b016d397 148
bce4c5fc 149def read_pipe_lines(c):
4addad22 150 if verbose:
6de040df
LD
151 sys.stderr.write('Reading pipe: %s\n' % str(c))
152
153 expand = isinstance(c, basestring)
154 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
155 pipe = p.stdout
b016d397 156 val = pipe.readlines()
6de040df
LD
157 if pipe.close() or p.wait():
158 die('Command failed: %s' % str(c))
b016d397
HWN
159
160 return val
caace111 161
2318121b
AK
162def p4_read_pipe_lines(c):
163 """Specifically invoke p4 on the command supplied. """
155af834 164 real_cmd = p4_build_cmd(c)
2318121b
AK
165 return read_pipe_lines(real_cmd)
166
8e9497c2
GG
167def p4_has_command(cmd):
168 """Ask p4 for help on this command. If it returns an error, the
169 command does not exist in this version of p4."""
170 real_cmd = p4_build_cmd(["help", cmd])
171 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
172 stderr=subprocess.PIPE)
173 p.communicate()
174 return p.returncode == 0
175
249da4c0
PW
176def p4_has_move_command():
177 """See if the move command exists, that it supports -k, and that
178 it has not been administratively disabled. The arguments
179 must be correct, but the filenames do not have to exist. Use
180 ones with wildcards so even if they exist, it will fail."""
181
182 if not p4_has_command("move"):
183 return False
184 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
185 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
186 (out, err) = p.communicate()
187 # return code will be 1 in either case
188 if err.find("Invalid option") >= 0:
189 return False
190 if err.find("disabled") >= 0:
191 return False
192 # assume it failed because @... was invalid changelist
193 return True
194
6754a299 195def system(cmd):
6de040df 196 expand = isinstance(cmd,basestring)
4addad22 197 if verbose:
6de040df 198 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c
BC
199 retcode = subprocess.call(cmd, shell=expand)
200 if retcode:
201 raise CalledProcessError(retcode, cmd)
6754a299 202
bf9320f1
AK
203def p4_system(cmd):
204 """Specifically invoke p4 as the system command. """
155af834 205 real_cmd = p4_build_cmd(cmd)
6de040df 206 expand = isinstance(real_cmd, basestring)
a235e85c
BC
207 retcode = subprocess.call(real_cmd, shell=expand)
208 if retcode:
209 raise CalledProcessError(retcode, real_cmd)
6de040df 210
7f0e5962
PW
211_p4_version_string = None
212def p4_version_string():
213 """Read the version string, showing just the last line, which
214 hopefully is the interesting version bit.
215
216 $ p4 -V
217 Perforce - The Fast Software Configuration Management System.
218 Copyright 1995-2011 Perforce Software. All rights reserved.
219 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
220 """
221 global _p4_version_string
222 if not _p4_version_string:
223 a = p4_read_pipe_lines(["-V"])
224 _p4_version_string = a[-1].rstrip()
225 return _p4_version_string
226
6de040df 227def p4_integrate(src, dest):
9d7d446a 228 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 229
8d7ec362 230def p4_sync(f, *options):
9d7d446a 231 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
232
233def p4_add(f):
9d7d446a
PW
234 # forcibly add file names with wildcards
235 if wildcard_present(f):
236 p4_system(["add", "-f", f])
237 else:
238 p4_system(["add", f])
6de040df
LD
239
240def p4_delete(f):
9d7d446a 241 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
242
243def p4_edit(f):
9d7d446a 244 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
245
246def p4_revert(f):
9d7d446a 247 p4_system(["revert", wildcard_encode(f)])
6de040df 248
9d7d446a
PW
249def p4_reopen(type, f):
250 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 251
8e9497c2
GG
252def p4_move(src, dest):
253 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
254
1051ef00
LD
255def p4_last_change():
256 results = p4CmdList(["changes", "-m", "1"])
257 return int(results[0]['change'])
258
18fa13d0
PW
259def p4_describe(change):
260 """Make sure it returns a valid result by checking for
261 the presence of field "time". Return a dict of the
262 results."""
263
264 ds = p4CmdList(["describe", "-s", str(change)])
265 if len(ds) != 1:
266 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
267
268 d = ds[0]
269
270 if "p4ExitCode" in d:
271 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
272 str(d)))
273 if "code" in d:
274 if d["code"] == "error":
275 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
276
277 if "time" not in d:
278 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
279
280 return d
281
9cffb8c8
PW
282#
283# Canonicalize the p4 type and return a tuple of the
284# base type, plus any modifiers. See "p4 help filetypes"
285# for a list and explanation.
286#
287def split_p4_type(p4type):
288
289 p4_filetypes_historical = {
290 "ctempobj": "binary+Sw",
291 "ctext": "text+C",
292 "cxtext": "text+Cx",
293 "ktext": "text+k",
294 "kxtext": "text+kx",
295 "ltext": "text+F",
296 "tempobj": "binary+FSw",
297 "ubinary": "binary+F",
298 "uresource": "resource+F",
299 "uxbinary": "binary+Fx",
300 "xbinary": "binary+x",
301 "xltext": "text+Fx",
302 "xtempobj": "binary+Swx",
303 "xtext": "text+x",
304 "xunicode": "unicode+x",
305 "xutf16": "utf16+x",
306 }
307 if p4type in p4_filetypes_historical:
308 p4type = p4_filetypes_historical[p4type]
309 mods = ""
310 s = p4type.split("+")
311 base = s[0]
312 mods = ""
313 if len(s) > 1:
314 mods = s[1]
315 return (base, mods)
b9fc6ea9 316
60df071c
LD
317#
318# return the raw p4 type of a file (text, text+ko, etc)
319#
79467e61
PW
320def p4_type(f):
321 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
322 return results[0]['headType']
323
324#
325# Given a type base and modifier, return a regexp matching
326# the keywords that can be expanded in the file
327#
328def p4_keywords_regexp_for_type(base, type_mods):
329 if base in ("text", "unicode", "binary"):
330 kwords = None
331 if "ko" in type_mods:
332 kwords = 'Id|Header'
333 elif "k" in type_mods:
334 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
335 else:
336 return None
337 pattern = r"""
338 \$ # Starts with a dollar, followed by...
339 (%s) # one of the keywords, followed by...
6b2bf41e 340 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
341 \$ # another dollar
342 """ % kwords
343 return pattern
344 else:
345 return None
346
347#
348# Given a file, return a regexp matching the possible
349# RCS keywords that will be expanded, or None for files
350# with kw expansion turned off.
351#
352def p4_keywords_regexp_for_file(file):
353 if not os.path.exists(file):
354 return None
355 else:
356 (type_base, type_mods) = split_p4_type(p4_type(file))
357 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 358
c65b670e
CP
359def setP4ExecBit(file, mode):
360 # Reopens an already open file and changes the execute bit to match
361 # the execute bit setting in the passed in mode.
362
363 p4Type = "+x"
364
365 if not isModeExec(mode):
366 p4Type = getP4OpenedType(file)
367 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
368 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
369 if p4Type[-1] == "+":
370 p4Type = p4Type[0:-1]
371
6de040df 372 p4_reopen(p4Type, file)
c65b670e
CP
373
374def getP4OpenedType(file):
375 # Returns the perforce file type for the given file.
376
9d7d446a 377 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 378 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
379 if match:
380 return match.group(1)
381 else:
f3e5ae4f 382 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 383
06804c76
LD
384# Return the set of all p4 labels
385def getP4Labels(depotPaths):
386 labels = set()
387 if isinstance(depotPaths,basestring):
388 depotPaths = [depotPaths]
389
390 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
391 label = l['label']
392 labels.add(label)
393
394 return labels
395
396# Return the set of all git tags
397def getGitTags():
398 gitTags = set()
399 for line in read_pipe_lines(["git", "tag"]):
400 tag = line.strip()
401 gitTags.add(tag)
402 return gitTags
403
b43b0a3c
CP
404def diffTreePattern():
405 # This is a simple generator for the diff tree regex pattern. This could be
406 # a class variable if this and parseDiffTreeEntry were a part of a class.
407 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
408 while True:
409 yield pattern
410
411def parseDiffTreeEntry(entry):
412 """Parses a single diff tree entry into its component elements.
413
414 See git-diff-tree(1) manpage for details about the format of the diff
415 output. This method returns a dictionary with the following elements:
416
417 src_mode - The mode of the source file
418 dst_mode - The mode of the destination file
419 src_sha1 - The sha1 for the source file
420 dst_sha1 - The sha1 fr the destination file
421 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
422 status_score - The score for the status (applicable for 'C' and 'R'
423 statuses). This is None if there is no score.
424 src - The path for the source file.
425 dst - The path for the destination file. This is only present for
426 copy or renames. If it is not present, this is None.
427
428 If the pattern is not matched, None is returned."""
429
430 match = diffTreePattern().next().match(entry)
431 if match:
432 return {
433 'src_mode': match.group(1),
434 'dst_mode': match.group(2),
435 'src_sha1': match.group(3),
436 'dst_sha1': match.group(4),
437 'status': match.group(5),
438 'status_score': match.group(6),
439 'src': match.group(7),
440 'dst': match.group(10)
441 }
442 return None
443
c65b670e
CP
444def isModeExec(mode):
445 # Returns True if the given git mode represents an executable file,
446 # otherwise False.
447 return mode[-3:] == "755"
448
449def isModeExecChanged(src_mode, dst_mode):
450 return isModeExec(src_mode) != isModeExec(dst_mode)
451
b932705b 452def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
453
454 if isinstance(cmd,basestring):
455 cmd = "-G " + cmd
456 expand = True
457 else:
458 cmd = ["-G"] + cmd
459 expand = False
460
461 cmd = p4_build_cmd(cmd)
6a49f8e2 462 if verbose:
6de040df 463 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
464
465 # Use a temporary file to avoid deadlocks without
466 # subprocess.communicate(), which would put another copy
467 # of stdout into memory.
468 stdin_file = None
469 if stdin is not None:
470 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
471 if isinstance(stdin,basestring):
472 stdin_file.write(stdin)
473 else:
474 for i in stdin:
475 stdin_file.write(i + '\n')
9f90c733
SL
476 stdin_file.flush()
477 stdin_file.seek(0)
478
6de040df
LD
479 p4 = subprocess.Popen(cmd,
480 shell=expand,
9f90c733
SL
481 stdin=stdin_file,
482 stdout=subprocess.PIPE)
86949eef
SH
483
484 result = []
485 try:
486 while True:
9f90c733 487 entry = marshal.load(p4.stdout)
c3f6163b
AG
488 if cb is not None:
489 cb(entry)
490 else:
491 result.append(entry)
86949eef
SH
492 except EOFError:
493 pass
9f90c733
SL
494 exitCode = p4.wait()
495 if exitCode != 0:
ac3e0d79
SH
496 entry = {}
497 entry["p4ExitCode"] = exitCode
498 result.append(entry)
86949eef
SH
499
500 return result
501
502def p4Cmd(cmd):
503 list = p4CmdList(cmd)
504 result = {}
505 for entry in list:
506 result.update(entry)
507 return result;
508
cb2c9db5
SH
509def p4Where(depotPath):
510 if not depotPath.endswith("/"):
511 depotPath += "/"
cd884106
VA
512 depotPathLong = depotPath + "..."
513 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
514 output = None
515 for entry in outputList:
75bc9573 516 if "depotFile" in entry:
cd884106
VA
517 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
518 # The base path always ends with "/...".
519 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
520 output = entry
521 break
522 elif "data" in entry:
523 data = entry.get("data")
524 space = data.find(" ")
525 if data[:space] == depotPath:
526 output = entry
527 break
7f705dc3
TAL
528 if output == None:
529 return ""
dc524036
SH
530 if output["code"] == "error":
531 return ""
cb2c9db5
SH
532 clientPath = ""
533 if "path" in output:
534 clientPath = output.get("path")
535 elif "data" in output:
536 data = output.get("data")
537 lastSpace = data.rfind(" ")
538 clientPath = data[lastSpace + 1:]
539
540 if clientPath.endswith("..."):
541 clientPath = clientPath[:-3]
542 return clientPath
543
86949eef 544def currentGitBranch():
b25b2065 545 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 546
4f5cf76a 547def isValidGitDir(path):
bb6e09b2
HWN
548 if (os.path.exists(path + "/HEAD")
549 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
550 return True;
551 return False
552
463e8af6 553def parseRevision(ref):
b25b2065 554 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 555
28755dba
PW
556def branchExists(ref):
557 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
558 ignore_error=True)
559 return len(rev) > 0
560
6ae8de88
SH
561def extractLogMessageFromGitCommit(commit):
562 logMessage = ""
b016d397
HWN
563
564 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 565 foundTitle = False
b016d397 566 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
567 if not foundTitle:
568 if len(log) == 1:
1c094184 569 foundTitle = True
6ae8de88
SH
570 continue
571
572 logMessage += log
573 return logMessage
574
bb6e09b2 575def extractSettingsGitLog(log):
6ae8de88
SH
576 values = {}
577 for line in log.split("\n"):
578 line = line.strip()
6326aa58
HWN
579 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
580 if not m:
581 continue
582
583 assignments = m.group(1).split (':')
584 for a in assignments:
585 vals = a.split ('=')
586 key = vals[0].strip()
587 val = ('='.join (vals[1:])).strip()
588 if val.endswith ('\"') and val.startswith('"'):
589 val = val[1:-1]
590
591 values[key] = val
592
845b42cb
SH
593 paths = values.get("depot-paths")
594 if not paths:
595 paths = values.get("depot-path")
a3fdd579
SH
596 if paths:
597 values['depot-paths'] = paths.split(',')
bb6e09b2 598 return values
6ae8de88 599
8136a639 600def gitBranchExists(branch):
bb6e09b2
HWN
601 proc = subprocess.Popen(["git", "rev-parse", branch],
602 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 603 return proc.wait() == 0;
8136a639 604
36bd8446 605_gitConfig = {}
b345d6c3 606
692e1796 607def gitConfig(key, typeSpecifier=None):
36bd8446 608 if not _gitConfig.has_key(key):
692e1796
LS
609 cmd = [ "git", "config" ]
610 if typeSpecifier:
611 cmd += [ typeSpecifier ]
612 cmd += [ key ]
b345d6c3
PW
613 s = read_pipe(cmd, ignore_error=True)
614 _gitConfig[key] = s.strip()
36bd8446 615 return _gitConfig[key]
01265103 616
0d609032
PW
617def gitConfigBool(key):
618 """Return a bool, using git config --bool. It is True only if the
619 variable is set to true, and False if set to false or not present
620 in the config."""
621
36bd8446 622 if not _gitConfig.has_key(key):
692e1796 623 _gitConfig[key] = gitConfig(key, '--bool') == "true"
36bd8446 624 return _gitConfig[key]
01265103 625
cb1dafdf
LS
626def gitConfigInt(key):
627 if not _gitConfig.has_key(key):
628 cmd = [ "git", "config", "--int", key ]
629 s = read_pipe(cmd, ignore_error=True)
630 v = s.strip()
631 try:
632 _gitConfig[key] = int(gitConfig(key, '--int'))
633 except ValueError:
634 _gitConfig[key] = None
635 return _gitConfig[key]
636
7199cf13
VA
637def gitConfigList(key):
638 if not _gitConfig.has_key(key):
2abba301
PW
639 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
640 _gitConfig[key] = s.strip().split(os.linesep)
7960e707
LS
641 if _gitConfig[key] == ['']:
642 _gitConfig[key] = []
7199cf13
VA
643 return _gitConfig[key]
644
2c8037ed
PW
645def p4BranchesInGit(branchesAreInRemotes=True):
646 """Find all the branches whose names start with "p4/", looking
647 in remotes or heads as specified by the argument. Return
648 a dictionary of { branch: revision } for each one found.
649 The branch names are the short names, without any
650 "p4/" prefix."""
651
062410bb
SH
652 branches = {}
653
654 cmdline = "git rev-parse --symbolic "
655 if branchesAreInRemotes:
2c8037ed 656 cmdline += "--remotes"
062410bb 657 else:
2c8037ed 658 cmdline += "--branches"
062410bb
SH
659
660 for line in read_pipe_lines(cmdline):
661 line = line.strip()
662
2c8037ed
PW
663 # only import to p4/
664 if not line.startswith('p4/'):
665 continue
666 # special symbolic ref to p4/master
667 if line == "p4/HEAD":
062410bb 668 continue
062410bb 669
2c8037ed
PW
670 # strip off p4/ prefix
671 branch = line[len("p4/"):]
062410bb
SH
672
673 branches[branch] = parseRevision(line)
2c8037ed 674
062410bb
SH
675 return branches
676
5a8e84cd
PW
677def branch_exists(branch):
678 """Make sure that the given ref name really exists."""
679
680 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
681 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
682 out, _ = p.communicate()
683 if p.returncode:
684 return False
685 # expect exactly one line of output: the branch name
686 return out.rstrip() == branch
687
9ceab363 688def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
689 branches = p4BranchesInGit()
690 # map from depot-path to branch name
691 branchByDepotPath = {}
692 for branch in branches.keys():
693 tip = branches[branch]
694 log = extractLogMessageFromGitCommit(tip)
695 settings = extractSettingsGitLog(log)
696 if settings.has_key("depot-paths"):
697 paths = ",".join(settings["depot-paths"])
698 branchByDepotPath[paths] = "remotes/p4/" + branch
699
27d2d811 700 settings = None
27d2d811
SH
701 parent = 0
702 while parent < 65535:
9ceab363 703 commit = head + "~%s" % parent
27d2d811
SH
704 log = extractLogMessageFromGitCommit(commit)
705 settings = extractSettingsGitLog(log)
86506fe5
SH
706 if settings.has_key("depot-paths"):
707 paths = ",".join(settings["depot-paths"])
708 if branchByDepotPath.has_key(paths):
709 return [branchByDepotPath[paths], settings]
27d2d811 710
86506fe5 711 parent = parent + 1
27d2d811 712
86506fe5 713 return ["", settings]
27d2d811 714
5ca44617
SH
715def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
716 if not silent:
717 print ("Creating/updating branch(es) in %s based on origin branch(es)"
718 % localRefPrefix)
719
720 originPrefix = "origin/p4/"
721
722 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
723 line = line.strip()
724 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
725 continue
726
727 headName = line[len(originPrefix):]
728 remoteHead = localRefPrefix + headName
729 originHead = line
730
731 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
732 if (not original.has_key('depot-paths')
733 or not original.has_key('change')):
734 continue
735
736 update = False
737 if not gitBranchExists(remoteHead):
738 if verbose:
739 print "creating %s" % remoteHead
740 update = True
741 else:
742 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
743 if settings.has_key('change') > 0:
744 if settings['depot-paths'] == original['depot-paths']:
745 originP4Change = int(original['change'])
746 p4Change = int(settings['change'])
747 if originP4Change > p4Change:
748 print ("%s (%s) is newer than %s (%s). "
749 "Updating p4 branch from origin."
750 % (originHead, originP4Change,
751 remoteHead, p4Change))
752 update = True
753 else:
754 print ("Ignoring: %s was imported from %s while "
755 "%s was imported from %s"
756 % (originHead, ','.join(original['depot-paths']),
757 remoteHead, ','.join(settings['depot-paths'])))
758
759 if update:
760 system("git update-ref %s %s" % (remoteHead, originHead))
761
762def originP4BranchesExist():
763 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
764
1051ef00
LD
765
766def p4ParseNumericChangeRange(parts):
767 changeStart = int(parts[0][1:])
768 if parts[1] == '#head':
769 changeEnd = p4_last_change()
770 else:
771 changeEnd = int(parts[1])
772
773 return (changeStart, changeEnd)
774
775def chooseBlockSize(blockSize):
776 if blockSize:
777 return blockSize
778 else:
779 return defaultBlockSize
780
781def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
4f6432d8 782 assert depotPaths
96b2d54a 783
1051ef00
LD
784 # Parse the change range into start and end. Try to find integer
785 # revision ranges as these can be broken up into blocks to avoid
786 # hitting server-side limits (maxrows, maxscanresults). But if
787 # that doesn't work, fall back to using the raw revision specifier
788 # strings, without using block mode.
789
96b2d54a 790 if changeRange is None or changeRange == '':
1051ef00
LD
791 changeStart = 1
792 changeEnd = p4_last_change()
793 block_size = chooseBlockSize(requestedBlockSize)
96b2d54a
LS
794 else:
795 parts = changeRange.split(',')
796 assert len(parts) == 2
1051ef00
LD
797 try:
798 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
799 block_size = chooseBlockSize(requestedBlockSize)
800 except:
801 changeStart = parts[0][1:]
802 changeEnd = parts[1]
803 if requestedBlockSize:
804 die("cannot use --changes-block-size with non-numeric revisions")
805 block_size = None
4f6432d8 806
96b2d54a 807 # Accumulate change numbers in a dictionary to avoid duplicates
b4b0ba06 808 changes = {}
96b2d54a
LS
809
810 for p in depotPaths:
811 # Retrieve changes a block at a time, to prevent running
1051ef00
LD
812 # into a MaxResults/MaxScanRows error from the server.
813
814 while True:
96b2d54a 815 cmd = ['changes']
1051ef00
LD
816
817 if block_size:
818 end = min(changeEnd, changeStart + block_size)
819 revisionRange = "%d,%d" % (changeStart, end)
820 else:
821 revisionRange = "%s,%s" % (changeStart, changeEnd)
822
823 cmd += ["%s...@%s" % (p, revisionRange)]
824
96b2d54a
LS
825 for line in p4_read_pipe_lines(cmd):
826 changeNum = int(line.split(" ")[1])
96b2d54a 827 changes[changeNum] = True
1051ef00
LD
828
829 if not block_size:
830 break
831
832 if end >= changeEnd:
833 break
834
835 changeStart = end + 1
4f6432d8 836
b4b0ba06
PW
837 changelist = changes.keys()
838 changelist.sort()
839 return changelist
4f6432d8 840
d53de8b9
TAL
841def p4PathStartsWith(path, prefix):
842 # This method tries to remedy a potential mixed-case issue:
843 #
844 # If UserA adds //depot/DirA/file1
845 # and UserB adds //depot/dira/file2
846 #
847 # we may or may not have a problem. If you have core.ignorecase=true,
848 # we treat DirA and dira as the same directory
0d609032 849 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
850 return path.lower().startswith(prefix.lower())
851 return path.startswith(prefix)
852
543987bd
PW
853def getClientSpec():
854 """Look at the p4 client spec, create a View() object that contains
855 all the mappings, and return it."""
856
857 specList = p4CmdList("client -o")
858 if len(specList) != 1:
859 die('Output from "client -o" is %d lines, expecting 1' %
860 len(specList))
861
862 # dictionary of all client parameters
863 entry = specList[0]
864
9d57c4a6
KS
865 # the //client/ name
866 client_name = entry["Client"]
867
543987bd
PW
868 # just the keys that start with "View"
869 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
870
871 # hold this new View
9d57c4a6 872 view = View(client_name)
543987bd
PW
873
874 # append the lines, in order, to the view
875 for view_num in range(len(view_keys)):
876 k = "View%d" % view_num
877 if k not in view_keys:
878 die("Expected view key %s missing" % k)
879 view.append(entry[k])
880
881 return view
882
883def getClientRoot():
884 """Grab the client directory."""
885
886 output = p4CmdList("client -o")
887 if len(output) != 1:
888 die('Output from "client -o" is %d lines, expecting 1' % len(output))
889
890 entry = output[0]
891 if "Root" not in entry:
892 die('Client has no "Root"')
893
894 return entry["Root"]
895
9d7d446a
PW
896#
897# P4 wildcards are not allowed in filenames. P4 complains
898# if you simply add them, but you can force it with "-f", in
899# which case it translates them into %xx encoding internally.
900#
901def wildcard_decode(path):
902 # Search for and fix just these four characters. Do % last so
903 # that fixing it does not inadvertently create new %-escapes.
904 # Cannot have * in a filename in windows; untested as to
905 # what p4 would do in such a case.
906 if not platform.system() == "Windows":
907 path = path.replace("%2A", "*")
908 path = path.replace("%23", "#") \
909 .replace("%40", "@") \
910 .replace("%25", "%")
911 return path
912
913def wildcard_encode(path):
914 # do % first to avoid double-encoding the %s introduced here
915 path = path.replace("%", "%25") \
916 .replace("*", "%2A") \
917 .replace("#", "%23") \
918 .replace("@", "%40")
919 return path
920
921def wildcard_present(path):
598354c0
BC
922 m = re.search("[*#@%]", path)
923 return m is not None
9d7d446a 924
b984733c
SH
925class Command:
926 def __init__(self):
927 self.usage = "usage: %prog [options]"
8910ac0e 928 self.needsGit = True
6a10b6aa 929 self.verbose = False
b984733c 930
3ea2cfd4
LD
931class P4UserMap:
932 def __init__(self):
933 self.userMapFromPerforceServer = False
affb474f
LD
934 self.myP4UserId = None
935
936 def p4UserId(self):
937 if self.myP4UserId:
938 return self.myP4UserId
939
940 results = p4CmdList("user -o")
941 for r in results:
942 if r.has_key('User'):
943 self.myP4UserId = r['User']
944 return r['User']
945 die("Could not find your p4 user id")
946
947 def p4UserIsMe(self, p4User):
948 # return True if the given p4 user is actually me
949 me = self.p4UserId()
950 if not p4User or p4User != me:
951 return False
952 else:
953 return True
3ea2cfd4
LD
954
955 def getUserCacheFilename(self):
956 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
957 return home + "/.gitp4-usercache.txt"
958
959 def getUserMapFromPerforceServer(self):
960 if self.userMapFromPerforceServer:
961 return
962 self.users = {}
963 self.emails = {}
964
965 for output in p4CmdList("users"):
966 if not output.has_key("User"):
967 continue
968 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
969 self.emails[output["Email"]] = output["User"]
970
971
972 s = ''
973 for (key, val) in self.users.items():
974 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
975
976 open(self.getUserCacheFilename(), "wb").write(s)
977 self.userMapFromPerforceServer = True
978
979 def loadUserMapFromCache(self):
980 self.users = {}
981 self.userMapFromPerforceServer = False
982 try:
983 cache = open(self.getUserCacheFilename(), "rb")
984 lines = cache.readlines()
985 cache.close()
986 for line in lines:
987 entry = line.strip().split("\t")
988 self.users[entry[0]] = entry[1]
989 except IOError:
990 self.getUserMapFromPerforceServer()
991
b984733c 992class P4Debug(Command):
86949eef 993 def __init__(self):
6ae8de88 994 Command.__init__(self)
6a10b6aa 995 self.options = []
c8c39116 996 self.description = "A tool to debug the output of p4 -G."
8910ac0e 997 self.needsGit = False
86949eef
SH
998
999 def run(self, args):
b1ce9447 1000 j = 0
6de040df 1001 for output in p4CmdList(args):
b1ce9447
HWN
1002 print 'Element: %d' % j
1003 j += 1
86949eef 1004 print output
b984733c 1005 return True
86949eef 1006
5834684d
SH
1007class P4RollBack(Command):
1008 def __init__(self):
1009 Command.__init__(self)
1010 self.options = [
0c66a783 1011 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
1012 ]
1013 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 1014 self.rollbackLocalBranches = False
5834684d
SH
1015
1016 def run(self, args):
1017 if len(args) != 1:
1018 return False
1019 maxChange = int(args[0])
0c66a783 1020
ad192f28 1021 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
1022 die("Problems executing p4");
1023
0c66a783
SH
1024 if self.rollbackLocalBranches:
1025 refPrefix = "refs/heads/"
b016d397 1026 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
1027 else:
1028 refPrefix = "refs/remotes/"
b016d397 1029 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
1030
1031 for line in lines:
1032 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
1033 line = line.strip()
1034 ref = refPrefix + line
5834684d 1035 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1036 settings = extractSettingsGitLog(log)
1037
1038 depotPaths = settings['depot-paths']
1039 change = settings['change']
1040
5834684d 1041 changed = False
52102d47 1042
6326aa58
HWN
1043 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1044 for p in depotPaths]))) == 0:
52102d47
SH
1045 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1046 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1047 continue
1048
bb6e09b2 1049 while change and int(change) > maxChange:
5834684d 1050 changed = True
52102d47
SH
1051 if self.verbose:
1052 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
1053 system("git update-ref %s \"%s^\"" % (ref, ref))
1054 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1055 settings = extractSettingsGitLog(log)
1056
1057
1058 depotPaths = settings['depot-paths']
1059 change = settings['change']
5834684d
SH
1060
1061 if changed:
52102d47 1062 print "%s rewound to %s" % (ref, change)
5834684d
SH
1063
1064 return True
1065
3ea2cfd4 1066class P4Submit(Command, P4UserMap):
6bbfd137
PW
1067
1068 conflict_behavior_choices = ("ask", "skip", "quit")
1069
4f5cf76a 1070 def __init__(self):
b984733c 1071 Command.__init__(self)
3ea2cfd4 1072 P4UserMap.__init__(self)
4f5cf76a 1073 self.options = [
4f5cf76a 1074 optparse.make_option("--origin", dest="origin"),
ae901090 1075 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1076 # preserve the user, requires relevant p4 permissions
1077 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1078 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1079 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1080 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1081 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1082 choices=self.conflict_behavior_choices),
1083 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
1084 ]
1085 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1086 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1087 self.origin = ""
ae901090 1088 self.detectRenames = False
0d609032 1089 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1090 self.dry_run = False
728b7ad8 1091 self.prepare_p4_only = False
6bbfd137 1092 self.conflict_behavior = None
f7baba8b 1093 self.isWindows = (platform.system() == "Windows")
06804c76 1094 self.exportLabels = False
249da4c0 1095 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1096 self.branch = None
4f5cf76a 1097
4f5cf76a
SH
1098 def check(self):
1099 if len(p4CmdList("opened ...")) > 0:
1100 die("You have files opened with perforce! Close them before starting the sync.")
1101
f19cb0a0
PW
1102 def separate_jobs_from_description(self, message):
1103 """Extract and return a possible Jobs field in the commit
1104 message. It goes into a separate section in the p4 change
1105 specification.
1106
1107 A jobs line starts with "Jobs:" and looks like a new field
1108 in a form. Values are white-space separated on the same
1109 line or on following lines that start with a tab.
1110
1111 This does not parse and extract the full git commit message
1112 like a p4 form. It just sees the Jobs: line as a marker
1113 to pass everything from then on directly into the p4 form,
1114 but outside the description section.
1115
1116 Return a tuple (stripped log message, jobs string)."""
1117
1118 m = re.search(r'^Jobs:', message, re.MULTILINE)
1119 if m is None:
1120 return (message, None)
1121
1122 jobtext = message[m.start():]
1123 stripped_message = message[:m.start()].rstrip()
1124 return (stripped_message, jobtext)
1125
1126 def prepareLogMessage(self, template, message, jobs):
1127 """Edits the template returned from "p4 change -o" to insert
1128 the message in the Description field, and the jobs text in
1129 the Jobs field."""
4f5cf76a
SH
1130 result = ""
1131
edae1e2f
SH
1132 inDescriptionSection = False
1133
4f5cf76a
SH
1134 for line in template.split("\n"):
1135 if line.startswith("#"):
1136 result += line + "\n"
1137 continue
1138
edae1e2f 1139 if inDescriptionSection:
c9dbab04 1140 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1141 inDescriptionSection = False
f19cb0a0
PW
1142 # insert Jobs section
1143 if jobs:
1144 result += jobs + "\n"
edae1e2f
SH
1145 else:
1146 continue
1147 else:
1148 if line.startswith("Description:"):
1149 inDescriptionSection = True
1150 line += "\n"
1151 for messageLine in message.split("\n"):
1152 line += "\t" + messageLine + "\n"
1153
1154 result += line + "\n"
4f5cf76a
SH
1155
1156 return result
1157
60df071c
LD
1158 def patchRCSKeywords(self, file, pattern):
1159 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1160 (handle, outFileName) = tempfile.mkstemp(dir='.')
1161 try:
1162 outFile = os.fdopen(handle, "w+")
1163 inFile = open(file, "r")
1164 regexp = re.compile(pattern, re.VERBOSE)
1165 for line in inFile.readlines():
1166 line = regexp.sub(r'$\1$', line)
1167 outFile.write(line)
1168 inFile.close()
1169 outFile.close()
1170 # Forcibly overwrite the original file
1171 os.unlink(file)
1172 shutil.move(outFileName, file)
1173 except:
1174 # cleanup our temporary file
1175 os.unlink(outFileName)
1176 print "Failed to strip RCS keywords in %s" % file
1177 raise
1178
1179 print "Patched up RCS keywords in %s" % file
1180
3ea2cfd4
LD
1181 def p4UserForCommit(self,id):
1182 # Return the tuple (perforce user,git email) for a given git commit id
1183 self.getUserMapFromPerforceServer()
9bf28855
PW
1184 gitEmail = read_pipe(["git", "log", "--max-count=1",
1185 "--format=%ae", id])
3ea2cfd4
LD
1186 gitEmail = gitEmail.strip()
1187 if not self.emails.has_key(gitEmail):
1188 return (None,gitEmail)
1189 else:
1190 return (self.emails[gitEmail],gitEmail)
1191
1192 def checkValidP4Users(self,commits):
1193 # check if any git authors cannot be mapped to p4 users
1194 for id in commits:
1195 (user,email) = self.p4UserForCommit(id)
1196 if not user:
1197 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1198 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1199 print "%s" % msg
1200 else:
1201 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1202
1203 def lastP4Changelist(self):
1204 # Get back the last changelist number submitted in this client spec. This
1205 # then gets used to patch up the username in the change. If the same
1206 # client spec is being used by multiple processes then this might go
1207 # wrong.
1208 results = p4CmdList("client -o") # find the current client
1209 client = None
1210 for r in results:
1211 if r.has_key('Client'):
1212 client = r['Client']
1213 break
1214 if not client:
1215 die("could not get client spec")
6de040df 1216 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1217 for r in results:
1218 if r.has_key('change'):
1219 return r['change']
1220 die("Could not get changelist number for last submit - cannot patch up user details")
1221
1222 def modifyChangelistUser(self, changelist, newUser):
1223 # fixup the user field of a changelist after it has been submitted.
1224 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1225 if len(changes) != 1:
1226 die("Bad output from p4 change modifying %s to user %s" %
1227 (changelist, newUser))
1228
1229 c = changes[0]
1230 if c['User'] == newUser: return # nothing to do
1231 c['User'] = newUser
1232 input = marshal.dumps(c)
1233
3ea2cfd4
LD
1234 result = p4CmdList("change -f -i", stdin=input)
1235 for r in result:
1236 if r.has_key('code'):
1237 if r['code'] == 'error':
1238 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1239 if r.has_key('data'):
1240 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1241 return
1242 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1243
1244 def canChangeChangelists(self):
1245 # check to see if we have p4 admin or super-user permissions, either of
1246 # which are required to modify changelists.
52a4880b 1247 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1248 for r in results:
1249 if r.has_key('perm'):
1250 if r['perm'] == 'admin':
1251 return 1
1252 if r['perm'] == 'super':
1253 return 1
1254 return 0
1255
ea99c3ae 1256 def prepareSubmitTemplate(self):
f19cb0a0
PW
1257 """Run "p4 change -o" to grab a change specification template.
1258 This does not use "p4 -G", as it is nice to keep the submission
1259 template in original order, since a human might edit it.
1260
1261 Remove lines in the Files section that show changes to files
1262 outside the depot path we're committing into."""
1263
ea99c3ae
SH
1264 template = ""
1265 inFilesSection = False
6de040df 1266 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1267 if line.endswith("\r\n"):
1268 line = line[:-2] + "\n"
ea99c3ae
SH
1269 if inFilesSection:
1270 if line.startswith("\t"):
1271 # path starts and ends with a tab
1272 path = line[1:]
1273 lastTab = path.rfind("\t")
1274 if lastTab != -1:
1275 path = path[:lastTab]
d53de8b9 1276 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
1277 continue
1278 else:
1279 inFilesSection = False
1280 else:
1281 if line.startswith("Files:"):
1282 inFilesSection = True
1283
1284 template += line
1285
1286 return template
1287
7c766e57
PW
1288 def edit_template(self, template_file):
1289 """Invoke the editor to let the user change the submission
1290 message. Return true if okay to continue with the submit."""
1291
1292 # if configured to skip the editing part, just submit
0d609032 1293 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1294 return True
1295
1296 # look at the modification time, to check later if the user saved
1297 # the file
1298 mtime = os.stat(template_file).st_mtime
1299
1300 # invoke the editor
f95ceaf0 1301 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1302 editor = os.environ.get("P4EDITOR")
1303 else:
1304 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1305 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1306
1307 # If the file was not saved, prompt to see if this patch should
1308 # be skipped. But skip this verification step if configured so.
0d609032 1309 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1310 return True
1311
d1652049
PW
1312 # modification time updated means user saved the file
1313 if os.stat(template_file).st_mtime > mtime:
1314 return True
1315
1316 while True:
1317 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1318 if response == 'y':
1319 return True
1320 if response == 'n':
1321 return False
7c766e57 1322
e2a892ee 1323 def get_diff_description(self, editedFiles, filesToAdd):
b4073bb3
MC
1324 # diff
1325 if os.environ.has_key("P4DIFF"):
1326 del(os.environ["P4DIFF"])
1327 diff = ""
1328 for editedFile in editedFiles:
1329 diff += p4_read_pipe(['diff', '-du',
1330 wildcard_encode(editedFile)])
1331
1332 # new file diff
1333 newdiff = ""
1334 for newFile in filesToAdd:
1335 newdiff += "==== new file ====\n"
1336 newdiff += "--- /dev/null\n"
1337 newdiff += "+++ %s\n" % newFile
1338 f = open(newFile, "r")
1339 for line in f.readlines():
1340 newdiff += "+" + line
1341 f.close()
1342
e2a892ee 1343 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1344
7cb5cbef 1345 def applyCommit(self, id):
67b0fe2e
PW
1346 """Apply one commit, return True if it succeeded."""
1347
1348 print "Applying", read_pipe(["git", "show", "-s",
1349 "--format=format:%h %s", id])
ae901090 1350
848de9c3 1351 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1352
84cb0003 1353 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1354 filesToAdd = set()
1355 filesToDelete = set()
d336c158 1356 editedFiles = set()
b6ad6dcc 1357 pureRenameCopy = set()
c65b670e 1358 filesToChangeExecBit = {}
60df071c 1359
4f5cf76a 1360 for line in diff:
b43b0a3c
CP
1361 diff = parseDiffTreeEntry(line)
1362 modifier = diff['status']
1363 path = diff['src']
4f5cf76a 1364 if modifier == "M":
6de040df 1365 p4_edit(path)
c65b670e
CP
1366 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1367 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1368 editedFiles.add(path)
4f5cf76a
SH
1369 elif modifier == "A":
1370 filesToAdd.add(path)
c65b670e 1371 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1372 if path in filesToDelete:
1373 filesToDelete.remove(path)
1374 elif modifier == "D":
1375 filesToDelete.add(path)
1376 if path in filesToAdd:
1377 filesToAdd.remove(path)
4fddb41b
VA
1378 elif modifier == "C":
1379 src, dest = diff['src'], diff['dst']
6de040df 1380 p4_integrate(src, dest)
b6ad6dcc 1381 pureRenameCopy.add(dest)
4fddb41b 1382 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1383 p4_edit(dest)
b6ad6dcc 1384 pureRenameCopy.discard(dest)
4fddb41b 1385 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1386 p4_edit(dest)
b6ad6dcc 1387 pureRenameCopy.discard(dest)
4fddb41b 1388 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1389 if self.isWindows:
1390 # turn off read-only attribute
1391 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1392 os.unlink(dest)
1393 editedFiles.add(dest)
d9a5f25b 1394 elif modifier == "R":
b43b0a3c 1395 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1396 if self.p4HasMoveCommand:
1397 p4_edit(src) # src must be open before move
1398 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1399 else:
8e9497c2
GG
1400 p4_integrate(src, dest)
1401 if diff['src_sha1'] != diff['dst_sha1']:
1402 p4_edit(dest)
1403 else:
1404 pureRenameCopy.add(dest)
c65b670e 1405 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1406 if not self.p4HasMoveCommand:
1407 p4_edit(dest) # with move: already open, writable
c65b670e 1408 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1409 if not self.p4HasMoveCommand:
d20f0f8e
PW
1410 if self.isWindows:
1411 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1412 os.unlink(dest)
1413 filesToDelete.add(src)
d9a5f25b 1414 editedFiles.add(dest)
4f5cf76a
SH
1415 else:
1416 die("unknown modifier %s for %s" % (modifier, path))
1417
749b668c 1418 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1419 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1420 tryPatchCmd = patchcmd + "--check -"
1421 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1422 patch_succeeded = True
51a2640a 1423
47a130b7 1424 if os.system(tryPatchCmd) != 0:
60df071c
LD
1425 fixed_rcs_keywords = False
1426 patch_succeeded = False
51a2640a 1427 print "Unfortunately applying the change failed!"
60df071c
LD
1428
1429 # Patch failed, maybe it's just RCS keyword woes. Look through
1430 # the patch to see if that's possible.
0d609032 1431 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1432 file = None
1433 pattern = None
1434 kwfiles = {}
1435 for file in editedFiles | filesToDelete:
1436 # did this file's delta contain RCS keywords?
1437 pattern = p4_keywords_regexp_for_file(file)
1438
1439 if pattern:
1440 # this file is a possibility...look for RCS keywords.
1441 regexp = re.compile(pattern, re.VERBOSE)
1442 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1443 if regexp.search(line):
1444 if verbose:
1445 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1446 kwfiles[file] = pattern
1447 break
1448
1449 for file in kwfiles:
1450 if verbose:
1451 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1452 # File is being deleted, so not open in p4. Must
1453 # disable the read-only bit on windows.
1454 if self.isWindows and file not in editedFiles:
1455 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1456 self.patchRCSKeywords(file, kwfiles[file])
1457 fixed_rcs_keywords = True
1458
1459 if fixed_rcs_keywords:
1460 print "Retrying the patch with RCS keywords cleaned up"
1461 if os.system(tryPatchCmd) == 0:
1462 patch_succeeded = True
1463
1464 if not patch_succeeded:
7e5dd9f2
PW
1465 for f in editedFiles:
1466 p4_revert(f)
7e5dd9f2 1467 return False
51a2640a 1468
55ac2ed6
PW
1469 #
1470 # Apply the patch for real, and do add/delete/+x handling.
1471 #
47a130b7 1472 system(applyPatchCmd)
4f5cf76a
SH
1473
1474 for f in filesToAdd:
6de040df 1475 p4_add(f)
4f5cf76a 1476 for f in filesToDelete:
6de040df
LD
1477 p4_revert(f)
1478 p4_delete(f)
4f5cf76a 1479
c65b670e
CP
1480 # Set/clear executable bits
1481 for f in filesToChangeExecBit.keys():
1482 mode = filesToChangeExecBit[f]
1483 setP4ExecBit(f, mode)
1484
55ac2ed6
PW
1485 #
1486 # Build p4 change description, starting with the contents
1487 # of the git commit message.
1488 #
0e36f2d7 1489 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1490 logMessage = logMessage.strip()
f19cb0a0 1491 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1492
ea99c3ae 1493 template = self.prepareSubmitTemplate()
f19cb0a0 1494 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1495
c47178d4 1496 if self.preserveUser:
55ac2ed6 1497 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1498
55ac2ed6
PW
1499 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1500 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1501 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1502 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1503
55ac2ed6 1504 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1505 if not self.prepare_p4_only:
1506 submitTemplate += separatorLine
e2a892ee 1507 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
55ac2ed6 1508
c47178d4 1509 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1510 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1511 if self.isWindows:
1512 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1513 tmpFile.write(submitTemplate)
c47178d4
PW
1514 tmpFile.close()
1515
728b7ad8
PW
1516 if self.prepare_p4_only:
1517 #
1518 # Leave the p4 tree prepared, and the submit template around
1519 # and let the user decide what to do next
1520 #
1521 print
1522 print "P4 workspace prepared for submission."
1523 print "To submit or revert, go to client workspace"
1524 print " " + self.clientPath
1525 print
1526 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1527 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1528 " \"git p4\"." % fileName
1529 print "You can delete the file \"%s\" when finished." % fileName
1530
1531 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1532 print "To preserve change ownership by user %s, you must\n" \
1533 "do \"p4 change -f <change>\" after submitting and\n" \
1534 "edit the User field."
1535 if pureRenameCopy:
1536 print "After submitting, renamed files must be re-synced."
1537 print "Invoke \"p4 sync -f\" on each of these files:"
1538 for f in pureRenameCopy:
1539 print " " + f
1540
1541 print
1542 print "To revert the changes, use \"p4 revert ...\", and delete"
1543 print "the submit template file \"%s\"" % fileName
1544 if filesToAdd:
1545 print "Since the commit adds new files, they must be deleted:"
1546 for f in filesToAdd:
1547 print " " + f
1548 print
1549 return True
1550
55ac2ed6
PW
1551 #
1552 # Let the user edit the change description, then submit it.
1553 #
c47178d4
PW
1554 if self.edit_template(fileName):
1555 # read the edited message and submit
5a41c16a 1556 ret = True
c47178d4
PW
1557 tmpFile = open(fileName, "rb")
1558 message = tmpFile.read()
e96e400f 1559 tmpFile.close()
c47178d4 1560 if self.isWindows:
e2a892ee
MC
1561 message = message.replace("\r\n", "\n")
1562 submitTemplate = message[:message.index(separatorLine)]
c47178d4 1563 p4_write_pipe(['submit', '-i'], submitTemplate)
e96e400f 1564
c47178d4
PW
1565 if self.preserveUser:
1566 if p4User:
1567 # Get last changelist number. Cannot easily get it from
1568 # the submit command output as the output is
1569 # unmarshalled.
1570 changelist = self.lastP4Changelist()
1571 self.modifyChangelistUser(changelist, p4User)
1572
1573 # The rename/copy happened by applying a patch that created a
1574 # new file. This leaves it writable, which confuses p4.
1575 for f in pureRenameCopy:
1576 p4_sync(f, "-f")
cdc7e388 1577
4f5cf76a 1578 else:
c47178d4 1579 # skip this patch
5a41c16a 1580 ret = False
c47178d4
PW
1581 print "Submission cancelled, undoing p4 changes."
1582 for f in editedFiles:
1583 p4_revert(f)
1584 for f in filesToAdd:
1585 p4_revert(f)
1586 os.remove(f)
df9c5453
PW
1587 for f in filesToDelete:
1588 p4_revert(f)
c47178d4
PW
1589
1590 os.remove(fileName)
5a41c16a 1591 return ret
4f5cf76a 1592
06804c76
LD
1593 # Export git tags as p4 labels. Create a p4 label and then tag
1594 # with that.
1595 def exportGitTags(self, gitTags):
c8942a22
LD
1596 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1597 if len(validLabelRegexp) == 0:
1598 validLabelRegexp = defaultLabelRegexp
1599 m = re.compile(validLabelRegexp)
06804c76
LD
1600
1601 for name in gitTags:
1602
1603 if not m.match(name):
1604 if verbose:
05a3cec5 1605 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1606 continue
1607
1608 # Get the p4 commit this corresponds to
c8942a22
LD
1609 logMessage = extractLogMessageFromGitCommit(name)
1610 values = extractSettingsGitLog(logMessage)
06804c76 1611
c8942a22 1612 if not values.has_key('change'):
06804c76
LD
1613 # a tag pointing to something not sent to p4; ignore
1614 if verbose:
1615 print "git tag %s does not give a p4 commit" % name
1616 continue
c8942a22
LD
1617 else:
1618 changelist = values['change']
06804c76
LD
1619
1620 # Get the tag details.
1621 inHeader = True
1622 isAnnotated = False
1623 body = []
1624 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1625 l = l.strip()
1626 if inHeader:
1627 if re.match(r'tag\s+', l):
1628 isAnnotated = True
1629 elif re.match(r'\s*$', l):
1630 inHeader = False
1631 continue
1632 else:
1633 body.append(l)
1634
1635 if not isAnnotated:
1636 body = ["lightweight tag imported by git p4\n"]
1637
1638 # Create the label - use the same view as the client spec we are using
1639 clientSpec = getClientSpec()
1640
1641 labelTemplate = "Label: %s\n" % name
1642 labelTemplate += "Description:\n"
1643 for b in body:
1644 labelTemplate += "\t" + b + "\n"
1645 labelTemplate += "View:\n"
9d57c4a6
KS
1646 for depot_side in clientSpec.mappings:
1647 labelTemplate += "\t%s\n" % depot_side
06804c76 1648
ef739f08
PW
1649 if self.dry_run:
1650 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1651 elif self.prepare_p4_only:
1652 print "Not creating p4 label %s for tag due to option" \
1653 " --prepare-p4-only" % name
ef739f08
PW
1654 else:
1655 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1656
ef739f08
PW
1657 # Use the label
1658 p4_system(["tag", "-l", name] +
9d57c4a6 1659 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1660
ef739f08
PW
1661 if verbose:
1662 print "created p4 label for tag %s" % name
06804c76 1663
4f5cf76a 1664 def run(self, args):
c9b50e63
SH
1665 if len(args) == 0:
1666 self.master = currentGitBranch()
4280e533 1667 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1668 die("Detecting current git branch failed!")
1669 elif len(args) == 1:
1670 self.master = args[0]
28755dba
PW
1671 if not branchExists(self.master):
1672 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1673 else:
1674 return False
1675
4c2d5d72
JX
1676 allowSubmit = gitConfig("git-p4.allowSubmit")
1677 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1678 die("%s is not in git-p4.allowSubmit" % self.master)
1679
27d2d811 1680 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1681 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1682 if len(self.origin) == 0:
1683 self.origin = upstream
a3fdd579 1684
3ea2cfd4
LD
1685 if self.preserveUser:
1686 if not self.canChangeChangelists():
1687 die("Cannot preserve user names without p4 super-user or admin permissions")
1688
6bbfd137
PW
1689 # if not set from the command line, try the config file
1690 if self.conflict_behavior is None:
1691 val = gitConfig("git-p4.conflict")
1692 if val:
1693 if val not in self.conflict_behavior_choices:
1694 die("Invalid value '%s' for config git-p4.conflict" % val)
1695 else:
1696 val = "ask"
1697 self.conflict_behavior = val
1698
a3fdd579
SH
1699 if self.verbose:
1700 print "Origin branch is " + self.origin
9512497b 1701
ea99c3ae 1702 if len(self.depotPath) == 0:
9512497b
SH
1703 print "Internal error: cannot locate perforce depot path from existing branches"
1704 sys.exit(128)
1705
543987bd 1706 self.useClientSpec = False
0d609032 1707 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1708 self.useClientSpec = True
1709 if self.useClientSpec:
1710 self.clientSpecDirs = getClientSpec()
9512497b 1711
cd884106
VA
1712 # Check for the existance of P4 branches
1713 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1714
1715 if self.useClientSpec and not branchesDetected:
543987bd
PW
1716 # all files are relative to the client spec
1717 self.clientPath = getClientRoot()
1718 else:
1719 self.clientPath = p4Where(self.depotPath)
9512497b 1720
543987bd
PW
1721 if self.clientPath == "":
1722 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1723
ea99c3ae 1724 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1725 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1726
0591cfa8 1727 # ensure the clientPath exists
8d7ec362 1728 new_client_dir = False
0591cfa8 1729 if not os.path.exists(self.clientPath):
8d7ec362 1730 new_client_dir = True
0591cfa8
GG
1731 os.makedirs(self.clientPath)
1732
bbd84863 1733 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1734 if self.dry_run:
1735 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1736 else:
ef739f08
PW
1737 print "Synchronizing p4 checkout..."
1738 if new_client_dir:
1739 # old one was destroyed, and maybe nobody told p4
1740 p4_sync("...", "-f")
1741 else:
1742 p4_sync("...")
4f5cf76a 1743 self.check()
4f5cf76a 1744
4c750c0d 1745 commits = []
c7d34884 1746 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
4c750c0d
SH
1747 commits.append(line.strip())
1748 commits.reverse()
4f5cf76a 1749
0d609032 1750 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1751 self.checkAuthorship = False
1752 else:
1753 self.checkAuthorship = True
1754
3ea2cfd4
LD
1755 if self.preserveUser:
1756 self.checkValidP4Users(commits)
1757
84cb0003
GG
1758 #
1759 # Build up a set of options to be passed to diff when
1760 # submitting each commit to p4.
1761 #
1762 if self.detectRenames:
1763 # command-line -M arg
1764 self.diffOpts = "-M"
1765 else:
1766 # If not explicitly set check the config variable
1767 detectRenames = gitConfig("git-p4.detectRenames")
1768
1769 if detectRenames.lower() == "false" or detectRenames == "":
1770 self.diffOpts = ""
1771 elif detectRenames.lower() == "true":
1772 self.diffOpts = "-M"
1773 else:
1774 self.diffOpts = "-M%s" % detectRenames
1775
1776 # no command-line arg for -C or --find-copies-harder, just
1777 # config variables
1778 detectCopies = gitConfig("git-p4.detectCopies")
1779 if detectCopies.lower() == "false" or detectCopies == "":
1780 pass
1781 elif detectCopies.lower() == "true":
1782 self.diffOpts += " -C"
1783 else:
1784 self.diffOpts += " -C%s" % detectCopies
1785
0d609032 1786 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1787 self.diffOpts += " --find-copies-harder"
1788
7e5dd9f2
PW
1789 #
1790 # Apply the commits, one at a time. On failure, ask if should
1791 # continue to try the rest of the patches, or quit.
1792 #
ef739f08
PW
1793 if self.dry_run:
1794 print "Would apply"
67b0fe2e 1795 applied = []
7e5dd9f2
PW
1796 last = len(commits) - 1
1797 for i, commit in enumerate(commits):
ef739f08
PW
1798 if self.dry_run:
1799 print " ", read_pipe(["git", "show", "-s",
1800 "--format=format:%h %s", commit])
1801 ok = True
1802 else:
1803 ok = self.applyCommit(commit)
67b0fe2e
PW
1804 if ok:
1805 applied.append(commit)
7e5dd9f2 1806 else:
728b7ad8
PW
1807 if self.prepare_p4_only and i < last:
1808 print "Processing only the first commit due to option" \
1809 " --prepare-p4-only"
1810 break
7e5dd9f2
PW
1811 if i < last:
1812 quit = False
1813 while True:
6bbfd137
PW
1814 # prompt for what to do, or use the option/variable
1815 if self.conflict_behavior == "ask":
1816 print "What do you want to do?"
1817 response = raw_input("[s]kip this commit but apply"
1818 " the rest, or [q]uit? ")
1819 if not response:
1820 continue
1821 elif self.conflict_behavior == "skip":
1822 response = "s"
1823 elif self.conflict_behavior == "quit":
1824 response = "q"
1825 else:
1826 die("Unknown conflict_behavior '%s'" %
1827 self.conflict_behavior)
1828
7e5dd9f2
PW
1829 if response[0] == "s":
1830 print "Skipping this commit, but applying the rest"
1831 break
1832 if response[0] == "q":
1833 print "Quitting"
1834 quit = True
1835 break
1836 if quit:
1837 break
4f5cf76a 1838
67b0fe2e 1839 chdir(self.oldWorkingDirectory)
4f5cf76a 1840
ef739f08
PW
1841 if self.dry_run:
1842 pass
728b7ad8
PW
1843 elif self.prepare_p4_only:
1844 pass
ef739f08 1845 elif len(commits) == len(applied):
67b0fe2e 1846 print "All commits applied!"
14594f4b 1847
4c750c0d 1848 sync = P4Sync()
44e8d26c
PW
1849 if self.branch:
1850 sync.branch = self.branch
4c750c0d 1851 sync.run([])
14594f4b 1852
4c750c0d
SH
1853 rebase = P4Rebase()
1854 rebase.rebase()
4f5cf76a 1855
67b0fe2e
PW
1856 else:
1857 if len(applied) == 0:
1858 print "No commits applied."
1859 else:
1860 print "Applied only the commits marked with '*':"
1861 for c in commits:
1862 if c in applied:
1863 star = "*"
1864 else:
1865 star = " "
1866 print star, read_pipe(["git", "show", "-s",
1867 "--format=format:%h %s", c])
1868 print "You will have to do 'git p4 sync' and rebase."
1869
0d609032 1870 if gitConfigBool("git-p4.exportLabels"):
06dcd152 1871 self.exportLabels = True
06804c76
LD
1872
1873 if self.exportLabels:
1874 p4Labels = getP4Labels(self.depotPath)
1875 gitTags = getGitTags()
1876
1877 missingGitTags = gitTags - p4Labels
1878 self.exportGitTags(missingGitTags)
1879
98e023de 1880 # exit with error unless everything applied perfectly
67b0fe2e
PW
1881 if len(commits) != len(applied):
1882 sys.exit(1)
1883
b984733c
SH
1884 return True
1885
ecb7cf98
PW
1886class View(object):
1887 """Represent a p4 view ("p4 help views"), and map files in a
1888 repo according to the view."""
1889
9d57c4a6 1890 def __init__(self, client_name):
ecb7cf98 1891 self.mappings = []
9d57c4a6
KS
1892 self.client_prefix = "//%s/" % client_name
1893 # cache results of "p4 where" to lookup client file locations
1894 self.client_spec_path_cache = {}
ecb7cf98
PW
1895
1896 def append(self, view_line):
1897 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
1898 sides. Append to self.mappings, preserving order. This
1899 is only needed for tag creation."""
ecb7cf98
PW
1900
1901 # Split the view line into exactly two words. P4 enforces
1902 # structure on these lines that simplifies this quite a bit.
1903 #
1904 # Either or both words may be double-quoted.
1905 # Single quotes do not matter.
1906 # Double-quote marks cannot occur inside the words.
1907 # A + or - prefix is also inside the quotes.
1908 # There are no quotes unless they contain a space.
1909 # The line is already white-space stripped.
1910 # The two words are separated by a single space.
1911 #
1912 if view_line[0] == '"':
1913 # First word is double quoted. Find its end.
1914 close_quote_index = view_line.find('"', 1)
1915 if close_quote_index <= 0:
1916 die("No first-word closing quote found: %s" % view_line)
1917 depot_side = view_line[1:close_quote_index]
1918 # skip closing quote and space
1919 rhs_index = close_quote_index + 1 + 1
1920 else:
1921 space_index = view_line.find(" ")
1922 if space_index <= 0:
1923 die("No word-splitting space found: %s" % view_line)
1924 depot_side = view_line[0:space_index]
1925 rhs_index = space_index + 1
1926
ecb7cf98 1927 # prefix + means overlay on previous mapping
ecb7cf98 1928 if depot_side.startswith("+"):
ecb7cf98
PW
1929 depot_side = depot_side[1:]
1930
9d57c4a6 1931 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
1932 exclude = False
1933 if depot_side.startswith("-"):
1934 exclude = True
1935 depot_side = depot_side[1:]
1936
9d57c4a6
KS
1937 if not exclude:
1938 self.mappings.append(depot_side)
ecb7cf98 1939
9d57c4a6
KS
1940 def convert_client_path(self, clientFile):
1941 # chop off //client/ part to make it relative
1942 if not clientFile.startswith(self.client_prefix):
1943 die("No prefix '%s' on clientFile '%s'" %
1944 (self.client_prefix, clientFile))
1945 return clientFile[len(self.client_prefix):]
ecb7cf98 1946
9d57c4a6
KS
1947 def update_client_spec_path_cache(self, files):
1948 """ Caching file paths by "p4 where" batch query """
ecb7cf98 1949
9d57c4a6
KS
1950 # List depot file paths exclude that already cached
1951 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 1952
9d57c4a6
KS
1953 if len(fileArgs) == 0:
1954 return # All files in cache
ecb7cf98 1955
9d57c4a6
KS
1956 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1957 for res in where_result:
1958 if "code" in res and res["code"] == "error":
1959 # assume error is "... file(s) not in client view"
1960 continue
1961 if "clientFile" not in res:
20005443 1962 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
1963 if "unmap" in res:
1964 # it will list all of them, but only one not unmap-ped
1965 continue
a0a50d87
LS
1966 if gitConfigBool("core.ignorecase"):
1967 res['depotFile'] = res['depotFile'].lower()
9d57c4a6 1968 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 1969
9d57c4a6
KS
1970 # not found files or unmap files set to ""
1971 for depotFile in fileArgs:
a0a50d87
LS
1972 if gitConfigBool("core.ignorecase"):
1973 depotFile = depotFile.lower()
9d57c4a6
KS
1974 if depotFile not in self.client_spec_path_cache:
1975 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 1976
9d57c4a6
KS
1977 def map_in_client(self, depot_path):
1978 """Return the relative location in the client where this
1979 depot file should live. Returns "" if the file should
1980 not be mapped in the client."""
ecb7cf98 1981
a0a50d87
LS
1982 if gitConfigBool("core.ignorecase"):
1983 depot_path = depot_path.lower()
1984
9d57c4a6
KS
1985 if depot_path in self.client_spec_path_cache:
1986 return self.client_spec_path_cache[depot_path]
1987
1988 die( "Error: %s is not found in client spec path" % depot_path )
1989 return ""
ecb7cf98 1990
3ea2cfd4 1991class P4Sync(Command, P4UserMap):
56c09345
PW
1992 delete_actions = ( "delete", "move/delete", "purge" )
1993
b984733c
SH
1994 def __init__(self):
1995 Command.__init__(self)
3ea2cfd4 1996 P4UserMap.__init__(self)
b984733c
SH
1997 self.options = [
1998 optparse.make_option("--branch", dest="branch"),
1999 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2000 optparse.make_option("--changesfile", dest="changesFile"),
2001 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2002 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2003 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2004 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2005 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
2006 optparse.make_option("--max-changes", dest="maxChanges",
2007 help="Maximum number of changes to import"),
2008 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2009 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 2010 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2011 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2012 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
2013 help="Only sync files that are included in the Perforce Client Spec"),
2014 optparse.make_option("-/", dest="cloneExclude",
2015 action="append", type="string",
2016 help="exclude depot path"),
b984733c
SH
2017 ]
2018 self.description = """Imports from Perforce into a git repository.\n
2019 example:
2020 //depot/my/project/ -- to import the current head
2021 //depot/my/project/@all -- to import everything
2022 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2023
2024 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2025
2026 self.usage += " //depot/path[@revRange]"
b984733c 2027 self.silent = False
1d7367dc
RG
2028 self.createdBranches = set()
2029 self.committedChanges = set()
569d1bd4 2030 self.branch = ""
b984733c 2031 self.detectBranches = False
cb53e1f8 2032 self.detectLabels = False
06804c76 2033 self.importLabels = False
b984733c 2034 self.changesFile = ""
01265103 2035 self.syncWithOrigin = True
a028a98e 2036 self.importIntoRemotes = True
01a9c9c5 2037 self.maxChanges = ""
1051ef00 2038 self.changes_block_size = None
8b41a97f 2039 self.keepRepoPath = False
6326aa58 2040 self.depotPaths = None
3c699645 2041 self.p4BranchesInGit = []
354081d5 2042 self.cloneExclude = []
3a70cdfa 2043 self.useClientSpec = False
a93d33ee 2044 self.useClientSpec_from_options = False
ecb7cf98 2045 self.clientSpecDirs = None
fed23693
VA
2046 self.tempBranches = []
2047 self.tempBranchLocation = "git-p4-tmp"
b984733c 2048
01265103
SH
2049 if gitConfig("git-p4.syncFromOrigin") == "false":
2050 self.syncWithOrigin = False
2051
51334bb0
LD
2052 # This is required for the "append" cloneExclude action
2053 def ensure_value(self, attr, value):
2054 if not hasattr(self, attr) or getattr(self, attr) is None:
2055 setattr(self, attr, value)
2056 return getattr(self, attr)
2057
fed23693
VA
2058 # Force a checkpoint in fast-import and wait for it to finish
2059 def checkpoint(self):
2060 self.gitStream.write("checkpoint\n\n")
2061 self.gitStream.write("progress checkpoint\n\n")
2062 out = self.gitOutput.readline()
2063 if self.verbose:
2064 print "checkpoint finished: " + out
2065
b984733c 2066 def extractFilesFromCommit(self, commit):
354081d5
TT
2067 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2068 for path in self.cloneExclude]
b984733c
SH
2069 files = []
2070 fnum = 0
2071 while commit.has_key("depotFile%s" % fnum):
2072 path = commit["depotFile%s" % fnum]
6326aa58 2073
354081d5 2074 if [p for p in self.cloneExclude
d53de8b9 2075 if p4PathStartsWith(path, p)]:
354081d5
TT
2076 found = False
2077 else:
2078 found = [p for p in self.depotPaths
d53de8b9 2079 if p4PathStartsWith(path, p)]
6326aa58 2080 if not found:
b984733c
SH
2081 fnum = fnum + 1
2082 continue
2083
2084 file = {}
2085 file["path"] = path
2086 file["rev"] = commit["rev%s" % fnum]
2087 file["action"] = commit["action%s" % fnum]
2088 file["type"] = commit["type%s" % fnum]
2089 files.append(file)
2090 fnum = fnum + 1
2091 return files
2092
6326aa58 2093 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2094 """When streaming files, this is called to map a p4 depot path
2095 to where it should go in git. The prefixes are either
2096 self.depotPaths, or self.branchPrefixes in the case of
2097 branch detection."""
2098
3952710b 2099 if self.useClientSpec:
21ef5df4
PW
2100 # branch detection moves files up a level (the branch name)
2101 # from what client spec interpretation gives
0d1696ef 2102 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2103 if self.detectBranches:
2104 for b in self.knownBranches:
2105 if path.startswith(b + "/"):
2106 path = path[len(b)+1:]
2107
2108 elif self.keepRepoPath:
2109 # Preserve everything in relative path name except leading
2110 # //depot/; just look at first prefix as they all should
2111 # be in the same depot.
2112 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2113 if p4PathStartsWith(path, depot):
2114 path = path[len(depot):]
3952710b 2115
0d1696ef 2116 else:
0d1696ef
PW
2117 for p in prefixes:
2118 if p4PathStartsWith(path, p):
2119 path = path[len(p):]
21ef5df4 2120 break
8b41a97f 2121
0d1696ef 2122 path = wildcard_decode(path)
6326aa58 2123 return path
6754a299 2124
71b112d4 2125 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2126 """Look at each depotFile in the commit to figure out to what
2127 branch it belongs."""
2128
9d57c4a6
KS
2129 if self.clientSpecDirs:
2130 files = self.extractFilesFromCommit(commit)
2131 self.clientSpecDirs.update_client_spec_path_cache(files)
2132
d5904674 2133 branches = {}
71b112d4
SH
2134 fnum = 0
2135 while commit.has_key("depotFile%s" % fnum):
2136 path = commit["depotFile%s" % fnum]
6326aa58 2137 found = [p for p in self.depotPaths
d53de8b9 2138 if p4PathStartsWith(path, p)]
6326aa58 2139 if not found:
71b112d4
SH
2140 fnum = fnum + 1
2141 continue
2142
2143 file = {}
2144 file["path"] = path
2145 file["rev"] = commit["rev%s" % fnum]
2146 file["action"] = commit["action%s" % fnum]
2147 file["type"] = commit["type%s" % fnum]
2148 fnum = fnum + 1
2149
21ef5df4
PW
2150 # start with the full relative path where this file would
2151 # go in a p4 client
2152 if self.useClientSpec:
2153 relPath = self.clientSpecDirs.map_in_client(path)
2154 else:
2155 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2156
4b97ffb1 2157 for branch in self.knownBranches.keys():
21ef5df4
PW
2158 # add a trailing slash so that a commit into qt/4.2foo
2159 # doesn't end up in qt/4.2, e.g.
6754a299 2160 if relPath.startswith(branch + "/"):
d5904674
SH
2161 if branch not in branches:
2162 branches[branch] = []
71b112d4 2163 branches[branch].append(file)
6555b2cc 2164 break
b984733c
SH
2165
2166 return branches
2167
b932705b
LD
2168 # output one file from the P4 stream
2169 # - helper for streamP4Files
2170
2171 def streamOneP4File(self, file, contents):
b932705b
LD
2172 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2173 if verbose:
2174 sys.stderr.write("%s\n" % relPath)
2175
9cffb8c8
PW
2176 (type_base, type_mods) = split_p4_type(file["type"])
2177
2178 git_mode = "100644"
2179 if "x" in type_mods:
2180 git_mode = "100755"
2181 if type_base == "symlink":
2182 git_mode = "120000"
1292df11
AJ
2183 # p4 print on a symlink sometimes contains "target\n";
2184 # if it does, remove the newline
b39c3612 2185 data = ''.join(contents)
40f846c3
PW
2186 if not data:
2187 # Some version of p4 allowed creating a symlink that pointed
2188 # to nothing. This causes p4 errors when checking out such
2189 # a change, and errors here too. Work around it by ignoring
2190 # the bad symlink; hopefully a future change fixes it.
2191 print "\nIgnoring empty symlink in %s" % file['depotFile']
2192 return
2193 elif data[-1] == '\n':
1292df11
AJ
2194 contents = [data[:-1]]
2195 else:
2196 contents = [data]
b932705b 2197
9cffb8c8 2198 if type_base == "utf16":
55aa5714
PW
2199 # p4 delivers different text in the python output to -G
2200 # than it does when using "print -o", or normal p4 client
2201 # operations. utf16 is converted to ascii or utf8, perhaps.
2202 # But ascii text saved as -t utf16 is completely mangled.
2203 # Invoke print -o to get the real contents.
7f0e5962
PW
2204 #
2205 # On windows, the newlines will always be mangled by print, so put
2206 # them back too. This is not needed to the cygwin windows version,
2207 # just the native "NT" type.
2208 #
f5f53f14 2209 text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
7f0e5962
PW
2210 if p4_version_string().find("/NT") >= 0:
2211 text = text.replace("\r\n", "\n")
55aa5714
PW
2212 contents = [ text ]
2213
9f7ef0ea
PW
2214 if type_base == "apple":
2215 # Apple filetype files will be streamed as a concatenation of
2216 # its appledouble header and the contents. This is useless
2217 # on both macs and non-macs. If using "print -q -o xx", it
2218 # will create "xx" with the data, and "%xx" with the header.
2219 # This is also not very useful.
2220 #
2221 # Ideally, someday, this script can learn how to generate
2222 # appledouble files directly and import those to git, but
2223 # non-mac machines can never find a use for apple filetype.
2224 print "\nIgnoring apple filetype file %s" % file['depotFile']
2225 return
2226
55aa5714
PW
2227 # Note that we do not try to de-mangle keywords on utf16 files,
2228 # even though in theory somebody may want that.
60df071c
LD
2229 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2230 if pattern:
2231 regexp = re.compile(pattern, re.VERBOSE)
2232 text = ''.join(contents)
2233 text = regexp.sub(r'$\1$', text)
2234 contents = [ text ]
b932705b 2235
9cffb8c8 2236 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
2237
2238 # total length...
2239 length = 0
2240 for d in contents:
2241 length = length + len(d)
2242
2243 self.gitStream.write("data %d\n" % length)
2244 for d in contents:
2245 self.gitStream.write(d)
2246 self.gitStream.write("\n")
2247
2248 def streamOneP4Deletion(self, file):
2249 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2250 if verbose:
2251 sys.stderr.write("delete %s\n" % relPath)
2252 self.gitStream.write("D %s\n" % relPath)
2253
2254 # handle another chunk of streaming data
2255 def streamP4FilesCb(self, marshalled):
2256
78189bea
PW
2257 # catch p4 errors and complain
2258 err = None
2259 if "code" in marshalled:
2260 if marshalled["code"] == "error":
2261 if "data" in marshalled:
2262 err = marshalled["data"].rstrip()
2263 if err:
2264 f = None
2265 if self.stream_have_file_info:
2266 if "depotFile" in self.stream_file:
2267 f = self.stream_file["depotFile"]
2268 # force a failure in fast-import, else an empty
2269 # commit will be made
2270 self.gitStream.write("\n")
2271 self.gitStream.write("die-now\n")
2272 self.gitStream.close()
2273 # ignore errors, but make sure it exits first
2274 self.importProcess.wait()
2275 if f:
2276 die("Error from p4 print for %s: %s" % (f, err))
2277 else:
2278 die("Error from p4 print: %s" % err)
2279
c3f6163b
AG
2280 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2281 # start of a new file - output the old one first
2282 self.streamOneP4File(self.stream_file, self.stream_contents)
2283 self.stream_file = {}
2284 self.stream_contents = []
2285 self.stream_have_file_info = False
b932705b 2286
c3f6163b
AG
2287 # pick up the new file information... for the
2288 # 'data' field we need to append to our array
2289 for k in marshalled.keys():
2290 if k == 'data':
2291 self.stream_contents.append(marshalled['data'])
2292 else:
2293 self.stream_file[k] = marshalled[k]
b932705b 2294
c3f6163b 2295 self.stream_have_file_info = True
b932705b
LD
2296
2297 # Stream directly from "p4 files" into "git fast-import"
2298 def streamP4Files(self, files):
30b5940b
SH
2299 filesForCommit = []
2300 filesToRead = []
b932705b 2301 filesToDelete = []
30b5940b 2302
3a70cdfa 2303 for f in files:
ecb7cf98
PW
2304 # if using a client spec, only add the files that have
2305 # a path in the client
2306 if self.clientSpecDirs:
2307 if self.clientSpecDirs.map_in_client(f['path']) == "":
2308 continue
3a70cdfa 2309
ecb7cf98
PW
2310 filesForCommit.append(f)
2311 if f['action'] in self.delete_actions:
2312 filesToDelete.append(f)
2313 else:
2314 filesToRead.append(f)
6a49f8e2 2315
b932705b
LD
2316 # deleted files...
2317 for f in filesToDelete:
2318 self.streamOneP4Deletion(f)
1b9a4684 2319
b932705b
LD
2320 if len(filesToRead) > 0:
2321 self.stream_file = {}
2322 self.stream_contents = []
2323 self.stream_have_file_info = False
8ff45f2a 2324
c3f6163b
AG
2325 # curry self argument
2326 def streamP4FilesCbSelf(entry):
2327 self.streamP4FilesCb(entry)
6a49f8e2 2328
6de040df
LD
2329 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2330
2331 p4CmdList(["-x", "-", "print"],
2332 stdin=fileArgs,
2333 cb=streamP4FilesCbSelf)
30b5940b 2334
b932705b
LD
2335 # do the last chunk
2336 if self.stream_file.has_key('depotFile'):
2337 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2338
affb474f
LD
2339 def make_email(self, userid):
2340 if userid in self.users:
2341 return self.users[userid]
2342 else:
2343 return "%s <a@b>" % userid
2344
06804c76
LD
2345 # Stream a p4 tag
2346 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2347 if verbose:
2348 print "writing tag %s for commit %s" % (labelName, commit)
2349 gitStream.write("tag %s\n" % labelName)
2350 gitStream.write("from %s\n" % commit)
2351
2352 if labelDetails.has_key('Owner'):
2353 owner = labelDetails["Owner"]
2354 else:
2355 owner = None
2356
2357 # Try to use the owner of the p4 label, or failing that,
2358 # the current p4 user id.
2359 if owner:
2360 email = self.make_email(owner)
2361 else:
2362 email = self.make_email(self.p4UserId())
2363 tagger = "%s %s %s" % (email, epoch, self.tz)
2364
2365 gitStream.write("tagger %s\n" % tagger)
2366
2367 print "labelDetails=",labelDetails
2368 if labelDetails.has_key('Description'):
2369 description = labelDetails['Description']
2370 else:
2371 description = 'Label from git p4'
2372
2373 gitStream.write("data %d\n" % len(description))
2374 gitStream.write(description)
2375 gitStream.write("\n")
2376
e63231e5 2377 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2378 epoch = details["time"]
2379 author = details["user"]
2380
4b97ffb1
SH
2381 if self.verbose:
2382 print "commit into %s" % branch
2383
96e07dd2
HWN
2384 # start with reading files; if that fails, we should not
2385 # create a commit.
2386 new_files = []
2387 for f in files:
e63231e5 2388 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2389 new_files.append (f)
2390 else:
afa1dd9a 2391 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2392
9d57c4a6
KS
2393 if self.clientSpecDirs:
2394 self.clientSpecDirs.update_client_spec_path_cache(files)
2395
b984733c 2396 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 2397# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2398 self.committedChanges.add(int(details["change"]))
2399 committer = ""
b607e71e
SH
2400 if author not in self.users:
2401 self.getUserMapFromPerforceServer()
affb474f 2402 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2403
2404 self.gitStream.write("committer %s\n" % committer)
2405
2406 self.gitStream.write("data <<EOT\n")
2407 self.gitStream.write(details["desc"])
e63231e5
PW
2408 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2409 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2410 if len(details['options']) > 0:
2411 self.gitStream.write(": options = %s" % details['options'])
2412 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2413
2414 if len(parent) > 0:
4b97ffb1
SH
2415 if self.verbose:
2416 print "parent %s" % parent
b984733c
SH
2417 self.gitStream.write("from %s\n" % parent)
2418
b932705b 2419 self.streamP4Files(new_files)
b984733c
SH
2420 self.gitStream.write("\n")
2421
1f4ba1cb
SH
2422 change = int(details["change"])
2423
9bda3a85 2424 if self.labels.has_key(change):
1f4ba1cb
SH
2425 label = self.labels[change]
2426 labelDetails = label[0]
2427 labelRevisions = label[1]
71b112d4
SH
2428 if self.verbose:
2429 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2430
6de040df 2431 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2432 for p in self.branchPrefixes])
1f4ba1cb
SH
2433
2434 if len(files) == len(labelRevisions):
2435
2436 cleanedFiles = {}
2437 for info in files:
56c09345 2438 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2439 continue
2440 cleanedFiles[info["depotFile"]] = info["rev"]
2441
2442 if cleanedFiles == labelRevisions:
06804c76 2443 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2444
2445 else:
a46668fa 2446 if not self.silent:
cebdf5af
HWN
2447 print ("Tag %s does not match with change %s: files do not match."
2448 % (labelDetails["label"], change))
1f4ba1cb
SH
2449
2450 else:
a46668fa 2451 if not self.silent:
cebdf5af
HWN
2452 print ("Tag %s does not match with change %s: file count is different."
2453 % (labelDetails["label"], change))
b984733c 2454
06804c76 2455 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2456 def getLabels(self):
2457 self.labels = {}
2458
52a4880b 2459 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2460 if len(l) > 0 and not self.silent:
183f8436 2461 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2462
2463 for output in l:
1f4ba1cb
SH
2464 label = output["label"]
2465 revisions = {}
2466 newestChange = 0
71b112d4
SH
2467 if self.verbose:
2468 print "Querying files for label %s" % label
6de040df
LD
2469 for file in p4CmdList(["files"] +
2470 ["%s...@%s" % (p, label)
2471 for p in self.depotPaths]):
1f4ba1cb
SH
2472 revisions[file["depotFile"]] = file["rev"]
2473 change = int(file["change"])
2474 if change > newestChange:
2475 newestChange = change
2476
9bda3a85
SH
2477 self.labels[newestChange] = [output, revisions]
2478
2479 if self.verbose:
2480 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2481
06804c76
LD
2482 # Import p4 labels as git tags. A direct mapping does not
2483 # exist, so assume that if all the files are at the same revision
2484 # then we can use that, or it's something more complicated we should
2485 # just ignore.
2486 def importP4Labels(self, stream, p4Labels):
2487 if verbose:
2488 print "import p4 labels: " + ' '.join(p4Labels)
2489
2490 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2491 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2492 if len(validLabelRegexp) == 0:
2493 validLabelRegexp = defaultLabelRegexp
2494 m = re.compile(validLabelRegexp)
2495
2496 for name in p4Labels:
2497 commitFound = False
2498
2499 if not m.match(name):
2500 if verbose:
2501 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2502 continue
2503
2504 if name in ignoredP4Labels:
2505 continue
2506
2507 labelDetails = p4CmdList(['label', "-o", name])[0]
2508
2509 # get the most recent changelist for each file in this label
2510 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2511 for p in self.depotPaths])
2512
2513 if change.has_key('change'):
2514 # find the corresponding git commit; take the oldest commit
2515 changelist = int(change['change'])
2516 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2517 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2518 if len(gitCommit) == 0:
2519 print "could not find git commit for changelist %d" % changelist
2520 else:
2521 gitCommit = gitCommit.strip()
2522 commitFound = True
2523 # Convert from p4 time format
2524 try:
2525 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2526 except ValueError:
a4e9054c 2527 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2528 tmwhen = 1
2529
2530 when = int(time.mktime(tmwhen))
2531 self.streamTag(stream, name, labelDetails, gitCommit, when)
2532 if verbose:
2533 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2534 else:
2535 if verbose:
2536 print "Label %s has no changelists - possibly deleted?" % name
2537
2538 if not commitFound:
2539 # We can't import this label; don't try again as it will get very
2540 # expensive repeatedly fetching all the files for labels that will
2541 # never be imported. If the label is moved in the future, the
2542 # ignore will need to be removed manually.
2543 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2544
86dff6b6
HWN
2545 def guessProjectName(self):
2546 for p in self.depotPaths:
6e5295c4
SH
2547 if p.endswith("/"):
2548 p = p[:-1]
2549 p = p[p.strip().rfind("/") + 1:]
2550 if not p.endswith("/"):
2551 p += "/"
2552 return p
86dff6b6 2553
4b97ffb1 2554 def getBranchMapping(self):
6555b2cc
SH
2555 lostAndFoundBranches = set()
2556
8ace74c0
VA
2557 user = gitConfig("git-p4.branchUser")
2558 if len(user) > 0:
2559 command = "branches -u %s" % user
2560 else:
2561 command = "branches"
2562
2563 for info in p4CmdList(command):
52a4880b 2564 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2565 viewIdx = 0
2566 while details.has_key("View%s" % viewIdx):
2567 paths = details["View%s" % viewIdx].split(" ")
2568 viewIdx = viewIdx + 1
2569 # require standard //depot/foo/... //depot/bar/... mapping
2570 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2571 continue
2572 source = paths[0]
2573 destination = paths[1]
6509e19c 2574 ## HACK
d53de8b9 2575 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2576 source = source[len(self.depotPaths[0]):-4]
2577 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2578
1a2edf4e
SH
2579 if destination in self.knownBranches:
2580 if not self.silent:
2581 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2582 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2583 continue
2584
6555b2cc
SH
2585 self.knownBranches[destination] = source
2586
2587 lostAndFoundBranches.discard(destination)
2588
29bdbac1 2589 if source not in self.knownBranches:
6555b2cc
SH
2590 lostAndFoundBranches.add(source)
2591
7199cf13
VA
2592 # Perforce does not strictly require branches to be defined, so we also
2593 # check git config for a branch list.
2594 #
2595 # Example of branch definition in git config file:
2596 # [git-p4]
2597 # branchList=main:branchA
2598 # branchList=main:branchB
2599 # branchList=branchA:branchC
2600 configBranches = gitConfigList("git-p4.branchList")
2601 for branch in configBranches:
2602 if branch:
2603 (source, destination) = branch.split(":")
2604 self.knownBranches[destination] = source
2605
2606 lostAndFoundBranches.discard(destination)
2607
2608 if source not in self.knownBranches:
2609 lostAndFoundBranches.add(source)
2610
6555b2cc
SH
2611
2612 for branch in lostAndFoundBranches:
2613 self.knownBranches[branch] = branch
29bdbac1 2614
38f9f5ec
SH
2615 def getBranchMappingFromGitBranches(self):
2616 branches = p4BranchesInGit(self.importIntoRemotes)
2617 for branch in branches.keys():
2618 if branch == "master":
2619 branch = "main"
2620 else:
2621 branch = branch[len(self.projectName):]
2622 self.knownBranches[branch] = branch
2623
bb6e09b2
HWN
2624 def updateOptionDict(self, d):
2625 option_keys = {}
2626 if self.keepRepoPath:
2627 option_keys['keepRepoPath'] = 1
2628
2629 d["options"] = ' '.join(sorted(option_keys.keys()))
2630
2631 def readOptions(self, d):
2632 self.keepRepoPath = (d.has_key('options')
2633 and ('keepRepoPath' in d['options']))
6326aa58 2634
8134f69c
SH
2635 def gitRefForBranch(self, branch):
2636 if branch == "main":
2637 return self.refPrefix + "master"
2638
2639 if len(branch) <= 0:
2640 return branch
2641
2642 return self.refPrefix + self.projectName + branch
2643
1ca3d710
SH
2644 def gitCommitByP4Change(self, ref, change):
2645 if self.verbose:
2646 print "looking in ref " + ref + " for change %s using bisect..." % change
2647
2648 earliestCommit = ""
2649 latestCommit = parseRevision(ref)
2650
2651 while True:
2652 if self.verbose:
2653 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2654 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2655 if len(next) == 0:
2656 if self.verbose:
2657 print "argh"
2658 return ""
2659 log = extractLogMessageFromGitCommit(next)
2660 settings = extractSettingsGitLog(log)
2661 currentChange = int(settings['change'])
2662 if self.verbose:
2663 print "current change %s" % currentChange
2664
2665 if currentChange == change:
2666 if self.verbose:
2667 print "found %s" % next
2668 return next
2669
2670 if currentChange < change:
2671 earliestCommit = "^%s" % next
2672 else:
2673 latestCommit = "%s" % next
2674
2675 return ""
2676
2677 def importNewBranch(self, branch, maxChange):
2678 # make fast-import flush all changes to disk and update the refs using the checkpoint
2679 # command so that we can try to find the branch parent in the git history
2680 self.gitStream.write("checkpoint\n\n");
2681 self.gitStream.flush();
2682 branchPrefix = self.depotPaths[0] + branch + "/"
2683 range = "@1,%s" % maxChange
2684 #print "prefix" + branchPrefix
96b2d54a 2685 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
2686 if len(changes) <= 0:
2687 return False
2688 firstChange = changes[0]
2689 #print "first change in branch: %s" % firstChange
2690 sourceBranch = self.knownBranches[branch]
2691 sourceDepotPath = self.depotPaths[0] + sourceBranch
2692 sourceRef = self.gitRefForBranch(sourceBranch)
2693 #print "source " + sourceBranch
2694
52a4880b 2695 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2696 #print "branch parent: %s" % branchParentChange
2697 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2698 if len(gitParent) > 0:
2699 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2700 #print "parent git commit: %s" % gitParent
2701
2702 self.importChanges(changes)
2703 return True
2704
fed23693
VA
2705 def searchParent(self, parent, branch, target):
2706 parentFound = False
c7d34884
PW
2707 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2708 "--no-merges", parent]):
fed23693
VA
2709 blob = blob.strip()
2710 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2711 parentFound = True
2712 if self.verbose:
2713 print "Found parent of %s in commit %s" % (branch, blob)
2714 break
2715 if parentFound:
2716 return blob
2717 else:
2718 return None
2719
e87f37ae
SH
2720 def importChanges(self, changes):
2721 cnt = 1
2722 for change in changes:
18fa13d0 2723 description = p4_describe(change)
e87f37ae
SH
2724 self.updateOptionDict(description)
2725
2726 if not self.silent:
2727 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2728 sys.stdout.flush()
2729 cnt = cnt + 1
2730
2731 try:
2732 if self.detectBranches:
2733 branches = self.splitFilesIntoBranches(description)
2734 for branch in branches.keys():
2735 ## HACK --hwn
2736 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 2737 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
2738
2739 parent = ""
2740
2741 filesForCommit = branches[branch]
2742
2743 if self.verbose:
2744 print "branch is %s" % branch
2745
2746 self.updatedBranches.add(branch)
2747
2748 if branch not in self.createdBranches:
2749 self.createdBranches.add(branch)
2750 parent = self.knownBranches[branch]
2751 if parent == branch:
2752 parent = ""
1ca3d710
SH
2753 else:
2754 fullBranch = self.projectName + branch
2755 if fullBranch not in self.p4BranchesInGit:
2756 if not self.silent:
2757 print("\n Importing new branch %s" % fullBranch);
2758 if self.importNewBranch(branch, change - 1):
2759 parent = ""
2760 self.p4BranchesInGit.append(fullBranch)
2761 if not self.silent:
2762 print("\n Resuming with change %s" % change);
2763
2764 if self.verbose:
2765 print "parent determined through known branches: %s" % parent
e87f37ae 2766
8134f69c
SH
2767 branch = self.gitRefForBranch(branch)
2768 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2769
2770 if self.verbose:
2771 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2772
2773 if len(parent) == 0 and branch in self.initialParents:
2774 parent = self.initialParents[branch]
2775 del self.initialParents[branch]
2776
fed23693
VA
2777 blob = None
2778 if len(parent) > 0:
4f9273d2 2779 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
2780 if self.verbose:
2781 print "Creating temporary branch: " + tempBranch
e63231e5 2782 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
2783 self.tempBranches.append(tempBranch)
2784 self.checkpoint()
2785 blob = self.searchParent(parent, branch, tempBranch)
2786 if blob:
e63231e5 2787 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
2788 else:
2789 if self.verbose:
2790 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 2791 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
2792 else:
2793 files = self.extractFilesFromCommit(description)
e63231e5 2794 self.commit(description, files, self.branch,
e87f37ae 2795 self.initialParent)
47497844 2796 # only needed once, to connect to the previous commit
e87f37ae
SH
2797 self.initialParent = ""
2798 except IOError:
2799 print self.gitError.read()
2800 sys.exit(1)
2801
c208a243
SH
2802 def importHeadRevision(self, revision):
2803 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2804
4e2e6ce4
PW
2805 details = {}
2806 details["user"] = "git perforce import user"
1494fcbb 2807 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2808 % (' '.join(self.depotPaths), revision))
2809 details["change"] = revision
2810 newestRevision = 0
2811
2812 fileCnt = 0
6de040df
LD
2813 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2814
2815 for info in p4CmdList(["files"] + fileArgs):
c208a243 2816
68b28593 2817 if 'code' in info and info['code'] == 'error':
c208a243
SH
2818 sys.stderr.write("p4 returned an error: %s\n"
2819 % info['data'])
d88e707f
PW
2820 if info['data'].find("must refer to client") >= 0:
2821 sys.stderr.write("This particular p4 error is misleading.\n")
2822 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2823 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2824 sys.exit(1)
68b28593
PW
2825 if 'p4ExitCode' in info:
2826 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2827 sys.exit(1)
2828
2829
2830 change = int(info["change"])
2831 if change > newestRevision:
2832 newestRevision = change
2833
56c09345 2834 if info["action"] in self.delete_actions:
c208a243
SH
2835 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2836 #fileCnt = fileCnt + 1
2837 continue
2838
2839 for prop in ["depotFile", "rev", "action", "type" ]:
2840 details["%s%s" % (prop, fileCnt)] = info[prop]
2841
2842 fileCnt = fileCnt + 1
2843
2844 details["change"] = newestRevision
4e2e6ce4 2845
9dcb9f24 2846 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 2847 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
2848 res = p4_describe(newestRevision)
2849 details["time"] = res["time"]
4e2e6ce4 2850
c208a243
SH
2851 self.updateOptionDict(details)
2852 try:
e63231e5 2853 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
2854 except IOError:
2855 print "IO error with git fast-import. Is your git version recent enough?"
2856 print self.gitError.read()
2857
2858
b984733c 2859 def run(self, args):
6326aa58 2860 self.depotPaths = []
179caebf 2861 self.changeRange = ""
6326aa58 2862 self.previousDepotPaths = []
991a2de4 2863 self.hasOrigin = False
ce6f33c8 2864
29bdbac1
SH
2865 # map from branch depot path to parent branch
2866 self.knownBranches = {}
2867 self.initialParents = {}
2868
a028a98e
SH
2869 if self.importIntoRemotes:
2870 self.refPrefix = "refs/remotes/p4/"
2871 else:
db775559 2872 self.refPrefix = "refs/heads/p4/"
a028a98e 2873
991a2de4
PW
2874 if self.syncWithOrigin:
2875 self.hasOrigin = originP4BranchesExist()
2876 if self.hasOrigin:
2877 if not self.silent:
2878 print 'Syncing with origin first, using "git fetch origin"'
2879 system("git fetch origin")
10f880f8 2880
5a8e84cd 2881 branch_arg_given = bool(self.branch)
569d1bd4 2882 if len(self.branch) == 0:
db775559 2883 self.branch = self.refPrefix + "master"
a028a98e 2884 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2885 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 2886 system("git branch -D p4")
967f72e2 2887
a93d33ee
PW
2888 # accept either the command-line option, or the configuration variable
2889 if self.useClientSpec:
2890 # will use this after clone to set the variable
2891 self.useClientSpec_from_options = True
2892 else:
0d609032 2893 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
2894 self.useClientSpec = True
2895 if self.useClientSpec:
543987bd 2896 self.clientSpecDirs = getClientSpec()
3a70cdfa 2897
6a49f8e2
HWN
2898 # TODO: should always look at previous commits,
2899 # merge with previous imports, if possible.
2900 if args == []:
d414c74a 2901 if self.hasOrigin:
5ca44617 2902 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
2903
2904 # branches holds mapping from branch name to sha1
2905 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
2906
2907 # restrict to just this one, disabling detect-branches
2908 if branch_arg_given:
2909 short = self.branch.split("/")[-1]
2910 if short in branches:
2911 self.p4BranchesInGit = [ short ]
2912 else:
2913 self.p4BranchesInGit = branches.keys()
abcd790f
SH
2914
2915 if len(self.p4BranchesInGit) > 1:
2916 if not self.silent:
2917 print "Importing from/into multiple branches"
2918 self.detectBranches = True
8c9e8b6e
PW
2919 for branch in branches.keys():
2920 self.initialParents[self.refPrefix + branch] = \
2921 branches[branch]
967f72e2 2922
29bdbac1
SH
2923 if self.verbose:
2924 print "branches: %s" % self.p4BranchesInGit
2925
2926 p4Change = 0
2927 for branch in self.p4BranchesInGit:
cebdf5af 2928 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2929
2930 settings = extractSettingsGitLog(logMsg)
29bdbac1 2931
bb6e09b2
HWN
2932 self.readOptions(settings)
2933 if (settings.has_key('depot-paths')
2934 and settings.has_key ('change')):
2935 change = int(settings['change']) + 1
29bdbac1
SH
2936 p4Change = max(p4Change, change)
2937
bb6e09b2
HWN
2938 depotPaths = sorted(settings['depot-paths'])
2939 if self.previousDepotPaths == []:
6326aa58 2940 self.previousDepotPaths = depotPaths
29bdbac1 2941 else:
6326aa58
HWN
2942 paths = []
2943 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2944 prev_list = prev.split("/")
2945 cur_list = cur.split("/")
2946 for i in range(0, min(len(cur_list), len(prev_list))):
2947 if cur_list[i] <> prev_list[i]:
583e1707 2948 i = i - 1
6326aa58
HWN
2949 break
2950
04d277b3 2951 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2952
2953 self.previousDepotPaths = paths
29bdbac1
SH
2954
2955 if p4Change > 0:
bb6e09b2 2956 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2957 self.changeRange = "@%s,#head" % p4Change
341dc1c1 2958 if not self.silent and not self.detectBranches:
967f72e2 2959 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2960
40d69ac3
PW
2961 # accept multiple ref name abbreviations:
2962 # refs/foo/bar/branch -> use it exactly
2963 # p4/branch -> prepend refs/remotes/ or refs/heads/
2964 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 2965 if not self.branch.startswith("refs/"):
40d69ac3
PW
2966 if self.importIntoRemotes:
2967 prepend = "refs/remotes/"
2968 else:
2969 prepend = "refs/heads/"
2970 if not self.branch.startswith("p4/"):
2971 prepend += "p4/"
2972 self.branch = prepend + self.branch
179caebf 2973
6326aa58 2974 if len(args) == 0 and self.depotPaths:
b984733c 2975 if not self.silent:
6326aa58 2976 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2977 else:
6326aa58 2978 if self.depotPaths and self.depotPaths != args:
cebdf5af 2979 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2980 "This doesn't work!" % (' '.join (self.depotPaths),
2981 ' '.join (args)))
b984733c 2982 sys.exit(1)
6326aa58 2983
bb6e09b2 2984 self.depotPaths = sorted(args)
b984733c 2985
1c49fc19 2986 revision = ""
b984733c 2987 self.users = {}
b984733c 2988
58c8bc7c
PW
2989 # Make sure no revision specifiers are used when --changesfile
2990 # is specified.
2991 bad_changesfile = False
2992 if len(self.changesFile) > 0:
2993 for p in self.depotPaths:
2994 if p.find("@") >= 0 or p.find("#") >= 0:
2995 bad_changesfile = True
2996 break
2997 if bad_changesfile:
2998 die("Option --changesfile is incompatible with revision specifiers")
2999
6326aa58
HWN
3000 newPaths = []
3001 for p in self.depotPaths:
3002 if p.find("@") != -1:
3003 atIdx = p.index("@")
3004 self.changeRange = p[atIdx:]
3005 if self.changeRange == "@all":
3006 self.changeRange = ""
6a49f8e2 3007 elif ',' not in self.changeRange:
1c49fc19 3008 revision = self.changeRange
6326aa58 3009 self.changeRange = ""
7fcff9de 3010 p = p[:atIdx]
6326aa58
HWN
3011 elif p.find("#") != -1:
3012 hashIdx = p.index("#")
1c49fc19 3013 revision = p[hashIdx:]
7fcff9de 3014 p = p[:hashIdx]
6326aa58 3015 elif self.previousDepotPaths == []:
58c8bc7c
PW
3016 # pay attention to changesfile, if given, else import
3017 # the entire p4 tree at the head revision
3018 if len(self.changesFile) == 0:
3019 revision = "#head"
6326aa58
HWN
3020
3021 p = re.sub ("\.\.\.$", "", p)
3022 if not p.endswith("/"):
3023 p += "/"
3024
3025 newPaths.append(p)
3026
3027 self.depotPaths = newPaths
3028
e63231e5
PW
3029 # --detect-branches may change this for each branch
3030 self.branchPrefixes = self.depotPaths
3031
b607e71e 3032 self.loadUserMapFromCache()
cb53e1f8
SH
3033 self.labels = {}
3034 if self.detectLabels:
3035 self.getLabels();
b984733c 3036
4b97ffb1 3037 if self.detectBranches:
df450923
SH
3038 ## FIXME - what's a P4 projectName ?
3039 self.projectName = self.guessProjectName()
3040
38f9f5ec
SH
3041 if self.hasOrigin:
3042 self.getBranchMappingFromGitBranches()
3043 else:
3044 self.getBranchMapping()
29bdbac1
SH
3045 if self.verbose:
3046 print "p4-git branches: %s" % self.p4BranchesInGit
3047 print "initial parents: %s" % self.initialParents
3048 for b in self.p4BranchesInGit:
3049 if b != "master":
6326aa58
HWN
3050
3051 ## FIXME
29bdbac1
SH
3052 b = b[len(self.projectName):]
3053 self.createdBranches.add(b)
4b97ffb1 3054
f291b4e3 3055 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3056
78189bea
PW
3057 self.importProcess = subprocess.Popen(["git", "fast-import"],
3058 stdin=subprocess.PIPE,
3059 stdout=subprocess.PIPE,
3060 stderr=subprocess.PIPE);
3061 self.gitOutput = self.importProcess.stdout
3062 self.gitStream = self.importProcess.stdin
3063 self.gitError = self.importProcess.stderr
b984733c 3064
1c49fc19 3065 if revision:
c208a243 3066 self.importHeadRevision(revision)
b984733c
SH
3067 else:
3068 changes = []
3069
0828ab14 3070 if len(self.changesFile) > 0:
b984733c 3071 output = open(self.changesFile).readlines()
1d7367dc 3072 changeSet = set()
b984733c
SH
3073 for line in output:
3074 changeSet.add(int(line))
3075
3076 for change in changeSet:
3077 changes.append(change)
3078
3079 changes.sort()
3080 else:
9dcb9f24
PW
3081 # catch "git p4 sync" with no new branches, in a repo that
3082 # does not have any existing p4 branches
5a8e84cd
PW
3083 if len(args) == 0:
3084 if not self.p4BranchesInGit:
3085 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3086
3087 # The default branch is master, unless --branch is used to
3088 # specify something else. Make sure it exists, or complain
3089 # nicely about how to use --branch.
3090 if not self.detectBranches:
3091 if not branch_exists(self.branch):
3092 if branch_arg_given:
3093 die("Error: branch %s does not exist." % self.branch)
3094 else:
3095 die("Error: no branch %s; perhaps specify one with --branch." %
3096 self.branch)
3097
29bdbac1 3098 if self.verbose:
86dff6b6 3099 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3100 self.changeRange)
96b2d54a 3101 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
b984733c 3102
01a9c9c5 3103 if len(self.maxChanges) > 0:
7fcff9de 3104 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3105
b984733c 3106 if len(changes) == 0:
0828ab14 3107 if not self.silent:
341dc1c1 3108 print "No changes to import!"
06804c76
LD
3109 else:
3110 if not self.silent and not self.detectBranches:
3111 print "Import destination: %s" % self.branch
3112
3113 self.updatedBranches = set()
b984733c 3114
47497844
PW
3115 if not self.detectBranches:
3116 if args:
3117 # start a new branch
3118 self.initialParent = ""
3119 else:
3120 # build on a previous revision
3121 self.initialParent = parseRevision(self.branch)
3122
06804c76 3123 self.importChanges(changes)
a9d1a27a 3124
06804c76
LD
3125 if not self.silent:
3126 print ""
3127 if len(self.updatedBranches) > 0:
3128 sys.stdout.write("Updated branches: ")
3129 for b in self.updatedBranches:
3130 sys.stdout.write("%s " % b)
3131 sys.stdout.write("\n")
341dc1c1 3132
0d609032 3133 if gitConfigBool("git-p4.importLabels"):
06dcd152 3134 self.importLabels = True
b984733c 3135
06804c76
LD
3136 if self.importLabels:
3137 p4Labels = getP4Labels(self.depotPaths)
3138 gitTags = getGitTags()
3139
3140 missingP4Labels = p4Labels - gitTags
3141 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3142
b984733c 3143 self.gitStream.close()
78189bea 3144 if self.importProcess.wait() != 0:
29bdbac1 3145 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3146 self.gitOutput.close()
3147 self.gitError.close()
3148
fed23693
VA
3149 # Cleanup temporary branches created during import
3150 if self.tempBranches != []:
3151 for branch in self.tempBranches:
3152 read_pipe("git update-ref -d %s" % branch)
3153 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3154
55d12437
PW
3155 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3156 # a convenient shortcut refname "p4".
3157 if self.importIntoRemotes:
3158 head_ref = self.refPrefix + "HEAD"
3159 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3160 system(["git", "symbolic-ref", head_ref, self.branch])
3161
b984733c
SH
3162 return True
3163
01ce1fe9
SH
3164class P4Rebase(Command):
3165 def __init__(self):
3166 Command.__init__(self)
06804c76
LD
3167 self.options = [
3168 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3169 ]
06804c76 3170 self.importLabels = False
cebdf5af
HWN
3171 self.description = ("Fetches the latest revision from perforce and "
3172 + "rebases the current work (branch) against it")
01ce1fe9
SH
3173
3174 def run(self, args):
3175 sync = P4Sync()
06804c76 3176 sync.importLabels = self.importLabels
01ce1fe9 3177 sync.run([])
d7e3868c 3178
14594f4b
SH
3179 return self.rebase()
3180
3181 def rebase(self):
36ee4ee4
SH
3182 if os.system("git update-index --refresh") != 0:
3183 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3184 if len(read_pipe("git diff-index HEAD --")) > 0:
f7e604ed 3185 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
36ee4ee4 3186
d7e3868c
SH
3187 [upstream, settings] = findUpstreamBranchPoint()
3188 if len(upstream) == 0:
3189 die("Cannot find upstream branchpoint for rebase")
3190
3191 # the branchpoint may be p4/foo~3, so strip off the parent
3192 upstream = re.sub("~[0-9]+$", "", upstream)
3193
3194 print "Rebasing the current branch onto %s" % upstream
b25b2065 3195 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 3196 system("git rebase %s" % upstream)
4e49d95e 3197 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
01ce1fe9
SH
3198 return True
3199
f9a3a4f7
SH
3200class P4Clone(P4Sync):
3201 def __init__(self):
3202 P4Sync.__init__(self)
3203 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 3204 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 3205 self.options += [
bb6e09b2
HWN
3206 optparse.make_option("--destination", dest="cloneDestination",
3207 action='store', default=None,
354081d5 3208 help="where to leave result of the clone"),
38200076
PW
3209 optparse.make_option("--bare", dest="cloneBare",
3210 action="store_true", default=False),
354081d5 3211 ]
bb6e09b2 3212 self.cloneDestination = None
f9a3a4f7 3213 self.needsGit = False
38200076 3214 self.cloneBare = False
f9a3a4f7 3215
6a49f8e2
HWN
3216 def defaultDestination(self, args):
3217 ## TODO: use common prefix of args?
3218 depotPath = args[0]
3219 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3220 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 3221 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
3222 depotDir = re.sub(r"/$", "", depotDir)
3223 return os.path.split(depotDir)[1]
3224
f9a3a4f7
SH
3225 def run(self, args):
3226 if len(args) < 1:
3227 return False
bb6e09b2
HWN
3228
3229 if self.keepRepoPath and not self.cloneDestination:
3230 sys.stderr.write("Must specify destination for --keep-path\n")
3231 sys.exit(1)
f9a3a4f7 3232
6326aa58 3233 depotPaths = args
5e100b5c
SH
3234
3235 if not self.cloneDestination and len(depotPaths) > 1:
3236 self.cloneDestination = depotPaths[-1]
3237 depotPaths = depotPaths[:-1]
3238
354081d5 3239 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
3240 for p in depotPaths:
3241 if not p.startswith("//"):
0f487d30 3242 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
6326aa58 3243 return False
f9a3a4f7 3244
bb6e09b2 3245 if not self.cloneDestination:
98ad4faf 3246 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 3247
86dff6b6 3248 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 3249
c3bf3f13
KG
3250 if not os.path.exists(self.cloneDestination):
3251 os.makedirs(self.cloneDestination)
053fd0c1 3252 chdir(self.cloneDestination)
38200076
PW
3253
3254 init_cmd = [ "git", "init" ]
3255 if self.cloneBare:
3256 init_cmd.append("--bare")
a235e85c
BC
3257 retcode = subprocess.call(init_cmd)
3258 if retcode:
3259 raise CalledProcessError(retcode, init_cmd)
38200076 3260
6326aa58 3261 if not P4Sync.run(self, depotPaths):
f9a3a4f7 3262 return False
c595956d
PW
3263
3264 # create a master branch and check out a work tree
3265 if gitBranchExists(self.branch):
3266 system([ "git", "branch", "master", self.branch ])
3267 if not self.cloneBare:
3268 system([ "git", "checkout", "-f" ])
3269 else:
3270 print 'Not checking out any branch, use ' \
3271 '"git checkout -q -b master <branch>"'
86dff6b6 3272
a93d33ee
PW
3273 # auto-set this variable if invoked with --use-client-spec
3274 if self.useClientSpec_from_options:
3275 system("git config --bool git-p4.useclientspec true")
3276
f9a3a4f7
SH
3277 return True
3278
09d89de2
SH
3279class P4Branches(Command):
3280 def __init__(self):
3281 Command.__init__(self)
3282 self.options = [ ]
3283 self.description = ("Shows the git branches that hold imports and their "
3284 + "corresponding perforce depot paths")
3285 self.verbose = False
3286
3287 def run(self, args):
5ca44617
SH
3288 if originP4BranchesExist():
3289 createOrUpdateBranchesFromOrigin()
3290
09d89de2
SH
3291 cmdline = "git rev-parse --symbolic "
3292 cmdline += " --remotes"
3293
3294 for line in read_pipe_lines(cmdline):
3295 line = line.strip()
3296
3297 if not line.startswith('p4/') or line == "p4/HEAD":
3298 continue
3299 branch = line
3300
3301 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3302 settings = extractSettingsGitLog(log)
3303
3304 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3305 return True
3306
b984733c
SH
3307class HelpFormatter(optparse.IndentedHelpFormatter):
3308 def __init__(self):
3309 optparse.IndentedHelpFormatter.__init__(self)
3310
3311 def format_description(self, description):
3312 if description:
3313 return description + "\n"
3314 else:
3315 return ""
4f5cf76a 3316
86949eef
SH
3317def printUsage(commands):
3318 print "usage: %s <command> [options]" % sys.argv[0]
3319 print ""
3320 print "valid commands: %s" % ", ".join(commands)
3321 print ""
3322 print "Try %s <command> --help for command specific help." % sys.argv[0]
3323 print ""
3324
3325commands = {
b86f7378
HWN
3326 "debug" : P4Debug,
3327 "submit" : P4Submit,
a9834f58 3328 "commit" : P4Submit,
b86f7378
HWN
3329 "sync" : P4Sync,
3330 "rebase" : P4Rebase,
3331 "clone" : P4Clone,
09d89de2
SH
3332 "rollback" : P4RollBack,
3333 "branches" : P4Branches
86949eef
SH
3334}
3335
86949eef 3336
bb6e09b2
HWN
3337def main():
3338 if len(sys.argv[1:]) == 0:
3339 printUsage(commands.keys())
3340 sys.exit(2)
4f5cf76a 3341
bb6e09b2
HWN
3342 cmdName = sys.argv[1]
3343 try:
b86f7378
HWN
3344 klass = commands[cmdName]
3345 cmd = klass()
bb6e09b2
HWN
3346 except KeyError:
3347 print "unknown command %s" % cmdName
3348 print ""
3349 printUsage(commands.keys())
3350 sys.exit(2)
3351
3352 options = cmd.options
b86f7378 3353 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
3354
3355 args = sys.argv[2:]
3356
b0ccc80d 3357 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
6a10b6aa
LD
3358 if cmd.needsGit:
3359 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 3360
6a10b6aa
LD
3361 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3362 options,
3363 description = cmd.description,
3364 formatter = HelpFormatter())
bb6e09b2 3365
6a10b6aa 3366 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
bb6e09b2
HWN
3367 global verbose
3368 verbose = cmd.verbose
3369 if cmd.needsGit:
b86f7378
HWN
3370 if cmd.gitdir == None:
3371 cmd.gitdir = os.path.abspath(".git")
3372 if not isValidGitDir(cmd.gitdir):
3373 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3374 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
3375 cdup = read_pipe("git rev-parse --show-cdup").strip()
3376 if len(cdup) > 0:
053fd0c1 3377 chdir(cdup);
e20a9e53 3378
b86f7378
HWN
3379 if not isValidGitDir(cmd.gitdir):
3380 if isValidGitDir(cmd.gitdir + "/.git"):
3381 cmd.gitdir += "/.git"
bb6e09b2 3382 else:
b86f7378 3383 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 3384
b86f7378 3385 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 3386
bb6e09b2
HWN
3387 if not cmd.run(args):
3388 parser.print_help()
09fca77b 3389 sys.exit(2)
4f5cf76a 3390
4f5cf76a 3391
bb6e09b2
HWN
3392if __name__ == '__main__':
3393 main()