git-p4: add tests for non-numeric revision range
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
8b41a97f 25
a235e85c
BC
26try:
27 from subprocess import CalledProcessError
28except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
4addad22 41verbose = False
86949eef 42
06804c76 43# Only labels/tags matching this will be imported/exported
c8942a22 44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753
AK
45
46def p4_build_cmd(cmd):
47 """Build a suitable p4 command line.
48
49 This consolidates building and returning a p4 command line into one
50 location. It means that hooking into the environment, or other configuration
51 can be done more easily.
52 """
6de040df 53 real_cmd = ["p4"]
abcaf073
AK
54
55 user = gitConfig("git-p4.user")
56 if len(user) > 0:
6de040df 57 real_cmd += ["-u",user]
abcaf073
AK
58
59 password = gitConfig("git-p4.password")
60 if len(password) > 0:
6de040df 61 real_cmd += ["-P", password]
abcaf073
AK
62
63 port = gitConfig("git-p4.port")
64 if len(port) > 0:
6de040df 65 real_cmd += ["-p", port]
abcaf073
AK
66
67 host = gitConfig("git-p4.host")
68 if len(host) > 0:
41799aa2 69 real_cmd += ["-H", host]
abcaf073
AK
70
71 client = gitConfig("git-p4.client")
72 if len(client) > 0:
6de040df 73 real_cmd += ["-c", client]
abcaf073 74
6de040df
LD
75
76 if isinstance(cmd,basestring):
77 real_cmd = ' '.join(real_cmd) + ' ' + cmd
78 else:
79 real_cmd += cmd
21a50753
AK
80 return real_cmd
81
bbd84863
MF
82def chdir(path, is_client_path=False):
83 """Do chdir to the given path, and set the PWD environment
84 variable for use by P4. It does not look at getcwd() output.
85 Since we're not using the shell, it is necessary to set the
86 PWD environment variable explicitly.
87
88 Normally, expand the path to force it to be absolute. This
89 addresses the use of relative path names inside P4 settings,
90 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
91 as given; it looks for .p4config using PWD.
92
93 If is_client_path, the path was handed to us directly by p4,
94 and may be a symbolic link. Do not call os.getcwd() in this
95 case, because it will cause p4 to think that PWD is not inside
96 the client path.
97 """
98
99 os.chdir(path)
100 if not is_client_path:
101 path = os.getcwd()
102 os.environ['PWD'] = path
053fd0c1 103
86dff6b6
HWN
104def die(msg):
105 if verbose:
106 raise Exception(msg)
107 else:
108 sys.stderr.write(msg + "\n")
109 sys.exit(1)
110
6de040df 111def write_pipe(c, stdin):
4addad22 112 if verbose:
6de040df 113 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 114
6de040df
LD
115 expand = isinstance(c,basestring)
116 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117 pipe = p.stdin
118 val = pipe.write(stdin)
119 pipe.close()
120 if p.wait():
121 die('Command failed: %s' % str(c))
b016d397
HWN
122
123 return val
124
6de040df 125def p4_write_pipe(c, stdin):
d9429194 126 real_cmd = p4_build_cmd(c)
6de040df 127 return write_pipe(real_cmd, stdin)
d9429194 128
4addad22
HWN
129def read_pipe(c, ignore_error=False):
130 if verbose:
6de040df 131 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 132
6de040df
LD
133 expand = isinstance(c,basestring)
134 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135 pipe = p.stdout
b016d397 136 val = pipe.read()
6de040df
LD
137 if p.wait() and not ignore_error:
138 die('Command failed: %s' % str(c))
b016d397
HWN
139
140 return val
141
d9429194
AK
142def p4_read_pipe(c, ignore_error=False):
143 real_cmd = p4_build_cmd(c)
144 return read_pipe(real_cmd, ignore_error)
b016d397 145
bce4c5fc 146def read_pipe_lines(c):
4addad22 147 if verbose:
6de040df
LD
148 sys.stderr.write('Reading pipe: %s\n' % str(c))
149
150 expand = isinstance(c, basestring)
151 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152 pipe = p.stdout
b016d397 153 val = pipe.readlines()
6de040df
LD
154 if pipe.close() or p.wait():
155 die('Command failed: %s' % str(c))
b016d397
HWN
156
157 return val
caace111 158
2318121b
AK
159def p4_read_pipe_lines(c):
160 """Specifically invoke p4 on the command supplied. """
155af834 161 real_cmd = p4_build_cmd(c)
2318121b
AK
162 return read_pipe_lines(real_cmd)
163
8e9497c2
GG
164def p4_has_command(cmd):
165 """Ask p4 for help on this command. If it returns an error, the
166 command does not exist in this version of p4."""
167 real_cmd = p4_build_cmd(["help", cmd])
168 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169 stderr=subprocess.PIPE)
170 p.communicate()
171 return p.returncode == 0
172
249da4c0
PW
173def p4_has_move_command():
174 """See if the move command exists, that it supports -k, and that
175 it has not been administratively disabled. The arguments
176 must be correct, but the filenames do not have to exist. Use
177 ones with wildcards so even if they exist, it will fail."""
178
179 if not p4_has_command("move"):
180 return False
181 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183 (out, err) = p.communicate()
184 # return code will be 1 in either case
185 if err.find("Invalid option") >= 0:
186 return False
187 if err.find("disabled") >= 0:
188 return False
189 # assume it failed because @... was invalid changelist
190 return True
191
6754a299 192def system(cmd):
6de040df 193 expand = isinstance(cmd,basestring)
4addad22 194 if verbose:
6de040df 195 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c
BC
196 retcode = subprocess.call(cmd, shell=expand)
197 if retcode:
198 raise CalledProcessError(retcode, cmd)
6754a299 199
bf9320f1
AK
200def p4_system(cmd):
201 """Specifically invoke p4 as the system command. """
155af834 202 real_cmd = p4_build_cmd(cmd)
6de040df 203 expand = isinstance(real_cmd, basestring)
a235e85c
BC
204 retcode = subprocess.call(real_cmd, shell=expand)
205 if retcode:
206 raise CalledProcessError(retcode, real_cmd)
6de040df 207
7f0e5962
PW
208_p4_version_string = None
209def p4_version_string():
210 """Read the version string, showing just the last line, which
211 hopefully is the interesting version bit.
212
213 $ p4 -V
214 Perforce - The Fast Software Configuration Management System.
215 Copyright 1995-2011 Perforce Software. All rights reserved.
216 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
217 """
218 global _p4_version_string
219 if not _p4_version_string:
220 a = p4_read_pipe_lines(["-V"])
221 _p4_version_string = a[-1].rstrip()
222 return _p4_version_string
223
6de040df 224def p4_integrate(src, dest):
9d7d446a 225 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 226
8d7ec362 227def p4_sync(f, *options):
9d7d446a 228 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
229
230def p4_add(f):
9d7d446a
PW
231 # forcibly add file names with wildcards
232 if wildcard_present(f):
233 p4_system(["add", "-f", f])
234 else:
235 p4_system(["add", f])
6de040df
LD
236
237def p4_delete(f):
9d7d446a 238 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
239
240def p4_edit(f):
9d7d446a 241 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
242
243def p4_revert(f):
9d7d446a 244 p4_system(["revert", wildcard_encode(f)])
6de040df 245
9d7d446a
PW
246def p4_reopen(type, f):
247 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 248
8e9497c2
GG
249def p4_move(src, dest):
250 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
251
18fa13d0
PW
252def p4_describe(change):
253 """Make sure it returns a valid result by checking for
254 the presence of field "time". Return a dict of the
255 results."""
256
257 ds = p4CmdList(["describe", "-s", str(change)])
258 if len(ds) != 1:
259 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
260
261 d = ds[0]
262
263 if "p4ExitCode" in d:
264 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265 str(d)))
266 if "code" in d:
267 if d["code"] == "error":
268 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
269
270 if "time" not in d:
271 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
272
273 return d
274
9cffb8c8
PW
275#
276# Canonicalize the p4 type and return a tuple of the
277# base type, plus any modifiers. See "p4 help filetypes"
278# for a list and explanation.
279#
280def split_p4_type(p4type):
281
282 p4_filetypes_historical = {
283 "ctempobj": "binary+Sw",
284 "ctext": "text+C",
285 "cxtext": "text+Cx",
286 "ktext": "text+k",
287 "kxtext": "text+kx",
288 "ltext": "text+F",
289 "tempobj": "binary+FSw",
290 "ubinary": "binary+F",
291 "uresource": "resource+F",
292 "uxbinary": "binary+Fx",
293 "xbinary": "binary+x",
294 "xltext": "text+Fx",
295 "xtempobj": "binary+Swx",
296 "xtext": "text+x",
297 "xunicode": "unicode+x",
298 "xutf16": "utf16+x",
299 }
300 if p4type in p4_filetypes_historical:
301 p4type = p4_filetypes_historical[p4type]
302 mods = ""
303 s = p4type.split("+")
304 base = s[0]
305 mods = ""
306 if len(s) > 1:
307 mods = s[1]
308 return (base, mods)
b9fc6ea9 309
60df071c
LD
310#
311# return the raw p4 type of a file (text, text+ko, etc)
312#
79467e61
PW
313def p4_type(f):
314 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
60df071c
LD
315 return results[0]['headType']
316
317#
318# Given a type base and modifier, return a regexp matching
319# the keywords that can be expanded in the file
320#
321def p4_keywords_regexp_for_type(base, type_mods):
322 if base in ("text", "unicode", "binary"):
323 kwords = None
324 if "ko" in type_mods:
325 kwords = 'Id|Header'
326 elif "k" in type_mods:
327 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328 else:
329 return None
330 pattern = r"""
331 \$ # Starts with a dollar, followed by...
332 (%s) # one of the keywords, followed by...
6b2bf41e 333 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
334 \$ # another dollar
335 """ % kwords
336 return pattern
337 else:
338 return None
339
340#
341# Given a file, return a regexp matching the possible
342# RCS keywords that will be expanded, or None for files
343# with kw expansion turned off.
344#
345def p4_keywords_regexp_for_file(file):
346 if not os.path.exists(file):
347 return None
348 else:
349 (type_base, type_mods) = split_p4_type(p4_type(file))
350 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 351
c65b670e
CP
352def setP4ExecBit(file, mode):
353 # Reopens an already open file and changes the execute bit to match
354 # the execute bit setting in the passed in mode.
355
356 p4Type = "+x"
357
358 if not isModeExec(mode):
359 p4Type = getP4OpenedType(file)
360 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362 if p4Type[-1] == "+":
363 p4Type = p4Type[0:-1]
364
6de040df 365 p4_reopen(p4Type, file)
c65b670e
CP
366
367def getP4OpenedType(file):
368 # Returns the perforce file type for the given file.
369
9d7d446a 370 result = p4_read_pipe(["opened", wildcard_encode(file)])
34a0dbfc 371 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
c65b670e
CP
372 if match:
373 return match.group(1)
374 else:
f3e5ae4f 375 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 376
06804c76
LD
377# Return the set of all p4 labels
378def getP4Labels(depotPaths):
379 labels = set()
380 if isinstance(depotPaths,basestring):
381 depotPaths = [depotPaths]
382
383 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384 label = l['label']
385 labels.add(label)
386
387 return labels
388
389# Return the set of all git tags
390def getGitTags():
391 gitTags = set()
392 for line in read_pipe_lines(["git", "tag"]):
393 tag = line.strip()
394 gitTags.add(tag)
395 return gitTags
396
b43b0a3c
CP
397def diffTreePattern():
398 # This is a simple generator for the diff tree regex pattern. This could be
399 # a class variable if this and parseDiffTreeEntry were a part of a class.
400 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401 while True:
402 yield pattern
403
404def parseDiffTreeEntry(entry):
405 """Parses a single diff tree entry into its component elements.
406
407 See git-diff-tree(1) manpage for details about the format of the diff
408 output. This method returns a dictionary with the following elements:
409
410 src_mode - The mode of the source file
411 dst_mode - The mode of the destination file
412 src_sha1 - The sha1 for the source file
413 dst_sha1 - The sha1 fr the destination file
414 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415 status_score - The score for the status (applicable for 'C' and 'R'
416 statuses). This is None if there is no score.
417 src - The path for the source file.
418 dst - The path for the destination file. This is only present for
419 copy or renames. If it is not present, this is None.
420
421 If the pattern is not matched, None is returned."""
422
423 match = diffTreePattern().next().match(entry)
424 if match:
425 return {
426 'src_mode': match.group(1),
427 'dst_mode': match.group(2),
428 'src_sha1': match.group(3),
429 'dst_sha1': match.group(4),
430 'status': match.group(5),
431 'status_score': match.group(6),
432 'src': match.group(7),
433 'dst': match.group(10)
434 }
435 return None
436
c65b670e
CP
437def isModeExec(mode):
438 # Returns True if the given git mode represents an executable file,
439 # otherwise False.
440 return mode[-3:] == "755"
441
442def isModeExecChanged(src_mode, dst_mode):
443 return isModeExec(src_mode) != isModeExec(dst_mode)
444
b932705b 445def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
446
447 if isinstance(cmd,basestring):
448 cmd = "-G " + cmd
449 expand = True
450 else:
451 cmd = ["-G"] + cmd
452 expand = False
453
454 cmd = p4_build_cmd(cmd)
6a49f8e2 455 if verbose:
6de040df 456 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
457
458 # Use a temporary file to avoid deadlocks without
459 # subprocess.communicate(), which would put another copy
460 # of stdout into memory.
461 stdin_file = None
462 if stdin is not None:
463 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
464 if isinstance(stdin,basestring):
465 stdin_file.write(stdin)
466 else:
467 for i in stdin:
468 stdin_file.write(i + '\n')
9f90c733
SL
469 stdin_file.flush()
470 stdin_file.seek(0)
471
6de040df
LD
472 p4 = subprocess.Popen(cmd,
473 shell=expand,
9f90c733
SL
474 stdin=stdin_file,
475 stdout=subprocess.PIPE)
86949eef
SH
476
477 result = []
478 try:
479 while True:
9f90c733 480 entry = marshal.load(p4.stdout)
c3f6163b
AG
481 if cb is not None:
482 cb(entry)
483 else:
484 result.append(entry)
86949eef
SH
485 except EOFError:
486 pass
9f90c733
SL
487 exitCode = p4.wait()
488 if exitCode != 0:
ac3e0d79
SH
489 entry = {}
490 entry["p4ExitCode"] = exitCode
491 result.append(entry)
86949eef
SH
492
493 return result
494
495def p4Cmd(cmd):
496 list = p4CmdList(cmd)
497 result = {}
498 for entry in list:
499 result.update(entry)
500 return result;
501
cb2c9db5
SH
502def p4Where(depotPath):
503 if not depotPath.endswith("/"):
504 depotPath += "/"
cd884106
VA
505 depotPathLong = depotPath + "..."
506 outputList = p4CmdList(["where", depotPathLong])
7f705dc3
TAL
507 output = None
508 for entry in outputList:
75bc9573 509 if "depotFile" in entry:
cd884106
VA
510 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
511 # The base path always ends with "/...".
512 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
75bc9573
TAL
513 output = entry
514 break
515 elif "data" in entry:
516 data = entry.get("data")
517 space = data.find(" ")
518 if data[:space] == depotPath:
519 output = entry
520 break
7f705dc3
TAL
521 if output == None:
522 return ""
dc524036
SH
523 if output["code"] == "error":
524 return ""
cb2c9db5
SH
525 clientPath = ""
526 if "path" in output:
527 clientPath = output.get("path")
528 elif "data" in output:
529 data = output.get("data")
530 lastSpace = data.rfind(" ")
531 clientPath = data[lastSpace + 1:]
532
533 if clientPath.endswith("..."):
534 clientPath = clientPath[:-3]
535 return clientPath
536
86949eef 537def currentGitBranch():
b25b2065 538 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 539
4f5cf76a 540def isValidGitDir(path):
bb6e09b2
HWN
541 if (os.path.exists(path + "/HEAD")
542 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
543 return True;
544 return False
545
463e8af6 546def parseRevision(ref):
b25b2065 547 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 548
28755dba
PW
549def branchExists(ref):
550 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
551 ignore_error=True)
552 return len(rev) > 0
553
6ae8de88
SH
554def extractLogMessageFromGitCommit(commit):
555 logMessage = ""
b016d397
HWN
556
557 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 558 foundTitle = False
b016d397 559 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
560 if not foundTitle:
561 if len(log) == 1:
1c094184 562 foundTitle = True
6ae8de88
SH
563 continue
564
565 logMessage += log
566 return logMessage
567
bb6e09b2 568def extractSettingsGitLog(log):
6ae8de88
SH
569 values = {}
570 for line in log.split("\n"):
571 line = line.strip()
6326aa58
HWN
572 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
573 if not m:
574 continue
575
576 assignments = m.group(1).split (':')
577 for a in assignments:
578 vals = a.split ('=')
579 key = vals[0].strip()
580 val = ('='.join (vals[1:])).strip()
581 if val.endswith ('\"') and val.startswith('"'):
582 val = val[1:-1]
583
584 values[key] = val
585
845b42cb
SH
586 paths = values.get("depot-paths")
587 if not paths:
588 paths = values.get("depot-path")
a3fdd579
SH
589 if paths:
590 values['depot-paths'] = paths.split(',')
bb6e09b2 591 return values
6ae8de88 592
8136a639 593def gitBranchExists(branch):
bb6e09b2
HWN
594 proc = subprocess.Popen(["git", "rev-parse", branch],
595 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 596 return proc.wait() == 0;
8136a639 597
36bd8446 598_gitConfig = {}
b345d6c3 599
0d609032 600def gitConfig(key):
36bd8446 601 if not _gitConfig.has_key(key):
0d609032 602 cmd = [ "git", "config", key ]
b345d6c3
PW
603 s = read_pipe(cmd, ignore_error=True)
604 _gitConfig[key] = s.strip()
36bd8446 605 return _gitConfig[key]
01265103 606
0d609032
PW
607def gitConfigBool(key):
608 """Return a bool, using git config --bool. It is True only if the
609 variable is set to true, and False if set to false or not present
610 in the config."""
611
36bd8446 612 if not _gitConfig.has_key(key):
0d609032
PW
613 cmd = [ "git", "config", "--bool", key ]
614 s = read_pipe(cmd, ignore_error=True)
615 v = s.strip()
616 _gitConfig[key] = v == "true"
36bd8446 617 return _gitConfig[key]
01265103 618
7199cf13
VA
619def gitConfigList(key):
620 if not _gitConfig.has_key(key):
2abba301
PW
621 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
622 _gitConfig[key] = s.strip().split(os.linesep)
7199cf13
VA
623 return _gitConfig[key]
624
2c8037ed
PW
625def p4BranchesInGit(branchesAreInRemotes=True):
626 """Find all the branches whose names start with "p4/", looking
627 in remotes or heads as specified by the argument. Return
628 a dictionary of { branch: revision } for each one found.
629 The branch names are the short names, without any
630 "p4/" prefix."""
631
062410bb
SH
632 branches = {}
633
634 cmdline = "git rev-parse --symbolic "
635 if branchesAreInRemotes:
2c8037ed 636 cmdline += "--remotes"
062410bb 637 else:
2c8037ed 638 cmdline += "--branches"
062410bb
SH
639
640 for line in read_pipe_lines(cmdline):
641 line = line.strip()
642
2c8037ed
PW
643 # only import to p4/
644 if not line.startswith('p4/'):
645 continue
646 # special symbolic ref to p4/master
647 if line == "p4/HEAD":
062410bb 648 continue
062410bb 649
2c8037ed
PW
650 # strip off p4/ prefix
651 branch = line[len("p4/"):]
062410bb
SH
652
653 branches[branch] = parseRevision(line)
2c8037ed 654
062410bb
SH
655 return branches
656
5a8e84cd
PW
657def branch_exists(branch):
658 """Make sure that the given ref name really exists."""
659
660 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
661 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
662 out, _ = p.communicate()
663 if p.returncode:
664 return False
665 # expect exactly one line of output: the branch name
666 return out.rstrip() == branch
667
9ceab363 668def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
669 branches = p4BranchesInGit()
670 # map from depot-path to branch name
671 branchByDepotPath = {}
672 for branch in branches.keys():
673 tip = branches[branch]
674 log = extractLogMessageFromGitCommit(tip)
675 settings = extractSettingsGitLog(log)
676 if settings.has_key("depot-paths"):
677 paths = ",".join(settings["depot-paths"])
678 branchByDepotPath[paths] = "remotes/p4/" + branch
679
27d2d811 680 settings = None
27d2d811
SH
681 parent = 0
682 while parent < 65535:
9ceab363 683 commit = head + "~%s" % parent
27d2d811
SH
684 log = extractLogMessageFromGitCommit(commit)
685 settings = extractSettingsGitLog(log)
86506fe5
SH
686 if settings.has_key("depot-paths"):
687 paths = ",".join(settings["depot-paths"])
688 if branchByDepotPath.has_key(paths):
689 return [branchByDepotPath[paths], settings]
27d2d811 690
86506fe5 691 parent = parent + 1
27d2d811 692
86506fe5 693 return ["", settings]
27d2d811 694
5ca44617
SH
695def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
696 if not silent:
697 print ("Creating/updating branch(es) in %s based on origin branch(es)"
698 % localRefPrefix)
699
700 originPrefix = "origin/p4/"
701
702 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
703 line = line.strip()
704 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
705 continue
706
707 headName = line[len(originPrefix):]
708 remoteHead = localRefPrefix + headName
709 originHead = line
710
711 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
712 if (not original.has_key('depot-paths')
713 or not original.has_key('change')):
714 continue
715
716 update = False
717 if not gitBranchExists(remoteHead):
718 if verbose:
719 print "creating %s" % remoteHead
720 update = True
721 else:
722 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
723 if settings.has_key('change') > 0:
724 if settings['depot-paths'] == original['depot-paths']:
725 originP4Change = int(original['change'])
726 p4Change = int(settings['change'])
727 if originP4Change > p4Change:
728 print ("%s (%s) is newer than %s (%s). "
729 "Updating p4 branch from origin."
730 % (originHead, originP4Change,
731 remoteHead, p4Change))
732 update = True
733 else:
734 print ("Ignoring: %s was imported from %s while "
735 "%s was imported from %s"
736 % (originHead, ','.join(original['depot-paths']),
737 remoteHead, ','.join(settings['depot-paths'])))
738
739 if update:
740 system("git update-ref %s %s" % (remoteHead, originHead))
741
742def originP4BranchesExist():
743 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
744
96b2d54a 745def p4ChangesForPaths(depotPaths, changeRange, block_size):
4f6432d8 746 assert depotPaths
96b2d54a
LS
747 assert block_size
748
749 # Parse the change range into start and end
750 if changeRange is None or changeRange == '':
751 changeStart = '@1'
752 changeEnd = '#head'
753 else:
754 parts = changeRange.split(',')
755 assert len(parts) == 2
756 changeStart = parts[0]
757 changeEnd = parts[1]
4f6432d8 758
96b2d54a 759 # Accumulate change numbers in a dictionary to avoid duplicates
b4b0ba06 760 changes = {}
96b2d54a
LS
761
762 for p in depotPaths:
763 # Retrieve changes a block at a time, to prevent running
764 # into a MaxScanRows error from the server.
765 start = changeStart
766 end = changeEnd
767 get_another_block = True
768 while get_another_block:
769 new_changes = []
770 cmd = ['changes']
771 cmd += ['-m', str(block_size)]
772 cmd += ["%s...%s,%s" % (p, start, end)]
773 for line in p4_read_pipe_lines(cmd):
774 changeNum = int(line.split(" ")[1])
775 new_changes.append(changeNum)
776 changes[changeNum] = True
777 if len(new_changes) == block_size:
778 get_another_block = True
779 end = '@' + str(min(new_changes))
780 else:
781 get_another_block = False
4f6432d8 782
b4b0ba06
PW
783 changelist = changes.keys()
784 changelist.sort()
785 return changelist
4f6432d8 786
d53de8b9
TAL
787def p4PathStartsWith(path, prefix):
788 # This method tries to remedy a potential mixed-case issue:
789 #
790 # If UserA adds //depot/DirA/file1
791 # and UserB adds //depot/dira/file2
792 #
793 # we may or may not have a problem. If you have core.ignorecase=true,
794 # we treat DirA and dira as the same directory
0d609032 795 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
796 return path.lower().startswith(prefix.lower())
797 return path.startswith(prefix)
798
543987bd
PW
799def getClientSpec():
800 """Look at the p4 client spec, create a View() object that contains
801 all the mappings, and return it."""
802
803 specList = p4CmdList("client -o")
804 if len(specList) != 1:
805 die('Output from "client -o" is %d lines, expecting 1' %
806 len(specList))
807
808 # dictionary of all client parameters
809 entry = specList[0]
810
9d57c4a6
KS
811 # the //client/ name
812 client_name = entry["Client"]
813
543987bd
PW
814 # just the keys that start with "View"
815 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
816
817 # hold this new View
9d57c4a6 818 view = View(client_name)
543987bd
PW
819
820 # append the lines, in order, to the view
821 for view_num in range(len(view_keys)):
822 k = "View%d" % view_num
823 if k not in view_keys:
824 die("Expected view key %s missing" % k)
825 view.append(entry[k])
826
827 return view
828
829def getClientRoot():
830 """Grab the client directory."""
831
832 output = p4CmdList("client -o")
833 if len(output) != 1:
834 die('Output from "client -o" is %d lines, expecting 1' % len(output))
835
836 entry = output[0]
837 if "Root" not in entry:
838 die('Client has no "Root"')
839
840 return entry["Root"]
841
9d7d446a
PW
842#
843# P4 wildcards are not allowed in filenames. P4 complains
844# if you simply add them, but you can force it with "-f", in
845# which case it translates them into %xx encoding internally.
846#
847def wildcard_decode(path):
848 # Search for and fix just these four characters. Do % last so
849 # that fixing it does not inadvertently create new %-escapes.
850 # Cannot have * in a filename in windows; untested as to
851 # what p4 would do in such a case.
852 if not platform.system() == "Windows":
853 path = path.replace("%2A", "*")
854 path = path.replace("%23", "#") \
855 .replace("%40", "@") \
856 .replace("%25", "%")
857 return path
858
859def wildcard_encode(path):
860 # do % first to avoid double-encoding the %s introduced here
861 path = path.replace("%", "%25") \
862 .replace("*", "%2A") \
863 .replace("#", "%23") \
864 .replace("@", "%40")
865 return path
866
867def wildcard_present(path):
598354c0
BC
868 m = re.search("[*#@%]", path)
869 return m is not None
9d7d446a 870
b984733c
SH
871class Command:
872 def __init__(self):
873 self.usage = "usage: %prog [options]"
8910ac0e 874 self.needsGit = True
6a10b6aa 875 self.verbose = False
b984733c 876
3ea2cfd4
LD
877class P4UserMap:
878 def __init__(self):
879 self.userMapFromPerforceServer = False
affb474f
LD
880 self.myP4UserId = None
881
882 def p4UserId(self):
883 if self.myP4UserId:
884 return self.myP4UserId
885
886 results = p4CmdList("user -o")
887 for r in results:
888 if r.has_key('User'):
889 self.myP4UserId = r['User']
890 return r['User']
891 die("Could not find your p4 user id")
892
893 def p4UserIsMe(self, p4User):
894 # return True if the given p4 user is actually me
895 me = self.p4UserId()
896 if not p4User or p4User != me:
897 return False
898 else:
899 return True
3ea2cfd4
LD
900
901 def getUserCacheFilename(self):
902 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
903 return home + "/.gitp4-usercache.txt"
904
905 def getUserMapFromPerforceServer(self):
906 if self.userMapFromPerforceServer:
907 return
908 self.users = {}
909 self.emails = {}
910
911 for output in p4CmdList("users"):
912 if not output.has_key("User"):
913 continue
914 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
915 self.emails[output["Email"]] = output["User"]
916
917
918 s = ''
919 for (key, val) in self.users.items():
920 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
921
922 open(self.getUserCacheFilename(), "wb").write(s)
923 self.userMapFromPerforceServer = True
924
925 def loadUserMapFromCache(self):
926 self.users = {}
927 self.userMapFromPerforceServer = False
928 try:
929 cache = open(self.getUserCacheFilename(), "rb")
930 lines = cache.readlines()
931 cache.close()
932 for line in lines:
933 entry = line.strip().split("\t")
934 self.users[entry[0]] = entry[1]
935 except IOError:
936 self.getUserMapFromPerforceServer()
937
b984733c 938class P4Debug(Command):
86949eef 939 def __init__(self):
6ae8de88 940 Command.__init__(self)
6a10b6aa 941 self.options = []
c8c39116 942 self.description = "A tool to debug the output of p4 -G."
8910ac0e 943 self.needsGit = False
86949eef
SH
944
945 def run(self, args):
b1ce9447 946 j = 0
6de040df 947 for output in p4CmdList(args):
b1ce9447
HWN
948 print 'Element: %d' % j
949 j += 1
86949eef 950 print output
b984733c 951 return True
86949eef 952
5834684d
SH
953class P4RollBack(Command):
954 def __init__(self):
955 Command.__init__(self)
956 self.options = [
0c66a783 957 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
958 ]
959 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 960 self.rollbackLocalBranches = False
5834684d
SH
961
962 def run(self, args):
963 if len(args) != 1:
964 return False
965 maxChange = int(args[0])
0c66a783 966
ad192f28 967 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
968 die("Problems executing p4");
969
0c66a783
SH
970 if self.rollbackLocalBranches:
971 refPrefix = "refs/heads/"
b016d397 972 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
973 else:
974 refPrefix = "refs/remotes/"
b016d397 975 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
976
977 for line in lines:
978 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
979 line = line.strip()
980 ref = refPrefix + line
5834684d 981 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
982 settings = extractSettingsGitLog(log)
983
984 depotPaths = settings['depot-paths']
985 change = settings['change']
986
5834684d 987 changed = False
52102d47 988
6326aa58
HWN
989 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
990 for p in depotPaths]))) == 0:
52102d47
SH
991 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
992 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
993 continue
994
bb6e09b2 995 while change and int(change) > maxChange:
5834684d 996 changed = True
52102d47
SH
997 if self.verbose:
998 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
999 system("git update-ref %s \"%s^\"" % (ref, ref))
1000 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
1001 settings = extractSettingsGitLog(log)
1002
1003
1004 depotPaths = settings['depot-paths']
1005 change = settings['change']
5834684d
SH
1006
1007 if changed:
52102d47 1008 print "%s rewound to %s" % (ref, change)
5834684d
SH
1009
1010 return True
1011
3ea2cfd4 1012class P4Submit(Command, P4UserMap):
6bbfd137
PW
1013
1014 conflict_behavior_choices = ("ask", "skip", "quit")
1015
4f5cf76a 1016 def __init__(self):
b984733c 1017 Command.__init__(self)
3ea2cfd4 1018 P4UserMap.__init__(self)
4f5cf76a 1019 self.options = [
4f5cf76a 1020 optparse.make_option("--origin", dest="origin"),
ae901090 1021 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
1022 # preserve the user, requires relevant p4 permissions
1023 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 1024 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 1025 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 1026 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 1027 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
1028 choices=self.conflict_behavior_choices),
1029 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
1030 ]
1031 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1032 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1033 self.origin = ""
ae901090 1034 self.detectRenames = False
0d609032 1035 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1036 self.dry_run = False
728b7ad8 1037 self.prepare_p4_only = False
6bbfd137 1038 self.conflict_behavior = None
f7baba8b 1039 self.isWindows = (platform.system() == "Windows")
06804c76 1040 self.exportLabels = False
249da4c0 1041 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1042 self.branch = None
4f5cf76a 1043
4f5cf76a
SH
1044 def check(self):
1045 if len(p4CmdList("opened ...")) > 0:
1046 die("You have files opened with perforce! Close them before starting the sync.")
1047
f19cb0a0
PW
1048 def separate_jobs_from_description(self, message):
1049 """Extract and return a possible Jobs field in the commit
1050 message. It goes into a separate section in the p4 change
1051 specification.
1052
1053 A jobs line starts with "Jobs:" and looks like a new field
1054 in a form. Values are white-space separated on the same
1055 line or on following lines that start with a tab.
1056
1057 This does not parse and extract the full git commit message
1058 like a p4 form. It just sees the Jobs: line as a marker
1059 to pass everything from then on directly into the p4 form,
1060 but outside the description section.
1061
1062 Return a tuple (stripped log message, jobs string)."""
1063
1064 m = re.search(r'^Jobs:', message, re.MULTILINE)
1065 if m is None:
1066 return (message, None)
1067
1068 jobtext = message[m.start():]
1069 stripped_message = message[:m.start()].rstrip()
1070 return (stripped_message, jobtext)
1071
1072 def prepareLogMessage(self, template, message, jobs):
1073 """Edits the template returned from "p4 change -o" to insert
1074 the message in the Description field, and the jobs text in
1075 the Jobs field."""
4f5cf76a
SH
1076 result = ""
1077
edae1e2f
SH
1078 inDescriptionSection = False
1079
4f5cf76a
SH
1080 for line in template.split("\n"):
1081 if line.startswith("#"):
1082 result += line + "\n"
1083 continue
1084
edae1e2f 1085 if inDescriptionSection:
c9dbab04 1086 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1087 inDescriptionSection = False
f19cb0a0
PW
1088 # insert Jobs section
1089 if jobs:
1090 result += jobs + "\n"
edae1e2f
SH
1091 else:
1092 continue
1093 else:
1094 if line.startswith("Description:"):
1095 inDescriptionSection = True
1096 line += "\n"
1097 for messageLine in message.split("\n"):
1098 line += "\t" + messageLine + "\n"
1099
1100 result += line + "\n"
4f5cf76a
SH
1101
1102 return result
1103
60df071c
LD
1104 def patchRCSKeywords(self, file, pattern):
1105 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1106 (handle, outFileName) = tempfile.mkstemp(dir='.')
1107 try:
1108 outFile = os.fdopen(handle, "w+")
1109 inFile = open(file, "r")
1110 regexp = re.compile(pattern, re.VERBOSE)
1111 for line in inFile.readlines():
1112 line = regexp.sub(r'$\1$', line)
1113 outFile.write(line)
1114 inFile.close()
1115 outFile.close()
1116 # Forcibly overwrite the original file
1117 os.unlink(file)
1118 shutil.move(outFileName, file)
1119 except:
1120 # cleanup our temporary file
1121 os.unlink(outFileName)
1122 print "Failed to strip RCS keywords in %s" % file
1123 raise
1124
1125 print "Patched up RCS keywords in %s" % file
1126
3ea2cfd4
LD
1127 def p4UserForCommit(self,id):
1128 # Return the tuple (perforce user,git email) for a given git commit id
1129 self.getUserMapFromPerforceServer()
9bf28855
PW
1130 gitEmail = read_pipe(["git", "log", "--max-count=1",
1131 "--format=%ae", id])
3ea2cfd4
LD
1132 gitEmail = gitEmail.strip()
1133 if not self.emails.has_key(gitEmail):
1134 return (None,gitEmail)
1135 else:
1136 return (self.emails[gitEmail],gitEmail)
1137
1138 def checkValidP4Users(self,commits):
1139 # check if any git authors cannot be mapped to p4 users
1140 for id in commits:
1141 (user,email) = self.p4UserForCommit(id)
1142 if not user:
1143 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1144 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1145 print "%s" % msg
1146 else:
1147 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1148
1149 def lastP4Changelist(self):
1150 # Get back the last changelist number submitted in this client spec. This
1151 # then gets used to patch up the username in the change. If the same
1152 # client spec is being used by multiple processes then this might go
1153 # wrong.
1154 results = p4CmdList("client -o") # find the current client
1155 client = None
1156 for r in results:
1157 if r.has_key('Client'):
1158 client = r['Client']
1159 break
1160 if not client:
1161 die("could not get client spec")
6de040df 1162 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1163 for r in results:
1164 if r.has_key('change'):
1165 return r['change']
1166 die("Could not get changelist number for last submit - cannot patch up user details")
1167
1168 def modifyChangelistUser(self, changelist, newUser):
1169 # fixup the user field of a changelist after it has been submitted.
1170 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1171 if len(changes) != 1:
1172 die("Bad output from p4 change modifying %s to user %s" %
1173 (changelist, newUser))
1174
1175 c = changes[0]
1176 if c['User'] == newUser: return # nothing to do
1177 c['User'] = newUser
1178 input = marshal.dumps(c)
1179
3ea2cfd4
LD
1180 result = p4CmdList("change -f -i", stdin=input)
1181 for r in result:
1182 if r.has_key('code'):
1183 if r['code'] == 'error':
1184 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1185 if r.has_key('data'):
1186 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1187 return
1188 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1189
1190 def canChangeChangelists(self):
1191 # check to see if we have p4 admin or super-user permissions, either of
1192 # which are required to modify changelists.
52a4880b 1193 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1194 for r in results:
1195 if r.has_key('perm'):
1196 if r['perm'] == 'admin':
1197 return 1
1198 if r['perm'] == 'super':
1199 return 1
1200 return 0
1201
ea99c3ae 1202 def prepareSubmitTemplate(self):
f19cb0a0
PW
1203 """Run "p4 change -o" to grab a change specification template.
1204 This does not use "p4 -G", as it is nice to keep the submission
1205 template in original order, since a human might edit it.
1206
1207 Remove lines in the Files section that show changes to files
1208 outside the depot path we're committing into."""
1209
ea99c3ae
SH
1210 template = ""
1211 inFilesSection = False
6de040df 1212 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1213 if line.endswith("\r\n"):
1214 line = line[:-2] + "\n"
ea99c3ae
SH
1215 if inFilesSection:
1216 if line.startswith("\t"):
1217 # path starts and ends with a tab
1218 path = line[1:]
1219 lastTab = path.rfind("\t")
1220 if lastTab != -1:
1221 path = path[:lastTab]
d53de8b9 1222 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
1223 continue
1224 else:
1225 inFilesSection = False
1226 else:
1227 if line.startswith("Files:"):
1228 inFilesSection = True
1229
1230 template += line
1231
1232 return template
1233
7c766e57
PW
1234 def edit_template(self, template_file):
1235 """Invoke the editor to let the user change the submission
1236 message. Return true if okay to continue with the submit."""
1237
1238 # if configured to skip the editing part, just submit
0d609032 1239 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1240 return True
1241
1242 # look at the modification time, to check later if the user saved
1243 # the file
1244 mtime = os.stat(template_file).st_mtime
1245
1246 # invoke the editor
f95ceaf0 1247 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1248 editor = os.environ.get("P4EDITOR")
1249 else:
1250 editor = read_pipe("git var GIT_EDITOR").strip()
2dade7a7 1251 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
7c766e57
PW
1252
1253 # If the file was not saved, prompt to see if this patch should
1254 # be skipped. But skip this verification step if configured so.
0d609032 1255 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1256 return True
1257
d1652049
PW
1258 # modification time updated means user saved the file
1259 if os.stat(template_file).st_mtime > mtime:
1260 return True
1261
1262 while True:
1263 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1264 if response == 'y':
1265 return True
1266 if response == 'n':
1267 return False
7c766e57 1268
e2a892ee 1269 def get_diff_description(self, editedFiles, filesToAdd):
b4073bb3
MC
1270 # diff
1271 if os.environ.has_key("P4DIFF"):
1272 del(os.environ["P4DIFF"])
1273 diff = ""
1274 for editedFile in editedFiles:
1275 diff += p4_read_pipe(['diff', '-du',
1276 wildcard_encode(editedFile)])
1277
1278 # new file diff
1279 newdiff = ""
1280 for newFile in filesToAdd:
1281 newdiff += "==== new file ====\n"
1282 newdiff += "--- /dev/null\n"
1283 newdiff += "+++ %s\n" % newFile
1284 f = open(newFile, "r")
1285 for line in f.readlines():
1286 newdiff += "+" + line
1287 f.close()
1288
e2a892ee 1289 return (diff + newdiff).replace('\r\n', '\n')
b4073bb3 1290
7cb5cbef 1291 def applyCommit(self, id):
67b0fe2e
PW
1292 """Apply one commit, return True if it succeeded."""
1293
1294 print "Applying", read_pipe(["git", "show", "-s",
1295 "--format=format:%h %s", id])
ae901090 1296
848de9c3 1297 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1298
84cb0003 1299 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1300 filesToAdd = set()
1301 filesToDelete = set()
d336c158 1302 editedFiles = set()
b6ad6dcc 1303 pureRenameCopy = set()
c65b670e 1304 filesToChangeExecBit = {}
60df071c 1305
4f5cf76a 1306 for line in diff:
b43b0a3c
CP
1307 diff = parseDiffTreeEntry(line)
1308 modifier = diff['status']
1309 path = diff['src']
4f5cf76a 1310 if modifier == "M":
6de040df 1311 p4_edit(path)
c65b670e
CP
1312 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1313 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1314 editedFiles.add(path)
4f5cf76a
SH
1315 elif modifier == "A":
1316 filesToAdd.add(path)
c65b670e 1317 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1318 if path in filesToDelete:
1319 filesToDelete.remove(path)
1320 elif modifier == "D":
1321 filesToDelete.add(path)
1322 if path in filesToAdd:
1323 filesToAdd.remove(path)
4fddb41b
VA
1324 elif modifier == "C":
1325 src, dest = diff['src'], diff['dst']
6de040df 1326 p4_integrate(src, dest)
b6ad6dcc 1327 pureRenameCopy.add(dest)
4fddb41b 1328 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1329 p4_edit(dest)
b6ad6dcc 1330 pureRenameCopy.discard(dest)
4fddb41b 1331 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1332 p4_edit(dest)
b6ad6dcc 1333 pureRenameCopy.discard(dest)
4fddb41b 1334 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1335 if self.isWindows:
1336 # turn off read-only attribute
1337 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1338 os.unlink(dest)
1339 editedFiles.add(dest)
d9a5f25b 1340 elif modifier == "R":
b43b0a3c 1341 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1342 if self.p4HasMoveCommand:
1343 p4_edit(src) # src must be open before move
1344 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1345 else:
8e9497c2
GG
1346 p4_integrate(src, dest)
1347 if diff['src_sha1'] != diff['dst_sha1']:
1348 p4_edit(dest)
1349 else:
1350 pureRenameCopy.add(dest)
c65b670e 1351 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1352 if not self.p4HasMoveCommand:
1353 p4_edit(dest) # with move: already open, writable
c65b670e 1354 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1355 if not self.p4HasMoveCommand:
d20f0f8e
PW
1356 if self.isWindows:
1357 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1358 os.unlink(dest)
1359 filesToDelete.add(src)
d9a5f25b 1360 editedFiles.add(dest)
4f5cf76a
SH
1361 else:
1362 die("unknown modifier %s for %s" % (modifier, path))
1363
749b668c 1364 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
47a130b7 1365 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1366 tryPatchCmd = patchcmd + "--check -"
1367 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1368 patch_succeeded = True
51a2640a 1369
47a130b7 1370 if os.system(tryPatchCmd) != 0:
60df071c
LD
1371 fixed_rcs_keywords = False
1372 patch_succeeded = False
51a2640a 1373 print "Unfortunately applying the change failed!"
60df071c
LD
1374
1375 # Patch failed, maybe it's just RCS keyword woes. Look through
1376 # the patch to see if that's possible.
0d609032 1377 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1378 file = None
1379 pattern = None
1380 kwfiles = {}
1381 for file in editedFiles | filesToDelete:
1382 # did this file's delta contain RCS keywords?
1383 pattern = p4_keywords_regexp_for_file(file)
1384
1385 if pattern:
1386 # this file is a possibility...look for RCS keywords.
1387 regexp = re.compile(pattern, re.VERBOSE)
1388 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1389 if regexp.search(line):
1390 if verbose:
1391 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1392 kwfiles[file] = pattern
1393 break
1394
1395 for file in kwfiles:
1396 if verbose:
1397 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1398 # File is being deleted, so not open in p4. Must
1399 # disable the read-only bit on windows.
1400 if self.isWindows and file not in editedFiles:
1401 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1402 self.patchRCSKeywords(file, kwfiles[file])
1403 fixed_rcs_keywords = True
1404
1405 if fixed_rcs_keywords:
1406 print "Retrying the patch with RCS keywords cleaned up"
1407 if os.system(tryPatchCmd) == 0:
1408 patch_succeeded = True
1409
1410 if not patch_succeeded:
7e5dd9f2
PW
1411 for f in editedFiles:
1412 p4_revert(f)
7e5dd9f2 1413 return False
51a2640a 1414
55ac2ed6
PW
1415 #
1416 # Apply the patch for real, and do add/delete/+x handling.
1417 #
47a130b7 1418 system(applyPatchCmd)
4f5cf76a
SH
1419
1420 for f in filesToAdd:
6de040df 1421 p4_add(f)
4f5cf76a 1422 for f in filesToDelete:
6de040df
LD
1423 p4_revert(f)
1424 p4_delete(f)
4f5cf76a 1425
c65b670e
CP
1426 # Set/clear executable bits
1427 for f in filesToChangeExecBit.keys():
1428 mode = filesToChangeExecBit[f]
1429 setP4ExecBit(f, mode)
1430
55ac2ed6
PW
1431 #
1432 # Build p4 change description, starting with the contents
1433 # of the git commit message.
1434 #
0e36f2d7 1435 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1436 logMessage = logMessage.strip()
f19cb0a0 1437 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1438
ea99c3ae 1439 template = self.prepareSubmitTemplate()
f19cb0a0 1440 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1441
c47178d4 1442 if self.preserveUser:
55ac2ed6 1443 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1444
55ac2ed6
PW
1445 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1446 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1447 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1448 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1449
55ac2ed6 1450 separatorLine = "######## everything below this line is just the diff #######\n"
b4073bb3
MC
1451 if not self.prepare_p4_only:
1452 submitTemplate += separatorLine
e2a892ee 1453 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
55ac2ed6 1454
c47178d4 1455 (handle, fileName) = tempfile.mkstemp()
e2a892ee 1456 tmpFile = os.fdopen(handle, "w+b")
c47178d4
PW
1457 if self.isWindows:
1458 submitTemplate = submitTemplate.replace("\n", "\r\n")
b4073bb3 1459 tmpFile.write(submitTemplate)
c47178d4
PW
1460 tmpFile.close()
1461
728b7ad8
PW
1462 if self.prepare_p4_only:
1463 #
1464 # Leave the p4 tree prepared, and the submit template around
1465 # and let the user decide what to do next
1466 #
1467 print
1468 print "P4 workspace prepared for submission."
1469 print "To submit or revert, go to client workspace"
1470 print " " + self.clientPath
1471 print
1472 print "To submit, use \"p4 submit\" to write a new description,"
10de86d0 1473 print "or \"p4 submit -i <%s\" to use the one prepared by" \
728b7ad8
PW
1474 " \"git p4\"." % fileName
1475 print "You can delete the file \"%s\" when finished." % fileName
1476
1477 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1478 print "To preserve change ownership by user %s, you must\n" \
1479 "do \"p4 change -f <change>\" after submitting and\n" \
1480 "edit the User field."
1481 if pureRenameCopy:
1482 print "After submitting, renamed files must be re-synced."
1483 print "Invoke \"p4 sync -f\" on each of these files:"
1484 for f in pureRenameCopy:
1485 print " " + f
1486
1487 print
1488 print "To revert the changes, use \"p4 revert ...\", and delete"
1489 print "the submit template file \"%s\"" % fileName
1490 if filesToAdd:
1491 print "Since the commit adds new files, they must be deleted:"
1492 for f in filesToAdd:
1493 print " " + f
1494 print
1495 return True
1496
55ac2ed6
PW
1497 #
1498 # Let the user edit the change description, then submit it.
1499 #
c47178d4
PW
1500 if self.edit_template(fileName):
1501 # read the edited message and submit
5a41c16a 1502 ret = True
c47178d4
PW
1503 tmpFile = open(fileName, "rb")
1504 message = tmpFile.read()
e96e400f 1505 tmpFile.close()
c47178d4 1506 if self.isWindows:
e2a892ee
MC
1507 message = message.replace("\r\n", "\n")
1508 submitTemplate = message[:message.index(separatorLine)]
c47178d4 1509 p4_write_pipe(['submit', '-i'], submitTemplate)
e96e400f 1510
c47178d4
PW
1511 if self.preserveUser:
1512 if p4User:
1513 # Get last changelist number. Cannot easily get it from
1514 # the submit command output as the output is
1515 # unmarshalled.
1516 changelist = self.lastP4Changelist()
1517 self.modifyChangelistUser(changelist, p4User)
1518
1519 # The rename/copy happened by applying a patch that created a
1520 # new file. This leaves it writable, which confuses p4.
1521 for f in pureRenameCopy:
1522 p4_sync(f, "-f")
cdc7e388 1523
4f5cf76a 1524 else:
c47178d4 1525 # skip this patch
5a41c16a 1526 ret = False
c47178d4
PW
1527 print "Submission cancelled, undoing p4 changes."
1528 for f in editedFiles:
1529 p4_revert(f)
1530 for f in filesToAdd:
1531 p4_revert(f)
1532 os.remove(f)
df9c5453
PW
1533 for f in filesToDelete:
1534 p4_revert(f)
c47178d4
PW
1535
1536 os.remove(fileName)
5a41c16a 1537 return ret
4f5cf76a 1538
06804c76
LD
1539 # Export git tags as p4 labels. Create a p4 label and then tag
1540 # with that.
1541 def exportGitTags(self, gitTags):
c8942a22
LD
1542 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1543 if len(validLabelRegexp) == 0:
1544 validLabelRegexp = defaultLabelRegexp
1545 m = re.compile(validLabelRegexp)
06804c76
LD
1546
1547 for name in gitTags:
1548
1549 if not m.match(name):
1550 if verbose:
05a3cec5 1551 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1552 continue
1553
1554 # Get the p4 commit this corresponds to
c8942a22
LD
1555 logMessage = extractLogMessageFromGitCommit(name)
1556 values = extractSettingsGitLog(logMessage)
06804c76 1557
c8942a22 1558 if not values.has_key('change'):
06804c76
LD
1559 # a tag pointing to something not sent to p4; ignore
1560 if verbose:
1561 print "git tag %s does not give a p4 commit" % name
1562 continue
c8942a22
LD
1563 else:
1564 changelist = values['change']
06804c76
LD
1565
1566 # Get the tag details.
1567 inHeader = True
1568 isAnnotated = False
1569 body = []
1570 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1571 l = l.strip()
1572 if inHeader:
1573 if re.match(r'tag\s+', l):
1574 isAnnotated = True
1575 elif re.match(r'\s*$', l):
1576 inHeader = False
1577 continue
1578 else:
1579 body.append(l)
1580
1581 if not isAnnotated:
1582 body = ["lightweight tag imported by git p4\n"]
1583
1584 # Create the label - use the same view as the client spec we are using
1585 clientSpec = getClientSpec()
1586
1587 labelTemplate = "Label: %s\n" % name
1588 labelTemplate += "Description:\n"
1589 for b in body:
1590 labelTemplate += "\t" + b + "\n"
1591 labelTemplate += "View:\n"
9d57c4a6
KS
1592 for depot_side in clientSpec.mappings:
1593 labelTemplate += "\t%s\n" % depot_side
06804c76 1594
ef739f08
PW
1595 if self.dry_run:
1596 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1597 elif self.prepare_p4_only:
1598 print "Not creating p4 label %s for tag due to option" \
1599 " --prepare-p4-only" % name
ef739f08
PW
1600 else:
1601 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1602
ef739f08
PW
1603 # Use the label
1604 p4_system(["tag", "-l", name] +
9d57c4a6 1605 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
06804c76 1606
ef739f08
PW
1607 if verbose:
1608 print "created p4 label for tag %s" % name
06804c76 1609
4f5cf76a 1610 def run(self, args):
c9b50e63
SH
1611 if len(args) == 0:
1612 self.master = currentGitBranch()
4280e533 1613 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1614 die("Detecting current git branch failed!")
1615 elif len(args) == 1:
1616 self.master = args[0]
28755dba
PW
1617 if not branchExists(self.master):
1618 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1619 else:
1620 return False
1621
4c2d5d72
JX
1622 allowSubmit = gitConfig("git-p4.allowSubmit")
1623 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1624 die("%s is not in git-p4.allowSubmit" % self.master)
1625
27d2d811 1626 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1627 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1628 if len(self.origin) == 0:
1629 self.origin = upstream
a3fdd579 1630
3ea2cfd4
LD
1631 if self.preserveUser:
1632 if not self.canChangeChangelists():
1633 die("Cannot preserve user names without p4 super-user or admin permissions")
1634
6bbfd137
PW
1635 # if not set from the command line, try the config file
1636 if self.conflict_behavior is None:
1637 val = gitConfig("git-p4.conflict")
1638 if val:
1639 if val not in self.conflict_behavior_choices:
1640 die("Invalid value '%s' for config git-p4.conflict" % val)
1641 else:
1642 val = "ask"
1643 self.conflict_behavior = val
1644
a3fdd579
SH
1645 if self.verbose:
1646 print "Origin branch is " + self.origin
9512497b 1647
ea99c3ae 1648 if len(self.depotPath) == 0:
9512497b
SH
1649 print "Internal error: cannot locate perforce depot path from existing branches"
1650 sys.exit(128)
1651
543987bd 1652 self.useClientSpec = False
0d609032 1653 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1654 self.useClientSpec = True
1655 if self.useClientSpec:
1656 self.clientSpecDirs = getClientSpec()
9512497b 1657
cd884106
VA
1658 # Check for the existance of P4 branches
1659 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1660
1661 if self.useClientSpec and not branchesDetected:
543987bd
PW
1662 # all files are relative to the client spec
1663 self.clientPath = getClientRoot()
1664 else:
1665 self.clientPath = p4Where(self.depotPath)
9512497b 1666
543987bd
PW
1667 if self.clientPath == "":
1668 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1669
ea99c3ae 1670 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1671 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1672
0591cfa8 1673 # ensure the clientPath exists
8d7ec362 1674 new_client_dir = False
0591cfa8 1675 if not os.path.exists(self.clientPath):
8d7ec362 1676 new_client_dir = True
0591cfa8
GG
1677 os.makedirs(self.clientPath)
1678
bbd84863 1679 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1680 if self.dry_run:
1681 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1682 else:
ef739f08
PW
1683 print "Synchronizing p4 checkout..."
1684 if new_client_dir:
1685 # old one was destroyed, and maybe nobody told p4
1686 p4_sync("...", "-f")
1687 else:
1688 p4_sync("...")
4f5cf76a 1689 self.check()
4f5cf76a 1690
4c750c0d 1691 commits = []
c7d34884 1692 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
4c750c0d
SH
1693 commits.append(line.strip())
1694 commits.reverse()
4f5cf76a 1695
0d609032 1696 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1697 self.checkAuthorship = False
1698 else:
1699 self.checkAuthorship = True
1700
3ea2cfd4
LD
1701 if self.preserveUser:
1702 self.checkValidP4Users(commits)
1703
84cb0003
GG
1704 #
1705 # Build up a set of options to be passed to diff when
1706 # submitting each commit to p4.
1707 #
1708 if self.detectRenames:
1709 # command-line -M arg
1710 self.diffOpts = "-M"
1711 else:
1712 # If not explicitly set check the config variable
1713 detectRenames = gitConfig("git-p4.detectRenames")
1714
1715 if detectRenames.lower() == "false" or detectRenames == "":
1716 self.diffOpts = ""
1717 elif detectRenames.lower() == "true":
1718 self.diffOpts = "-M"
1719 else:
1720 self.diffOpts = "-M%s" % detectRenames
1721
1722 # no command-line arg for -C or --find-copies-harder, just
1723 # config variables
1724 detectCopies = gitConfig("git-p4.detectCopies")
1725 if detectCopies.lower() == "false" or detectCopies == "":
1726 pass
1727 elif detectCopies.lower() == "true":
1728 self.diffOpts += " -C"
1729 else:
1730 self.diffOpts += " -C%s" % detectCopies
1731
0d609032 1732 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1733 self.diffOpts += " --find-copies-harder"
1734
7e5dd9f2
PW
1735 #
1736 # Apply the commits, one at a time. On failure, ask if should
1737 # continue to try the rest of the patches, or quit.
1738 #
ef739f08
PW
1739 if self.dry_run:
1740 print "Would apply"
67b0fe2e 1741 applied = []
7e5dd9f2
PW
1742 last = len(commits) - 1
1743 for i, commit in enumerate(commits):
ef739f08
PW
1744 if self.dry_run:
1745 print " ", read_pipe(["git", "show", "-s",
1746 "--format=format:%h %s", commit])
1747 ok = True
1748 else:
1749 ok = self.applyCommit(commit)
67b0fe2e
PW
1750 if ok:
1751 applied.append(commit)
7e5dd9f2 1752 else:
728b7ad8
PW
1753 if self.prepare_p4_only and i < last:
1754 print "Processing only the first commit due to option" \
1755 " --prepare-p4-only"
1756 break
7e5dd9f2
PW
1757 if i < last:
1758 quit = False
1759 while True:
6bbfd137
PW
1760 # prompt for what to do, or use the option/variable
1761 if self.conflict_behavior == "ask":
1762 print "What do you want to do?"
1763 response = raw_input("[s]kip this commit but apply"
1764 " the rest, or [q]uit? ")
1765 if not response:
1766 continue
1767 elif self.conflict_behavior == "skip":
1768 response = "s"
1769 elif self.conflict_behavior == "quit":
1770 response = "q"
1771 else:
1772 die("Unknown conflict_behavior '%s'" %
1773 self.conflict_behavior)
1774
7e5dd9f2
PW
1775 if response[0] == "s":
1776 print "Skipping this commit, but applying the rest"
1777 break
1778 if response[0] == "q":
1779 print "Quitting"
1780 quit = True
1781 break
1782 if quit:
1783 break
4f5cf76a 1784
67b0fe2e 1785 chdir(self.oldWorkingDirectory)
4f5cf76a 1786
ef739f08
PW
1787 if self.dry_run:
1788 pass
728b7ad8
PW
1789 elif self.prepare_p4_only:
1790 pass
ef739f08 1791 elif len(commits) == len(applied):
67b0fe2e 1792 print "All commits applied!"
14594f4b 1793
4c750c0d 1794 sync = P4Sync()
44e8d26c
PW
1795 if self.branch:
1796 sync.branch = self.branch
4c750c0d 1797 sync.run([])
14594f4b 1798
4c750c0d
SH
1799 rebase = P4Rebase()
1800 rebase.rebase()
4f5cf76a 1801
67b0fe2e
PW
1802 else:
1803 if len(applied) == 0:
1804 print "No commits applied."
1805 else:
1806 print "Applied only the commits marked with '*':"
1807 for c in commits:
1808 if c in applied:
1809 star = "*"
1810 else:
1811 star = " "
1812 print star, read_pipe(["git", "show", "-s",
1813 "--format=format:%h %s", c])
1814 print "You will have to do 'git p4 sync' and rebase."
1815
0d609032 1816 if gitConfigBool("git-p4.exportLabels"):
06dcd152 1817 self.exportLabels = True
06804c76
LD
1818
1819 if self.exportLabels:
1820 p4Labels = getP4Labels(self.depotPath)
1821 gitTags = getGitTags()
1822
1823 missingGitTags = gitTags - p4Labels
1824 self.exportGitTags(missingGitTags)
1825
98e023de 1826 # exit with error unless everything applied perfectly
67b0fe2e
PW
1827 if len(commits) != len(applied):
1828 sys.exit(1)
1829
b984733c
SH
1830 return True
1831
ecb7cf98
PW
1832class View(object):
1833 """Represent a p4 view ("p4 help views"), and map files in a
1834 repo according to the view."""
1835
9d57c4a6 1836 def __init__(self, client_name):
ecb7cf98 1837 self.mappings = []
9d57c4a6
KS
1838 self.client_prefix = "//%s/" % client_name
1839 # cache results of "p4 where" to lookup client file locations
1840 self.client_spec_path_cache = {}
ecb7cf98
PW
1841
1842 def append(self, view_line):
1843 """Parse a view line, splitting it into depot and client
9d57c4a6
KS
1844 sides. Append to self.mappings, preserving order. This
1845 is only needed for tag creation."""
ecb7cf98
PW
1846
1847 # Split the view line into exactly two words. P4 enforces
1848 # structure on these lines that simplifies this quite a bit.
1849 #
1850 # Either or both words may be double-quoted.
1851 # Single quotes do not matter.
1852 # Double-quote marks cannot occur inside the words.
1853 # A + or - prefix is also inside the quotes.
1854 # There are no quotes unless they contain a space.
1855 # The line is already white-space stripped.
1856 # The two words are separated by a single space.
1857 #
1858 if view_line[0] == '"':
1859 # First word is double quoted. Find its end.
1860 close_quote_index = view_line.find('"', 1)
1861 if close_quote_index <= 0:
1862 die("No first-word closing quote found: %s" % view_line)
1863 depot_side = view_line[1:close_quote_index]
1864 # skip closing quote and space
1865 rhs_index = close_quote_index + 1 + 1
1866 else:
1867 space_index = view_line.find(" ")
1868 if space_index <= 0:
1869 die("No word-splitting space found: %s" % view_line)
1870 depot_side = view_line[0:space_index]
1871 rhs_index = space_index + 1
1872
ecb7cf98 1873 # prefix + means overlay on previous mapping
ecb7cf98 1874 if depot_side.startswith("+"):
ecb7cf98
PW
1875 depot_side = depot_side[1:]
1876
9d57c4a6 1877 # prefix - means exclude this path, leave out of mappings
ecb7cf98
PW
1878 exclude = False
1879 if depot_side.startswith("-"):
1880 exclude = True
1881 depot_side = depot_side[1:]
1882
9d57c4a6
KS
1883 if not exclude:
1884 self.mappings.append(depot_side)
ecb7cf98 1885
9d57c4a6
KS
1886 def convert_client_path(self, clientFile):
1887 # chop off //client/ part to make it relative
1888 if not clientFile.startswith(self.client_prefix):
1889 die("No prefix '%s' on clientFile '%s'" %
1890 (self.client_prefix, clientFile))
1891 return clientFile[len(self.client_prefix):]
ecb7cf98 1892
9d57c4a6
KS
1893 def update_client_spec_path_cache(self, files):
1894 """ Caching file paths by "p4 where" batch query """
ecb7cf98 1895
9d57c4a6
KS
1896 # List depot file paths exclude that already cached
1897 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
ecb7cf98 1898
9d57c4a6
KS
1899 if len(fileArgs) == 0:
1900 return # All files in cache
ecb7cf98 1901
9d57c4a6
KS
1902 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1903 for res in where_result:
1904 if "code" in res and res["code"] == "error":
1905 # assume error is "... file(s) not in client view"
1906 continue
1907 if "clientFile" not in res:
20005443 1908 die("No clientFile in 'p4 where' output")
9d57c4a6
KS
1909 if "unmap" in res:
1910 # it will list all of them, but only one not unmap-ped
1911 continue
1912 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
ecb7cf98 1913
9d57c4a6
KS
1914 # not found files or unmap files set to ""
1915 for depotFile in fileArgs:
1916 if depotFile not in self.client_spec_path_cache:
1917 self.client_spec_path_cache[depotFile] = ""
ecb7cf98 1918
9d57c4a6
KS
1919 def map_in_client(self, depot_path):
1920 """Return the relative location in the client where this
1921 depot file should live. Returns "" if the file should
1922 not be mapped in the client."""
ecb7cf98 1923
9d57c4a6
KS
1924 if depot_path in self.client_spec_path_cache:
1925 return self.client_spec_path_cache[depot_path]
1926
1927 die( "Error: %s is not found in client spec path" % depot_path )
1928 return ""
ecb7cf98 1929
3ea2cfd4 1930class P4Sync(Command, P4UserMap):
56c09345
PW
1931 delete_actions = ( "delete", "move/delete", "purge" )
1932
b984733c
SH
1933 def __init__(self):
1934 Command.__init__(self)
3ea2cfd4 1935 P4UserMap.__init__(self)
b984733c
SH
1936 self.options = [
1937 optparse.make_option("--branch", dest="branch"),
1938 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1939 optparse.make_option("--changesfile", dest="changesFile"),
1940 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 1941 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 1942 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
1943 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1944 help="Import into refs/heads/ , not refs/remotes"),
96b2d54a
LS
1945 optparse.make_option("--max-changes", dest="maxChanges",
1946 help="Maximum number of changes to import"),
1947 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1948 help="Internal block size to use when iteratively calling p4 changes"),
86dff6b6 1949 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
1950 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1951 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
51334bb0
LD
1952 help="Only sync files that are included in the Perforce Client Spec"),
1953 optparse.make_option("-/", dest="cloneExclude",
1954 action="append", type="string",
1955 help="exclude depot path"),
b984733c
SH
1956 ]
1957 self.description = """Imports from Perforce into a git repository.\n
1958 example:
1959 //depot/my/project/ -- to import the current head
1960 //depot/my/project/@all -- to import everything
1961 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1962
1963 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1964
1965 self.usage += " //depot/path[@revRange]"
b984733c 1966 self.silent = False
1d7367dc
RG
1967 self.createdBranches = set()
1968 self.committedChanges = set()
569d1bd4 1969 self.branch = ""
b984733c 1970 self.detectBranches = False
cb53e1f8 1971 self.detectLabels = False
06804c76 1972 self.importLabels = False
b984733c 1973 self.changesFile = ""
01265103 1974 self.syncWithOrigin = True
a028a98e 1975 self.importIntoRemotes = True
01a9c9c5 1976 self.maxChanges = ""
96b2d54a 1977 self.changes_block_size = 500
8b41a97f 1978 self.keepRepoPath = False
6326aa58 1979 self.depotPaths = None
3c699645 1980 self.p4BranchesInGit = []
354081d5 1981 self.cloneExclude = []
3a70cdfa 1982 self.useClientSpec = False
a93d33ee 1983 self.useClientSpec_from_options = False
ecb7cf98 1984 self.clientSpecDirs = None
fed23693
VA
1985 self.tempBranches = []
1986 self.tempBranchLocation = "git-p4-tmp"
b984733c 1987
01265103
SH
1988 if gitConfig("git-p4.syncFromOrigin") == "false":
1989 self.syncWithOrigin = False
1990
51334bb0
LD
1991 # This is required for the "append" cloneExclude action
1992 def ensure_value(self, attr, value):
1993 if not hasattr(self, attr) or getattr(self, attr) is None:
1994 setattr(self, attr, value)
1995 return getattr(self, attr)
1996
fed23693
VA
1997 # Force a checkpoint in fast-import and wait for it to finish
1998 def checkpoint(self):
1999 self.gitStream.write("checkpoint\n\n")
2000 self.gitStream.write("progress checkpoint\n\n")
2001 out = self.gitOutput.readline()
2002 if self.verbose:
2003 print "checkpoint finished: " + out
2004
b984733c 2005 def extractFilesFromCommit(self, commit):
354081d5
TT
2006 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2007 for path in self.cloneExclude]
b984733c
SH
2008 files = []
2009 fnum = 0
2010 while commit.has_key("depotFile%s" % fnum):
2011 path = commit["depotFile%s" % fnum]
6326aa58 2012
354081d5 2013 if [p for p in self.cloneExclude
d53de8b9 2014 if p4PathStartsWith(path, p)]:
354081d5
TT
2015 found = False
2016 else:
2017 found = [p for p in self.depotPaths
d53de8b9 2018 if p4PathStartsWith(path, p)]
6326aa58 2019 if not found:
b984733c
SH
2020 fnum = fnum + 1
2021 continue
2022
2023 file = {}
2024 file["path"] = path
2025 file["rev"] = commit["rev%s" % fnum]
2026 file["action"] = commit["action%s" % fnum]
2027 file["type"] = commit["type%s" % fnum]
2028 files.append(file)
2029 fnum = fnum + 1
2030 return files
2031
6326aa58 2032 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2033 """When streaming files, this is called to map a p4 depot path
2034 to where it should go in git. The prefixes are either
2035 self.depotPaths, or self.branchPrefixes in the case of
2036 branch detection."""
2037
3952710b 2038 if self.useClientSpec:
21ef5df4
PW
2039 # branch detection moves files up a level (the branch name)
2040 # from what client spec interpretation gives
0d1696ef 2041 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2042 if self.detectBranches:
2043 for b in self.knownBranches:
2044 if path.startswith(b + "/"):
2045 path = path[len(b)+1:]
2046
2047 elif self.keepRepoPath:
2048 # Preserve everything in relative path name except leading
2049 # //depot/; just look at first prefix as they all should
2050 # be in the same depot.
2051 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2052 if p4PathStartsWith(path, depot):
2053 path = path[len(depot):]
3952710b 2054
0d1696ef 2055 else:
0d1696ef
PW
2056 for p in prefixes:
2057 if p4PathStartsWith(path, p):
2058 path = path[len(p):]
21ef5df4 2059 break
8b41a97f 2060
0d1696ef 2061 path = wildcard_decode(path)
6326aa58 2062 return path
6754a299 2063
71b112d4 2064 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2065 """Look at each depotFile in the commit to figure out to what
2066 branch it belongs."""
2067
9d57c4a6
KS
2068 if self.clientSpecDirs:
2069 files = self.extractFilesFromCommit(commit)
2070 self.clientSpecDirs.update_client_spec_path_cache(files)
2071
d5904674 2072 branches = {}
71b112d4
SH
2073 fnum = 0
2074 while commit.has_key("depotFile%s" % fnum):
2075 path = commit["depotFile%s" % fnum]
6326aa58 2076 found = [p for p in self.depotPaths
d53de8b9 2077 if p4PathStartsWith(path, p)]
6326aa58 2078 if not found:
71b112d4
SH
2079 fnum = fnum + 1
2080 continue
2081
2082 file = {}
2083 file["path"] = path
2084 file["rev"] = commit["rev%s" % fnum]
2085 file["action"] = commit["action%s" % fnum]
2086 file["type"] = commit["type%s" % fnum]
2087 fnum = fnum + 1
2088
21ef5df4
PW
2089 # start with the full relative path where this file would
2090 # go in a p4 client
2091 if self.useClientSpec:
2092 relPath = self.clientSpecDirs.map_in_client(path)
2093 else:
2094 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2095
4b97ffb1 2096 for branch in self.knownBranches.keys():
21ef5df4
PW
2097 # add a trailing slash so that a commit into qt/4.2foo
2098 # doesn't end up in qt/4.2, e.g.
6754a299 2099 if relPath.startswith(branch + "/"):
d5904674
SH
2100 if branch not in branches:
2101 branches[branch] = []
71b112d4 2102 branches[branch].append(file)
6555b2cc 2103 break
b984733c
SH
2104
2105 return branches
2106
b932705b
LD
2107 # output one file from the P4 stream
2108 # - helper for streamP4Files
2109
2110 def streamOneP4File(self, file, contents):
b932705b
LD
2111 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2112 if verbose:
2113 sys.stderr.write("%s\n" % relPath)
2114
9cffb8c8
PW
2115 (type_base, type_mods) = split_p4_type(file["type"])
2116
2117 git_mode = "100644"
2118 if "x" in type_mods:
2119 git_mode = "100755"
2120 if type_base == "symlink":
2121 git_mode = "120000"
1292df11
AJ
2122 # p4 print on a symlink sometimes contains "target\n";
2123 # if it does, remove the newline
b39c3612 2124 data = ''.join(contents)
40f846c3
PW
2125 if not data:
2126 # Some version of p4 allowed creating a symlink that pointed
2127 # to nothing. This causes p4 errors when checking out such
2128 # a change, and errors here too. Work around it by ignoring
2129 # the bad symlink; hopefully a future change fixes it.
2130 print "\nIgnoring empty symlink in %s" % file['depotFile']
2131 return
2132 elif data[-1] == '\n':
1292df11
AJ
2133 contents = [data[:-1]]
2134 else:
2135 contents = [data]
b932705b 2136
9cffb8c8 2137 if type_base == "utf16":
55aa5714
PW
2138 # p4 delivers different text in the python output to -G
2139 # than it does when using "print -o", or normal p4 client
2140 # operations. utf16 is converted to ascii or utf8, perhaps.
2141 # But ascii text saved as -t utf16 is completely mangled.
2142 # Invoke print -o to get the real contents.
7f0e5962
PW
2143 #
2144 # On windows, the newlines will always be mangled by print, so put
2145 # them back too. This is not needed to the cygwin windows version,
2146 # just the native "NT" type.
2147 #
6de040df 2148 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
7f0e5962
PW
2149 if p4_version_string().find("/NT") >= 0:
2150 text = text.replace("\r\n", "\n")
55aa5714
PW
2151 contents = [ text ]
2152
9f7ef0ea
PW
2153 if type_base == "apple":
2154 # Apple filetype files will be streamed as a concatenation of
2155 # its appledouble header and the contents. This is useless
2156 # on both macs and non-macs. If using "print -q -o xx", it
2157 # will create "xx" with the data, and "%xx" with the header.
2158 # This is also not very useful.
2159 #
2160 # Ideally, someday, this script can learn how to generate
2161 # appledouble files directly and import those to git, but
2162 # non-mac machines can never find a use for apple filetype.
2163 print "\nIgnoring apple filetype file %s" % file['depotFile']
2164 return
2165
55aa5714
PW
2166 # Note that we do not try to de-mangle keywords on utf16 files,
2167 # even though in theory somebody may want that.
60df071c
LD
2168 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2169 if pattern:
2170 regexp = re.compile(pattern, re.VERBOSE)
2171 text = ''.join(contents)
2172 text = regexp.sub(r'$\1$', text)
2173 contents = [ text ]
b932705b 2174
9cffb8c8 2175 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
2176
2177 # total length...
2178 length = 0
2179 for d in contents:
2180 length = length + len(d)
2181
2182 self.gitStream.write("data %d\n" % length)
2183 for d in contents:
2184 self.gitStream.write(d)
2185 self.gitStream.write("\n")
2186
2187 def streamOneP4Deletion(self, file):
2188 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2189 if verbose:
2190 sys.stderr.write("delete %s\n" % relPath)
2191 self.gitStream.write("D %s\n" % relPath)
2192
2193 # handle another chunk of streaming data
2194 def streamP4FilesCb(self, marshalled):
2195
78189bea
PW
2196 # catch p4 errors and complain
2197 err = None
2198 if "code" in marshalled:
2199 if marshalled["code"] == "error":
2200 if "data" in marshalled:
2201 err = marshalled["data"].rstrip()
2202 if err:
2203 f = None
2204 if self.stream_have_file_info:
2205 if "depotFile" in self.stream_file:
2206 f = self.stream_file["depotFile"]
2207 # force a failure in fast-import, else an empty
2208 # commit will be made
2209 self.gitStream.write("\n")
2210 self.gitStream.write("die-now\n")
2211 self.gitStream.close()
2212 # ignore errors, but make sure it exits first
2213 self.importProcess.wait()
2214 if f:
2215 die("Error from p4 print for %s: %s" % (f, err))
2216 else:
2217 die("Error from p4 print: %s" % err)
2218
c3f6163b
AG
2219 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2220 # start of a new file - output the old one first
2221 self.streamOneP4File(self.stream_file, self.stream_contents)
2222 self.stream_file = {}
2223 self.stream_contents = []
2224 self.stream_have_file_info = False
b932705b 2225
c3f6163b
AG
2226 # pick up the new file information... for the
2227 # 'data' field we need to append to our array
2228 for k in marshalled.keys():
2229 if k == 'data':
2230 self.stream_contents.append(marshalled['data'])
2231 else:
2232 self.stream_file[k] = marshalled[k]
b932705b 2233
c3f6163b 2234 self.stream_have_file_info = True
b932705b
LD
2235
2236 # Stream directly from "p4 files" into "git fast-import"
2237 def streamP4Files(self, files):
30b5940b
SH
2238 filesForCommit = []
2239 filesToRead = []
b932705b 2240 filesToDelete = []
30b5940b 2241
3a70cdfa 2242 for f in files:
ecb7cf98
PW
2243 # if using a client spec, only add the files that have
2244 # a path in the client
2245 if self.clientSpecDirs:
2246 if self.clientSpecDirs.map_in_client(f['path']) == "":
2247 continue
3a70cdfa 2248
ecb7cf98
PW
2249 filesForCommit.append(f)
2250 if f['action'] in self.delete_actions:
2251 filesToDelete.append(f)
2252 else:
2253 filesToRead.append(f)
6a49f8e2 2254
b932705b
LD
2255 # deleted files...
2256 for f in filesToDelete:
2257 self.streamOneP4Deletion(f)
1b9a4684 2258
b932705b
LD
2259 if len(filesToRead) > 0:
2260 self.stream_file = {}
2261 self.stream_contents = []
2262 self.stream_have_file_info = False
8ff45f2a 2263
c3f6163b
AG
2264 # curry self argument
2265 def streamP4FilesCbSelf(entry):
2266 self.streamP4FilesCb(entry)
6a49f8e2 2267
6de040df
LD
2268 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2269
2270 p4CmdList(["-x", "-", "print"],
2271 stdin=fileArgs,
2272 cb=streamP4FilesCbSelf)
30b5940b 2273
b932705b
LD
2274 # do the last chunk
2275 if self.stream_file.has_key('depotFile'):
2276 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2277
affb474f
LD
2278 def make_email(self, userid):
2279 if userid in self.users:
2280 return self.users[userid]
2281 else:
2282 return "%s <a@b>" % userid
2283
06804c76
LD
2284 # Stream a p4 tag
2285 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2286 if verbose:
2287 print "writing tag %s for commit %s" % (labelName, commit)
2288 gitStream.write("tag %s\n" % labelName)
2289 gitStream.write("from %s\n" % commit)
2290
2291 if labelDetails.has_key('Owner'):
2292 owner = labelDetails["Owner"]
2293 else:
2294 owner = None
2295
2296 # Try to use the owner of the p4 label, or failing that,
2297 # the current p4 user id.
2298 if owner:
2299 email = self.make_email(owner)
2300 else:
2301 email = self.make_email(self.p4UserId())
2302 tagger = "%s %s %s" % (email, epoch, self.tz)
2303
2304 gitStream.write("tagger %s\n" % tagger)
2305
2306 print "labelDetails=",labelDetails
2307 if labelDetails.has_key('Description'):
2308 description = labelDetails['Description']
2309 else:
2310 description = 'Label from git p4'
2311
2312 gitStream.write("data %d\n" % len(description))
2313 gitStream.write(description)
2314 gitStream.write("\n")
2315
e63231e5 2316 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2317 epoch = details["time"]
2318 author = details["user"]
2319
4b97ffb1
SH
2320 if self.verbose:
2321 print "commit into %s" % branch
2322
96e07dd2
HWN
2323 # start with reading files; if that fails, we should not
2324 # create a commit.
2325 new_files = []
2326 for f in files:
e63231e5 2327 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2328 new_files.append (f)
2329 else:
afa1dd9a 2330 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2331
9d57c4a6
KS
2332 if self.clientSpecDirs:
2333 self.clientSpecDirs.update_client_spec_path_cache(files)
2334
b984733c 2335 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 2336# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2337 self.committedChanges.add(int(details["change"]))
2338 committer = ""
b607e71e
SH
2339 if author not in self.users:
2340 self.getUserMapFromPerforceServer()
affb474f 2341 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2342
2343 self.gitStream.write("committer %s\n" % committer)
2344
2345 self.gitStream.write("data <<EOT\n")
2346 self.gitStream.write(details["desc"])
e63231e5
PW
2347 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2348 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2349 if len(details['options']) > 0:
2350 self.gitStream.write(": options = %s" % details['options'])
2351 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2352
2353 if len(parent) > 0:
4b97ffb1
SH
2354 if self.verbose:
2355 print "parent %s" % parent
b984733c
SH
2356 self.gitStream.write("from %s\n" % parent)
2357
b932705b 2358 self.streamP4Files(new_files)
b984733c
SH
2359 self.gitStream.write("\n")
2360
1f4ba1cb
SH
2361 change = int(details["change"])
2362
9bda3a85 2363 if self.labels.has_key(change):
1f4ba1cb
SH
2364 label = self.labels[change]
2365 labelDetails = label[0]
2366 labelRevisions = label[1]
71b112d4
SH
2367 if self.verbose:
2368 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2369
6de040df 2370 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2371 for p in self.branchPrefixes])
1f4ba1cb
SH
2372
2373 if len(files) == len(labelRevisions):
2374
2375 cleanedFiles = {}
2376 for info in files:
56c09345 2377 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2378 continue
2379 cleanedFiles[info["depotFile"]] = info["rev"]
2380
2381 if cleanedFiles == labelRevisions:
06804c76 2382 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2383
2384 else:
a46668fa 2385 if not self.silent:
cebdf5af
HWN
2386 print ("Tag %s does not match with change %s: files do not match."
2387 % (labelDetails["label"], change))
1f4ba1cb
SH
2388
2389 else:
a46668fa 2390 if not self.silent:
cebdf5af
HWN
2391 print ("Tag %s does not match with change %s: file count is different."
2392 % (labelDetails["label"], change))
b984733c 2393
06804c76 2394 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2395 def getLabels(self):
2396 self.labels = {}
2397
52a4880b 2398 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2399 if len(l) > 0 and not self.silent:
183f8436 2400 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2401
2402 for output in l:
1f4ba1cb
SH
2403 label = output["label"]
2404 revisions = {}
2405 newestChange = 0
71b112d4
SH
2406 if self.verbose:
2407 print "Querying files for label %s" % label
6de040df
LD
2408 for file in p4CmdList(["files"] +
2409 ["%s...@%s" % (p, label)
2410 for p in self.depotPaths]):
1f4ba1cb
SH
2411 revisions[file["depotFile"]] = file["rev"]
2412 change = int(file["change"])
2413 if change > newestChange:
2414 newestChange = change
2415
9bda3a85
SH
2416 self.labels[newestChange] = [output, revisions]
2417
2418 if self.verbose:
2419 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2420
06804c76
LD
2421 # Import p4 labels as git tags. A direct mapping does not
2422 # exist, so assume that if all the files are at the same revision
2423 # then we can use that, or it's something more complicated we should
2424 # just ignore.
2425 def importP4Labels(self, stream, p4Labels):
2426 if verbose:
2427 print "import p4 labels: " + ' '.join(p4Labels)
2428
2429 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2430 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2431 if len(validLabelRegexp) == 0:
2432 validLabelRegexp = defaultLabelRegexp
2433 m = re.compile(validLabelRegexp)
2434
2435 for name in p4Labels:
2436 commitFound = False
2437
2438 if not m.match(name):
2439 if verbose:
2440 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2441 continue
2442
2443 if name in ignoredP4Labels:
2444 continue
2445
2446 labelDetails = p4CmdList(['label', "-o", name])[0]
2447
2448 # get the most recent changelist for each file in this label
2449 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2450 for p in self.depotPaths])
2451
2452 if change.has_key('change'):
2453 # find the corresponding git commit; take the oldest commit
2454 changelist = int(change['change'])
2455 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2456 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2457 if len(gitCommit) == 0:
2458 print "could not find git commit for changelist %d" % changelist
2459 else:
2460 gitCommit = gitCommit.strip()
2461 commitFound = True
2462 # Convert from p4 time format
2463 try:
2464 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2465 except ValueError:
a4e9054c 2466 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2467 tmwhen = 1
2468
2469 when = int(time.mktime(tmwhen))
2470 self.streamTag(stream, name, labelDetails, gitCommit, when)
2471 if verbose:
2472 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2473 else:
2474 if verbose:
2475 print "Label %s has no changelists - possibly deleted?" % name
2476
2477 if not commitFound:
2478 # We can't import this label; don't try again as it will get very
2479 # expensive repeatedly fetching all the files for labels that will
2480 # never be imported. If the label is moved in the future, the
2481 # ignore will need to be removed manually.
2482 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2483
86dff6b6
HWN
2484 def guessProjectName(self):
2485 for p in self.depotPaths:
6e5295c4
SH
2486 if p.endswith("/"):
2487 p = p[:-1]
2488 p = p[p.strip().rfind("/") + 1:]
2489 if not p.endswith("/"):
2490 p += "/"
2491 return p
86dff6b6 2492
4b97ffb1 2493 def getBranchMapping(self):
6555b2cc
SH
2494 lostAndFoundBranches = set()
2495
8ace74c0
VA
2496 user = gitConfig("git-p4.branchUser")
2497 if len(user) > 0:
2498 command = "branches -u %s" % user
2499 else:
2500 command = "branches"
2501
2502 for info in p4CmdList(command):
52a4880b 2503 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2504 viewIdx = 0
2505 while details.has_key("View%s" % viewIdx):
2506 paths = details["View%s" % viewIdx].split(" ")
2507 viewIdx = viewIdx + 1
2508 # require standard //depot/foo/... //depot/bar/... mapping
2509 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2510 continue
2511 source = paths[0]
2512 destination = paths[1]
6509e19c 2513 ## HACK
d53de8b9 2514 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2515 source = source[len(self.depotPaths[0]):-4]
2516 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2517
1a2edf4e
SH
2518 if destination in self.knownBranches:
2519 if not self.silent:
2520 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2521 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2522 continue
2523
6555b2cc
SH
2524 self.knownBranches[destination] = source
2525
2526 lostAndFoundBranches.discard(destination)
2527
29bdbac1 2528 if source not in self.knownBranches:
6555b2cc
SH
2529 lostAndFoundBranches.add(source)
2530
7199cf13
VA
2531 # Perforce does not strictly require branches to be defined, so we also
2532 # check git config for a branch list.
2533 #
2534 # Example of branch definition in git config file:
2535 # [git-p4]
2536 # branchList=main:branchA
2537 # branchList=main:branchB
2538 # branchList=branchA:branchC
2539 configBranches = gitConfigList("git-p4.branchList")
2540 for branch in configBranches:
2541 if branch:
2542 (source, destination) = branch.split(":")
2543 self.knownBranches[destination] = source
2544
2545 lostAndFoundBranches.discard(destination)
2546
2547 if source not in self.knownBranches:
2548 lostAndFoundBranches.add(source)
2549
6555b2cc
SH
2550
2551 for branch in lostAndFoundBranches:
2552 self.knownBranches[branch] = branch
29bdbac1 2553
38f9f5ec
SH
2554 def getBranchMappingFromGitBranches(self):
2555 branches = p4BranchesInGit(self.importIntoRemotes)
2556 for branch in branches.keys():
2557 if branch == "master":
2558 branch = "main"
2559 else:
2560 branch = branch[len(self.projectName):]
2561 self.knownBranches[branch] = branch
2562
bb6e09b2
HWN
2563 def updateOptionDict(self, d):
2564 option_keys = {}
2565 if self.keepRepoPath:
2566 option_keys['keepRepoPath'] = 1
2567
2568 d["options"] = ' '.join(sorted(option_keys.keys()))
2569
2570 def readOptions(self, d):
2571 self.keepRepoPath = (d.has_key('options')
2572 and ('keepRepoPath' in d['options']))
6326aa58 2573
8134f69c
SH
2574 def gitRefForBranch(self, branch):
2575 if branch == "main":
2576 return self.refPrefix + "master"
2577
2578 if len(branch) <= 0:
2579 return branch
2580
2581 return self.refPrefix + self.projectName + branch
2582
1ca3d710
SH
2583 def gitCommitByP4Change(self, ref, change):
2584 if self.verbose:
2585 print "looking in ref " + ref + " for change %s using bisect..." % change
2586
2587 earliestCommit = ""
2588 latestCommit = parseRevision(ref)
2589
2590 while True:
2591 if self.verbose:
2592 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2593 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2594 if len(next) == 0:
2595 if self.verbose:
2596 print "argh"
2597 return ""
2598 log = extractLogMessageFromGitCommit(next)
2599 settings = extractSettingsGitLog(log)
2600 currentChange = int(settings['change'])
2601 if self.verbose:
2602 print "current change %s" % currentChange
2603
2604 if currentChange == change:
2605 if self.verbose:
2606 print "found %s" % next
2607 return next
2608
2609 if currentChange < change:
2610 earliestCommit = "^%s" % next
2611 else:
2612 latestCommit = "%s" % next
2613
2614 return ""
2615
2616 def importNewBranch(self, branch, maxChange):
2617 # make fast-import flush all changes to disk and update the refs using the checkpoint
2618 # command so that we can try to find the branch parent in the git history
2619 self.gitStream.write("checkpoint\n\n");
2620 self.gitStream.flush();
2621 branchPrefix = self.depotPaths[0] + branch + "/"
2622 range = "@1,%s" % maxChange
2623 #print "prefix" + branchPrefix
96b2d54a 2624 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
1ca3d710
SH
2625 if len(changes) <= 0:
2626 return False
2627 firstChange = changes[0]
2628 #print "first change in branch: %s" % firstChange
2629 sourceBranch = self.knownBranches[branch]
2630 sourceDepotPath = self.depotPaths[0] + sourceBranch
2631 sourceRef = self.gitRefForBranch(sourceBranch)
2632 #print "source " + sourceBranch
2633
52a4880b 2634 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2635 #print "branch parent: %s" % branchParentChange
2636 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2637 if len(gitParent) > 0:
2638 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2639 #print "parent git commit: %s" % gitParent
2640
2641 self.importChanges(changes)
2642 return True
2643
fed23693
VA
2644 def searchParent(self, parent, branch, target):
2645 parentFound = False
c7d34884
PW
2646 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2647 "--no-merges", parent]):
fed23693
VA
2648 blob = blob.strip()
2649 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2650 parentFound = True
2651 if self.verbose:
2652 print "Found parent of %s in commit %s" % (branch, blob)
2653 break
2654 if parentFound:
2655 return blob
2656 else:
2657 return None
2658
e87f37ae
SH
2659 def importChanges(self, changes):
2660 cnt = 1
2661 for change in changes:
18fa13d0 2662 description = p4_describe(change)
e87f37ae
SH
2663 self.updateOptionDict(description)
2664
2665 if not self.silent:
2666 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2667 sys.stdout.flush()
2668 cnt = cnt + 1
2669
2670 try:
2671 if self.detectBranches:
2672 branches = self.splitFilesIntoBranches(description)
2673 for branch in branches.keys():
2674 ## HACK --hwn
2675 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 2676 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
2677
2678 parent = ""
2679
2680 filesForCommit = branches[branch]
2681
2682 if self.verbose:
2683 print "branch is %s" % branch
2684
2685 self.updatedBranches.add(branch)
2686
2687 if branch not in self.createdBranches:
2688 self.createdBranches.add(branch)
2689 parent = self.knownBranches[branch]
2690 if parent == branch:
2691 parent = ""
1ca3d710
SH
2692 else:
2693 fullBranch = self.projectName + branch
2694 if fullBranch not in self.p4BranchesInGit:
2695 if not self.silent:
2696 print("\n Importing new branch %s" % fullBranch);
2697 if self.importNewBranch(branch, change - 1):
2698 parent = ""
2699 self.p4BranchesInGit.append(fullBranch)
2700 if not self.silent:
2701 print("\n Resuming with change %s" % change);
2702
2703 if self.verbose:
2704 print "parent determined through known branches: %s" % parent
e87f37ae 2705
8134f69c
SH
2706 branch = self.gitRefForBranch(branch)
2707 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2708
2709 if self.verbose:
2710 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2711
2712 if len(parent) == 0 and branch in self.initialParents:
2713 parent = self.initialParents[branch]
2714 del self.initialParents[branch]
2715
fed23693
VA
2716 blob = None
2717 if len(parent) > 0:
4f9273d2 2718 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
2719 if self.verbose:
2720 print "Creating temporary branch: " + tempBranch
e63231e5 2721 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
2722 self.tempBranches.append(tempBranch)
2723 self.checkpoint()
2724 blob = self.searchParent(parent, branch, tempBranch)
2725 if blob:
e63231e5 2726 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
2727 else:
2728 if self.verbose:
2729 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 2730 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
2731 else:
2732 files = self.extractFilesFromCommit(description)
e63231e5 2733 self.commit(description, files, self.branch,
e87f37ae 2734 self.initialParent)
47497844 2735 # only needed once, to connect to the previous commit
e87f37ae
SH
2736 self.initialParent = ""
2737 except IOError:
2738 print self.gitError.read()
2739 sys.exit(1)
2740
c208a243
SH
2741 def importHeadRevision(self, revision):
2742 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2743
4e2e6ce4
PW
2744 details = {}
2745 details["user"] = "git perforce import user"
1494fcbb 2746 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2747 % (' '.join(self.depotPaths), revision))
2748 details["change"] = revision
2749 newestRevision = 0
2750
2751 fileCnt = 0
6de040df
LD
2752 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2753
2754 for info in p4CmdList(["files"] + fileArgs):
c208a243 2755
68b28593 2756 if 'code' in info and info['code'] == 'error':
c208a243
SH
2757 sys.stderr.write("p4 returned an error: %s\n"
2758 % info['data'])
d88e707f
PW
2759 if info['data'].find("must refer to client") >= 0:
2760 sys.stderr.write("This particular p4 error is misleading.\n")
2761 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2762 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2763 sys.exit(1)
68b28593
PW
2764 if 'p4ExitCode' in info:
2765 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2766 sys.exit(1)
2767
2768
2769 change = int(info["change"])
2770 if change > newestRevision:
2771 newestRevision = change
2772
56c09345 2773 if info["action"] in self.delete_actions:
c208a243
SH
2774 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2775 #fileCnt = fileCnt + 1
2776 continue
2777
2778 for prop in ["depotFile", "rev", "action", "type" ]:
2779 details["%s%s" % (prop, fileCnt)] = info[prop]
2780
2781 fileCnt = fileCnt + 1
2782
2783 details["change"] = newestRevision
4e2e6ce4 2784
9dcb9f24 2785 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 2786 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
2787 res = p4_describe(newestRevision)
2788 details["time"] = res["time"]
4e2e6ce4 2789
c208a243
SH
2790 self.updateOptionDict(details)
2791 try:
e63231e5 2792 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
2793 except IOError:
2794 print "IO error with git fast-import. Is your git version recent enough?"
2795 print self.gitError.read()
2796
2797
b984733c 2798 def run(self, args):
6326aa58 2799 self.depotPaths = []
179caebf 2800 self.changeRange = ""
6326aa58 2801 self.previousDepotPaths = []
991a2de4 2802 self.hasOrigin = False
ce6f33c8 2803
29bdbac1
SH
2804 # map from branch depot path to parent branch
2805 self.knownBranches = {}
2806 self.initialParents = {}
2807
a028a98e
SH
2808 if self.importIntoRemotes:
2809 self.refPrefix = "refs/remotes/p4/"
2810 else:
db775559 2811 self.refPrefix = "refs/heads/p4/"
a028a98e 2812
991a2de4
PW
2813 if self.syncWithOrigin:
2814 self.hasOrigin = originP4BranchesExist()
2815 if self.hasOrigin:
2816 if not self.silent:
2817 print 'Syncing with origin first, using "git fetch origin"'
2818 system("git fetch origin")
10f880f8 2819
5a8e84cd 2820 branch_arg_given = bool(self.branch)
569d1bd4 2821 if len(self.branch) == 0:
db775559 2822 self.branch = self.refPrefix + "master"
a028a98e 2823 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2824 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 2825 system("git branch -D p4")
967f72e2 2826
a93d33ee
PW
2827 # accept either the command-line option, or the configuration variable
2828 if self.useClientSpec:
2829 # will use this after clone to set the variable
2830 self.useClientSpec_from_options = True
2831 else:
0d609032 2832 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
2833 self.useClientSpec = True
2834 if self.useClientSpec:
543987bd 2835 self.clientSpecDirs = getClientSpec()
3a70cdfa 2836
6a49f8e2
HWN
2837 # TODO: should always look at previous commits,
2838 # merge with previous imports, if possible.
2839 if args == []:
d414c74a 2840 if self.hasOrigin:
5ca44617 2841 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
2842
2843 # branches holds mapping from branch name to sha1
2844 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
2845
2846 # restrict to just this one, disabling detect-branches
2847 if branch_arg_given:
2848 short = self.branch.split("/")[-1]
2849 if short in branches:
2850 self.p4BranchesInGit = [ short ]
2851 else:
2852 self.p4BranchesInGit = branches.keys()
abcd790f
SH
2853
2854 if len(self.p4BranchesInGit) > 1:
2855 if not self.silent:
2856 print "Importing from/into multiple branches"
2857 self.detectBranches = True
8c9e8b6e
PW
2858 for branch in branches.keys():
2859 self.initialParents[self.refPrefix + branch] = \
2860 branches[branch]
967f72e2 2861
29bdbac1
SH
2862 if self.verbose:
2863 print "branches: %s" % self.p4BranchesInGit
2864
2865 p4Change = 0
2866 for branch in self.p4BranchesInGit:
cebdf5af 2867 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2868
2869 settings = extractSettingsGitLog(logMsg)
29bdbac1 2870
bb6e09b2
HWN
2871 self.readOptions(settings)
2872 if (settings.has_key('depot-paths')
2873 and settings.has_key ('change')):
2874 change = int(settings['change']) + 1
29bdbac1
SH
2875 p4Change = max(p4Change, change)
2876
bb6e09b2
HWN
2877 depotPaths = sorted(settings['depot-paths'])
2878 if self.previousDepotPaths == []:
6326aa58 2879 self.previousDepotPaths = depotPaths
29bdbac1 2880 else:
6326aa58
HWN
2881 paths = []
2882 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2883 prev_list = prev.split("/")
2884 cur_list = cur.split("/")
2885 for i in range(0, min(len(cur_list), len(prev_list))):
2886 if cur_list[i] <> prev_list[i]:
583e1707 2887 i = i - 1
6326aa58
HWN
2888 break
2889
04d277b3 2890 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2891
2892 self.previousDepotPaths = paths
29bdbac1
SH
2893
2894 if p4Change > 0:
bb6e09b2 2895 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2896 self.changeRange = "@%s,#head" % p4Change
341dc1c1 2897 if not self.silent and not self.detectBranches:
967f72e2 2898 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2899
40d69ac3
PW
2900 # accept multiple ref name abbreviations:
2901 # refs/foo/bar/branch -> use it exactly
2902 # p4/branch -> prepend refs/remotes/ or refs/heads/
2903 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 2904 if not self.branch.startswith("refs/"):
40d69ac3
PW
2905 if self.importIntoRemotes:
2906 prepend = "refs/remotes/"
2907 else:
2908 prepend = "refs/heads/"
2909 if not self.branch.startswith("p4/"):
2910 prepend += "p4/"
2911 self.branch = prepend + self.branch
179caebf 2912
6326aa58 2913 if len(args) == 0 and self.depotPaths:
b984733c 2914 if not self.silent:
6326aa58 2915 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2916 else:
6326aa58 2917 if self.depotPaths and self.depotPaths != args:
cebdf5af 2918 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2919 "This doesn't work!" % (' '.join (self.depotPaths),
2920 ' '.join (args)))
b984733c 2921 sys.exit(1)
6326aa58 2922
bb6e09b2 2923 self.depotPaths = sorted(args)
b984733c 2924
1c49fc19 2925 revision = ""
b984733c 2926 self.users = {}
b984733c 2927
58c8bc7c
PW
2928 # Make sure no revision specifiers are used when --changesfile
2929 # is specified.
2930 bad_changesfile = False
2931 if len(self.changesFile) > 0:
2932 for p in self.depotPaths:
2933 if p.find("@") >= 0 or p.find("#") >= 0:
2934 bad_changesfile = True
2935 break
2936 if bad_changesfile:
2937 die("Option --changesfile is incompatible with revision specifiers")
2938
6326aa58
HWN
2939 newPaths = []
2940 for p in self.depotPaths:
2941 if p.find("@") != -1:
2942 atIdx = p.index("@")
2943 self.changeRange = p[atIdx:]
2944 if self.changeRange == "@all":
2945 self.changeRange = ""
6a49f8e2 2946 elif ',' not in self.changeRange:
1c49fc19 2947 revision = self.changeRange
6326aa58 2948 self.changeRange = ""
7fcff9de 2949 p = p[:atIdx]
6326aa58
HWN
2950 elif p.find("#") != -1:
2951 hashIdx = p.index("#")
1c49fc19 2952 revision = p[hashIdx:]
7fcff9de 2953 p = p[:hashIdx]
6326aa58 2954 elif self.previousDepotPaths == []:
58c8bc7c
PW
2955 # pay attention to changesfile, if given, else import
2956 # the entire p4 tree at the head revision
2957 if len(self.changesFile) == 0:
2958 revision = "#head"
6326aa58
HWN
2959
2960 p = re.sub ("\.\.\.$", "", p)
2961 if not p.endswith("/"):
2962 p += "/"
2963
2964 newPaths.append(p)
2965
2966 self.depotPaths = newPaths
2967
e63231e5
PW
2968 # --detect-branches may change this for each branch
2969 self.branchPrefixes = self.depotPaths
2970
b607e71e 2971 self.loadUserMapFromCache()
cb53e1f8
SH
2972 self.labels = {}
2973 if self.detectLabels:
2974 self.getLabels();
b984733c 2975
4b97ffb1 2976 if self.detectBranches:
df450923
SH
2977 ## FIXME - what's a P4 projectName ?
2978 self.projectName = self.guessProjectName()
2979
38f9f5ec
SH
2980 if self.hasOrigin:
2981 self.getBranchMappingFromGitBranches()
2982 else:
2983 self.getBranchMapping()
29bdbac1
SH
2984 if self.verbose:
2985 print "p4-git branches: %s" % self.p4BranchesInGit
2986 print "initial parents: %s" % self.initialParents
2987 for b in self.p4BranchesInGit:
2988 if b != "master":
6326aa58
HWN
2989
2990 ## FIXME
29bdbac1
SH
2991 b = b[len(self.projectName):]
2992 self.createdBranches.add(b)
4b97ffb1 2993
f291b4e3 2994 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 2995
78189bea
PW
2996 self.importProcess = subprocess.Popen(["git", "fast-import"],
2997 stdin=subprocess.PIPE,
2998 stdout=subprocess.PIPE,
2999 stderr=subprocess.PIPE);
3000 self.gitOutput = self.importProcess.stdout
3001 self.gitStream = self.importProcess.stdin
3002 self.gitError = self.importProcess.stderr
b984733c 3003
1c49fc19 3004 if revision:
c208a243 3005 self.importHeadRevision(revision)
b984733c
SH
3006 else:
3007 changes = []
3008
0828ab14 3009 if len(self.changesFile) > 0:
b984733c 3010 output = open(self.changesFile).readlines()
1d7367dc 3011 changeSet = set()
b984733c
SH
3012 for line in output:
3013 changeSet.add(int(line))
3014
3015 for change in changeSet:
3016 changes.append(change)
3017
3018 changes.sort()
3019 else:
9dcb9f24
PW
3020 # catch "git p4 sync" with no new branches, in a repo that
3021 # does not have any existing p4 branches
5a8e84cd
PW
3022 if len(args) == 0:
3023 if not self.p4BranchesInGit:
3024 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3025
3026 # The default branch is master, unless --branch is used to
3027 # specify something else. Make sure it exists, or complain
3028 # nicely about how to use --branch.
3029 if not self.detectBranches:
3030 if not branch_exists(self.branch):
3031 if branch_arg_given:
3032 die("Error: branch %s does not exist." % self.branch)
3033 else:
3034 die("Error: no branch %s; perhaps specify one with --branch." %
3035 self.branch)
3036
29bdbac1 3037 if self.verbose:
86dff6b6 3038 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3039 self.changeRange)
96b2d54a 3040 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
b984733c 3041
01a9c9c5 3042 if len(self.maxChanges) > 0:
7fcff9de 3043 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3044
b984733c 3045 if len(changes) == 0:
0828ab14 3046 if not self.silent:
341dc1c1 3047 print "No changes to import!"
06804c76
LD
3048 else:
3049 if not self.silent and not self.detectBranches:
3050 print "Import destination: %s" % self.branch
3051
3052 self.updatedBranches = set()
b984733c 3053
47497844
PW
3054 if not self.detectBranches:
3055 if args:
3056 # start a new branch
3057 self.initialParent = ""
3058 else:
3059 # build on a previous revision
3060 self.initialParent = parseRevision(self.branch)
3061
06804c76 3062 self.importChanges(changes)
a9d1a27a 3063
06804c76
LD
3064 if not self.silent:
3065 print ""
3066 if len(self.updatedBranches) > 0:
3067 sys.stdout.write("Updated branches: ")
3068 for b in self.updatedBranches:
3069 sys.stdout.write("%s " % b)
3070 sys.stdout.write("\n")
341dc1c1 3071
0d609032 3072 if gitConfigBool("git-p4.importLabels"):
06dcd152 3073 self.importLabels = True
b984733c 3074
06804c76
LD
3075 if self.importLabels:
3076 p4Labels = getP4Labels(self.depotPaths)
3077 gitTags = getGitTags()
3078
3079 missingP4Labels = p4Labels - gitTags
3080 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3081
b984733c 3082 self.gitStream.close()
78189bea 3083 if self.importProcess.wait() != 0:
29bdbac1 3084 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3085 self.gitOutput.close()
3086 self.gitError.close()
3087
fed23693
VA
3088 # Cleanup temporary branches created during import
3089 if self.tempBranches != []:
3090 for branch in self.tempBranches:
3091 read_pipe("git update-ref -d %s" % branch)
3092 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3093
55d12437
PW
3094 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3095 # a convenient shortcut refname "p4".
3096 if self.importIntoRemotes:
3097 head_ref = self.refPrefix + "HEAD"
3098 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3099 system(["git", "symbolic-ref", head_ref, self.branch])
3100
b984733c
SH
3101 return True
3102
01ce1fe9
SH
3103class P4Rebase(Command):
3104 def __init__(self):
3105 Command.__init__(self)
06804c76
LD
3106 self.options = [
3107 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3108 ]
06804c76 3109 self.importLabels = False
cebdf5af
HWN
3110 self.description = ("Fetches the latest revision from perforce and "
3111 + "rebases the current work (branch) against it")
01ce1fe9
SH
3112
3113 def run(self, args):
3114 sync = P4Sync()
06804c76 3115 sync.importLabels = self.importLabels
01ce1fe9 3116 sync.run([])
d7e3868c 3117
14594f4b
SH
3118 return self.rebase()
3119
3120 def rebase(self):
36ee4ee4
SH
3121 if os.system("git update-index --refresh") != 0:
3122 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3123 if len(read_pipe("git diff-index HEAD --")) > 0:
f7e604ed 3124 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
36ee4ee4 3125
d7e3868c
SH
3126 [upstream, settings] = findUpstreamBranchPoint()
3127 if len(upstream) == 0:
3128 die("Cannot find upstream branchpoint for rebase")
3129
3130 # the branchpoint may be p4/foo~3, so strip off the parent
3131 upstream = re.sub("~[0-9]+$", "", upstream)
3132
3133 print "Rebasing the current branch onto %s" % upstream
b25b2065 3134 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 3135 system("git rebase %s" % upstream)
4e49d95e 3136 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
01ce1fe9
SH
3137 return True
3138
f9a3a4f7
SH
3139class P4Clone(P4Sync):
3140 def __init__(self):
3141 P4Sync.__init__(self)
3142 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 3143 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 3144 self.options += [
bb6e09b2
HWN
3145 optparse.make_option("--destination", dest="cloneDestination",
3146 action='store', default=None,
354081d5 3147 help="where to leave result of the clone"),
38200076
PW
3148 optparse.make_option("--bare", dest="cloneBare",
3149 action="store_true", default=False),
354081d5 3150 ]
bb6e09b2 3151 self.cloneDestination = None
f9a3a4f7 3152 self.needsGit = False
38200076 3153 self.cloneBare = False
f9a3a4f7 3154
6a49f8e2
HWN
3155 def defaultDestination(self, args):
3156 ## TODO: use common prefix of args?
3157 depotPath = args[0]
3158 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3159 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 3160 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
3161 depotDir = re.sub(r"/$", "", depotDir)
3162 return os.path.split(depotDir)[1]
3163
f9a3a4f7
SH
3164 def run(self, args):
3165 if len(args) < 1:
3166 return False
bb6e09b2
HWN
3167
3168 if self.keepRepoPath and not self.cloneDestination:
3169 sys.stderr.write("Must specify destination for --keep-path\n")
3170 sys.exit(1)
f9a3a4f7 3171
6326aa58 3172 depotPaths = args
5e100b5c
SH
3173
3174 if not self.cloneDestination and len(depotPaths) > 1:
3175 self.cloneDestination = depotPaths[-1]
3176 depotPaths = depotPaths[:-1]
3177
354081d5 3178 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
3179 for p in depotPaths:
3180 if not p.startswith("//"):
0f487d30 3181 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
6326aa58 3182 return False
f9a3a4f7 3183
bb6e09b2 3184 if not self.cloneDestination:
98ad4faf 3185 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 3186
86dff6b6 3187 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 3188
c3bf3f13
KG
3189 if not os.path.exists(self.cloneDestination):
3190 os.makedirs(self.cloneDestination)
053fd0c1 3191 chdir(self.cloneDestination)
38200076
PW
3192
3193 init_cmd = [ "git", "init" ]
3194 if self.cloneBare:
3195 init_cmd.append("--bare")
a235e85c
BC
3196 retcode = subprocess.call(init_cmd)
3197 if retcode:
3198 raise CalledProcessError(retcode, init_cmd)
38200076 3199
6326aa58 3200 if not P4Sync.run(self, depotPaths):
f9a3a4f7 3201 return False
c595956d
PW
3202
3203 # create a master branch and check out a work tree
3204 if gitBranchExists(self.branch):
3205 system([ "git", "branch", "master", self.branch ])
3206 if not self.cloneBare:
3207 system([ "git", "checkout", "-f" ])
3208 else:
3209 print 'Not checking out any branch, use ' \
3210 '"git checkout -q -b master <branch>"'
86dff6b6 3211
a93d33ee
PW
3212 # auto-set this variable if invoked with --use-client-spec
3213 if self.useClientSpec_from_options:
3214 system("git config --bool git-p4.useclientspec true")
3215
f9a3a4f7
SH
3216 return True
3217
09d89de2
SH
3218class P4Branches(Command):
3219 def __init__(self):
3220 Command.__init__(self)
3221 self.options = [ ]
3222 self.description = ("Shows the git branches that hold imports and their "
3223 + "corresponding perforce depot paths")
3224 self.verbose = False
3225
3226 def run(self, args):
5ca44617
SH
3227 if originP4BranchesExist():
3228 createOrUpdateBranchesFromOrigin()
3229
09d89de2
SH
3230 cmdline = "git rev-parse --symbolic "
3231 cmdline += " --remotes"
3232
3233 for line in read_pipe_lines(cmdline):
3234 line = line.strip()
3235
3236 if not line.startswith('p4/') or line == "p4/HEAD":
3237 continue
3238 branch = line
3239
3240 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3241 settings = extractSettingsGitLog(log)
3242
3243 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3244 return True
3245
b984733c
SH
3246class HelpFormatter(optparse.IndentedHelpFormatter):
3247 def __init__(self):
3248 optparse.IndentedHelpFormatter.__init__(self)
3249
3250 def format_description(self, description):
3251 if description:
3252 return description + "\n"
3253 else:
3254 return ""
4f5cf76a 3255
86949eef
SH
3256def printUsage(commands):
3257 print "usage: %s <command> [options]" % sys.argv[0]
3258 print ""
3259 print "valid commands: %s" % ", ".join(commands)
3260 print ""
3261 print "Try %s <command> --help for command specific help." % sys.argv[0]
3262 print ""
3263
3264commands = {
b86f7378
HWN
3265 "debug" : P4Debug,
3266 "submit" : P4Submit,
a9834f58 3267 "commit" : P4Submit,
b86f7378
HWN
3268 "sync" : P4Sync,
3269 "rebase" : P4Rebase,
3270 "clone" : P4Clone,
09d89de2
SH
3271 "rollback" : P4RollBack,
3272 "branches" : P4Branches
86949eef
SH
3273}
3274
86949eef 3275
bb6e09b2
HWN
3276def main():
3277 if len(sys.argv[1:]) == 0:
3278 printUsage(commands.keys())
3279 sys.exit(2)
4f5cf76a 3280
bb6e09b2
HWN
3281 cmdName = sys.argv[1]
3282 try:
b86f7378
HWN
3283 klass = commands[cmdName]
3284 cmd = klass()
bb6e09b2
HWN
3285 except KeyError:
3286 print "unknown command %s" % cmdName
3287 print ""
3288 printUsage(commands.keys())
3289 sys.exit(2)
3290
3291 options = cmd.options
b86f7378 3292 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
3293
3294 args = sys.argv[2:]
3295
b0ccc80d 3296 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
6a10b6aa
LD
3297 if cmd.needsGit:
3298 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 3299
6a10b6aa
LD
3300 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3301 options,
3302 description = cmd.description,
3303 formatter = HelpFormatter())
bb6e09b2 3304
6a10b6aa 3305 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
bb6e09b2
HWN
3306 global verbose
3307 verbose = cmd.verbose
3308 if cmd.needsGit:
b86f7378
HWN
3309 if cmd.gitdir == None:
3310 cmd.gitdir = os.path.abspath(".git")
3311 if not isValidGitDir(cmd.gitdir):
3312 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3313 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
3314 cdup = read_pipe("git rev-parse --show-cdup").strip()
3315 if len(cdup) > 0:
053fd0c1 3316 chdir(cdup);
e20a9e53 3317
b86f7378
HWN
3318 if not isValidGitDir(cmd.gitdir):
3319 if isValidGitDir(cmd.gitdir + "/.git"):
3320 cmd.gitdir += "/.git"
bb6e09b2 3321 else:
b86f7378 3322 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 3323
b86f7378 3324 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 3325
bb6e09b2
HWN
3326 if not cmd.run(args):
3327 parser.print_help()
09fca77b 3328 sys.exit(2)
4f5cf76a 3329
4f5cf76a 3330
bb6e09b2
HWN
3331if __name__ == '__main__':
3332 main()