git p4 test: should honor symlink in p4 client root
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
8b41a97f 25
a235e85c
BC
26try:
27 from subprocess import CalledProcessError
28except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
4addad22 41verbose = False
86949eef 42
06804c76 43# Only labels/tags matching this will be imported/exported
c8942a22 44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753
AK
45
46def p4_build_cmd(cmd):
47 """Build a suitable p4 command line.
48
49 This consolidates building and returning a p4 command line into one
50 location. It means that hooking into the environment, or other configuration
51 can be done more easily.
52 """
6de040df 53 real_cmd = ["p4"]
abcaf073
AK
54
55 user = gitConfig("git-p4.user")
56 if len(user) > 0:
6de040df 57 real_cmd += ["-u",user]
abcaf073
AK
58
59 password = gitConfig("git-p4.password")
60 if len(password) > 0:
6de040df 61 real_cmd += ["-P", password]
abcaf073
AK
62
63 port = gitConfig("git-p4.port")
64 if len(port) > 0:
6de040df 65 real_cmd += ["-p", port]
abcaf073
AK
66
67 host = gitConfig("git-p4.host")
68 if len(host) > 0:
41799aa2 69 real_cmd += ["-H", host]
abcaf073
AK
70
71 client = gitConfig("git-p4.client")
72 if len(client) > 0:
6de040df 73 real_cmd += ["-c", client]
abcaf073 74
6de040df
LD
75
76 if isinstance(cmd,basestring):
77 real_cmd = ' '.join(real_cmd) + ' ' + cmd
78 else:
79 real_cmd += cmd
21a50753
AK
80 return real_cmd
81
053fd0c1 82def chdir(dir):
6de040df 83 # P4 uses the PWD environment variable rather than getcwd(). Since we're
bf1d68ff
GG
84 # not using the shell, we have to set it ourselves. This path could
85 # be relative, so go there first, then figure out where we ended up.
053fd0c1 86 os.chdir(dir)
bf1d68ff 87 os.environ['PWD'] = os.getcwd()
053fd0c1 88
86dff6b6
HWN
89def die(msg):
90 if verbose:
91 raise Exception(msg)
92 else:
93 sys.stderr.write(msg + "\n")
94 sys.exit(1)
95
6de040df 96def write_pipe(c, stdin):
4addad22 97 if verbose:
6de040df 98 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 99
6de040df
LD
100 expand = isinstance(c,basestring)
101 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
102 pipe = p.stdin
103 val = pipe.write(stdin)
104 pipe.close()
105 if p.wait():
106 die('Command failed: %s' % str(c))
b016d397
HWN
107
108 return val
109
6de040df 110def p4_write_pipe(c, stdin):
d9429194 111 real_cmd = p4_build_cmd(c)
6de040df 112 return write_pipe(real_cmd, stdin)
d9429194 113
4addad22
HWN
114def read_pipe(c, ignore_error=False):
115 if verbose:
6de040df 116 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 117
6de040df
LD
118 expand = isinstance(c,basestring)
119 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
120 pipe = p.stdout
b016d397 121 val = pipe.read()
6de040df
LD
122 if p.wait() and not ignore_error:
123 die('Command failed: %s' % str(c))
b016d397
HWN
124
125 return val
126
d9429194
AK
127def p4_read_pipe(c, ignore_error=False):
128 real_cmd = p4_build_cmd(c)
129 return read_pipe(real_cmd, ignore_error)
b016d397 130
bce4c5fc 131def read_pipe_lines(c):
4addad22 132 if verbose:
6de040df
LD
133 sys.stderr.write('Reading pipe: %s\n' % str(c))
134
135 expand = isinstance(c, basestring)
136 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
137 pipe = p.stdout
b016d397 138 val = pipe.readlines()
6de040df
LD
139 if pipe.close() or p.wait():
140 die('Command failed: %s' % str(c))
b016d397
HWN
141
142 return val
caace111 143
2318121b
AK
144def p4_read_pipe_lines(c):
145 """Specifically invoke p4 on the command supplied. """
155af834 146 real_cmd = p4_build_cmd(c)
2318121b
AK
147 return read_pipe_lines(real_cmd)
148
8e9497c2
GG
149def p4_has_command(cmd):
150 """Ask p4 for help on this command. If it returns an error, the
151 command does not exist in this version of p4."""
152 real_cmd = p4_build_cmd(["help", cmd])
153 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
154 stderr=subprocess.PIPE)
155 p.communicate()
156 return p.returncode == 0
157
249da4c0
PW
158def p4_has_move_command():
159 """See if the move command exists, that it supports -k, and that
160 it has not been administratively disabled. The arguments
161 must be correct, but the filenames do not have to exist. Use
162 ones with wildcards so even if they exist, it will fail."""
163
164 if not p4_has_command("move"):
165 return False
166 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
167 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
168 (out, err) = p.communicate()
169 # return code will be 1 in either case
170 if err.find("Invalid option") >= 0:
171 return False
172 if err.find("disabled") >= 0:
173 return False
174 # assume it failed because @... was invalid changelist
175 return True
176
6754a299 177def system(cmd):
6de040df 178 expand = isinstance(cmd,basestring)
4addad22 179 if verbose:
6de040df 180 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c
BC
181 retcode = subprocess.call(cmd, shell=expand)
182 if retcode:
183 raise CalledProcessError(retcode, cmd)
6754a299 184
bf9320f1
AK
185def p4_system(cmd):
186 """Specifically invoke p4 as the system command. """
155af834 187 real_cmd = p4_build_cmd(cmd)
6de040df 188 expand = isinstance(real_cmd, basestring)
a235e85c
BC
189 retcode = subprocess.call(real_cmd, shell=expand)
190 if retcode:
191 raise CalledProcessError(retcode, real_cmd)
6de040df 192
7f0e5962
PW
193_p4_version_string = None
194def p4_version_string():
195 """Read the version string, showing just the last line, which
196 hopefully is the interesting version bit.
197
198 $ p4 -V
199 Perforce - The Fast Software Configuration Management System.
200 Copyright 1995-2011 Perforce Software. All rights reserved.
201 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
202 """
203 global _p4_version_string
204 if not _p4_version_string:
205 a = p4_read_pipe_lines(["-V"])
206 _p4_version_string = a[-1].rstrip()
207 return _p4_version_string
208
6de040df 209def p4_integrate(src, dest):
9d7d446a 210 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 211
8d7ec362 212def p4_sync(f, *options):
9d7d446a 213 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
214
215def p4_add(f):
9d7d446a
PW
216 # forcibly add file names with wildcards
217 if wildcard_present(f):
218 p4_system(["add", "-f", f])
219 else:
220 p4_system(["add", f])
6de040df
LD
221
222def p4_delete(f):
9d7d446a 223 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
224
225def p4_edit(f):
9d7d446a 226 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
227
228def p4_revert(f):
9d7d446a 229 p4_system(["revert", wildcard_encode(f)])
6de040df 230
9d7d446a
PW
231def p4_reopen(type, f):
232 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 233
8e9497c2
GG
234def p4_move(src, dest):
235 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
236
18fa13d0
PW
237def p4_describe(change):
238 """Make sure it returns a valid result by checking for
239 the presence of field "time". Return a dict of the
240 results."""
241
242 ds = p4CmdList(["describe", "-s", str(change)])
243 if len(ds) != 1:
244 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
245
246 d = ds[0]
247
248 if "p4ExitCode" in d:
249 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
250 str(d)))
251 if "code" in d:
252 if d["code"] == "error":
253 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
254
255 if "time" not in d:
256 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
257
258 return d
259
9cffb8c8
PW
260#
261# Canonicalize the p4 type and return a tuple of the
262# base type, plus any modifiers. See "p4 help filetypes"
263# for a list and explanation.
264#
265def split_p4_type(p4type):
266
267 p4_filetypes_historical = {
268 "ctempobj": "binary+Sw",
269 "ctext": "text+C",
270 "cxtext": "text+Cx",
271 "ktext": "text+k",
272 "kxtext": "text+kx",
273 "ltext": "text+F",
274 "tempobj": "binary+FSw",
275 "ubinary": "binary+F",
276 "uresource": "resource+F",
277 "uxbinary": "binary+Fx",
278 "xbinary": "binary+x",
279 "xltext": "text+Fx",
280 "xtempobj": "binary+Swx",
281 "xtext": "text+x",
282 "xunicode": "unicode+x",
283 "xutf16": "utf16+x",
284 }
285 if p4type in p4_filetypes_historical:
286 p4type = p4_filetypes_historical[p4type]
287 mods = ""
288 s = p4type.split("+")
289 base = s[0]
290 mods = ""
291 if len(s) > 1:
292 mods = s[1]
293 return (base, mods)
b9fc6ea9 294
60df071c
LD
295#
296# return the raw p4 type of a file (text, text+ko, etc)
297#
298def p4_type(file):
299 results = p4CmdList(["fstat", "-T", "headType", file])
300 return results[0]['headType']
301
302#
303# Given a type base and modifier, return a regexp matching
304# the keywords that can be expanded in the file
305#
306def p4_keywords_regexp_for_type(base, type_mods):
307 if base in ("text", "unicode", "binary"):
308 kwords = None
309 if "ko" in type_mods:
310 kwords = 'Id|Header'
311 elif "k" in type_mods:
312 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
313 else:
314 return None
315 pattern = r"""
316 \$ # Starts with a dollar, followed by...
317 (%s) # one of the keywords, followed by...
6b2bf41e 318 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
319 \$ # another dollar
320 """ % kwords
321 return pattern
322 else:
323 return None
324
325#
326# Given a file, return a regexp matching the possible
327# RCS keywords that will be expanded, or None for files
328# with kw expansion turned off.
329#
330def p4_keywords_regexp_for_file(file):
331 if not os.path.exists(file):
332 return None
333 else:
334 (type_base, type_mods) = split_p4_type(p4_type(file))
335 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 336
c65b670e
CP
337def setP4ExecBit(file, mode):
338 # Reopens an already open file and changes the execute bit to match
339 # the execute bit setting in the passed in mode.
340
341 p4Type = "+x"
342
343 if not isModeExec(mode):
344 p4Type = getP4OpenedType(file)
345 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
346 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
347 if p4Type[-1] == "+":
348 p4Type = p4Type[0:-1]
349
6de040df 350 p4_reopen(p4Type, file)
c65b670e
CP
351
352def getP4OpenedType(file):
353 # Returns the perforce file type for the given file.
354
9d7d446a 355 result = p4_read_pipe(["opened", wildcard_encode(file)])
f3e5ae4f 356 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
357 if match:
358 return match.group(1)
359 else:
f3e5ae4f 360 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 361
06804c76
LD
362# Return the set of all p4 labels
363def getP4Labels(depotPaths):
364 labels = set()
365 if isinstance(depotPaths,basestring):
366 depotPaths = [depotPaths]
367
368 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
369 label = l['label']
370 labels.add(label)
371
372 return labels
373
374# Return the set of all git tags
375def getGitTags():
376 gitTags = set()
377 for line in read_pipe_lines(["git", "tag"]):
378 tag = line.strip()
379 gitTags.add(tag)
380 return gitTags
381
b43b0a3c
CP
382def diffTreePattern():
383 # This is a simple generator for the diff tree regex pattern. This could be
384 # a class variable if this and parseDiffTreeEntry were a part of a class.
385 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
386 while True:
387 yield pattern
388
389def parseDiffTreeEntry(entry):
390 """Parses a single diff tree entry into its component elements.
391
392 See git-diff-tree(1) manpage for details about the format of the diff
393 output. This method returns a dictionary with the following elements:
394
395 src_mode - The mode of the source file
396 dst_mode - The mode of the destination file
397 src_sha1 - The sha1 for the source file
398 dst_sha1 - The sha1 fr the destination file
399 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
400 status_score - The score for the status (applicable for 'C' and 'R'
401 statuses). This is None if there is no score.
402 src - The path for the source file.
403 dst - The path for the destination file. This is only present for
404 copy or renames. If it is not present, this is None.
405
406 If the pattern is not matched, None is returned."""
407
408 match = diffTreePattern().next().match(entry)
409 if match:
410 return {
411 'src_mode': match.group(1),
412 'dst_mode': match.group(2),
413 'src_sha1': match.group(3),
414 'dst_sha1': match.group(4),
415 'status': match.group(5),
416 'status_score': match.group(6),
417 'src': match.group(7),
418 'dst': match.group(10)
419 }
420 return None
421
c65b670e
CP
422def isModeExec(mode):
423 # Returns True if the given git mode represents an executable file,
424 # otherwise False.
425 return mode[-3:] == "755"
426
427def isModeExecChanged(src_mode, dst_mode):
428 return isModeExec(src_mode) != isModeExec(dst_mode)
429
b932705b 430def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
431
432 if isinstance(cmd,basestring):
433 cmd = "-G " + cmd
434 expand = True
435 else:
436 cmd = ["-G"] + cmd
437 expand = False
438
439 cmd = p4_build_cmd(cmd)
6a49f8e2 440 if verbose:
6de040df 441 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
442
443 # Use a temporary file to avoid deadlocks without
444 # subprocess.communicate(), which would put another copy
445 # of stdout into memory.
446 stdin_file = None
447 if stdin is not None:
448 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
449 if isinstance(stdin,basestring):
450 stdin_file.write(stdin)
451 else:
452 for i in stdin:
453 stdin_file.write(i + '\n')
9f90c733
SL
454 stdin_file.flush()
455 stdin_file.seek(0)
456
6de040df
LD
457 p4 = subprocess.Popen(cmd,
458 shell=expand,
9f90c733
SL
459 stdin=stdin_file,
460 stdout=subprocess.PIPE)
86949eef
SH
461
462 result = []
463 try:
464 while True:
9f90c733 465 entry = marshal.load(p4.stdout)
c3f6163b
AG
466 if cb is not None:
467 cb(entry)
468 else:
469 result.append(entry)
86949eef
SH
470 except EOFError:
471 pass
9f90c733
SL
472 exitCode = p4.wait()
473 if exitCode != 0:
ac3e0d79
SH
474 entry = {}
475 entry["p4ExitCode"] = exitCode
476 result.append(entry)
86949eef
SH
477
478 return result
479
480def p4Cmd(cmd):
481 list = p4CmdList(cmd)
482 result = {}
483 for entry in list:
484 result.update(entry)
485 return result;
486
cb2c9db5
SH
487def p4Where(depotPath):
488 if not depotPath.endswith("/"):
489 depotPath += "/"
7f705dc3 490 depotPath = depotPath + "..."
6de040df 491 outputList = p4CmdList(["where", depotPath])
7f705dc3
TAL
492 output = None
493 for entry in outputList:
75bc9573
TAL
494 if "depotFile" in entry:
495 if entry["depotFile"] == depotPath:
496 output = entry
497 break
498 elif "data" in entry:
499 data = entry.get("data")
500 space = data.find(" ")
501 if data[:space] == depotPath:
502 output = entry
503 break
7f705dc3
TAL
504 if output == None:
505 return ""
dc524036
SH
506 if output["code"] == "error":
507 return ""
cb2c9db5
SH
508 clientPath = ""
509 if "path" in output:
510 clientPath = output.get("path")
511 elif "data" in output:
512 data = output.get("data")
513 lastSpace = data.rfind(" ")
514 clientPath = data[lastSpace + 1:]
515
516 if clientPath.endswith("..."):
517 clientPath = clientPath[:-3]
518 return clientPath
519
86949eef 520def currentGitBranch():
b25b2065 521 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 522
4f5cf76a 523def isValidGitDir(path):
bb6e09b2
HWN
524 if (os.path.exists(path + "/HEAD")
525 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
526 return True;
527 return False
528
463e8af6 529def parseRevision(ref):
b25b2065 530 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 531
28755dba
PW
532def branchExists(ref):
533 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
534 ignore_error=True)
535 return len(rev) > 0
536
6ae8de88
SH
537def extractLogMessageFromGitCommit(commit):
538 logMessage = ""
b016d397
HWN
539
540 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 541 foundTitle = False
b016d397 542 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
543 if not foundTitle:
544 if len(log) == 1:
1c094184 545 foundTitle = True
6ae8de88
SH
546 continue
547
548 logMessage += log
549 return logMessage
550
bb6e09b2 551def extractSettingsGitLog(log):
6ae8de88
SH
552 values = {}
553 for line in log.split("\n"):
554 line = line.strip()
6326aa58
HWN
555 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
556 if not m:
557 continue
558
559 assignments = m.group(1).split (':')
560 for a in assignments:
561 vals = a.split ('=')
562 key = vals[0].strip()
563 val = ('='.join (vals[1:])).strip()
564 if val.endswith ('\"') and val.startswith('"'):
565 val = val[1:-1]
566
567 values[key] = val
568
845b42cb
SH
569 paths = values.get("depot-paths")
570 if not paths:
571 paths = values.get("depot-path")
a3fdd579
SH
572 if paths:
573 values['depot-paths'] = paths.split(',')
bb6e09b2 574 return values
6ae8de88 575
8136a639 576def gitBranchExists(branch):
bb6e09b2
HWN
577 proc = subprocess.Popen(["git", "rev-parse", branch],
578 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 579 return proc.wait() == 0;
8136a639 580
36bd8446 581_gitConfig = {}
b345d6c3 582
0d609032 583def gitConfig(key):
36bd8446 584 if not _gitConfig.has_key(key):
0d609032 585 cmd = [ "git", "config", key ]
b345d6c3
PW
586 s = read_pipe(cmd, ignore_error=True)
587 _gitConfig[key] = s.strip()
36bd8446 588 return _gitConfig[key]
01265103 589
0d609032
PW
590def gitConfigBool(key):
591 """Return a bool, using git config --bool. It is True only if the
592 variable is set to true, and False if set to false or not present
593 in the config."""
594
36bd8446 595 if not _gitConfig.has_key(key):
0d609032
PW
596 cmd = [ "git", "config", "--bool", key ]
597 s = read_pipe(cmd, ignore_error=True)
598 v = s.strip()
599 _gitConfig[key] = v == "true"
36bd8446 600 return _gitConfig[key]
01265103 601
7199cf13
VA
602def gitConfigList(key):
603 if not _gitConfig.has_key(key):
2abba301
PW
604 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
605 _gitConfig[key] = s.strip().split(os.linesep)
7199cf13
VA
606 return _gitConfig[key]
607
2c8037ed
PW
608def p4BranchesInGit(branchesAreInRemotes=True):
609 """Find all the branches whose names start with "p4/", looking
610 in remotes or heads as specified by the argument. Return
611 a dictionary of { branch: revision } for each one found.
612 The branch names are the short names, without any
613 "p4/" prefix."""
614
062410bb
SH
615 branches = {}
616
617 cmdline = "git rev-parse --symbolic "
618 if branchesAreInRemotes:
2c8037ed 619 cmdline += "--remotes"
062410bb 620 else:
2c8037ed 621 cmdline += "--branches"
062410bb
SH
622
623 for line in read_pipe_lines(cmdline):
624 line = line.strip()
625
2c8037ed
PW
626 # only import to p4/
627 if not line.startswith('p4/'):
628 continue
629 # special symbolic ref to p4/master
630 if line == "p4/HEAD":
062410bb 631 continue
062410bb 632
2c8037ed
PW
633 # strip off p4/ prefix
634 branch = line[len("p4/"):]
062410bb
SH
635
636 branches[branch] = parseRevision(line)
2c8037ed 637
062410bb
SH
638 return branches
639
5a8e84cd
PW
640def branch_exists(branch):
641 """Make sure that the given ref name really exists."""
642
643 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
644 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
645 out, _ = p.communicate()
646 if p.returncode:
647 return False
648 # expect exactly one line of output: the branch name
649 return out.rstrip() == branch
650
9ceab363 651def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
652 branches = p4BranchesInGit()
653 # map from depot-path to branch name
654 branchByDepotPath = {}
655 for branch in branches.keys():
656 tip = branches[branch]
657 log = extractLogMessageFromGitCommit(tip)
658 settings = extractSettingsGitLog(log)
659 if settings.has_key("depot-paths"):
660 paths = ",".join(settings["depot-paths"])
661 branchByDepotPath[paths] = "remotes/p4/" + branch
662
27d2d811 663 settings = None
27d2d811
SH
664 parent = 0
665 while parent < 65535:
9ceab363 666 commit = head + "~%s" % parent
27d2d811
SH
667 log = extractLogMessageFromGitCommit(commit)
668 settings = extractSettingsGitLog(log)
86506fe5
SH
669 if settings.has_key("depot-paths"):
670 paths = ",".join(settings["depot-paths"])
671 if branchByDepotPath.has_key(paths):
672 return [branchByDepotPath[paths], settings]
27d2d811 673
86506fe5 674 parent = parent + 1
27d2d811 675
86506fe5 676 return ["", settings]
27d2d811 677
5ca44617
SH
678def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
679 if not silent:
680 print ("Creating/updating branch(es) in %s based on origin branch(es)"
681 % localRefPrefix)
682
683 originPrefix = "origin/p4/"
684
685 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
686 line = line.strip()
687 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
688 continue
689
690 headName = line[len(originPrefix):]
691 remoteHead = localRefPrefix + headName
692 originHead = line
693
694 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
695 if (not original.has_key('depot-paths')
696 or not original.has_key('change')):
697 continue
698
699 update = False
700 if not gitBranchExists(remoteHead):
701 if verbose:
702 print "creating %s" % remoteHead
703 update = True
704 else:
705 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
706 if settings.has_key('change') > 0:
707 if settings['depot-paths'] == original['depot-paths']:
708 originP4Change = int(original['change'])
709 p4Change = int(settings['change'])
710 if originP4Change > p4Change:
711 print ("%s (%s) is newer than %s (%s). "
712 "Updating p4 branch from origin."
713 % (originHead, originP4Change,
714 remoteHead, p4Change))
715 update = True
716 else:
717 print ("Ignoring: %s was imported from %s while "
718 "%s was imported from %s"
719 % (originHead, ','.join(original['depot-paths']),
720 remoteHead, ','.join(settings['depot-paths'])))
721
722 if update:
723 system("git update-ref %s %s" % (remoteHead, originHead))
724
725def originP4BranchesExist():
726 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
727
4f6432d8
SH
728def p4ChangesForPaths(depotPaths, changeRange):
729 assert depotPaths
6de040df
LD
730 cmd = ['changes']
731 for p in depotPaths:
732 cmd += ["%s...%s" % (p, changeRange)]
733 output = p4_read_pipe_lines(cmd)
4f6432d8 734
b4b0ba06 735 changes = {}
4f6432d8 736 for line in output:
c3f6163b
AG
737 changeNum = int(line.split(" ")[1])
738 changes[changeNum] = True
4f6432d8 739
b4b0ba06
PW
740 changelist = changes.keys()
741 changelist.sort()
742 return changelist
4f6432d8 743
d53de8b9
TAL
744def p4PathStartsWith(path, prefix):
745 # This method tries to remedy a potential mixed-case issue:
746 #
747 # If UserA adds //depot/DirA/file1
748 # and UserB adds //depot/dira/file2
749 #
750 # we may or may not have a problem. If you have core.ignorecase=true,
751 # we treat DirA and dira as the same directory
0d609032 752 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
753 return path.lower().startswith(prefix.lower())
754 return path.startswith(prefix)
755
543987bd
PW
756def getClientSpec():
757 """Look at the p4 client spec, create a View() object that contains
758 all the mappings, and return it."""
759
760 specList = p4CmdList("client -o")
761 if len(specList) != 1:
762 die('Output from "client -o" is %d lines, expecting 1' %
763 len(specList))
764
765 # dictionary of all client parameters
766 entry = specList[0]
767
768 # just the keys that start with "View"
769 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
770
771 # hold this new View
772 view = View()
773
774 # append the lines, in order, to the view
775 for view_num in range(len(view_keys)):
776 k = "View%d" % view_num
777 if k not in view_keys:
778 die("Expected view key %s missing" % k)
779 view.append(entry[k])
780
781 return view
782
783def getClientRoot():
784 """Grab the client directory."""
785
786 output = p4CmdList("client -o")
787 if len(output) != 1:
788 die('Output from "client -o" is %d lines, expecting 1' % len(output))
789
790 entry = output[0]
791 if "Root" not in entry:
792 die('Client has no "Root"')
793
794 return entry["Root"]
795
9d7d446a
PW
796#
797# P4 wildcards are not allowed in filenames. P4 complains
798# if you simply add them, but you can force it with "-f", in
799# which case it translates them into %xx encoding internally.
800#
801def wildcard_decode(path):
802 # Search for and fix just these four characters. Do % last so
803 # that fixing it does not inadvertently create new %-escapes.
804 # Cannot have * in a filename in windows; untested as to
805 # what p4 would do in such a case.
806 if not platform.system() == "Windows":
807 path = path.replace("%2A", "*")
808 path = path.replace("%23", "#") \
809 .replace("%40", "@") \
810 .replace("%25", "%")
811 return path
812
813def wildcard_encode(path):
814 # do % first to avoid double-encoding the %s introduced here
815 path = path.replace("%", "%25") \
816 .replace("*", "%2A") \
817 .replace("#", "%23") \
818 .replace("@", "%40")
819 return path
820
821def wildcard_present(path):
598354c0
BC
822 m = re.search("[*#@%]", path)
823 return m is not None
9d7d446a 824
b984733c
SH
825class Command:
826 def __init__(self):
827 self.usage = "usage: %prog [options]"
8910ac0e 828 self.needsGit = True
6a10b6aa 829 self.verbose = False
b984733c 830
3ea2cfd4
LD
831class P4UserMap:
832 def __init__(self):
833 self.userMapFromPerforceServer = False
affb474f
LD
834 self.myP4UserId = None
835
836 def p4UserId(self):
837 if self.myP4UserId:
838 return self.myP4UserId
839
840 results = p4CmdList("user -o")
841 for r in results:
842 if r.has_key('User'):
843 self.myP4UserId = r['User']
844 return r['User']
845 die("Could not find your p4 user id")
846
847 def p4UserIsMe(self, p4User):
848 # return True if the given p4 user is actually me
849 me = self.p4UserId()
850 if not p4User or p4User != me:
851 return False
852 else:
853 return True
3ea2cfd4
LD
854
855 def getUserCacheFilename(self):
856 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
857 return home + "/.gitp4-usercache.txt"
858
859 def getUserMapFromPerforceServer(self):
860 if self.userMapFromPerforceServer:
861 return
862 self.users = {}
863 self.emails = {}
864
865 for output in p4CmdList("users"):
866 if not output.has_key("User"):
867 continue
868 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
869 self.emails[output["Email"]] = output["User"]
870
871
872 s = ''
873 for (key, val) in self.users.items():
874 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
875
876 open(self.getUserCacheFilename(), "wb").write(s)
877 self.userMapFromPerforceServer = True
878
879 def loadUserMapFromCache(self):
880 self.users = {}
881 self.userMapFromPerforceServer = False
882 try:
883 cache = open(self.getUserCacheFilename(), "rb")
884 lines = cache.readlines()
885 cache.close()
886 for line in lines:
887 entry = line.strip().split("\t")
888 self.users[entry[0]] = entry[1]
889 except IOError:
890 self.getUserMapFromPerforceServer()
891
b984733c 892class P4Debug(Command):
86949eef 893 def __init__(self):
6ae8de88 894 Command.__init__(self)
6a10b6aa 895 self.options = []
c8c39116 896 self.description = "A tool to debug the output of p4 -G."
8910ac0e 897 self.needsGit = False
86949eef
SH
898
899 def run(self, args):
b1ce9447 900 j = 0
6de040df 901 for output in p4CmdList(args):
b1ce9447
HWN
902 print 'Element: %d' % j
903 j += 1
86949eef 904 print output
b984733c 905 return True
86949eef 906
5834684d
SH
907class P4RollBack(Command):
908 def __init__(self):
909 Command.__init__(self)
910 self.options = [
0c66a783 911 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
912 ]
913 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 914 self.rollbackLocalBranches = False
5834684d
SH
915
916 def run(self, args):
917 if len(args) != 1:
918 return False
919 maxChange = int(args[0])
0c66a783 920
ad192f28 921 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
922 die("Problems executing p4");
923
0c66a783
SH
924 if self.rollbackLocalBranches:
925 refPrefix = "refs/heads/"
b016d397 926 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
927 else:
928 refPrefix = "refs/remotes/"
b016d397 929 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
930
931 for line in lines:
932 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
933 line = line.strip()
934 ref = refPrefix + line
5834684d 935 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
936 settings = extractSettingsGitLog(log)
937
938 depotPaths = settings['depot-paths']
939 change = settings['change']
940
5834684d 941 changed = False
52102d47 942
6326aa58
HWN
943 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
944 for p in depotPaths]))) == 0:
52102d47
SH
945 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
946 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
947 continue
948
bb6e09b2 949 while change and int(change) > maxChange:
5834684d 950 changed = True
52102d47
SH
951 if self.verbose:
952 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
953 system("git update-ref %s \"%s^\"" % (ref, ref))
954 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
955 settings = extractSettingsGitLog(log)
956
957
958 depotPaths = settings['depot-paths']
959 change = settings['change']
5834684d
SH
960
961 if changed:
52102d47 962 print "%s rewound to %s" % (ref, change)
5834684d
SH
963
964 return True
965
3ea2cfd4 966class P4Submit(Command, P4UserMap):
6bbfd137
PW
967
968 conflict_behavior_choices = ("ask", "skip", "quit")
969
4f5cf76a 970 def __init__(self):
b984733c 971 Command.__init__(self)
3ea2cfd4 972 P4UserMap.__init__(self)
4f5cf76a 973 self.options = [
4f5cf76a 974 optparse.make_option("--origin", dest="origin"),
ae901090 975 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
976 # preserve the user, requires relevant p4 permissions
977 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 978 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 979 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 980 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 981 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
982 choices=self.conflict_behavior_choices),
983 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
984 ]
985 self.description = "Submit changes from git to the perforce depot."
c9b50e63 986 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 987 self.origin = ""
ae901090 988 self.detectRenames = False
0d609032 989 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 990 self.dry_run = False
728b7ad8 991 self.prepare_p4_only = False
6bbfd137 992 self.conflict_behavior = None
f7baba8b 993 self.isWindows = (platform.system() == "Windows")
06804c76 994 self.exportLabels = False
249da4c0 995 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 996 self.branch = None
4f5cf76a 997
4f5cf76a
SH
998 def check(self):
999 if len(p4CmdList("opened ...")) > 0:
1000 die("You have files opened with perforce! Close them before starting the sync.")
1001
f19cb0a0
PW
1002 def separate_jobs_from_description(self, message):
1003 """Extract and return a possible Jobs field in the commit
1004 message. It goes into a separate section in the p4 change
1005 specification.
1006
1007 A jobs line starts with "Jobs:" and looks like a new field
1008 in a form. Values are white-space separated on the same
1009 line or on following lines that start with a tab.
1010
1011 This does not parse and extract the full git commit message
1012 like a p4 form. It just sees the Jobs: line as a marker
1013 to pass everything from then on directly into the p4 form,
1014 but outside the description section.
1015
1016 Return a tuple (stripped log message, jobs string)."""
1017
1018 m = re.search(r'^Jobs:', message, re.MULTILINE)
1019 if m is None:
1020 return (message, None)
1021
1022 jobtext = message[m.start():]
1023 stripped_message = message[:m.start()].rstrip()
1024 return (stripped_message, jobtext)
1025
1026 def prepareLogMessage(self, template, message, jobs):
1027 """Edits the template returned from "p4 change -o" to insert
1028 the message in the Description field, and the jobs text in
1029 the Jobs field."""
4f5cf76a
SH
1030 result = ""
1031
edae1e2f
SH
1032 inDescriptionSection = False
1033
4f5cf76a
SH
1034 for line in template.split("\n"):
1035 if line.startswith("#"):
1036 result += line + "\n"
1037 continue
1038
edae1e2f 1039 if inDescriptionSection:
c9dbab04 1040 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1041 inDescriptionSection = False
f19cb0a0
PW
1042 # insert Jobs section
1043 if jobs:
1044 result += jobs + "\n"
edae1e2f
SH
1045 else:
1046 continue
1047 else:
1048 if line.startswith("Description:"):
1049 inDescriptionSection = True
1050 line += "\n"
1051 for messageLine in message.split("\n"):
1052 line += "\t" + messageLine + "\n"
1053
1054 result += line + "\n"
4f5cf76a
SH
1055
1056 return result
1057
60df071c
LD
1058 def patchRCSKeywords(self, file, pattern):
1059 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1060 (handle, outFileName) = tempfile.mkstemp(dir='.')
1061 try:
1062 outFile = os.fdopen(handle, "w+")
1063 inFile = open(file, "r")
1064 regexp = re.compile(pattern, re.VERBOSE)
1065 for line in inFile.readlines():
1066 line = regexp.sub(r'$\1$', line)
1067 outFile.write(line)
1068 inFile.close()
1069 outFile.close()
1070 # Forcibly overwrite the original file
1071 os.unlink(file)
1072 shutil.move(outFileName, file)
1073 except:
1074 # cleanup our temporary file
1075 os.unlink(outFileName)
1076 print "Failed to strip RCS keywords in %s" % file
1077 raise
1078
1079 print "Patched up RCS keywords in %s" % file
1080
3ea2cfd4
LD
1081 def p4UserForCommit(self,id):
1082 # Return the tuple (perforce user,git email) for a given git commit id
1083 self.getUserMapFromPerforceServer()
9bf28855
PW
1084 gitEmail = read_pipe(["git", "log", "--max-count=1",
1085 "--format=%ae", id])
3ea2cfd4
LD
1086 gitEmail = gitEmail.strip()
1087 if not self.emails.has_key(gitEmail):
1088 return (None,gitEmail)
1089 else:
1090 return (self.emails[gitEmail],gitEmail)
1091
1092 def checkValidP4Users(self,commits):
1093 # check if any git authors cannot be mapped to p4 users
1094 for id in commits:
1095 (user,email) = self.p4UserForCommit(id)
1096 if not user:
1097 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1098 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1099 print "%s" % msg
1100 else:
1101 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1102
1103 def lastP4Changelist(self):
1104 # Get back the last changelist number submitted in this client spec. This
1105 # then gets used to patch up the username in the change. If the same
1106 # client spec is being used by multiple processes then this might go
1107 # wrong.
1108 results = p4CmdList("client -o") # find the current client
1109 client = None
1110 for r in results:
1111 if r.has_key('Client'):
1112 client = r['Client']
1113 break
1114 if not client:
1115 die("could not get client spec")
6de040df 1116 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1117 for r in results:
1118 if r.has_key('change'):
1119 return r['change']
1120 die("Could not get changelist number for last submit - cannot patch up user details")
1121
1122 def modifyChangelistUser(self, changelist, newUser):
1123 # fixup the user field of a changelist after it has been submitted.
1124 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1125 if len(changes) != 1:
1126 die("Bad output from p4 change modifying %s to user %s" %
1127 (changelist, newUser))
1128
1129 c = changes[0]
1130 if c['User'] == newUser: return # nothing to do
1131 c['User'] = newUser
1132 input = marshal.dumps(c)
1133
3ea2cfd4
LD
1134 result = p4CmdList("change -f -i", stdin=input)
1135 for r in result:
1136 if r.has_key('code'):
1137 if r['code'] == 'error':
1138 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1139 if r.has_key('data'):
1140 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1141 return
1142 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1143
1144 def canChangeChangelists(self):
1145 # check to see if we have p4 admin or super-user permissions, either of
1146 # which are required to modify changelists.
52a4880b 1147 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1148 for r in results:
1149 if r.has_key('perm'):
1150 if r['perm'] == 'admin':
1151 return 1
1152 if r['perm'] == 'super':
1153 return 1
1154 return 0
1155
ea99c3ae 1156 def prepareSubmitTemplate(self):
f19cb0a0
PW
1157 """Run "p4 change -o" to grab a change specification template.
1158 This does not use "p4 -G", as it is nice to keep the submission
1159 template in original order, since a human might edit it.
1160
1161 Remove lines in the Files section that show changes to files
1162 outside the depot path we're committing into."""
1163
ea99c3ae
SH
1164 template = ""
1165 inFilesSection = False
6de040df 1166 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1167 if line.endswith("\r\n"):
1168 line = line[:-2] + "\n"
ea99c3ae
SH
1169 if inFilesSection:
1170 if line.startswith("\t"):
1171 # path starts and ends with a tab
1172 path = line[1:]
1173 lastTab = path.rfind("\t")
1174 if lastTab != -1:
1175 path = path[:lastTab]
d53de8b9 1176 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
1177 continue
1178 else:
1179 inFilesSection = False
1180 else:
1181 if line.startswith("Files:"):
1182 inFilesSection = True
1183
1184 template += line
1185
1186 return template
1187
7c766e57
PW
1188 def edit_template(self, template_file):
1189 """Invoke the editor to let the user change the submission
1190 message. Return true if okay to continue with the submit."""
1191
1192 # if configured to skip the editing part, just submit
0d609032 1193 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1194 return True
1195
1196 # look at the modification time, to check later if the user saved
1197 # the file
1198 mtime = os.stat(template_file).st_mtime
1199
1200 # invoke the editor
f95ceaf0 1201 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1202 editor = os.environ.get("P4EDITOR")
1203 else:
1204 editor = read_pipe("git var GIT_EDITOR").strip()
1205 system(editor + " " + template_file)
1206
1207 # If the file was not saved, prompt to see if this patch should
1208 # be skipped. But skip this verification step if configured so.
0d609032 1209 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1210 return True
1211
d1652049
PW
1212 # modification time updated means user saved the file
1213 if os.stat(template_file).st_mtime > mtime:
1214 return True
1215
1216 while True:
1217 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1218 if response == 'y':
1219 return True
1220 if response == 'n':
1221 return False
7c766e57 1222
7cb5cbef 1223 def applyCommit(self, id):
67b0fe2e
PW
1224 """Apply one commit, return True if it succeeded."""
1225
1226 print "Applying", read_pipe(["git", "show", "-s",
1227 "--format=format:%h %s", id])
ae901090 1228
848de9c3 1229 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1230
84cb0003 1231 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1232 filesToAdd = set()
1233 filesToDelete = set()
d336c158 1234 editedFiles = set()
b6ad6dcc 1235 pureRenameCopy = set()
c65b670e 1236 filesToChangeExecBit = {}
60df071c 1237
4f5cf76a 1238 for line in diff:
b43b0a3c
CP
1239 diff = parseDiffTreeEntry(line)
1240 modifier = diff['status']
1241 path = diff['src']
4f5cf76a 1242 if modifier == "M":
6de040df 1243 p4_edit(path)
c65b670e
CP
1244 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1245 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1246 editedFiles.add(path)
4f5cf76a
SH
1247 elif modifier == "A":
1248 filesToAdd.add(path)
c65b670e 1249 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1250 if path in filesToDelete:
1251 filesToDelete.remove(path)
1252 elif modifier == "D":
1253 filesToDelete.add(path)
1254 if path in filesToAdd:
1255 filesToAdd.remove(path)
4fddb41b
VA
1256 elif modifier == "C":
1257 src, dest = diff['src'], diff['dst']
6de040df 1258 p4_integrate(src, dest)
b6ad6dcc 1259 pureRenameCopy.add(dest)
4fddb41b 1260 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1261 p4_edit(dest)
b6ad6dcc 1262 pureRenameCopy.discard(dest)
4fddb41b 1263 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1264 p4_edit(dest)
b6ad6dcc 1265 pureRenameCopy.discard(dest)
4fddb41b 1266 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1267 if self.isWindows:
1268 # turn off read-only attribute
1269 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1270 os.unlink(dest)
1271 editedFiles.add(dest)
d9a5f25b 1272 elif modifier == "R":
b43b0a3c 1273 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1274 if self.p4HasMoveCommand:
1275 p4_edit(src) # src must be open before move
1276 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1277 else:
8e9497c2
GG
1278 p4_integrate(src, dest)
1279 if diff['src_sha1'] != diff['dst_sha1']:
1280 p4_edit(dest)
1281 else:
1282 pureRenameCopy.add(dest)
c65b670e 1283 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1284 if not self.p4HasMoveCommand:
1285 p4_edit(dest) # with move: already open, writable
c65b670e 1286 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1287 if not self.p4HasMoveCommand:
d20f0f8e
PW
1288 if self.isWindows:
1289 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1290 os.unlink(dest)
1291 filesToDelete.add(src)
d9a5f25b 1292 editedFiles.add(dest)
4f5cf76a
SH
1293 else:
1294 die("unknown modifier %s for %s" % (modifier, path))
1295
0e36f2d7 1296 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 1297 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1298 tryPatchCmd = patchcmd + "--check -"
1299 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1300 patch_succeeded = True
51a2640a 1301
47a130b7 1302 if os.system(tryPatchCmd) != 0:
60df071c
LD
1303 fixed_rcs_keywords = False
1304 patch_succeeded = False
51a2640a 1305 print "Unfortunately applying the change failed!"
60df071c
LD
1306
1307 # Patch failed, maybe it's just RCS keyword woes. Look through
1308 # the patch to see if that's possible.
0d609032 1309 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1310 file = None
1311 pattern = None
1312 kwfiles = {}
1313 for file in editedFiles | filesToDelete:
1314 # did this file's delta contain RCS keywords?
1315 pattern = p4_keywords_regexp_for_file(file)
1316
1317 if pattern:
1318 # this file is a possibility...look for RCS keywords.
1319 regexp = re.compile(pattern, re.VERBOSE)
1320 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1321 if regexp.search(line):
1322 if verbose:
1323 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1324 kwfiles[file] = pattern
1325 break
1326
1327 for file in kwfiles:
1328 if verbose:
1329 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1330 # File is being deleted, so not open in p4. Must
1331 # disable the read-only bit on windows.
1332 if self.isWindows and file not in editedFiles:
1333 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1334 self.patchRCSKeywords(file, kwfiles[file])
1335 fixed_rcs_keywords = True
1336
1337 if fixed_rcs_keywords:
1338 print "Retrying the patch with RCS keywords cleaned up"
1339 if os.system(tryPatchCmd) == 0:
1340 patch_succeeded = True
1341
1342 if not patch_succeeded:
7e5dd9f2
PW
1343 for f in editedFiles:
1344 p4_revert(f)
7e5dd9f2 1345 return False
51a2640a 1346
55ac2ed6
PW
1347 #
1348 # Apply the patch for real, and do add/delete/+x handling.
1349 #
47a130b7 1350 system(applyPatchCmd)
4f5cf76a
SH
1351
1352 for f in filesToAdd:
6de040df 1353 p4_add(f)
4f5cf76a 1354 for f in filesToDelete:
6de040df
LD
1355 p4_revert(f)
1356 p4_delete(f)
4f5cf76a 1357
c65b670e
CP
1358 # Set/clear executable bits
1359 for f in filesToChangeExecBit.keys():
1360 mode = filesToChangeExecBit[f]
1361 setP4ExecBit(f, mode)
1362
55ac2ed6
PW
1363 #
1364 # Build p4 change description, starting with the contents
1365 # of the git commit message.
1366 #
0e36f2d7 1367 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1368 logMessage = logMessage.strip()
f19cb0a0 1369 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1370
ea99c3ae 1371 template = self.prepareSubmitTemplate()
f19cb0a0 1372 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1373
c47178d4 1374 if self.preserveUser:
55ac2ed6 1375 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1376
55ac2ed6
PW
1377 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1378 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1379 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1380 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1381
55ac2ed6
PW
1382 separatorLine = "######## everything below this line is just the diff #######\n"
1383
1384 # diff
c47178d4
PW
1385 if os.environ.has_key("P4DIFF"):
1386 del(os.environ["P4DIFF"])
1387 diff = ""
1388 for editedFile in editedFiles:
1389 diff += p4_read_pipe(['diff', '-du',
1390 wildcard_encode(editedFile)])
1391
55ac2ed6 1392 # new file diff
c47178d4
PW
1393 newdiff = ""
1394 for newFile in filesToAdd:
1395 newdiff += "==== new file ====\n"
1396 newdiff += "--- /dev/null\n"
1397 newdiff += "+++ %s\n" % newFile
1398 f = open(newFile, "r")
1399 for line in f.readlines():
1400 newdiff += "+" + line
1401 f.close()
1402
55ac2ed6 1403 # change description file: submitTemplate, separatorLine, diff, newdiff
c47178d4
PW
1404 (handle, fileName) = tempfile.mkstemp()
1405 tmpFile = os.fdopen(handle, "w+")
1406 if self.isWindows:
1407 submitTemplate = submitTemplate.replace("\n", "\r\n")
1408 separatorLine = separatorLine.replace("\n", "\r\n")
1409 newdiff = newdiff.replace("\n", "\r\n")
1410 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1411 tmpFile.close()
1412
728b7ad8
PW
1413 if self.prepare_p4_only:
1414 #
1415 # Leave the p4 tree prepared, and the submit template around
1416 # and let the user decide what to do next
1417 #
1418 print
1419 print "P4 workspace prepared for submission."
1420 print "To submit or revert, go to client workspace"
1421 print " " + self.clientPath
1422 print
1423 print "To submit, use \"p4 submit\" to write a new description,"
1424 print "or \"p4 submit -i %s\" to use the one prepared by" \
1425 " \"git p4\"." % fileName
1426 print "You can delete the file \"%s\" when finished." % fileName
1427
1428 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1429 print "To preserve change ownership by user %s, you must\n" \
1430 "do \"p4 change -f <change>\" after submitting and\n" \
1431 "edit the User field."
1432 if pureRenameCopy:
1433 print "After submitting, renamed files must be re-synced."
1434 print "Invoke \"p4 sync -f\" on each of these files:"
1435 for f in pureRenameCopy:
1436 print " " + f
1437
1438 print
1439 print "To revert the changes, use \"p4 revert ...\", and delete"
1440 print "the submit template file \"%s\"" % fileName
1441 if filesToAdd:
1442 print "Since the commit adds new files, they must be deleted:"
1443 for f in filesToAdd:
1444 print " " + f
1445 print
1446 return True
1447
55ac2ed6
PW
1448 #
1449 # Let the user edit the change description, then submit it.
1450 #
c47178d4
PW
1451 if self.edit_template(fileName):
1452 # read the edited message and submit
5a41c16a 1453 ret = True
c47178d4
PW
1454 tmpFile = open(fileName, "rb")
1455 message = tmpFile.read()
e96e400f 1456 tmpFile.close()
c47178d4
PW
1457 submitTemplate = message[:message.index(separatorLine)]
1458 if self.isWindows:
1459 submitTemplate = submitTemplate.replace("\r\n", "\n")
1460 p4_write_pipe(['submit', '-i'], submitTemplate)
e96e400f 1461
c47178d4
PW
1462 if self.preserveUser:
1463 if p4User:
1464 # Get last changelist number. Cannot easily get it from
1465 # the submit command output as the output is
1466 # unmarshalled.
1467 changelist = self.lastP4Changelist()
1468 self.modifyChangelistUser(changelist, p4User)
1469
1470 # The rename/copy happened by applying a patch that created a
1471 # new file. This leaves it writable, which confuses p4.
1472 for f in pureRenameCopy:
1473 p4_sync(f, "-f")
cdc7e388 1474
4f5cf76a 1475 else:
c47178d4 1476 # skip this patch
5a41c16a 1477 ret = False
c47178d4
PW
1478 print "Submission cancelled, undoing p4 changes."
1479 for f in editedFiles:
1480 p4_revert(f)
1481 for f in filesToAdd:
1482 p4_revert(f)
1483 os.remove(f)
df9c5453
PW
1484 for f in filesToDelete:
1485 p4_revert(f)
c47178d4
PW
1486
1487 os.remove(fileName)
5a41c16a 1488 return ret
4f5cf76a 1489
06804c76
LD
1490 # Export git tags as p4 labels. Create a p4 label and then tag
1491 # with that.
1492 def exportGitTags(self, gitTags):
c8942a22
LD
1493 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1494 if len(validLabelRegexp) == 0:
1495 validLabelRegexp = defaultLabelRegexp
1496 m = re.compile(validLabelRegexp)
06804c76
LD
1497
1498 for name in gitTags:
1499
1500 if not m.match(name):
1501 if verbose:
05a3cec5 1502 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1503 continue
1504
1505 # Get the p4 commit this corresponds to
c8942a22
LD
1506 logMessage = extractLogMessageFromGitCommit(name)
1507 values = extractSettingsGitLog(logMessage)
06804c76 1508
c8942a22 1509 if not values.has_key('change'):
06804c76
LD
1510 # a tag pointing to something not sent to p4; ignore
1511 if verbose:
1512 print "git tag %s does not give a p4 commit" % name
1513 continue
c8942a22
LD
1514 else:
1515 changelist = values['change']
06804c76
LD
1516
1517 # Get the tag details.
1518 inHeader = True
1519 isAnnotated = False
1520 body = []
1521 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1522 l = l.strip()
1523 if inHeader:
1524 if re.match(r'tag\s+', l):
1525 isAnnotated = True
1526 elif re.match(r'\s*$', l):
1527 inHeader = False
1528 continue
1529 else:
1530 body.append(l)
1531
1532 if not isAnnotated:
1533 body = ["lightweight tag imported by git p4\n"]
1534
1535 # Create the label - use the same view as the client spec we are using
1536 clientSpec = getClientSpec()
1537
1538 labelTemplate = "Label: %s\n" % name
1539 labelTemplate += "Description:\n"
1540 for b in body:
1541 labelTemplate += "\t" + b + "\n"
1542 labelTemplate += "View:\n"
1543 for mapping in clientSpec.mappings:
1544 labelTemplate += "\t%s\n" % mapping.depot_side.path
1545
ef739f08
PW
1546 if self.dry_run:
1547 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1548 elif self.prepare_p4_only:
1549 print "Not creating p4 label %s for tag due to option" \
1550 " --prepare-p4-only" % name
ef739f08
PW
1551 else:
1552 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1553
ef739f08
PW
1554 # Use the label
1555 p4_system(["tag", "-l", name] +
1556 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
06804c76 1557
ef739f08
PW
1558 if verbose:
1559 print "created p4 label for tag %s" % name
06804c76 1560
4f5cf76a 1561 def run(self, args):
c9b50e63
SH
1562 if len(args) == 0:
1563 self.master = currentGitBranch()
4280e533 1564 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1565 die("Detecting current git branch failed!")
1566 elif len(args) == 1:
1567 self.master = args[0]
28755dba
PW
1568 if not branchExists(self.master):
1569 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1570 else:
1571 return False
1572
4c2d5d72
JX
1573 allowSubmit = gitConfig("git-p4.allowSubmit")
1574 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1575 die("%s is not in git-p4.allowSubmit" % self.master)
1576
27d2d811 1577 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1578 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1579 if len(self.origin) == 0:
1580 self.origin = upstream
a3fdd579 1581
3ea2cfd4
LD
1582 if self.preserveUser:
1583 if not self.canChangeChangelists():
1584 die("Cannot preserve user names without p4 super-user or admin permissions")
1585
6bbfd137
PW
1586 # if not set from the command line, try the config file
1587 if self.conflict_behavior is None:
1588 val = gitConfig("git-p4.conflict")
1589 if val:
1590 if val not in self.conflict_behavior_choices:
1591 die("Invalid value '%s' for config git-p4.conflict" % val)
1592 else:
1593 val = "ask"
1594 self.conflict_behavior = val
1595
a3fdd579
SH
1596 if self.verbose:
1597 print "Origin branch is " + self.origin
9512497b 1598
ea99c3ae 1599 if len(self.depotPath) == 0:
9512497b
SH
1600 print "Internal error: cannot locate perforce depot path from existing branches"
1601 sys.exit(128)
1602
543987bd 1603 self.useClientSpec = False
0d609032 1604 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1605 self.useClientSpec = True
1606 if self.useClientSpec:
1607 self.clientSpecDirs = getClientSpec()
9512497b 1608
543987bd
PW
1609 if self.useClientSpec:
1610 # all files are relative to the client spec
1611 self.clientPath = getClientRoot()
1612 else:
1613 self.clientPath = p4Where(self.depotPath)
9512497b 1614
543987bd
PW
1615 if self.clientPath == "":
1616 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1617
ea99c3ae 1618 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1619 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1620
0591cfa8 1621 # ensure the clientPath exists
8d7ec362 1622 new_client_dir = False
0591cfa8 1623 if not os.path.exists(self.clientPath):
8d7ec362 1624 new_client_dir = True
0591cfa8
GG
1625 os.makedirs(self.clientPath)
1626
053fd0c1 1627 chdir(self.clientPath)
ef739f08
PW
1628 if self.dry_run:
1629 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1630 else:
ef739f08
PW
1631 print "Synchronizing p4 checkout..."
1632 if new_client_dir:
1633 # old one was destroyed, and maybe nobody told p4
1634 p4_sync("...", "-f")
1635 else:
1636 p4_sync("...")
4f5cf76a 1637 self.check()
4f5cf76a 1638
4c750c0d 1639 commits = []
c7d34884 1640 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
4c750c0d
SH
1641 commits.append(line.strip())
1642 commits.reverse()
4f5cf76a 1643
0d609032 1644 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1645 self.checkAuthorship = False
1646 else:
1647 self.checkAuthorship = True
1648
3ea2cfd4
LD
1649 if self.preserveUser:
1650 self.checkValidP4Users(commits)
1651
84cb0003
GG
1652 #
1653 # Build up a set of options to be passed to diff when
1654 # submitting each commit to p4.
1655 #
1656 if self.detectRenames:
1657 # command-line -M arg
1658 self.diffOpts = "-M"
1659 else:
1660 # If not explicitly set check the config variable
1661 detectRenames = gitConfig("git-p4.detectRenames")
1662
1663 if detectRenames.lower() == "false" or detectRenames == "":
1664 self.diffOpts = ""
1665 elif detectRenames.lower() == "true":
1666 self.diffOpts = "-M"
1667 else:
1668 self.diffOpts = "-M%s" % detectRenames
1669
1670 # no command-line arg for -C or --find-copies-harder, just
1671 # config variables
1672 detectCopies = gitConfig("git-p4.detectCopies")
1673 if detectCopies.lower() == "false" or detectCopies == "":
1674 pass
1675 elif detectCopies.lower() == "true":
1676 self.diffOpts += " -C"
1677 else:
1678 self.diffOpts += " -C%s" % detectCopies
1679
0d609032 1680 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1681 self.diffOpts += " --find-copies-harder"
1682
7e5dd9f2
PW
1683 #
1684 # Apply the commits, one at a time. On failure, ask if should
1685 # continue to try the rest of the patches, or quit.
1686 #
ef739f08
PW
1687 if self.dry_run:
1688 print "Would apply"
67b0fe2e 1689 applied = []
7e5dd9f2
PW
1690 last = len(commits) - 1
1691 for i, commit in enumerate(commits):
ef739f08
PW
1692 if self.dry_run:
1693 print " ", read_pipe(["git", "show", "-s",
1694 "--format=format:%h %s", commit])
1695 ok = True
1696 else:
1697 ok = self.applyCommit(commit)
67b0fe2e
PW
1698 if ok:
1699 applied.append(commit)
7e5dd9f2 1700 else:
728b7ad8
PW
1701 if self.prepare_p4_only and i < last:
1702 print "Processing only the first commit due to option" \
1703 " --prepare-p4-only"
1704 break
7e5dd9f2
PW
1705 if i < last:
1706 quit = False
1707 while True:
6bbfd137
PW
1708 # prompt for what to do, or use the option/variable
1709 if self.conflict_behavior == "ask":
1710 print "What do you want to do?"
1711 response = raw_input("[s]kip this commit but apply"
1712 " the rest, or [q]uit? ")
1713 if not response:
1714 continue
1715 elif self.conflict_behavior == "skip":
1716 response = "s"
1717 elif self.conflict_behavior == "quit":
1718 response = "q"
1719 else:
1720 die("Unknown conflict_behavior '%s'" %
1721 self.conflict_behavior)
1722
7e5dd9f2
PW
1723 if response[0] == "s":
1724 print "Skipping this commit, but applying the rest"
1725 break
1726 if response[0] == "q":
1727 print "Quitting"
1728 quit = True
1729 break
1730 if quit:
1731 break
4f5cf76a 1732
67b0fe2e 1733 chdir(self.oldWorkingDirectory)
4f5cf76a 1734
ef739f08
PW
1735 if self.dry_run:
1736 pass
728b7ad8
PW
1737 elif self.prepare_p4_only:
1738 pass
ef739f08 1739 elif len(commits) == len(applied):
67b0fe2e 1740 print "All commits applied!"
14594f4b 1741
4c750c0d 1742 sync = P4Sync()
44e8d26c
PW
1743 if self.branch:
1744 sync.branch = self.branch
4c750c0d 1745 sync.run([])
14594f4b 1746
4c750c0d
SH
1747 rebase = P4Rebase()
1748 rebase.rebase()
4f5cf76a 1749
67b0fe2e
PW
1750 else:
1751 if len(applied) == 0:
1752 print "No commits applied."
1753 else:
1754 print "Applied only the commits marked with '*':"
1755 for c in commits:
1756 if c in applied:
1757 star = "*"
1758 else:
1759 star = " "
1760 print star, read_pipe(["git", "show", "-s",
1761 "--format=format:%h %s", c])
1762 print "You will have to do 'git p4 sync' and rebase."
1763
0d609032 1764 if gitConfigBool("git-p4.exportLabels"):
06dcd152 1765 self.exportLabels = True
06804c76
LD
1766
1767 if self.exportLabels:
1768 p4Labels = getP4Labels(self.depotPath)
1769 gitTags = getGitTags()
1770
1771 missingGitTags = gitTags - p4Labels
1772 self.exportGitTags(missingGitTags)
1773
67b0fe2e
PW
1774 # exit with error unless everything applied perfecly
1775 if len(commits) != len(applied):
1776 sys.exit(1)
1777
b984733c
SH
1778 return True
1779
ecb7cf98
PW
1780class View(object):
1781 """Represent a p4 view ("p4 help views"), and map files in a
1782 repo according to the view."""
1783
1784 class Path(object):
1785 """A depot or client path, possibly containing wildcards.
1786 The only one supported is ... at the end, currently.
1787 Initialize with the full path, with //depot or //client."""
1788
1789 def __init__(self, path, is_depot):
1790 self.path = path
1791 self.is_depot = is_depot
1792 self.find_wildcards()
1793 # remember the prefix bit, useful for relative mappings
1794 m = re.match("(//[^/]+/)", self.path)
1795 if not m:
1796 die("Path %s does not start with //prefix/" % self.path)
1797 prefix = m.group(1)
1798 if not self.is_depot:
1799 # strip //client/ on client paths
1800 self.path = self.path[len(prefix):]
1801
1802 def find_wildcards(self):
1803 """Make sure wildcards are valid, and set up internal
1804 variables."""
1805
1806 self.ends_triple_dot = False
1807 # There are three wildcards allowed in p4 views
1808 # (see "p4 help views"). This code knows how to
1809 # handle "..." (only at the end), but cannot deal with
1810 # "%%n" or "*". Only check the depot_side, as p4 should
1811 # validate that the client_side matches too.
1812 if re.search(r'%%[1-9]', self.path):
1813 die("Can't handle %%n wildcards in view: %s" % self.path)
1814 if self.path.find("*") >= 0:
1815 die("Can't handle * wildcards in view: %s" % self.path)
1816 triple_dot_index = self.path.find("...")
1817 if triple_dot_index >= 0:
43b82bd9
PW
1818 if triple_dot_index != len(self.path) - 3:
1819 die("Can handle only single ... wildcard, at end: %s" %
ecb7cf98
PW
1820 self.path)
1821 self.ends_triple_dot = True
1822
1823 def ensure_compatible(self, other_path):
1824 """Make sure the wildcards agree."""
1825 if self.ends_triple_dot != other_path.ends_triple_dot:
1826 die("Both paths must end with ... if either does;\n" +
1827 "paths: %s %s" % (self.path, other_path.path))
1828
1829 def match_wildcards(self, test_path):
1830 """See if this test_path matches us, and fill in the value
1831 of the wildcards if so. Returns a tuple of
1832 (True|False, wildcards[]). For now, only the ... at end
1833 is supported, so at most one wildcard."""
1834 if self.ends_triple_dot:
1835 dotless = self.path[:-3]
1836 if test_path.startswith(dotless):
1837 wildcard = test_path[len(dotless):]
1838 return (True, [ wildcard ])
1839 else:
1840 if test_path == self.path:
1841 return (True, [])
1842 return (False, [])
1843
1844 def match(self, test_path):
1845 """Just return if it matches; don't bother with the wildcards."""
1846 b, _ = self.match_wildcards(test_path)
1847 return b
1848
1849 def fill_in_wildcards(self, wildcards):
1850 """Return the relative path, with the wildcards filled in
1851 if there are any."""
1852 if self.ends_triple_dot:
1853 return self.path[:-3] + wildcards[0]
1854 else:
1855 return self.path
1856
1857 class Mapping(object):
1858 def __init__(self, depot_side, client_side, overlay, exclude):
1859 # depot_side is without the trailing /... if it had one
1860 self.depot_side = View.Path(depot_side, is_depot=True)
1861 self.client_side = View.Path(client_side, is_depot=False)
1862 self.overlay = overlay # started with "+"
1863 self.exclude = exclude # started with "-"
1864 assert not (self.overlay and self.exclude)
1865 self.depot_side.ensure_compatible(self.client_side)
1866
1867 def __str__(self):
1868 c = " "
1869 if self.overlay:
1870 c = "+"
1871 if self.exclude:
1872 c = "-"
1873 return "View.Mapping: %s%s -> %s" % \
329afb8e 1874 (c, self.depot_side.path, self.client_side.path)
ecb7cf98
PW
1875
1876 def map_depot_to_client(self, depot_path):
1877 """Calculate the client path if using this mapping on the
1878 given depot path; does not consider the effect of other
1879 mappings in a view. Even excluded mappings are returned."""
1880 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1881 if not matches:
1882 return ""
1883 client_path = self.client_side.fill_in_wildcards(wildcards)
1884 return client_path
1885
1886 #
1887 # View methods
1888 #
1889 def __init__(self):
1890 self.mappings = []
1891
1892 def append(self, view_line):
1893 """Parse a view line, splitting it into depot and client
1894 sides. Append to self.mappings, preserving order."""
1895
1896 # Split the view line into exactly two words. P4 enforces
1897 # structure on these lines that simplifies this quite a bit.
1898 #
1899 # Either or both words may be double-quoted.
1900 # Single quotes do not matter.
1901 # Double-quote marks cannot occur inside the words.
1902 # A + or - prefix is also inside the quotes.
1903 # There are no quotes unless they contain a space.
1904 # The line is already white-space stripped.
1905 # The two words are separated by a single space.
1906 #
1907 if view_line[0] == '"':
1908 # First word is double quoted. Find its end.
1909 close_quote_index = view_line.find('"', 1)
1910 if close_quote_index <= 0:
1911 die("No first-word closing quote found: %s" % view_line)
1912 depot_side = view_line[1:close_quote_index]
1913 # skip closing quote and space
1914 rhs_index = close_quote_index + 1 + 1
1915 else:
1916 space_index = view_line.find(" ")
1917 if space_index <= 0:
1918 die("No word-splitting space found: %s" % view_line)
1919 depot_side = view_line[0:space_index]
1920 rhs_index = space_index + 1
1921
1922 if view_line[rhs_index] == '"':
1923 # Second word is double quoted. Make sure there is a
1924 # double quote at the end too.
1925 if not view_line.endswith('"'):
1926 die("View line with rhs quote should end with one: %s" %
1927 view_line)
1928 # skip the quotes
1929 client_side = view_line[rhs_index+1:-1]
1930 else:
1931 client_side = view_line[rhs_index:]
1932
1933 # prefix + means overlay on previous mapping
1934 overlay = False
1935 if depot_side.startswith("+"):
1936 overlay = True
1937 depot_side = depot_side[1:]
1938
1939 # prefix - means exclude this path
1940 exclude = False
1941 if depot_side.startswith("-"):
1942 exclude = True
1943 depot_side = depot_side[1:]
1944
1945 m = View.Mapping(depot_side, client_side, overlay, exclude)
1946 self.mappings.append(m)
1947
1948 def map_in_client(self, depot_path):
1949 """Return the relative location in the client where this
1950 depot file should live. Returns "" if the file should
1951 not be mapped in the client."""
1952
1953 paths_filled = []
1954 client_path = ""
1955
1956 # look at later entries first
1957 for m in self.mappings[::-1]:
1958
1959 # see where will this path end up in the client
1960 p = m.map_depot_to_client(depot_path)
1961
1962 if p == "":
1963 # Depot path does not belong in client. Must remember
1964 # this, as previous items should not cause files to
1965 # exist in this path either. Remember that the list is
1966 # being walked from the end, which has higher precedence.
1967 # Overlap mappings do not exclude previous mappings.
1968 if not m.overlay:
1969 paths_filled.append(m.client_side)
1970
1971 else:
1972 # This mapping matched; no need to search any further.
1973 # But, the mapping could be rejected if the client path
6ee9a999
PW
1974 # has already been claimed by an earlier mapping (i.e.
1975 # one later in the list, which we are walking backwards).
ecb7cf98
PW
1976 already_mapped_in_client = False
1977 for f in paths_filled:
1978 # this is View.Path.match
1979 if f.match(p):
1980 already_mapped_in_client = True
1981 break
1982 if not already_mapped_in_client:
1983 # Include this file, unless it is from a line that
1984 # explicitly said to exclude it.
1985 if not m.exclude:
1986 client_path = p
1987
1988 # a match, even if rejected, always stops the search
1989 break
1990
1991 return client_path
1992
3ea2cfd4 1993class P4Sync(Command, P4UserMap):
56c09345
PW
1994 delete_actions = ( "delete", "move/delete", "purge" )
1995
b984733c
SH
1996 def __init__(self):
1997 Command.__init__(self)
3ea2cfd4 1998 P4UserMap.__init__(self)
b984733c
SH
1999 self.options = [
2000 optparse.make_option("--branch", dest="branch"),
2001 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2002 optparse.make_option("--changesfile", dest="changesFile"),
2003 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2004 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2005 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2006 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2007 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 2008 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 2009 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2010 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2011 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2012 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
2013 ]
2014 self.description = """Imports from Perforce into a git repository.\n
2015 example:
2016 //depot/my/project/ -- to import the current head
2017 //depot/my/project/@all -- to import everything
2018 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2019
2020 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2021
2022 self.usage += " //depot/path[@revRange]"
b984733c 2023 self.silent = False
1d7367dc
RG
2024 self.createdBranches = set()
2025 self.committedChanges = set()
569d1bd4 2026 self.branch = ""
b984733c 2027 self.detectBranches = False
cb53e1f8 2028 self.detectLabels = False
06804c76 2029 self.importLabels = False
b984733c 2030 self.changesFile = ""
01265103 2031 self.syncWithOrigin = True
a028a98e 2032 self.importIntoRemotes = True
01a9c9c5 2033 self.maxChanges = ""
8b41a97f 2034 self.keepRepoPath = False
6326aa58 2035 self.depotPaths = None
3c699645 2036 self.p4BranchesInGit = []
354081d5 2037 self.cloneExclude = []
3a70cdfa 2038 self.useClientSpec = False
a93d33ee 2039 self.useClientSpec_from_options = False
ecb7cf98 2040 self.clientSpecDirs = None
fed23693
VA
2041 self.tempBranches = []
2042 self.tempBranchLocation = "git-p4-tmp"
b984733c 2043
01265103
SH
2044 if gitConfig("git-p4.syncFromOrigin") == "false":
2045 self.syncWithOrigin = False
2046
fed23693
VA
2047 # Force a checkpoint in fast-import and wait for it to finish
2048 def checkpoint(self):
2049 self.gitStream.write("checkpoint\n\n")
2050 self.gitStream.write("progress checkpoint\n\n")
2051 out = self.gitOutput.readline()
2052 if self.verbose:
2053 print "checkpoint finished: " + out
2054
b984733c 2055 def extractFilesFromCommit(self, commit):
354081d5
TT
2056 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2057 for path in self.cloneExclude]
b984733c
SH
2058 files = []
2059 fnum = 0
2060 while commit.has_key("depotFile%s" % fnum):
2061 path = commit["depotFile%s" % fnum]
6326aa58 2062
354081d5 2063 if [p for p in self.cloneExclude
d53de8b9 2064 if p4PathStartsWith(path, p)]:
354081d5
TT
2065 found = False
2066 else:
2067 found = [p for p in self.depotPaths
d53de8b9 2068 if p4PathStartsWith(path, p)]
6326aa58 2069 if not found:
b984733c
SH
2070 fnum = fnum + 1
2071 continue
2072
2073 file = {}
2074 file["path"] = path
2075 file["rev"] = commit["rev%s" % fnum]
2076 file["action"] = commit["action%s" % fnum]
2077 file["type"] = commit["type%s" % fnum]
2078 files.append(file)
2079 fnum = fnum + 1
2080 return files
2081
6326aa58 2082 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2083 """When streaming files, this is called to map a p4 depot path
2084 to where it should go in git. The prefixes are either
2085 self.depotPaths, or self.branchPrefixes in the case of
2086 branch detection."""
2087
3952710b 2088 if self.useClientSpec:
21ef5df4
PW
2089 # branch detection moves files up a level (the branch name)
2090 # from what client spec interpretation gives
0d1696ef 2091 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2092 if self.detectBranches:
2093 for b in self.knownBranches:
2094 if path.startswith(b + "/"):
2095 path = path[len(b)+1:]
2096
2097 elif self.keepRepoPath:
2098 # Preserve everything in relative path name except leading
2099 # //depot/; just look at first prefix as they all should
2100 # be in the same depot.
2101 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2102 if p4PathStartsWith(path, depot):
2103 path = path[len(depot):]
3952710b 2104
0d1696ef 2105 else:
0d1696ef
PW
2106 for p in prefixes:
2107 if p4PathStartsWith(path, p):
2108 path = path[len(p):]
21ef5df4 2109 break
8b41a97f 2110
0d1696ef 2111 path = wildcard_decode(path)
6326aa58 2112 return path
6754a299 2113
71b112d4 2114 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2115 """Look at each depotFile in the commit to figure out to what
2116 branch it belongs."""
2117
d5904674 2118 branches = {}
71b112d4
SH
2119 fnum = 0
2120 while commit.has_key("depotFile%s" % fnum):
2121 path = commit["depotFile%s" % fnum]
6326aa58 2122 found = [p for p in self.depotPaths
d53de8b9 2123 if p4PathStartsWith(path, p)]
6326aa58 2124 if not found:
71b112d4
SH
2125 fnum = fnum + 1
2126 continue
2127
2128 file = {}
2129 file["path"] = path
2130 file["rev"] = commit["rev%s" % fnum]
2131 file["action"] = commit["action%s" % fnum]
2132 file["type"] = commit["type%s" % fnum]
2133 fnum = fnum + 1
2134
21ef5df4
PW
2135 # start with the full relative path where this file would
2136 # go in a p4 client
2137 if self.useClientSpec:
2138 relPath = self.clientSpecDirs.map_in_client(path)
2139 else:
2140 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2141
4b97ffb1 2142 for branch in self.knownBranches.keys():
21ef5df4
PW
2143 # add a trailing slash so that a commit into qt/4.2foo
2144 # doesn't end up in qt/4.2, e.g.
6754a299 2145 if relPath.startswith(branch + "/"):
d5904674
SH
2146 if branch not in branches:
2147 branches[branch] = []
71b112d4 2148 branches[branch].append(file)
6555b2cc 2149 break
b984733c
SH
2150
2151 return branches
2152
b932705b
LD
2153 # output one file from the P4 stream
2154 # - helper for streamP4Files
2155
2156 def streamOneP4File(self, file, contents):
b932705b
LD
2157 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2158 if verbose:
2159 sys.stderr.write("%s\n" % relPath)
2160
9cffb8c8
PW
2161 (type_base, type_mods) = split_p4_type(file["type"])
2162
2163 git_mode = "100644"
2164 if "x" in type_mods:
2165 git_mode = "100755"
2166 if type_base == "symlink":
2167 git_mode = "120000"
2168 # p4 print on a symlink contains "target\n"; remove the newline
b39c3612
EP
2169 data = ''.join(contents)
2170 contents = [data[:-1]]
b932705b 2171
9cffb8c8 2172 if type_base == "utf16":
55aa5714
PW
2173 # p4 delivers different text in the python output to -G
2174 # than it does when using "print -o", or normal p4 client
2175 # operations. utf16 is converted to ascii or utf8, perhaps.
2176 # But ascii text saved as -t utf16 is completely mangled.
2177 # Invoke print -o to get the real contents.
7f0e5962
PW
2178 #
2179 # On windows, the newlines will always be mangled by print, so put
2180 # them back too. This is not needed to the cygwin windows version,
2181 # just the native "NT" type.
2182 #
6de040df 2183 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
7f0e5962
PW
2184 if p4_version_string().find("/NT") >= 0:
2185 text = text.replace("\r\n", "\n")
55aa5714
PW
2186 contents = [ text ]
2187
9f7ef0ea
PW
2188 if type_base == "apple":
2189 # Apple filetype files will be streamed as a concatenation of
2190 # its appledouble header and the contents. This is useless
2191 # on both macs and non-macs. If using "print -q -o xx", it
2192 # will create "xx" with the data, and "%xx" with the header.
2193 # This is also not very useful.
2194 #
2195 # Ideally, someday, this script can learn how to generate
2196 # appledouble files directly and import those to git, but
2197 # non-mac machines can never find a use for apple filetype.
2198 print "\nIgnoring apple filetype file %s" % file['depotFile']
2199 return
2200
55aa5714
PW
2201 # Note that we do not try to de-mangle keywords on utf16 files,
2202 # even though in theory somebody may want that.
60df071c
LD
2203 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2204 if pattern:
2205 regexp = re.compile(pattern, re.VERBOSE)
2206 text = ''.join(contents)
2207 text = regexp.sub(r'$\1$', text)
2208 contents = [ text ]
b932705b 2209
9cffb8c8 2210 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
2211
2212 # total length...
2213 length = 0
2214 for d in contents:
2215 length = length + len(d)
2216
2217 self.gitStream.write("data %d\n" % length)
2218 for d in contents:
2219 self.gitStream.write(d)
2220 self.gitStream.write("\n")
2221
2222 def streamOneP4Deletion(self, file):
2223 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2224 if verbose:
2225 sys.stderr.write("delete %s\n" % relPath)
2226 self.gitStream.write("D %s\n" % relPath)
2227
2228 # handle another chunk of streaming data
2229 def streamP4FilesCb(self, marshalled):
2230
78189bea
PW
2231 # catch p4 errors and complain
2232 err = None
2233 if "code" in marshalled:
2234 if marshalled["code"] == "error":
2235 if "data" in marshalled:
2236 err = marshalled["data"].rstrip()
2237 if err:
2238 f = None
2239 if self.stream_have_file_info:
2240 if "depotFile" in self.stream_file:
2241 f = self.stream_file["depotFile"]
2242 # force a failure in fast-import, else an empty
2243 # commit will be made
2244 self.gitStream.write("\n")
2245 self.gitStream.write("die-now\n")
2246 self.gitStream.close()
2247 # ignore errors, but make sure it exits first
2248 self.importProcess.wait()
2249 if f:
2250 die("Error from p4 print for %s: %s" % (f, err))
2251 else:
2252 die("Error from p4 print: %s" % err)
2253
c3f6163b
AG
2254 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2255 # start of a new file - output the old one first
2256 self.streamOneP4File(self.stream_file, self.stream_contents)
2257 self.stream_file = {}
2258 self.stream_contents = []
2259 self.stream_have_file_info = False
b932705b 2260
c3f6163b
AG
2261 # pick up the new file information... for the
2262 # 'data' field we need to append to our array
2263 for k in marshalled.keys():
2264 if k == 'data':
2265 self.stream_contents.append(marshalled['data'])
2266 else:
2267 self.stream_file[k] = marshalled[k]
b932705b 2268
c3f6163b 2269 self.stream_have_file_info = True
b932705b
LD
2270
2271 # Stream directly from "p4 files" into "git fast-import"
2272 def streamP4Files(self, files):
30b5940b
SH
2273 filesForCommit = []
2274 filesToRead = []
b932705b 2275 filesToDelete = []
30b5940b 2276
3a70cdfa 2277 for f in files:
ecb7cf98
PW
2278 # if using a client spec, only add the files that have
2279 # a path in the client
2280 if self.clientSpecDirs:
2281 if self.clientSpecDirs.map_in_client(f['path']) == "":
2282 continue
3a70cdfa 2283
ecb7cf98
PW
2284 filesForCommit.append(f)
2285 if f['action'] in self.delete_actions:
2286 filesToDelete.append(f)
2287 else:
2288 filesToRead.append(f)
6a49f8e2 2289
b932705b
LD
2290 # deleted files...
2291 for f in filesToDelete:
2292 self.streamOneP4Deletion(f)
1b9a4684 2293
b932705b
LD
2294 if len(filesToRead) > 0:
2295 self.stream_file = {}
2296 self.stream_contents = []
2297 self.stream_have_file_info = False
8ff45f2a 2298
c3f6163b
AG
2299 # curry self argument
2300 def streamP4FilesCbSelf(entry):
2301 self.streamP4FilesCb(entry)
6a49f8e2 2302
6de040df
LD
2303 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2304
2305 p4CmdList(["-x", "-", "print"],
2306 stdin=fileArgs,
2307 cb=streamP4FilesCbSelf)
30b5940b 2308
b932705b
LD
2309 # do the last chunk
2310 if self.stream_file.has_key('depotFile'):
2311 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2312
affb474f
LD
2313 def make_email(self, userid):
2314 if userid in self.users:
2315 return self.users[userid]
2316 else:
2317 return "%s <a@b>" % userid
2318
06804c76
LD
2319 # Stream a p4 tag
2320 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2321 if verbose:
2322 print "writing tag %s for commit %s" % (labelName, commit)
2323 gitStream.write("tag %s\n" % labelName)
2324 gitStream.write("from %s\n" % commit)
2325
2326 if labelDetails.has_key('Owner'):
2327 owner = labelDetails["Owner"]
2328 else:
2329 owner = None
2330
2331 # Try to use the owner of the p4 label, or failing that,
2332 # the current p4 user id.
2333 if owner:
2334 email = self.make_email(owner)
2335 else:
2336 email = self.make_email(self.p4UserId())
2337 tagger = "%s %s %s" % (email, epoch, self.tz)
2338
2339 gitStream.write("tagger %s\n" % tagger)
2340
2341 print "labelDetails=",labelDetails
2342 if labelDetails.has_key('Description'):
2343 description = labelDetails['Description']
2344 else:
2345 description = 'Label from git p4'
2346
2347 gitStream.write("data %d\n" % len(description))
2348 gitStream.write(description)
2349 gitStream.write("\n")
2350
e63231e5 2351 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2352 epoch = details["time"]
2353 author = details["user"]
2354
4b97ffb1
SH
2355 if self.verbose:
2356 print "commit into %s" % branch
2357
96e07dd2
HWN
2358 # start with reading files; if that fails, we should not
2359 # create a commit.
2360 new_files = []
2361 for f in files:
e63231e5 2362 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2363 new_files.append (f)
2364 else:
afa1dd9a 2365 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2366
b984733c 2367 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 2368# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2369 self.committedChanges.add(int(details["change"]))
2370 committer = ""
b607e71e
SH
2371 if author not in self.users:
2372 self.getUserMapFromPerforceServer()
affb474f 2373 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2374
2375 self.gitStream.write("committer %s\n" % committer)
2376
2377 self.gitStream.write("data <<EOT\n")
2378 self.gitStream.write(details["desc"])
e63231e5
PW
2379 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2380 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2381 if len(details['options']) > 0:
2382 self.gitStream.write(": options = %s" % details['options'])
2383 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2384
2385 if len(parent) > 0:
4b97ffb1
SH
2386 if self.verbose:
2387 print "parent %s" % parent
b984733c
SH
2388 self.gitStream.write("from %s\n" % parent)
2389
b932705b 2390 self.streamP4Files(new_files)
b984733c
SH
2391 self.gitStream.write("\n")
2392
1f4ba1cb
SH
2393 change = int(details["change"])
2394
9bda3a85 2395 if self.labels.has_key(change):
1f4ba1cb
SH
2396 label = self.labels[change]
2397 labelDetails = label[0]
2398 labelRevisions = label[1]
71b112d4
SH
2399 if self.verbose:
2400 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2401
6de040df 2402 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2403 for p in self.branchPrefixes])
1f4ba1cb
SH
2404
2405 if len(files) == len(labelRevisions):
2406
2407 cleanedFiles = {}
2408 for info in files:
56c09345 2409 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2410 continue
2411 cleanedFiles[info["depotFile"]] = info["rev"]
2412
2413 if cleanedFiles == labelRevisions:
06804c76 2414 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2415
2416 else:
a46668fa 2417 if not self.silent:
cebdf5af
HWN
2418 print ("Tag %s does not match with change %s: files do not match."
2419 % (labelDetails["label"], change))
1f4ba1cb
SH
2420
2421 else:
a46668fa 2422 if not self.silent:
cebdf5af
HWN
2423 print ("Tag %s does not match with change %s: file count is different."
2424 % (labelDetails["label"], change))
b984733c 2425
06804c76 2426 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2427 def getLabels(self):
2428 self.labels = {}
2429
52a4880b 2430 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2431 if len(l) > 0 and not self.silent:
183f8436 2432 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2433
2434 for output in l:
1f4ba1cb
SH
2435 label = output["label"]
2436 revisions = {}
2437 newestChange = 0
71b112d4
SH
2438 if self.verbose:
2439 print "Querying files for label %s" % label
6de040df
LD
2440 for file in p4CmdList(["files"] +
2441 ["%s...@%s" % (p, label)
2442 for p in self.depotPaths]):
1f4ba1cb
SH
2443 revisions[file["depotFile"]] = file["rev"]
2444 change = int(file["change"])
2445 if change > newestChange:
2446 newestChange = change
2447
9bda3a85
SH
2448 self.labels[newestChange] = [output, revisions]
2449
2450 if self.verbose:
2451 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2452
06804c76
LD
2453 # Import p4 labels as git tags. A direct mapping does not
2454 # exist, so assume that if all the files are at the same revision
2455 # then we can use that, or it's something more complicated we should
2456 # just ignore.
2457 def importP4Labels(self, stream, p4Labels):
2458 if verbose:
2459 print "import p4 labels: " + ' '.join(p4Labels)
2460
2461 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2462 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2463 if len(validLabelRegexp) == 0:
2464 validLabelRegexp = defaultLabelRegexp
2465 m = re.compile(validLabelRegexp)
2466
2467 for name in p4Labels:
2468 commitFound = False
2469
2470 if not m.match(name):
2471 if verbose:
2472 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2473 continue
2474
2475 if name in ignoredP4Labels:
2476 continue
2477
2478 labelDetails = p4CmdList(['label', "-o", name])[0]
2479
2480 # get the most recent changelist for each file in this label
2481 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2482 for p in self.depotPaths])
2483
2484 if change.has_key('change'):
2485 # find the corresponding git commit; take the oldest commit
2486 changelist = int(change['change'])
2487 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2488 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2489 if len(gitCommit) == 0:
2490 print "could not find git commit for changelist %d" % changelist
2491 else:
2492 gitCommit = gitCommit.strip()
2493 commitFound = True
2494 # Convert from p4 time format
2495 try:
2496 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2497 except ValueError:
a4e9054c 2498 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2499 tmwhen = 1
2500
2501 when = int(time.mktime(tmwhen))
2502 self.streamTag(stream, name, labelDetails, gitCommit, when)
2503 if verbose:
2504 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2505 else:
2506 if verbose:
2507 print "Label %s has no changelists - possibly deleted?" % name
2508
2509 if not commitFound:
2510 # We can't import this label; don't try again as it will get very
2511 # expensive repeatedly fetching all the files for labels that will
2512 # never be imported. If the label is moved in the future, the
2513 # ignore will need to be removed manually.
2514 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2515
86dff6b6
HWN
2516 def guessProjectName(self):
2517 for p in self.depotPaths:
6e5295c4
SH
2518 if p.endswith("/"):
2519 p = p[:-1]
2520 p = p[p.strip().rfind("/") + 1:]
2521 if not p.endswith("/"):
2522 p += "/"
2523 return p
86dff6b6 2524
4b97ffb1 2525 def getBranchMapping(self):
6555b2cc
SH
2526 lostAndFoundBranches = set()
2527
8ace74c0
VA
2528 user = gitConfig("git-p4.branchUser")
2529 if len(user) > 0:
2530 command = "branches -u %s" % user
2531 else:
2532 command = "branches"
2533
2534 for info in p4CmdList(command):
52a4880b 2535 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2536 viewIdx = 0
2537 while details.has_key("View%s" % viewIdx):
2538 paths = details["View%s" % viewIdx].split(" ")
2539 viewIdx = viewIdx + 1
2540 # require standard //depot/foo/... //depot/bar/... mapping
2541 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2542 continue
2543 source = paths[0]
2544 destination = paths[1]
6509e19c 2545 ## HACK
d53de8b9 2546 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2547 source = source[len(self.depotPaths[0]):-4]
2548 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2549
1a2edf4e
SH
2550 if destination in self.knownBranches:
2551 if not self.silent:
2552 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2553 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2554 continue
2555
6555b2cc
SH
2556 self.knownBranches[destination] = source
2557
2558 lostAndFoundBranches.discard(destination)
2559
29bdbac1 2560 if source not in self.knownBranches:
6555b2cc
SH
2561 lostAndFoundBranches.add(source)
2562
7199cf13
VA
2563 # Perforce does not strictly require branches to be defined, so we also
2564 # check git config for a branch list.
2565 #
2566 # Example of branch definition in git config file:
2567 # [git-p4]
2568 # branchList=main:branchA
2569 # branchList=main:branchB
2570 # branchList=branchA:branchC
2571 configBranches = gitConfigList("git-p4.branchList")
2572 for branch in configBranches:
2573 if branch:
2574 (source, destination) = branch.split(":")
2575 self.knownBranches[destination] = source
2576
2577 lostAndFoundBranches.discard(destination)
2578
2579 if source not in self.knownBranches:
2580 lostAndFoundBranches.add(source)
2581
6555b2cc
SH
2582
2583 for branch in lostAndFoundBranches:
2584 self.knownBranches[branch] = branch
29bdbac1 2585
38f9f5ec
SH
2586 def getBranchMappingFromGitBranches(self):
2587 branches = p4BranchesInGit(self.importIntoRemotes)
2588 for branch in branches.keys():
2589 if branch == "master":
2590 branch = "main"
2591 else:
2592 branch = branch[len(self.projectName):]
2593 self.knownBranches[branch] = branch
2594
bb6e09b2
HWN
2595 def updateOptionDict(self, d):
2596 option_keys = {}
2597 if self.keepRepoPath:
2598 option_keys['keepRepoPath'] = 1
2599
2600 d["options"] = ' '.join(sorted(option_keys.keys()))
2601
2602 def readOptions(self, d):
2603 self.keepRepoPath = (d.has_key('options')
2604 and ('keepRepoPath' in d['options']))
6326aa58 2605
8134f69c
SH
2606 def gitRefForBranch(self, branch):
2607 if branch == "main":
2608 return self.refPrefix + "master"
2609
2610 if len(branch) <= 0:
2611 return branch
2612
2613 return self.refPrefix + self.projectName + branch
2614
1ca3d710
SH
2615 def gitCommitByP4Change(self, ref, change):
2616 if self.verbose:
2617 print "looking in ref " + ref + " for change %s using bisect..." % change
2618
2619 earliestCommit = ""
2620 latestCommit = parseRevision(ref)
2621
2622 while True:
2623 if self.verbose:
2624 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2625 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2626 if len(next) == 0:
2627 if self.verbose:
2628 print "argh"
2629 return ""
2630 log = extractLogMessageFromGitCommit(next)
2631 settings = extractSettingsGitLog(log)
2632 currentChange = int(settings['change'])
2633 if self.verbose:
2634 print "current change %s" % currentChange
2635
2636 if currentChange == change:
2637 if self.verbose:
2638 print "found %s" % next
2639 return next
2640
2641 if currentChange < change:
2642 earliestCommit = "^%s" % next
2643 else:
2644 latestCommit = "%s" % next
2645
2646 return ""
2647
2648 def importNewBranch(self, branch, maxChange):
2649 # make fast-import flush all changes to disk and update the refs using the checkpoint
2650 # command so that we can try to find the branch parent in the git history
2651 self.gitStream.write("checkpoint\n\n");
2652 self.gitStream.flush();
2653 branchPrefix = self.depotPaths[0] + branch + "/"
2654 range = "@1,%s" % maxChange
2655 #print "prefix" + branchPrefix
2656 changes = p4ChangesForPaths([branchPrefix], range)
2657 if len(changes) <= 0:
2658 return False
2659 firstChange = changes[0]
2660 #print "first change in branch: %s" % firstChange
2661 sourceBranch = self.knownBranches[branch]
2662 sourceDepotPath = self.depotPaths[0] + sourceBranch
2663 sourceRef = self.gitRefForBranch(sourceBranch)
2664 #print "source " + sourceBranch
2665
52a4880b 2666 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2667 #print "branch parent: %s" % branchParentChange
2668 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2669 if len(gitParent) > 0:
2670 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2671 #print "parent git commit: %s" % gitParent
2672
2673 self.importChanges(changes)
2674 return True
2675
fed23693
VA
2676 def searchParent(self, parent, branch, target):
2677 parentFound = False
c7d34884
PW
2678 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2679 "--no-merges", parent]):
fed23693
VA
2680 blob = blob.strip()
2681 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2682 parentFound = True
2683 if self.verbose:
2684 print "Found parent of %s in commit %s" % (branch, blob)
2685 break
2686 if parentFound:
2687 return blob
2688 else:
2689 return None
2690
e87f37ae
SH
2691 def importChanges(self, changes):
2692 cnt = 1
2693 for change in changes:
18fa13d0 2694 description = p4_describe(change)
e87f37ae
SH
2695 self.updateOptionDict(description)
2696
2697 if not self.silent:
2698 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2699 sys.stdout.flush()
2700 cnt = cnt + 1
2701
2702 try:
2703 if self.detectBranches:
2704 branches = self.splitFilesIntoBranches(description)
2705 for branch in branches.keys():
2706 ## HACK --hwn
2707 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 2708 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
2709
2710 parent = ""
2711
2712 filesForCommit = branches[branch]
2713
2714 if self.verbose:
2715 print "branch is %s" % branch
2716
2717 self.updatedBranches.add(branch)
2718
2719 if branch not in self.createdBranches:
2720 self.createdBranches.add(branch)
2721 parent = self.knownBranches[branch]
2722 if parent == branch:
2723 parent = ""
1ca3d710
SH
2724 else:
2725 fullBranch = self.projectName + branch
2726 if fullBranch not in self.p4BranchesInGit:
2727 if not self.silent:
2728 print("\n Importing new branch %s" % fullBranch);
2729 if self.importNewBranch(branch, change - 1):
2730 parent = ""
2731 self.p4BranchesInGit.append(fullBranch)
2732 if not self.silent:
2733 print("\n Resuming with change %s" % change);
2734
2735 if self.verbose:
2736 print "parent determined through known branches: %s" % parent
e87f37ae 2737
8134f69c
SH
2738 branch = self.gitRefForBranch(branch)
2739 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2740
2741 if self.verbose:
2742 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2743
2744 if len(parent) == 0 and branch in self.initialParents:
2745 parent = self.initialParents[branch]
2746 del self.initialParents[branch]
2747
fed23693
VA
2748 blob = None
2749 if len(parent) > 0:
4f9273d2 2750 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
2751 if self.verbose:
2752 print "Creating temporary branch: " + tempBranch
e63231e5 2753 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
2754 self.tempBranches.append(tempBranch)
2755 self.checkpoint()
2756 blob = self.searchParent(parent, branch, tempBranch)
2757 if blob:
e63231e5 2758 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
2759 else:
2760 if self.verbose:
2761 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 2762 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
2763 else:
2764 files = self.extractFilesFromCommit(description)
e63231e5 2765 self.commit(description, files, self.branch,
e87f37ae 2766 self.initialParent)
47497844 2767 # only needed once, to connect to the previous commit
e87f37ae
SH
2768 self.initialParent = ""
2769 except IOError:
2770 print self.gitError.read()
2771 sys.exit(1)
2772
c208a243
SH
2773 def importHeadRevision(self, revision):
2774 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2775
4e2e6ce4
PW
2776 details = {}
2777 details["user"] = "git perforce import user"
1494fcbb 2778 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2779 % (' '.join(self.depotPaths), revision))
2780 details["change"] = revision
2781 newestRevision = 0
2782
2783 fileCnt = 0
6de040df
LD
2784 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2785
2786 for info in p4CmdList(["files"] + fileArgs):
c208a243 2787
68b28593 2788 if 'code' in info and info['code'] == 'error':
c208a243
SH
2789 sys.stderr.write("p4 returned an error: %s\n"
2790 % info['data'])
d88e707f
PW
2791 if info['data'].find("must refer to client") >= 0:
2792 sys.stderr.write("This particular p4 error is misleading.\n")
2793 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2794 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2795 sys.exit(1)
68b28593
PW
2796 if 'p4ExitCode' in info:
2797 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2798 sys.exit(1)
2799
2800
2801 change = int(info["change"])
2802 if change > newestRevision:
2803 newestRevision = change
2804
56c09345 2805 if info["action"] in self.delete_actions:
c208a243
SH
2806 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2807 #fileCnt = fileCnt + 1
2808 continue
2809
2810 for prop in ["depotFile", "rev", "action", "type" ]:
2811 details["%s%s" % (prop, fileCnt)] = info[prop]
2812
2813 fileCnt = fileCnt + 1
2814
2815 details["change"] = newestRevision
4e2e6ce4 2816
9dcb9f24 2817 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 2818 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
2819 res = p4_describe(newestRevision)
2820 details["time"] = res["time"]
4e2e6ce4 2821
c208a243
SH
2822 self.updateOptionDict(details)
2823 try:
e63231e5 2824 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
2825 except IOError:
2826 print "IO error with git fast-import. Is your git version recent enough?"
2827 print self.gitError.read()
2828
2829
b984733c 2830 def run(self, args):
6326aa58 2831 self.depotPaths = []
179caebf 2832 self.changeRange = ""
6326aa58 2833 self.previousDepotPaths = []
991a2de4 2834 self.hasOrigin = False
ce6f33c8 2835
29bdbac1
SH
2836 # map from branch depot path to parent branch
2837 self.knownBranches = {}
2838 self.initialParents = {}
2839
a028a98e
SH
2840 if self.importIntoRemotes:
2841 self.refPrefix = "refs/remotes/p4/"
2842 else:
db775559 2843 self.refPrefix = "refs/heads/p4/"
a028a98e 2844
991a2de4
PW
2845 if self.syncWithOrigin:
2846 self.hasOrigin = originP4BranchesExist()
2847 if self.hasOrigin:
2848 if not self.silent:
2849 print 'Syncing with origin first, using "git fetch origin"'
2850 system("git fetch origin")
10f880f8 2851
5a8e84cd 2852 branch_arg_given = bool(self.branch)
569d1bd4 2853 if len(self.branch) == 0:
db775559 2854 self.branch = self.refPrefix + "master"
a028a98e 2855 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2856 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 2857 system("git branch -D p4")
967f72e2 2858
a93d33ee
PW
2859 # accept either the command-line option, or the configuration variable
2860 if self.useClientSpec:
2861 # will use this after clone to set the variable
2862 self.useClientSpec_from_options = True
2863 else:
0d609032 2864 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
2865 self.useClientSpec = True
2866 if self.useClientSpec:
543987bd 2867 self.clientSpecDirs = getClientSpec()
3a70cdfa 2868
6a49f8e2
HWN
2869 # TODO: should always look at previous commits,
2870 # merge with previous imports, if possible.
2871 if args == []:
d414c74a 2872 if self.hasOrigin:
5ca44617 2873 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
2874
2875 # branches holds mapping from branch name to sha1
2876 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
2877
2878 # restrict to just this one, disabling detect-branches
2879 if branch_arg_given:
2880 short = self.branch.split("/")[-1]
2881 if short in branches:
2882 self.p4BranchesInGit = [ short ]
2883 else:
2884 self.p4BranchesInGit = branches.keys()
abcd790f
SH
2885
2886 if len(self.p4BranchesInGit) > 1:
2887 if not self.silent:
2888 print "Importing from/into multiple branches"
2889 self.detectBranches = True
8c9e8b6e
PW
2890 for branch in branches.keys():
2891 self.initialParents[self.refPrefix + branch] = \
2892 branches[branch]
967f72e2 2893
29bdbac1
SH
2894 if self.verbose:
2895 print "branches: %s" % self.p4BranchesInGit
2896
2897 p4Change = 0
2898 for branch in self.p4BranchesInGit:
cebdf5af 2899 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2900
2901 settings = extractSettingsGitLog(logMsg)
29bdbac1 2902
bb6e09b2
HWN
2903 self.readOptions(settings)
2904 if (settings.has_key('depot-paths')
2905 and settings.has_key ('change')):
2906 change = int(settings['change']) + 1
29bdbac1
SH
2907 p4Change = max(p4Change, change)
2908
bb6e09b2
HWN
2909 depotPaths = sorted(settings['depot-paths'])
2910 if self.previousDepotPaths == []:
6326aa58 2911 self.previousDepotPaths = depotPaths
29bdbac1 2912 else:
6326aa58
HWN
2913 paths = []
2914 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2915 prev_list = prev.split("/")
2916 cur_list = cur.split("/")
2917 for i in range(0, min(len(cur_list), len(prev_list))):
2918 if cur_list[i] <> prev_list[i]:
583e1707 2919 i = i - 1
6326aa58
HWN
2920 break
2921
04d277b3 2922 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2923
2924 self.previousDepotPaths = paths
29bdbac1
SH
2925
2926 if p4Change > 0:
bb6e09b2 2927 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2928 self.changeRange = "@%s,#head" % p4Change
341dc1c1 2929 if not self.silent and not self.detectBranches:
967f72e2 2930 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2931
40d69ac3
PW
2932 # accept multiple ref name abbreviations:
2933 # refs/foo/bar/branch -> use it exactly
2934 # p4/branch -> prepend refs/remotes/ or refs/heads/
2935 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 2936 if not self.branch.startswith("refs/"):
40d69ac3
PW
2937 if self.importIntoRemotes:
2938 prepend = "refs/remotes/"
2939 else:
2940 prepend = "refs/heads/"
2941 if not self.branch.startswith("p4/"):
2942 prepend += "p4/"
2943 self.branch = prepend + self.branch
179caebf 2944
6326aa58 2945 if len(args) == 0 and self.depotPaths:
b984733c 2946 if not self.silent:
6326aa58 2947 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2948 else:
6326aa58 2949 if self.depotPaths and self.depotPaths != args:
cebdf5af 2950 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2951 "This doesn't work!" % (' '.join (self.depotPaths),
2952 ' '.join (args)))
b984733c 2953 sys.exit(1)
6326aa58 2954
bb6e09b2 2955 self.depotPaths = sorted(args)
b984733c 2956
1c49fc19 2957 revision = ""
b984733c 2958 self.users = {}
b984733c 2959
58c8bc7c
PW
2960 # Make sure no revision specifiers are used when --changesfile
2961 # is specified.
2962 bad_changesfile = False
2963 if len(self.changesFile) > 0:
2964 for p in self.depotPaths:
2965 if p.find("@") >= 0 or p.find("#") >= 0:
2966 bad_changesfile = True
2967 break
2968 if bad_changesfile:
2969 die("Option --changesfile is incompatible with revision specifiers")
2970
6326aa58
HWN
2971 newPaths = []
2972 for p in self.depotPaths:
2973 if p.find("@") != -1:
2974 atIdx = p.index("@")
2975 self.changeRange = p[atIdx:]
2976 if self.changeRange == "@all":
2977 self.changeRange = ""
6a49f8e2 2978 elif ',' not in self.changeRange:
1c49fc19 2979 revision = self.changeRange
6326aa58 2980 self.changeRange = ""
7fcff9de 2981 p = p[:atIdx]
6326aa58
HWN
2982 elif p.find("#") != -1:
2983 hashIdx = p.index("#")
1c49fc19 2984 revision = p[hashIdx:]
7fcff9de 2985 p = p[:hashIdx]
6326aa58 2986 elif self.previousDepotPaths == []:
58c8bc7c
PW
2987 # pay attention to changesfile, if given, else import
2988 # the entire p4 tree at the head revision
2989 if len(self.changesFile) == 0:
2990 revision = "#head"
6326aa58
HWN
2991
2992 p = re.sub ("\.\.\.$", "", p)
2993 if not p.endswith("/"):
2994 p += "/"
2995
2996 newPaths.append(p)
2997
2998 self.depotPaths = newPaths
2999
e63231e5
PW
3000 # --detect-branches may change this for each branch
3001 self.branchPrefixes = self.depotPaths
3002
b607e71e 3003 self.loadUserMapFromCache()
cb53e1f8
SH
3004 self.labels = {}
3005 if self.detectLabels:
3006 self.getLabels();
b984733c 3007
4b97ffb1 3008 if self.detectBranches:
df450923
SH
3009 ## FIXME - what's a P4 projectName ?
3010 self.projectName = self.guessProjectName()
3011
38f9f5ec
SH
3012 if self.hasOrigin:
3013 self.getBranchMappingFromGitBranches()
3014 else:
3015 self.getBranchMapping()
29bdbac1
SH
3016 if self.verbose:
3017 print "p4-git branches: %s" % self.p4BranchesInGit
3018 print "initial parents: %s" % self.initialParents
3019 for b in self.p4BranchesInGit:
3020 if b != "master":
6326aa58
HWN
3021
3022 ## FIXME
29bdbac1
SH
3023 b = b[len(self.projectName):]
3024 self.createdBranches.add(b)
4b97ffb1 3025
f291b4e3 3026 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3027
78189bea
PW
3028 self.importProcess = subprocess.Popen(["git", "fast-import"],
3029 stdin=subprocess.PIPE,
3030 stdout=subprocess.PIPE,
3031 stderr=subprocess.PIPE);
3032 self.gitOutput = self.importProcess.stdout
3033 self.gitStream = self.importProcess.stdin
3034 self.gitError = self.importProcess.stderr
b984733c 3035
1c49fc19 3036 if revision:
c208a243 3037 self.importHeadRevision(revision)
b984733c
SH
3038 else:
3039 changes = []
3040
0828ab14 3041 if len(self.changesFile) > 0:
b984733c 3042 output = open(self.changesFile).readlines()
1d7367dc 3043 changeSet = set()
b984733c
SH
3044 for line in output:
3045 changeSet.add(int(line))
3046
3047 for change in changeSet:
3048 changes.append(change)
3049
3050 changes.sort()
3051 else:
9dcb9f24
PW
3052 # catch "git p4 sync" with no new branches, in a repo that
3053 # does not have any existing p4 branches
5a8e84cd
PW
3054 if len(args) == 0:
3055 if not self.p4BranchesInGit:
3056 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3057
3058 # The default branch is master, unless --branch is used to
3059 # specify something else. Make sure it exists, or complain
3060 # nicely about how to use --branch.
3061 if not self.detectBranches:
3062 if not branch_exists(self.branch):
3063 if branch_arg_given:
3064 die("Error: branch %s does not exist." % self.branch)
3065 else:
3066 die("Error: no branch %s; perhaps specify one with --branch." %
3067 self.branch)
3068
29bdbac1 3069 if self.verbose:
86dff6b6 3070 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3071 self.changeRange)
4f6432d8 3072 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 3073
01a9c9c5 3074 if len(self.maxChanges) > 0:
7fcff9de 3075 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3076
b984733c 3077 if len(changes) == 0:
0828ab14 3078 if not self.silent:
341dc1c1 3079 print "No changes to import!"
06804c76
LD
3080 else:
3081 if not self.silent and not self.detectBranches:
3082 print "Import destination: %s" % self.branch
3083
3084 self.updatedBranches = set()
b984733c 3085
47497844
PW
3086 if not self.detectBranches:
3087 if args:
3088 # start a new branch
3089 self.initialParent = ""
3090 else:
3091 # build on a previous revision
3092 self.initialParent = parseRevision(self.branch)
3093
06804c76 3094 self.importChanges(changes)
a9d1a27a 3095
06804c76
LD
3096 if not self.silent:
3097 print ""
3098 if len(self.updatedBranches) > 0:
3099 sys.stdout.write("Updated branches: ")
3100 for b in self.updatedBranches:
3101 sys.stdout.write("%s " % b)
3102 sys.stdout.write("\n")
341dc1c1 3103
0d609032 3104 if gitConfigBool("git-p4.importLabels"):
06dcd152 3105 self.importLabels = True
b984733c 3106
06804c76
LD
3107 if self.importLabels:
3108 p4Labels = getP4Labels(self.depotPaths)
3109 gitTags = getGitTags()
3110
3111 missingP4Labels = p4Labels - gitTags
3112 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3113
b984733c 3114 self.gitStream.close()
78189bea 3115 if self.importProcess.wait() != 0:
29bdbac1 3116 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3117 self.gitOutput.close()
3118 self.gitError.close()
3119
fed23693
VA
3120 # Cleanup temporary branches created during import
3121 if self.tempBranches != []:
3122 for branch in self.tempBranches:
3123 read_pipe("git update-ref -d %s" % branch)
3124 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3125
55d12437
PW
3126 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3127 # a convenient shortcut refname "p4".
3128 if self.importIntoRemotes:
3129 head_ref = self.refPrefix + "HEAD"
3130 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3131 system(["git", "symbolic-ref", head_ref, self.branch])
3132
b984733c
SH
3133 return True
3134
01ce1fe9
SH
3135class P4Rebase(Command):
3136 def __init__(self):
3137 Command.__init__(self)
06804c76
LD
3138 self.options = [
3139 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3140 ]
06804c76 3141 self.importLabels = False
cebdf5af
HWN
3142 self.description = ("Fetches the latest revision from perforce and "
3143 + "rebases the current work (branch) against it")
01ce1fe9
SH
3144
3145 def run(self, args):
3146 sync = P4Sync()
06804c76 3147 sync.importLabels = self.importLabels
01ce1fe9 3148 sync.run([])
d7e3868c 3149
14594f4b
SH
3150 return self.rebase()
3151
3152 def rebase(self):
36ee4ee4
SH
3153 if os.system("git update-index --refresh") != 0:
3154 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3155 if len(read_pipe("git diff-index HEAD --")) > 0:
3156 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3157
d7e3868c
SH
3158 [upstream, settings] = findUpstreamBranchPoint()
3159 if len(upstream) == 0:
3160 die("Cannot find upstream branchpoint for rebase")
3161
3162 # the branchpoint may be p4/foo~3, so strip off the parent
3163 upstream = re.sub("~[0-9]+$", "", upstream)
3164
3165 print "Rebasing the current branch onto %s" % upstream
b25b2065 3166 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 3167 system("git rebase %s" % upstream)
1f52af6c 3168 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
3169 return True
3170
f9a3a4f7
SH
3171class P4Clone(P4Sync):
3172 def __init__(self):
3173 P4Sync.__init__(self)
3174 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 3175 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 3176 self.options += [
bb6e09b2
HWN
3177 optparse.make_option("--destination", dest="cloneDestination",
3178 action='store', default=None,
354081d5
TT
3179 help="where to leave result of the clone"),
3180 optparse.make_option("-/", dest="cloneExclude",
3181 action="append", type="string",
38200076
PW
3182 help="exclude depot path"),
3183 optparse.make_option("--bare", dest="cloneBare",
3184 action="store_true", default=False),
354081d5 3185 ]
bb6e09b2 3186 self.cloneDestination = None
f9a3a4f7 3187 self.needsGit = False
38200076 3188 self.cloneBare = False
f9a3a4f7 3189
354081d5
TT
3190 # This is required for the "append" cloneExclude action
3191 def ensure_value(self, attr, value):
3192 if not hasattr(self, attr) or getattr(self, attr) is None:
3193 setattr(self, attr, value)
3194 return getattr(self, attr)
3195
6a49f8e2
HWN
3196 def defaultDestination(self, args):
3197 ## TODO: use common prefix of args?
3198 depotPath = args[0]
3199 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3200 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 3201 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
3202 depotDir = re.sub(r"/$", "", depotDir)
3203 return os.path.split(depotDir)[1]
3204
f9a3a4f7
SH
3205 def run(self, args):
3206 if len(args) < 1:
3207 return False
bb6e09b2
HWN
3208
3209 if self.keepRepoPath and not self.cloneDestination:
3210 sys.stderr.write("Must specify destination for --keep-path\n")
3211 sys.exit(1)
f9a3a4f7 3212
6326aa58 3213 depotPaths = args
5e100b5c
SH
3214
3215 if not self.cloneDestination and len(depotPaths) > 1:
3216 self.cloneDestination = depotPaths[-1]
3217 depotPaths = depotPaths[:-1]
3218
354081d5 3219 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
3220 for p in depotPaths:
3221 if not p.startswith("//"):
0f487d30 3222 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
6326aa58 3223 return False
f9a3a4f7 3224
bb6e09b2 3225 if not self.cloneDestination:
98ad4faf 3226 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 3227
86dff6b6 3228 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 3229
c3bf3f13
KG
3230 if not os.path.exists(self.cloneDestination):
3231 os.makedirs(self.cloneDestination)
053fd0c1 3232 chdir(self.cloneDestination)
38200076
PW
3233
3234 init_cmd = [ "git", "init" ]
3235 if self.cloneBare:
3236 init_cmd.append("--bare")
a235e85c
BC
3237 retcode = subprocess.call(init_cmd)
3238 if retcode:
3239 raise CalledProcessError(retcode, init_cmd)
38200076 3240
6326aa58 3241 if not P4Sync.run(self, depotPaths):
f9a3a4f7 3242 return False
c595956d
PW
3243
3244 # create a master branch and check out a work tree
3245 if gitBranchExists(self.branch):
3246 system([ "git", "branch", "master", self.branch ])
3247 if not self.cloneBare:
3248 system([ "git", "checkout", "-f" ])
3249 else:
3250 print 'Not checking out any branch, use ' \
3251 '"git checkout -q -b master <branch>"'
86dff6b6 3252
a93d33ee
PW
3253 # auto-set this variable if invoked with --use-client-spec
3254 if self.useClientSpec_from_options:
3255 system("git config --bool git-p4.useclientspec true")
3256
f9a3a4f7
SH
3257 return True
3258
09d89de2
SH
3259class P4Branches(Command):
3260 def __init__(self):
3261 Command.__init__(self)
3262 self.options = [ ]
3263 self.description = ("Shows the git branches that hold imports and their "
3264 + "corresponding perforce depot paths")
3265 self.verbose = False
3266
3267 def run(self, args):
5ca44617
SH
3268 if originP4BranchesExist():
3269 createOrUpdateBranchesFromOrigin()
3270
09d89de2
SH
3271 cmdline = "git rev-parse --symbolic "
3272 cmdline += " --remotes"
3273
3274 for line in read_pipe_lines(cmdline):
3275 line = line.strip()
3276
3277 if not line.startswith('p4/') or line == "p4/HEAD":
3278 continue
3279 branch = line
3280
3281 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3282 settings = extractSettingsGitLog(log)
3283
3284 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3285 return True
3286
b984733c
SH
3287class HelpFormatter(optparse.IndentedHelpFormatter):
3288 def __init__(self):
3289 optparse.IndentedHelpFormatter.__init__(self)
3290
3291 def format_description(self, description):
3292 if description:
3293 return description + "\n"
3294 else:
3295 return ""
4f5cf76a 3296
86949eef
SH
3297def printUsage(commands):
3298 print "usage: %s <command> [options]" % sys.argv[0]
3299 print ""
3300 print "valid commands: %s" % ", ".join(commands)
3301 print ""
3302 print "Try %s <command> --help for command specific help." % sys.argv[0]
3303 print ""
3304
3305commands = {
b86f7378
HWN
3306 "debug" : P4Debug,
3307 "submit" : P4Submit,
a9834f58 3308 "commit" : P4Submit,
b86f7378
HWN
3309 "sync" : P4Sync,
3310 "rebase" : P4Rebase,
3311 "clone" : P4Clone,
09d89de2
SH
3312 "rollback" : P4RollBack,
3313 "branches" : P4Branches
86949eef
SH
3314}
3315
86949eef 3316
bb6e09b2
HWN
3317def main():
3318 if len(sys.argv[1:]) == 0:
3319 printUsage(commands.keys())
3320 sys.exit(2)
4f5cf76a 3321
bb6e09b2
HWN
3322 cmdName = sys.argv[1]
3323 try:
b86f7378
HWN
3324 klass = commands[cmdName]
3325 cmd = klass()
bb6e09b2
HWN
3326 except KeyError:
3327 print "unknown command %s" % cmdName
3328 print ""
3329 printUsage(commands.keys())
3330 sys.exit(2)
3331
3332 options = cmd.options
b86f7378 3333 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
3334
3335 args = sys.argv[2:]
3336
b0ccc80d 3337 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
6a10b6aa
LD
3338 if cmd.needsGit:
3339 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 3340
6a10b6aa
LD
3341 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3342 options,
3343 description = cmd.description,
3344 formatter = HelpFormatter())
bb6e09b2 3345
6a10b6aa 3346 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
bb6e09b2
HWN
3347 global verbose
3348 verbose = cmd.verbose
3349 if cmd.needsGit:
b86f7378
HWN
3350 if cmd.gitdir == None:
3351 cmd.gitdir = os.path.abspath(".git")
3352 if not isValidGitDir(cmd.gitdir):
3353 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3354 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
3355 cdup = read_pipe("git rev-parse --show-cdup").strip()
3356 if len(cdup) > 0:
053fd0c1 3357 chdir(cdup);
e20a9e53 3358
b86f7378
HWN
3359 if not isValidGitDir(cmd.gitdir):
3360 if isValidGitDir(cmd.gitdir + "/.git"):
3361 cmd.gitdir += "/.git"
bb6e09b2 3362 else:
b86f7378 3363 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 3364
b86f7378 3365 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 3366
bb6e09b2
HWN
3367 if not cmd.run(args):
3368 parser.print_help()
09fca77b 3369 sys.exit(2)
4f5cf76a 3370
4f5cf76a 3371
bb6e09b2
HWN
3372if __name__ == '__main__':
3373 main()