git p4 test: sanitize P4CHARSET
[git/git.git] / git-p4.py
CommitLineData
86949eef
SH
1#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
c8cbbee9
SH
5# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
83dce55a 7# 2007 Trolltech ASA
86949eef
SH
8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
a33faf28
ER
10import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
f629fa59
PW
15import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
d20f0f8e 24import stat
8b41a97f 25
a235e85c
BC
26try:
27 from subprocess import CalledProcessError
28except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
4addad22 41verbose = False
86949eef 42
06804c76 43# Only labels/tags matching this will be imported/exported
c8942a22 44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
21a50753
AK
45
46def p4_build_cmd(cmd):
47 """Build a suitable p4 command line.
48
49 This consolidates building and returning a p4 command line into one
50 location. It means that hooking into the environment, or other configuration
51 can be done more easily.
52 """
6de040df 53 real_cmd = ["p4"]
abcaf073
AK
54
55 user = gitConfig("git-p4.user")
56 if len(user) > 0:
6de040df 57 real_cmd += ["-u",user]
abcaf073
AK
58
59 password = gitConfig("git-p4.password")
60 if len(password) > 0:
6de040df 61 real_cmd += ["-P", password]
abcaf073
AK
62
63 port = gitConfig("git-p4.port")
64 if len(port) > 0:
6de040df 65 real_cmd += ["-p", port]
abcaf073
AK
66
67 host = gitConfig("git-p4.host")
68 if len(host) > 0:
41799aa2 69 real_cmd += ["-H", host]
abcaf073
AK
70
71 client = gitConfig("git-p4.client")
72 if len(client) > 0:
6de040df 73 real_cmd += ["-c", client]
abcaf073 74
6de040df
LD
75
76 if isinstance(cmd,basestring):
77 real_cmd = ' '.join(real_cmd) + ' ' + cmd
78 else:
79 real_cmd += cmd
21a50753
AK
80 return real_cmd
81
bbd84863
MF
82def chdir(path, is_client_path=False):
83 """Do chdir to the given path, and set the PWD environment
84 variable for use by P4. It does not look at getcwd() output.
85 Since we're not using the shell, it is necessary to set the
86 PWD environment variable explicitly.
87
88 Normally, expand the path to force it to be absolute. This
89 addresses the use of relative path names inside P4 settings,
90 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
91 as given; it looks for .p4config using PWD.
92
93 If is_client_path, the path was handed to us directly by p4,
94 and may be a symbolic link. Do not call os.getcwd() in this
95 case, because it will cause p4 to think that PWD is not inside
96 the client path.
97 """
98
99 os.chdir(path)
100 if not is_client_path:
101 path = os.getcwd()
102 os.environ['PWD'] = path
053fd0c1 103
86dff6b6
HWN
104def die(msg):
105 if verbose:
106 raise Exception(msg)
107 else:
108 sys.stderr.write(msg + "\n")
109 sys.exit(1)
110
6de040df 111def write_pipe(c, stdin):
4addad22 112 if verbose:
6de040df 113 sys.stderr.write('Writing pipe: %s\n' % str(c))
b016d397 114
6de040df
LD
115 expand = isinstance(c,basestring)
116 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117 pipe = p.stdin
118 val = pipe.write(stdin)
119 pipe.close()
120 if p.wait():
121 die('Command failed: %s' % str(c))
b016d397
HWN
122
123 return val
124
6de040df 125def p4_write_pipe(c, stdin):
d9429194 126 real_cmd = p4_build_cmd(c)
6de040df 127 return write_pipe(real_cmd, stdin)
d9429194 128
4addad22
HWN
129def read_pipe(c, ignore_error=False):
130 if verbose:
6de040df 131 sys.stderr.write('Reading pipe: %s\n' % str(c))
8b41a97f 132
6de040df
LD
133 expand = isinstance(c,basestring)
134 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135 pipe = p.stdout
b016d397 136 val = pipe.read()
6de040df
LD
137 if p.wait() and not ignore_error:
138 die('Command failed: %s' % str(c))
b016d397
HWN
139
140 return val
141
d9429194
AK
142def p4_read_pipe(c, ignore_error=False):
143 real_cmd = p4_build_cmd(c)
144 return read_pipe(real_cmd, ignore_error)
b016d397 145
bce4c5fc 146def read_pipe_lines(c):
4addad22 147 if verbose:
6de040df
LD
148 sys.stderr.write('Reading pipe: %s\n' % str(c))
149
150 expand = isinstance(c, basestring)
151 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152 pipe = p.stdout
b016d397 153 val = pipe.readlines()
6de040df
LD
154 if pipe.close() or p.wait():
155 die('Command failed: %s' % str(c))
b016d397
HWN
156
157 return val
caace111 158
2318121b
AK
159def p4_read_pipe_lines(c):
160 """Specifically invoke p4 on the command supplied. """
155af834 161 real_cmd = p4_build_cmd(c)
2318121b
AK
162 return read_pipe_lines(real_cmd)
163
8e9497c2
GG
164def p4_has_command(cmd):
165 """Ask p4 for help on this command. If it returns an error, the
166 command does not exist in this version of p4."""
167 real_cmd = p4_build_cmd(["help", cmd])
168 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169 stderr=subprocess.PIPE)
170 p.communicate()
171 return p.returncode == 0
172
249da4c0
PW
173def p4_has_move_command():
174 """See if the move command exists, that it supports -k, and that
175 it has not been administratively disabled. The arguments
176 must be correct, but the filenames do not have to exist. Use
177 ones with wildcards so even if they exist, it will fail."""
178
179 if not p4_has_command("move"):
180 return False
181 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183 (out, err) = p.communicate()
184 # return code will be 1 in either case
185 if err.find("Invalid option") >= 0:
186 return False
187 if err.find("disabled") >= 0:
188 return False
189 # assume it failed because @... was invalid changelist
190 return True
191
6754a299 192def system(cmd):
6de040df 193 expand = isinstance(cmd,basestring)
4addad22 194 if verbose:
6de040df 195 sys.stderr.write("executing %s\n" % str(cmd))
a235e85c
BC
196 retcode = subprocess.call(cmd, shell=expand)
197 if retcode:
198 raise CalledProcessError(retcode, cmd)
6754a299 199
bf9320f1
AK
200def p4_system(cmd):
201 """Specifically invoke p4 as the system command. """
155af834 202 real_cmd = p4_build_cmd(cmd)
6de040df 203 expand = isinstance(real_cmd, basestring)
a235e85c
BC
204 retcode = subprocess.call(real_cmd, shell=expand)
205 if retcode:
206 raise CalledProcessError(retcode, real_cmd)
6de040df 207
7f0e5962
PW
208_p4_version_string = None
209def p4_version_string():
210 """Read the version string, showing just the last line, which
211 hopefully is the interesting version bit.
212
213 $ p4 -V
214 Perforce - The Fast Software Configuration Management System.
215 Copyright 1995-2011 Perforce Software. All rights reserved.
216 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
217 """
218 global _p4_version_string
219 if not _p4_version_string:
220 a = p4_read_pipe_lines(["-V"])
221 _p4_version_string = a[-1].rstrip()
222 return _p4_version_string
223
6de040df 224def p4_integrate(src, dest):
9d7d446a 225 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
6de040df 226
8d7ec362 227def p4_sync(f, *options):
9d7d446a 228 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
6de040df
LD
229
230def p4_add(f):
9d7d446a
PW
231 # forcibly add file names with wildcards
232 if wildcard_present(f):
233 p4_system(["add", "-f", f])
234 else:
235 p4_system(["add", f])
6de040df
LD
236
237def p4_delete(f):
9d7d446a 238 p4_system(["delete", wildcard_encode(f)])
6de040df
LD
239
240def p4_edit(f):
9d7d446a 241 p4_system(["edit", wildcard_encode(f)])
6de040df
LD
242
243def p4_revert(f):
9d7d446a 244 p4_system(["revert", wildcard_encode(f)])
6de040df 245
9d7d446a
PW
246def p4_reopen(type, f):
247 p4_system(["reopen", "-t", type, wildcard_encode(f)])
bf9320f1 248
8e9497c2
GG
249def p4_move(src, dest):
250 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
251
18fa13d0
PW
252def p4_describe(change):
253 """Make sure it returns a valid result by checking for
254 the presence of field "time". Return a dict of the
255 results."""
256
257 ds = p4CmdList(["describe", "-s", str(change)])
258 if len(ds) != 1:
259 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
260
261 d = ds[0]
262
263 if "p4ExitCode" in d:
264 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265 str(d)))
266 if "code" in d:
267 if d["code"] == "error":
268 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
269
270 if "time" not in d:
271 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
272
273 return d
274
9cffb8c8
PW
275#
276# Canonicalize the p4 type and return a tuple of the
277# base type, plus any modifiers. See "p4 help filetypes"
278# for a list and explanation.
279#
280def split_p4_type(p4type):
281
282 p4_filetypes_historical = {
283 "ctempobj": "binary+Sw",
284 "ctext": "text+C",
285 "cxtext": "text+Cx",
286 "ktext": "text+k",
287 "kxtext": "text+kx",
288 "ltext": "text+F",
289 "tempobj": "binary+FSw",
290 "ubinary": "binary+F",
291 "uresource": "resource+F",
292 "uxbinary": "binary+Fx",
293 "xbinary": "binary+x",
294 "xltext": "text+Fx",
295 "xtempobj": "binary+Swx",
296 "xtext": "text+x",
297 "xunicode": "unicode+x",
298 "xutf16": "utf16+x",
299 }
300 if p4type in p4_filetypes_historical:
301 p4type = p4_filetypes_historical[p4type]
302 mods = ""
303 s = p4type.split("+")
304 base = s[0]
305 mods = ""
306 if len(s) > 1:
307 mods = s[1]
308 return (base, mods)
b9fc6ea9 309
60df071c
LD
310#
311# return the raw p4 type of a file (text, text+ko, etc)
312#
313def p4_type(file):
314 results = p4CmdList(["fstat", "-T", "headType", file])
315 return results[0]['headType']
316
317#
318# Given a type base and modifier, return a regexp matching
319# the keywords that can be expanded in the file
320#
321def p4_keywords_regexp_for_type(base, type_mods):
322 if base in ("text", "unicode", "binary"):
323 kwords = None
324 if "ko" in type_mods:
325 kwords = 'Id|Header'
326 elif "k" in type_mods:
327 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328 else:
329 return None
330 pattern = r"""
331 \$ # Starts with a dollar, followed by...
332 (%s) # one of the keywords, followed by...
6b2bf41e 333 (:[^$\n]+)? # possibly an old expansion, followed by...
60df071c
LD
334 \$ # another dollar
335 """ % kwords
336 return pattern
337 else:
338 return None
339
340#
341# Given a file, return a regexp matching the possible
342# RCS keywords that will be expanded, or None for files
343# with kw expansion turned off.
344#
345def p4_keywords_regexp_for_file(file):
346 if not os.path.exists(file):
347 return None
348 else:
349 (type_base, type_mods) = split_p4_type(p4_type(file))
350 return p4_keywords_regexp_for_type(type_base, type_mods)
b9fc6ea9 351
c65b670e
CP
352def setP4ExecBit(file, mode):
353 # Reopens an already open file and changes the execute bit to match
354 # the execute bit setting in the passed in mode.
355
356 p4Type = "+x"
357
358 if not isModeExec(mode):
359 p4Type = getP4OpenedType(file)
360 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362 if p4Type[-1] == "+":
363 p4Type = p4Type[0:-1]
364
6de040df 365 p4_reopen(p4Type, file)
c65b670e
CP
366
367def getP4OpenedType(file):
368 # Returns the perforce file type for the given file.
369
9d7d446a 370 result = p4_read_pipe(["opened", wildcard_encode(file)])
f3e5ae4f 371 match = re.match(".*\((.+)\)\r?$", result)
c65b670e
CP
372 if match:
373 return match.group(1)
374 else:
f3e5ae4f 375 die("Could not determine file type for %s (result: '%s')" % (file, result))
c65b670e 376
06804c76
LD
377# Return the set of all p4 labels
378def getP4Labels(depotPaths):
379 labels = set()
380 if isinstance(depotPaths,basestring):
381 depotPaths = [depotPaths]
382
383 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384 label = l['label']
385 labels.add(label)
386
387 return labels
388
389# Return the set of all git tags
390def getGitTags():
391 gitTags = set()
392 for line in read_pipe_lines(["git", "tag"]):
393 tag = line.strip()
394 gitTags.add(tag)
395 return gitTags
396
b43b0a3c
CP
397def diffTreePattern():
398 # This is a simple generator for the diff tree regex pattern. This could be
399 # a class variable if this and parseDiffTreeEntry were a part of a class.
400 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401 while True:
402 yield pattern
403
404def parseDiffTreeEntry(entry):
405 """Parses a single diff tree entry into its component elements.
406
407 See git-diff-tree(1) manpage for details about the format of the diff
408 output. This method returns a dictionary with the following elements:
409
410 src_mode - The mode of the source file
411 dst_mode - The mode of the destination file
412 src_sha1 - The sha1 for the source file
413 dst_sha1 - The sha1 fr the destination file
414 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415 status_score - The score for the status (applicable for 'C' and 'R'
416 statuses). This is None if there is no score.
417 src - The path for the source file.
418 dst - The path for the destination file. This is only present for
419 copy or renames. If it is not present, this is None.
420
421 If the pattern is not matched, None is returned."""
422
423 match = diffTreePattern().next().match(entry)
424 if match:
425 return {
426 'src_mode': match.group(1),
427 'dst_mode': match.group(2),
428 'src_sha1': match.group(3),
429 'dst_sha1': match.group(4),
430 'status': match.group(5),
431 'status_score': match.group(6),
432 'src': match.group(7),
433 'dst': match.group(10)
434 }
435 return None
436
c65b670e
CP
437def isModeExec(mode):
438 # Returns True if the given git mode represents an executable file,
439 # otherwise False.
440 return mode[-3:] == "755"
441
442def isModeExecChanged(src_mode, dst_mode):
443 return isModeExec(src_mode) != isModeExec(dst_mode)
444
b932705b 445def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
6de040df
LD
446
447 if isinstance(cmd,basestring):
448 cmd = "-G " + cmd
449 expand = True
450 else:
451 cmd = ["-G"] + cmd
452 expand = False
453
454 cmd = p4_build_cmd(cmd)
6a49f8e2 455 if verbose:
6de040df 456 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
9f90c733
SL
457
458 # Use a temporary file to avoid deadlocks without
459 # subprocess.communicate(), which would put another copy
460 # of stdout into memory.
461 stdin_file = None
462 if stdin is not None:
463 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
6de040df
LD
464 if isinstance(stdin,basestring):
465 stdin_file.write(stdin)
466 else:
467 for i in stdin:
468 stdin_file.write(i + '\n')
9f90c733
SL
469 stdin_file.flush()
470 stdin_file.seek(0)
471
6de040df
LD
472 p4 = subprocess.Popen(cmd,
473 shell=expand,
9f90c733
SL
474 stdin=stdin_file,
475 stdout=subprocess.PIPE)
86949eef
SH
476
477 result = []
478 try:
479 while True:
9f90c733 480 entry = marshal.load(p4.stdout)
c3f6163b
AG
481 if cb is not None:
482 cb(entry)
483 else:
484 result.append(entry)
86949eef
SH
485 except EOFError:
486 pass
9f90c733
SL
487 exitCode = p4.wait()
488 if exitCode != 0:
ac3e0d79
SH
489 entry = {}
490 entry["p4ExitCode"] = exitCode
491 result.append(entry)
86949eef
SH
492
493 return result
494
495def p4Cmd(cmd):
496 list = p4CmdList(cmd)
497 result = {}
498 for entry in list:
499 result.update(entry)
500 return result;
501
cb2c9db5
SH
502def p4Where(depotPath):
503 if not depotPath.endswith("/"):
504 depotPath += "/"
7f705dc3 505 depotPath = depotPath + "..."
6de040df 506 outputList = p4CmdList(["where", depotPath])
7f705dc3
TAL
507 output = None
508 for entry in outputList:
75bc9573
TAL
509 if "depotFile" in entry:
510 if entry["depotFile"] == depotPath:
511 output = entry
512 break
513 elif "data" in entry:
514 data = entry.get("data")
515 space = data.find(" ")
516 if data[:space] == depotPath:
517 output = entry
518 break
7f705dc3
TAL
519 if output == None:
520 return ""
dc524036
SH
521 if output["code"] == "error":
522 return ""
cb2c9db5
SH
523 clientPath = ""
524 if "path" in output:
525 clientPath = output.get("path")
526 elif "data" in output:
527 data = output.get("data")
528 lastSpace = data.rfind(" ")
529 clientPath = data[lastSpace + 1:]
530
531 if clientPath.endswith("..."):
532 clientPath = clientPath[:-3]
533 return clientPath
534
86949eef 535def currentGitBranch():
b25b2065 536 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
86949eef 537
4f5cf76a 538def isValidGitDir(path):
bb6e09b2
HWN
539 if (os.path.exists(path + "/HEAD")
540 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
4f5cf76a
SH
541 return True;
542 return False
543
463e8af6 544def parseRevision(ref):
b25b2065 545 return read_pipe("git rev-parse %s" % ref).strip()
463e8af6 546
28755dba
PW
547def branchExists(ref):
548 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
549 ignore_error=True)
550 return len(rev) > 0
551
6ae8de88
SH
552def extractLogMessageFromGitCommit(commit):
553 logMessage = ""
b016d397
HWN
554
555 ## fixme: title is first line of commit, not 1st paragraph.
6ae8de88 556 foundTitle = False
b016d397 557 for log in read_pipe_lines("git cat-file commit %s" % commit):
6ae8de88
SH
558 if not foundTitle:
559 if len(log) == 1:
1c094184 560 foundTitle = True
6ae8de88
SH
561 continue
562
563 logMessage += log
564 return logMessage
565
bb6e09b2 566def extractSettingsGitLog(log):
6ae8de88
SH
567 values = {}
568 for line in log.split("\n"):
569 line = line.strip()
6326aa58
HWN
570 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
571 if not m:
572 continue
573
574 assignments = m.group(1).split (':')
575 for a in assignments:
576 vals = a.split ('=')
577 key = vals[0].strip()
578 val = ('='.join (vals[1:])).strip()
579 if val.endswith ('\"') and val.startswith('"'):
580 val = val[1:-1]
581
582 values[key] = val
583
845b42cb
SH
584 paths = values.get("depot-paths")
585 if not paths:
586 paths = values.get("depot-path")
a3fdd579
SH
587 if paths:
588 values['depot-paths'] = paths.split(',')
bb6e09b2 589 return values
6ae8de88 590
8136a639 591def gitBranchExists(branch):
bb6e09b2
HWN
592 proc = subprocess.Popen(["git", "rev-parse", branch],
593 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
caace111 594 return proc.wait() == 0;
8136a639 595
36bd8446 596_gitConfig = {}
b345d6c3 597
0d609032 598def gitConfig(key):
36bd8446 599 if not _gitConfig.has_key(key):
0d609032 600 cmd = [ "git", "config", key ]
b345d6c3
PW
601 s = read_pipe(cmd, ignore_error=True)
602 _gitConfig[key] = s.strip()
36bd8446 603 return _gitConfig[key]
01265103 604
0d609032
PW
605def gitConfigBool(key):
606 """Return a bool, using git config --bool. It is True only if the
607 variable is set to true, and False if set to false or not present
608 in the config."""
609
36bd8446 610 if not _gitConfig.has_key(key):
0d609032
PW
611 cmd = [ "git", "config", "--bool", key ]
612 s = read_pipe(cmd, ignore_error=True)
613 v = s.strip()
614 _gitConfig[key] = v == "true"
36bd8446 615 return _gitConfig[key]
01265103 616
7199cf13
VA
617def gitConfigList(key):
618 if not _gitConfig.has_key(key):
2abba301
PW
619 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
620 _gitConfig[key] = s.strip().split(os.linesep)
7199cf13
VA
621 return _gitConfig[key]
622
2c8037ed
PW
623def p4BranchesInGit(branchesAreInRemotes=True):
624 """Find all the branches whose names start with "p4/", looking
625 in remotes or heads as specified by the argument. Return
626 a dictionary of { branch: revision } for each one found.
627 The branch names are the short names, without any
628 "p4/" prefix."""
629
062410bb
SH
630 branches = {}
631
632 cmdline = "git rev-parse --symbolic "
633 if branchesAreInRemotes:
2c8037ed 634 cmdline += "--remotes"
062410bb 635 else:
2c8037ed 636 cmdline += "--branches"
062410bb
SH
637
638 for line in read_pipe_lines(cmdline):
639 line = line.strip()
640
2c8037ed
PW
641 # only import to p4/
642 if not line.startswith('p4/'):
643 continue
644 # special symbolic ref to p4/master
645 if line == "p4/HEAD":
062410bb 646 continue
062410bb 647
2c8037ed
PW
648 # strip off p4/ prefix
649 branch = line[len("p4/"):]
062410bb
SH
650
651 branches[branch] = parseRevision(line)
2c8037ed 652
062410bb
SH
653 return branches
654
5a8e84cd
PW
655def branch_exists(branch):
656 """Make sure that the given ref name really exists."""
657
658 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
659 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
660 out, _ = p.communicate()
661 if p.returncode:
662 return False
663 # expect exactly one line of output: the branch name
664 return out.rstrip() == branch
665
9ceab363 666def findUpstreamBranchPoint(head = "HEAD"):
86506fe5
SH
667 branches = p4BranchesInGit()
668 # map from depot-path to branch name
669 branchByDepotPath = {}
670 for branch in branches.keys():
671 tip = branches[branch]
672 log = extractLogMessageFromGitCommit(tip)
673 settings = extractSettingsGitLog(log)
674 if settings.has_key("depot-paths"):
675 paths = ",".join(settings["depot-paths"])
676 branchByDepotPath[paths] = "remotes/p4/" + branch
677
27d2d811 678 settings = None
27d2d811
SH
679 parent = 0
680 while parent < 65535:
9ceab363 681 commit = head + "~%s" % parent
27d2d811
SH
682 log = extractLogMessageFromGitCommit(commit)
683 settings = extractSettingsGitLog(log)
86506fe5
SH
684 if settings.has_key("depot-paths"):
685 paths = ",".join(settings["depot-paths"])
686 if branchByDepotPath.has_key(paths):
687 return [branchByDepotPath[paths], settings]
27d2d811 688
86506fe5 689 parent = parent + 1
27d2d811 690
86506fe5 691 return ["", settings]
27d2d811 692
5ca44617
SH
693def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
694 if not silent:
695 print ("Creating/updating branch(es) in %s based on origin branch(es)"
696 % localRefPrefix)
697
698 originPrefix = "origin/p4/"
699
700 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
701 line = line.strip()
702 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
703 continue
704
705 headName = line[len(originPrefix):]
706 remoteHead = localRefPrefix + headName
707 originHead = line
708
709 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
710 if (not original.has_key('depot-paths')
711 or not original.has_key('change')):
712 continue
713
714 update = False
715 if not gitBranchExists(remoteHead):
716 if verbose:
717 print "creating %s" % remoteHead
718 update = True
719 else:
720 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
721 if settings.has_key('change') > 0:
722 if settings['depot-paths'] == original['depot-paths']:
723 originP4Change = int(original['change'])
724 p4Change = int(settings['change'])
725 if originP4Change > p4Change:
726 print ("%s (%s) is newer than %s (%s). "
727 "Updating p4 branch from origin."
728 % (originHead, originP4Change,
729 remoteHead, p4Change))
730 update = True
731 else:
732 print ("Ignoring: %s was imported from %s while "
733 "%s was imported from %s"
734 % (originHead, ','.join(original['depot-paths']),
735 remoteHead, ','.join(settings['depot-paths'])))
736
737 if update:
738 system("git update-ref %s %s" % (remoteHead, originHead))
739
740def originP4BranchesExist():
741 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
742
4f6432d8
SH
743def p4ChangesForPaths(depotPaths, changeRange):
744 assert depotPaths
6de040df
LD
745 cmd = ['changes']
746 for p in depotPaths:
747 cmd += ["%s...%s" % (p, changeRange)]
748 output = p4_read_pipe_lines(cmd)
4f6432d8 749
b4b0ba06 750 changes = {}
4f6432d8 751 for line in output:
c3f6163b
AG
752 changeNum = int(line.split(" ")[1])
753 changes[changeNum] = True
4f6432d8 754
b4b0ba06
PW
755 changelist = changes.keys()
756 changelist.sort()
757 return changelist
4f6432d8 758
d53de8b9
TAL
759def p4PathStartsWith(path, prefix):
760 # This method tries to remedy a potential mixed-case issue:
761 #
762 # If UserA adds //depot/DirA/file1
763 # and UserB adds //depot/dira/file2
764 #
765 # we may or may not have a problem. If you have core.ignorecase=true,
766 # we treat DirA and dira as the same directory
0d609032 767 if gitConfigBool("core.ignorecase"):
d53de8b9
TAL
768 return path.lower().startswith(prefix.lower())
769 return path.startswith(prefix)
770
543987bd
PW
771def getClientSpec():
772 """Look at the p4 client spec, create a View() object that contains
773 all the mappings, and return it."""
774
775 specList = p4CmdList("client -o")
776 if len(specList) != 1:
777 die('Output from "client -o" is %d lines, expecting 1' %
778 len(specList))
779
780 # dictionary of all client parameters
781 entry = specList[0]
782
783 # just the keys that start with "View"
784 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
785
786 # hold this new View
787 view = View()
788
789 # append the lines, in order, to the view
790 for view_num in range(len(view_keys)):
791 k = "View%d" % view_num
792 if k not in view_keys:
793 die("Expected view key %s missing" % k)
794 view.append(entry[k])
795
796 return view
797
798def getClientRoot():
799 """Grab the client directory."""
800
801 output = p4CmdList("client -o")
802 if len(output) != 1:
803 die('Output from "client -o" is %d lines, expecting 1' % len(output))
804
805 entry = output[0]
806 if "Root" not in entry:
807 die('Client has no "Root"')
808
809 return entry["Root"]
810
9d7d446a
PW
811#
812# P4 wildcards are not allowed in filenames. P4 complains
813# if you simply add them, but you can force it with "-f", in
814# which case it translates them into %xx encoding internally.
815#
816def wildcard_decode(path):
817 # Search for and fix just these four characters. Do % last so
818 # that fixing it does not inadvertently create new %-escapes.
819 # Cannot have * in a filename in windows; untested as to
820 # what p4 would do in such a case.
821 if not platform.system() == "Windows":
822 path = path.replace("%2A", "*")
823 path = path.replace("%23", "#") \
824 .replace("%40", "@") \
825 .replace("%25", "%")
826 return path
827
828def wildcard_encode(path):
829 # do % first to avoid double-encoding the %s introduced here
830 path = path.replace("%", "%25") \
831 .replace("*", "%2A") \
832 .replace("#", "%23") \
833 .replace("@", "%40")
834 return path
835
836def wildcard_present(path):
598354c0
BC
837 m = re.search("[*#@%]", path)
838 return m is not None
9d7d446a 839
b984733c
SH
840class Command:
841 def __init__(self):
842 self.usage = "usage: %prog [options]"
8910ac0e 843 self.needsGit = True
6a10b6aa 844 self.verbose = False
b984733c 845
3ea2cfd4
LD
846class P4UserMap:
847 def __init__(self):
848 self.userMapFromPerforceServer = False
affb474f
LD
849 self.myP4UserId = None
850
851 def p4UserId(self):
852 if self.myP4UserId:
853 return self.myP4UserId
854
855 results = p4CmdList("user -o")
856 for r in results:
857 if r.has_key('User'):
858 self.myP4UserId = r['User']
859 return r['User']
860 die("Could not find your p4 user id")
861
862 def p4UserIsMe(self, p4User):
863 # return True if the given p4 user is actually me
864 me = self.p4UserId()
865 if not p4User or p4User != me:
866 return False
867 else:
868 return True
3ea2cfd4
LD
869
870 def getUserCacheFilename(self):
871 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
872 return home + "/.gitp4-usercache.txt"
873
874 def getUserMapFromPerforceServer(self):
875 if self.userMapFromPerforceServer:
876 return
877 self.users = {}
878 self.emails = {}
879
880 for output in p4CmdList("users"):
881 if not output.has_key("User"):
882 continue
883 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
884 self.emails[output["Email"]] = output["User"]
885
886
887 s = ''
888 for (key, val) in self.users.items():
889 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
890
891 open(self.getUserCacheFilename(), "wb").write(s)
892 self.userMapFromPerforceServer = True
893
894 def loadUserMapFromCache(self):
895 self.users = {}
896 self.userMapFromPerforceServer = False
897 try:
898 cache = open(self.getUserCacheFilename(), "rb")
899 lines = cache.readlines()
900 cache.close()
901 for line in lines:
902 entry = line.strip().split("\t")
903 self.users[entry[0]] = entry[1]
904 except IOError:
905 self.getUserMapFromPerforceServer()
906
b984733c 907class P4Debug(Command):
86949eef 908 def __init__(self):
6ae8de88 909 Command.__init__(self)
6a10b6aa 910 self.options = []
c8c39116 911 self.description = "A tool to debug the output of p4 -G."
8910ac0e 912 self.needsGit = False
86949eef
SH
913
914 def run(self, args):
b1ce9447 915 j = 0
6de040df 916 for output in p4CmdList(args):
b1ce9447
HWN
917 print 'Element: %d' % j
918 j += 1
86949eef 919 print output
b984733c 920 return True
86949eef 921
5834684d
SH
922class P4RollBack(Command):
923 def __init__(self):
924 Command.__init__(self)
925 self.options = [
0c66a783 926 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
5834684d
SH
927 ]
928 self.description = "A tool to debug the multi-branch import. Don't use :)"
0c66a783 929 self.rollbackLocalBranches = False
5834684d
SH
930
931 def run(self, args):
932 if len(args) != 1:
933 return False
934 maxChange = int(args[0])
0c66a783 935
ad192f28 936 if "p4ExitCode" in p4Cmd("changes -m 1"):
66a2f523
SH
937 die("Problems executing p4");
938
0c66a783
SH
939 if self.rollbackLocalBranches:
940 refPrefix = "refs/heads/"
b016d397 941 lines = read_pipe_lines("git rev-parse --symbolic --branches")
0c66a783
SH
942 else:
943 refPrefix = "refs/remotes/"
b016d397 944 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
0c66a783
SH
945
946 for line in lines:
947 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
b25b2065
HWN
948 line = line.strip()
949 ref = refPrefix + line
5834684d 950 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
951 settings = extractSettingsGitLog(log)
952
953 depotPaths = settings['depot-paths']
954 change = settings['change']
955
5834684d 956 changed = False
52102d47 957
6326aa58
HWN
958 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
959 for p in depotPaths]))) == 0:
52102d47
SH
960 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
961 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
962 continue
963
bb6e09b2 964 while change and int(change) > maxChange:
5834684d 965 changed = True
52102d47
SH
966 if self.verbose:
967 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
5834684d
SH
968 system("git update-ref %s \"%s^\"" % (ref, ref))
969 log = extractLogMessageFromGitCommit(ref)
bb6e09b2
HWN
970 settings = extractSettingsGitLog(log)
971
972
973 depotPaths = settings['depot-paths']
974 change = settings['change']
5834684d
SH
975
976 if changed:
52102d47 977 print "%s rewound to %s" % (ref, change)
5834684d
SH
978
979 return True
980
3ea2cfd4 981class P4Submit(Command, P4UserMap):
6bbfd137
PW
982
983 conflict_behavior_choices = ("ask", "skip", "quit")
984
4f5cf76a 985 def __init__(self):
b984733c 986 Command.__init__(self)
3ea2cfd4 987 P4UserMap.__init__(self)
4f5cf76a 988 self.options = [
4f5cf76a 989 optparse.make_option("--origin", dest="origin"),
ae901090 990 optparse.make_option("-M", dest="detectRenames", action="store_true"),
3ea2cfd4
LD
991 # preserve the user, requires relevant p4 permissions
992 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
06804c76 993 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
ef739f08 994 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
728b7ad8 995 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
6bbfd137 996 optparse.make_option("--conflict", dest="conflict_behavior",
44e8d26c
PW
997 choices=self.conflict_behavior_choices),
998 optparse.make_option("--branch", dest="branch"),
4f5cf76a
SH
999 ]
1000 self.description = "Submit changes from git to the perforce depot."
c9b50e63 1001 self.usage += " [name of git branch to submit into perforce depot]"
9512497b 1002 self.origin = ""
ae901090 1003 self.detectRenames = False
0d609032 1004 self.preserveUser = gitConfigBool("git-p4.preserveUser")
ef739f08 1005 self.dry_run = False
728b7ad8 1006 self.prepare_p4_only = False
6bbfd137 1007 self.conflict_behavior = None
f7baba8b 1008 self.isWindows = (platform.system() == "Windows")
06804c76 1009 self.exportLabels = False
249da4c0 1010 self.p4HasMoveCommand = p4_has_move_command()
44e8d26c 1011 self.branch = None
4f5cf76a 1012
4f5cf76a
SH
1013 def check(self):
1014 if len(p4CmdList("opened ...")) > 0:
1015 die("You have files opened with perforce! Close them before starting the sync.")
1016
f19cb0a0
PW
1017 def separate_jobs_from_description(self, message):
1018 """Extract and return a possible Jobs field in the commit
1019 message. It goes into a separate section in the p4 change
1020 specification.
1021
1022 A jobs line starts with "Jobs:" and looks like a new field
1023 in a form. Values are white-space separated on the same
1024 line or on following lines that start with a tab.
1025
1026 This does not parse and extract the full git commit message
1027 like a p4 form. It just sees the Jobs: line as a marker
1028 to pass everything from then on directly into the p4 form,
1029 but outside the description section.
1030
1031 Return a tuple (stripped log message, jobs string)."""
1032
1033 m = re.search(r'^Jobs:', message, re.MULTILINE)
1034 if m is None:
1035 return (message, None)
1036
1037 jobtext = message[m.start():]
1038 stripped_message = message[:m.start()].rstrip()
1039 return (stripped_message, jobtext)
1040
1041 def prepareLogMessage(self, template, message, jobs):
1042 """Edits the template returned from "p4 change -o" to insert
1043 the message in the Description field, and the jobs text in
1044 the Jobs field."""
4f5cf76a
SH
1045 result = ""
1046
edae1e2f
SH
1047 inDescriptionSection = False
1048
4f5cf76a
SH
1049 for line in template.split("\n"):
1050 if line.startswith("#"):
1051 result += line + "\n"
1052 continue
1053
edae1e2f 1054 if inDescriptionSection:
c9dbab04 1055 if line.startswith("Files:") or line.startswith("Jobs:"):
edae1e2f 1056 inDescriptionSection = False
f19cb0a0
PW
1057 # insert Jobs section
1058 if jobs:
1059 result += jobs + "\n"
edae1e2f
SH
1060 else:
1061 continue
1062 else:
1063 if line.startswith("Description:"):
1064 inDescriptionSection = True
1065 line += "\n"
1066 for messageLine in message.split("\n"):
1067 line += "\t" + messageLine + "\n"
1068
1069 result += line + "\n"
4f5cf76a
SH
1070
1071 return result
1072
60df071c
LD
1073 def patchRCSKeywords(self, file, pattern):
1074 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1075 (handle, outFileName) = tempfile.mkstemp(dir='.')
1076 try:
1077 outFile = os.fdopen(handle, "w+")
1078 inFile = open(file, "r")
1079 regexp = re.compile(pattern, re.VERBOSE)
1080 for line in inFile.readlines():
1081 line = regexp.sub(r'$\1$', line)
1082 outFile.write(line)
1083 inFile.close()
1084 outFile.close()
1085 # Forcibly overwrite the original file
1086 os.unlink(file)
1087 shutil.move(outFileName, file)
1088 except:
1089 # cleanup our temporary file
1090 os.unlink(outFileName)
1091 print "Failed to strip RCS keywords in %s" % file
1092 raise
1093
1094 print "Patched up RCS keywords in %s" % file
1095
3ea2cfd4
LD
1096 def p4UserForCommit(self,id):
1097 # Return the tuple (perforce user,git email) for a given git commit id
1098 self.getUserMapFromPerforceServer()
9bf28855
PW
1099 gitEmail = read_pipe(["git", "log", "--max-count=1",
1100 "--format=%ae", id])
3ea2cfd4
LD
1101 gitEmail = gitEmail.strip()
1102 if not self.emails.has_key(gitEmail):
1103 return (None,gitEmail)
1104 else:
1105 return (self.emails[gitEmail],gitEmail)
1106
1107 def checkValidP4Users(self,commits):
1108 # check if any git authors cannot be mapped to p4 users
1109 for id in commits:
1110 (user,email) = self.p4UserForCommit(id)
1111 if not user:
1112 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
0d609032 1113 if gitConfigBool("git-p4.allowMissingP4Users"):
3ea2cfd4
LD
1114 print "%s" % msg
1115 else:
1116 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1117
1118 def lastP4Changelist(self):
1119 # Get back the last changelist number submitted in this client spec. This
1120 # then gets used to patch up the username in the change. If the same
1121 # client spec is being used by multiple processes then this might go
1122 # wrong.
1123 results = p4CmdList("client -o") # find the current client
1124 client = None
1125 for r in results:
1126 if r.has_key('Client'):
1127 client = r['Client']
1128 break
1129 if not client:
1130 die("could not get client spec")
6de040df 1131 results = p4CmdList(["changes", "-c", client, "-m", "1"])
3ea2cfd4
LD
1132 for r in results:
1133 if r.has_key('change'):
1134 return r['change']
1135 die("Could not get changelist number for last submit - cannot patch up user details")
1136
1137 def modifyChangelistUser(self, changelist, newUser):
1138 # fixup the user field of a changelist after it has been submitted.
1139 changes = p4CmdList("change -o %s" % changelist)
ecdba36d
LD
1140 if len(changes) != 1:
1141 die("Bad output from p4 change modifying %s to user %s" %
1142 (changelist, newUser))
1143
1144 c = changes[0]
1145 if c['User'] == newUser: return # nothing to do
1146 c['User'] = newUser
1147 input = marshal.dumps(c)
1148
3ea2cfd4
LD
1149 result = p4CmdList("change -f -i", stdin=input)
1150 for r in result:
1151 if r.has_key('code'):
1152 if r['code'] == 'error':
1153 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1154 if r.has_key('data'):
1155 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1156 return
1157 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1158
1159 def canChangeChangelists(self):
1160 # check to see if we have p4 admin or super-user permissions, either of
1161 # which are required to modify changelists.
52a4880b 1162 results = p4CmdList(["protects", self.depotPath])
3ea2cfd4
LD
1163 for r in results:
1164 if r.has_key('perm'):
1165 if r['perm'] == 'admin':
1166 return 1
1167 if r['perm'] == 'super':
1168 return 1
1169 return 0
1170
ea99c3ae 1171 def prepareSubmitTemplate(self):
f19cb0a0
PW
1172 """Run "p4 change -o" to grab a change specification template.
1173 This does not use "p4 -G", as it is nice to keep the submission
1174 template in original order, since a human might edit it.
1175
1176 Remove lines in the Files section that show changes to files
1177 outside the depot path we're committing into."""
1178
ea99c3ae
SH
1179 template = ""
1180 inFilesSection = False
6de040df 1181 for line in p4_read_pipe_lines(['change', '-o']):
f3e5ae4f
MSO
1182 if line.endswith("\r\n"):
1183 line = line[:-2] + "\n"
ea99c3ae
SH
1184 if inFilesSection:
1185 if line.startswith("\t"):
1186 # path starts and ends with a tab
1187 path = line[1:]
1188 lastTab = path.rfind("\t")
1189 if lastTab != -1:
1190 path = path[:lastTab]
d53de8b9 1191 if not p4PathStartsWith(path, self.depotPath):
ea99c3ae
SH
1192 continue
1193 else:
1194 inFilesSection = False
1195 else:
1196 if line.startswith("Files:"):
1197 inFilesSection = True
1198
1199 template += line
1200
1201 return template
1202
7c766e57
PW
1203 def edit_template(self, template_file):
1204 """Invoke the editor to let the user change the submission
1205 message. Return true if okay to continue with the submit."""
1206
1207 # if configured to skip the editing part, just submit
0d609032 1208 if gitConfigBool("git-p4.skipSubmitEdit"):
7c766e57
PW
1209 return True
1210
1211 # look at the modification time, to check later if the user saved
1212 # the file
1213 mtime = os.stat(template_file).st_mtime
1214
1215 # invoke the editor
f95ceaf0 1216 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
7c766e57
PW
1217 editor = os.environ.get("P4EDITOR")
1218 else:
1219 editor = read_pipe("git var GIT_EDITOR").strip()
1220 system(editor + " " + template_file)
1221
1222 # If the file was not saved, prompt to see if this patch should
1223 # be skipped. But skip this verification step if configured so.
0d609032 1224 if gitConfigBool("git-p4.skipSubmitEditCheck"):
7c766e57
PW
1225 return True
1226
d1652049
PW
1227 # modification time updated means user saved the file
1228 if os.stat(template_file).st_mtime > mtime:
1229 return True
1230
1231 while True:
1232 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1233 if response == 'y':
1234 return True
1235 if response == 'n':
1236 return False
7c766e57 1237
7cb5cbef 1238 def applyCommit(self, id):
67b0fe2e
PW
1239 """Apply one commit, return True if it succeeded."""
1240
1241 print "Applying", read_pipe(["git", "show", "-s",
1242 "--format=format:%h %s", id])
ae901090 1243
848de9c3 1244 (p4User, gitEmail) = self.p4UserForCommit(id)
3ea2cfd4 1245
84cb0003 1246 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
4f5cf76a
SH
1247 filesToAdd = set()
1248 filesToDelete = set()
d336c158 1249 editedFiles = set()
b6ad6dcc 1250 pureRenameCopy = set()
c65b670e 1251 filesToChangeExecBit = {}
60df071c 1252
4f5cf76a 1253 for line in diff:
b43b0a3c
CP
1254 diff = parseDiffTreeEntry(line)
1255 modifier = diff['status']
1256 path = diff['src']
4f5cf76a 1257 if modifier == "M":
6de040df 1258 p4_edit(path)
c65b670e
CP
1259 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1260 filesToChangeExecBit[path] = diff['dst_mode']
d336c158 1261 editedFiles.add(path)
4f5cf76a
SH
1262 elif modifier == "A":
1263 filesToAdd.add(path)
c65b670e 1264 filesToChangeExecBit[path] = diff['dst_mode']
4f5cf76a
SH
1265 if path in filesToDelete:
1266 filesToDelete.remove(path)
1267 elif modifier == "D":
1268 filesToDelete.add(path)
1269 if path in filesToAdd:
1270 filesToAdd.remove(path)
4fddb41b
VA
1271 elif modifier == "C":
1272 src, dest = diff['src'], diff['dst']
6de040df 1273 p4_integrate(src, dest)
b6ad6dcc 1274 pureRenameCopy.add(dest)
4fddb41b 1275 if diff['src_sha1'] != diff['dst_sha1']:
6de040df 1276 p4_edit(dest)
b6ad6dcc 1277 pureRenameCopy.discard(dest)
4fddb41b 1278 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
6de040df 1279 p4_edit(dest)
b6ad6dcc 1280 pureRenameCopy.discard(dest)
4fddb41b 1281 filesToChangeExecBit[dest] = diff['dst_mode']
d20f0f8e
PW
1282 if self.isWindows:
1283 # turn off read-only attribute
1284 os.chmod(dest, stat.S_IWRITE)
4fddb41b
VA
1285 os.unlink(dest)
1286 editedFiles.add(dest)
d9a5f25b 1287 elif modifier == "R":
b43b0a3c 1288 src, dest = diff['src'], diff['dst']
8e9497c2
GG
1289 if self.p4HasMoveCommand:
1290 p4_edit(src) # src must be open before move
1291 p4_move(src, dest) # opens for (move/delete, move/add)
b6ad6dcc 1292 else:
8e9497c2
GG
1293 p4_integrate(src, dest)
1294 if diff['src_sha1'] != diff['dst_sha1']:
1295 p4_edit(dest)
1296 else:
1297 pureRenameCopy.add(dest)
c65b670e 1298 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
8e9497c2
GG
1299 if not self.p4HasMoveCommand:
1300 p4_edit(dest) # with move: already open, writable
c65b670e 1301 filesToChangeExecBit[dest] = diff['dst_mode']
8e9497c2 1302 if not self.p4HasMoveCommand:
d20f0f8e
PW
1303 if self.isWindows:
1304 os.chmod(dest, stat.S_IWRITE)
8e9497c2
GG
1305 os.unlink(dest)
1306 filesToDelete.add(src)
d9a5f25b 1307 editedFiles.add(dest)
4f5cf76a
SH
1308 else:
1309 die("unknown modifier %s for %s" % (modifier, path))
1310
0e36f2d7 1311 diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
47a130b7 1312 patchcmd = diffcmd + " | git apply "
c1b296b9
SH
1313 tryPatchCmd = patchcmd + "--check -"
1314 applyPatchCmd = patchcmd + "--check --apply -"
60df071c 1315 patch_succeeded = True
51a2640a 1316
47a130b7 1317 if os.system(tryPatchCmd) != 0:
60df071c
LD
1318 fixed_rcs_keywords = False
1319 patch_succeeded = False
51a2640a 1320 print "Unfortunately applying the change failed!"
60df071c
LD
1321
1322 # Patch failed, maybe it's just RCS keyword woes. Look through
1323 # the patch to see if that's possible.
0d609032 1324 if gitConfigBool("git-p4.attemptRCSCleanup"):
60df071c
LD
1325 file = None
1326 pattern = None
1327 kwfiles = {}
1328 for file in editedFiles | filesToDelete:
1329 # did this file's delta contain RCS keywords?
1330 pattern = p4_keywords_regexp_for_file(file)
1331
1332 if pattern:
1333 # this file is a possibility...look for RCS keywords.
1334 regexp = re.compile(pattern, re.VERBOSE)
1335 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1336 if regexp.search(line):
1337 if verbose:
1338 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1339 kwfiles[file] = pattern
1340 break
1341
1342 for file in kwfiles:
1343 if verbose:
1344 print "zapping %s with %s" % (line,pattern)
d20f0f8e
PW
1345 # File is being deleted, so not open in p4. Must
1346 # disable the read-only bit on windows.
1347 if self.isWindows and file not in editedFiles:
1348 os.chmod(file, stat.S_IWRITE)
60df071c
LD
1349 self.patchRCSKeywords(file, kwfiles[file])
1350 fixed_rcs_keywords = True
1351
1352 if fixed_rcs_keywords:
1353 print "Retrying the patch with RCS keywords cleaned up"
1354 if os.system(tryPatchCmd) == 0:
1355 patch_succeeded = True
1356
1357 if not patch_succeeded:
7e5dd9f2
PW
1358 for f in editedFiles:
1359 p4_revert(f)
7e5dd9f2 1360 return False
51a2640a 1361
55ac2ed6
PW
1362 #
1363 # Apply the patch for real, and do add/delete/+x handling.
1364 #
47a130b7 1365 system(applyPatchCmd)
4f5cf76a
SH
1366
1367 for f in filesToAdd:
6de040df 1368 p4_add(f)
4f5cf76a 1369 for f in filesToDelete:
6de040df
LD
1370 p4_revert(f)
1371 p4_delete(f)
4f5cf76a 1372
c65b670e
CP
1373 # Set/clear executable bits
1374 for f in filesToChangeExecBit.keys():
1375 mode = filesToChangeExecBit[f]
1376 setP4ExecBit(f, mode)
1377
55ac2ed6
PW
1378 #
1379 # Build p4 change description, starting with the contents
1380 # of the git commit message.
1381 #
0e36f2d7 1382 logMessage = extractLogMessageFromGitCommit(id)
0e36f2d7 1383 logMessage = logMessage.strip()
f19cb0a0 1384 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
4f5cf76a 1385
ea99c3ae 1386 template = self.prepareSubmitTemplate()
f19cb0a0 1387 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
ecdba36d 1388
c47178d4 1389 if self.preserveUser:
55ac2ed6 1390 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
c47178d4 1391
55ac2ed6
PW
1392 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1393 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1394 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1395 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
c47178d4 1396
55ac2ed6
PW
1397 separatorLine = "######## everything below this line is just the diff #######\n"
1398
1399 # diff
c47178d4
PW
1400 if os.environ.has_key("P4DIFF"):
1401 del(os.environ["P4DIFF"])
1402 diff = ""
1403 for editedFile in editedFiles:
1404 diff += p4_read_pipe(['diff', '-du',
1405 wildcard_encode(editedFile)])
1406
55ac2ed6 1407 # new file diff
c47178d4
PW
1408 newdiff = ""
1409 for newFile in filesToAdd:
1410 newdiff += "==== new file ====\n"
1411 newdiff += "--- /dev/null\n"
1412 newdiff += "+++ %s\n" % newFile
1413 f = open(newFile, "r")
1414 for line in f.readlines():
1415 newdiff += "+" + line
1416 f.close()
1417
55ac2ed6 1418 # change description file: submitTemplate, separatorLine, diff, newdiff
c47178d4
PW
1419 (handle, fileName) = tempfile.mkstemp()
1420 tmpFile = os.fdopen(handle, "w+")
1421 if self.isWindows:
1422 submitTemplate = submitTemplate.replace("\n", "\r\n")
1423 separatorLine = separatorLine.replace("\n", "\r\n")
1424 newdiff = newdiff.replace("\n", "\r\n")
1425 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1426 tmpFile.close()
1427
728b7ad8
PW
1428 if self.prepare_p4_only:
1429 #
1430 # Leave the p4 tree prepared, and the submit template around
1431 # and let the user decide what to do next
1432 #
1433 print
1434 print "P4 workspace prepared for submission."
1435 print "To submit or revert, go to client workspace"
1436 print " " + self.clientPath
1437 print
1438 print "To submit, use \"p4 submit\" to write a new description,"
1439 print "or \"p4 submit -i %s\" to use the one prepared by" \
1440 " \"git p4\"." % fileName
1441 print "You can delete the file \"%s\" when finished." % fileName
1442
1443 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1444 print "To preserve change ownership by user %s, you must\n" \
1445 "do \"p4 change -f <change>\" after submitting and\n" \
1446 "edit the User field."
1447 if pureRenameCopy:
1448 print "After submitting, renamed files must be re-synced."
1449 print "Invoke \"p4 sync -f\" on each of these files:"
1450 for f in pureRenameCopy:
1451 print " " + f
1452
1453 print
1454 print "To revert the changes, use \"p4 revert ...\", and delete"
1455 print "the submit template file \"%s\"" % fileName
1456 if filesToAdd:
1457 print "Since the commit adds new files, they must be deleted:"
1458 for f in filesToAdd:
1459 print " " + f
1460 print
1461 return True
1462
55ac2ed6
PW
1463 #
1464 # Let the user edit the change description, then submit it.
1465 #
c47178d4
PW
1466 if self.edit_template(fileName):
1467 # read the edited message and submit
5a41c16a 1468 ret = True
c47178d4
PW
1469 tmpFile = open(fileName, "rb")
1470 message = tmpFile.read()
e96e400f 1471 tmpFile.close()
c47178d4
PW
1472 submitTemplate = message[:message.index(separatorLine)]
1473 if self.isWindows:
1474 submitTemplate = submitTemplate.replace("\r\n", "\n")
1475 p4_write_pipe(['submit', '-i'], submitTemplate)
e96e400f 1476
c47178d4
PW
1477 if self.preserveUser:
1478 if p4User:
1479 # Get last changelist number. Cannot easily get it from
1480 # the submit command output as the output is
1481 # unmarshalled.
1482 changelist = self.lastP4Changelist()
1483 self.modifyChangelistUser(changelist, p4User)
1484
1485 # The rename/copy happened by applying a patch that created a
1486 # new file. This leaves it writable, which confuses p4.
1487 for f in pureRenameCopy:
1488 p4_sync(f, "-f")
cdc7e388 1489
4f5cf76a 1490 else:
c47178d4 1491 # skip this patch
5a41c16a 1492 ret = False
c47178d4
PW
1493 print "Submission cancelled, undoing p4 changes."
1494 for f in editedFiles:
1495 p4_revert(f)
1496 for f in filesToAdd:
1497 p4_revert(f)
1498 os.remove(f)
df9c5453
PW
1499 for f in filesToDelete:
1500 p4_revert(f)
c47178d4
PW
1501
1502 os.remove(fileName)
5a41c16a 1503 return ret
4f5cf76a 1504
06804c76
LD
1505 # Export git tags as p4 labels. Create a p4 label and then tag
1506 # with that.
1507 def exportGitTags(self, gitTags):
c8942a22
LD
1508 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1509 if len(validLabelRegexp) == 0:
1510 validLabelRegexp = defaultLabelRegexp
1511 m = re.compile(validLabelRegexp)
06804c76
LD
1512
1513 for name in gitTags:
1514
1515 if not m.match(name):
1516 if verbose:
05a3cec5 1517 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
06804c76
LD
1518 continue
1519
1520 # Get the p4 commit this corresponds to
c8942a22
LD
1521 logMessage = extractLogMessageFromGitCommit(name)
1522 values = extractSettingsGitLog(logMessage)
06804c76 1523
c8942a22 1524 if not values.has_key('change'):
06804c76
LD
1525 # a tag pointing to something not sent to p4; ignore
1526 if verbose:
1527 print "git tag %s does not give a p4 commit" % name
1528 continue
c8942a22
LD
1529 else:
1530 changelist = values['change']
06804c76
LD
1531
1532 # Get the tag details.
1533 inHeader = True
1534 isAnnotated = False
1535 body = []
1536 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1537 l = l.strip()
1538 if inHeader:
1539 if re.match(r'tag\s+', l):
1540 isAnnotated = True
1541 elif re.match(r'\s*$', l):
1542 inHeader = False
1543 continue
1544 else:
1545 body.append(l)
1546
1547 if not isAnnotated:
1548 body = ["lightweight tag imported by git p4\n"]
1549
1550 # Create the label - use the same view as the client spec we are using
1551 clientSpec = getClientSpec()
1552
1553 labelTemplate = "Label: %s\n" % name
1554 labelTemplate += "Description:\n"
1555 for b in body:
1556 labelTemplate += "\t" + b + "\n"
1557 labelTemplate += "View:\n"
1558 for mapping in clientSpec.mappings:
1559 labelTemplate += "\t%s\n" % mapping.depot_side.path
1560
ef739f08
PW
1561 if self.dry_run:
1562 print "Would create p4 label %s for tag" % name
728b7ad8
PW
1563 elif self.prepare_p4_only:
1564 print "Not creating p4 label %s for tag due to option" \
1565 " --prepare-p4-only" % name
ef739f08
PW
1566 else:
1567 p4_write_pipe(["label", "-i"], labelTemplate)
06804c76 1568
ef739f08
PW
1569 # Use the label
1570 p4_system(["tag", "-l", name] +
1571 ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
06804c76 1572
ef739f08
PW
1573 if verbose:
1574 print "created p4 label for tag %s" % name
06804c76 1575
4f5cf76a 1576 def run(self, args):
c9b50e63
SH
1577 if len(args) == 0:
1578 self.master = currentGitBranch()
4280e533 1579 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
c9b50e63
SH
1580 die("Detecting current git branch failed!")
1581 elif len(args) == 1:
1582 self.master = args[0]
28755dba
PW
1583 if not branchExists(self.master):
1584 die("Branch %s does not exist" % self.master)
c9b50e63
SH
1585 else:
1586 return False
1587
4c2d5d72
JX
1588 allowSubmit = gitConfig("git-p4.allowSubmit")
1589 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1590 die("%s is not in git-p4.allowSubmit" % self.master)
1591
27d2d811 1592 [upstream, settings] = findUpstreamBranchPoint()
ea99c3ae 1593 self.depotPath = settings['depot-paths'][0]
27d2d811
SH
1594 if len(self.origin) == 0:
1595 self.origin = upstream
a3fdd579 1596
3ea2cfd4
LD
1597 if self.preserveUser:
1598 if not self.canChangeChangelists():
1599 die("Cannot preserve user names without p4 super-user or admin permissions")
1600
6bbfd137
PW
1601 # if not set from the command line, try the config file
1602 if self.conflict_behavior is None:
1603 val = gitConfig("git-p4.conflict")
1604 if val:
1605 if val not in self.conflict_behavior_choices:
1606 die("Invalid value '%s' for config git-p4.conflict" % val)
1607 else:
1608 val = "ask"
1609 self.conflict_behavior = val
1610
a3fdd579
SH
1611 if self.verbose:
1612 print "Origin branch is " + self.origin
9512497b 1613
ea99c3ae 1614 if len(self.depotPath) == 0:
9512497b
SH
1615 print "Internal error: cannot locate perforce depot path from existing branches"
1616 sys.exit(128)
1617
543987bd 1618 self.useClientSpec = False
0d609032 1619 if gitConfigBool("git-p4.useclientspec"):
543987bd
PW
1620 self.useClientSpec = True
1621 if self.useClientSpec:
1622 self.clientSpecDirs = getClientSpec()
9512497b 1623
543987bd
PW
1624 if self.useClientSpec:
1625 # all files are relative to the client spec
1626 self.clientPath = getClientRoot()
1627 else:
1628 self.clientPath = p4Where(self.depotPath)
9512497b 1629
543987bd
PW
1630 if self.clientPath == "":
1631 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
9512497b 1632
ea99c3ae 1633 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
7944f142 1634 self.oldWorkingDirectory = os.getcwd()
c1b296b9 1635
0591cfa8 1636 # ensure the clientPath exists
8d7ec362 1637 new_client_dir = False
0591cfa8 1638 if not os.path.exists(self.clientPath):
8d7ec362 1639 new_client_dir = True
0591cfa8
GG
1640 os.makedirs(self.clientPath)
1641
bbd84863 1642 chdir(self.clientPath, is_client_path=True)
ef739f08
PW
1643 if self.dry_run:
1644 print "Would synchronize p4 checkout in %s" % self.clientPath
8d7ec362 1645 else:
ef739f08
PW
1646 print "Synchronizing p4 checkout..."
1647 if new_client_dir:
1648 # old one was destroyed, and maybe nobody told p4
1649 p4_sync("...", "-f")
1650 else:
1651 p4_sync("...")
4f5cf76a 1652 self.check()
4f5cf76a 1653
4c750c0d 1654 commits = []
c7d34884 1655 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
4c750c0d
SH
1656 commits.append(line.strip())
1657 commits.reverse()
4f5cf76a 1658
0d609032 1659 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
848de9c3
LD
1660 self.checkAuthorship = False
1661 else:
1662 self.checkAuthorship = True
1663
3ea2cfd4
LD
1664 if self.preserveUser:
1665 self.checkValidP4Users(commits)
1666
84cb0003
GG
1667 #
1668 # Build up a set of options to be passed to diff when
1669 # submitting each commit to p4.
1670 #
1671 if self.detectRenames:
1672 # command-line -M arg
1673 self.diffOpts = "-M"
1674 else:
1675 # If not explicitly set check the config variable
1676 detectRenames = gitConfig("git-p4.detectRenames")
1677
1678 if detectRenames.lower() == "false" or detectRenames == "":
1679 self.diffOpts = ""
1680 elif detectRenames.lower() == "true":
1681 self.diffOpts = "-M"
1682 else:
1683 self.diffOpts = "-M%s" % detectRenames
1684
1685 # no command-line arg for -C or --find-copies-harder, just
1686 # config variables
1687 detectCopies = gitConfig("git-p4.detectCopies")
1688 if detectCopies.lower() == "false" or detectCopies == "":
1689 pass
1690 elif detectCopies.lower() == "true":
1691 self.diffOpts += " -C"
1692 else:
1693 self.diffOpts += " -C%s" % detectCopies
1694
0d609032 1695 if gitConfigBool("git-p4.detectCopiesHarder"):
84cb0003
GG
1696 self.diffOpts += " --find-copies-harder"
1697
7e5dd9f2
PW
1698 #
1699 # Apply the commits, one at a time. On failure, ask if should
1700 # continue to try the rest of the patches, or quit.
1701 #
ef739f08
PW
1702 if self.dry_run:
1703 print "Would apply"
67b0fe2e 1704 applied = []
7e5dd9f2
PW
1705 last = len(commits) - 1
1706 for i, commit in enumerate(commits):
ef739f08
PW
1707 if self.dry_run:
1708 print " ", read_pipe(["git", "show", "-s",
1709 "--format=format:%h %s", commit])
1710 ok = True
1711 else:
1712 ok = self.applyCommit(commit)
67b0fe2e
PW
1713 if ok:
1714 applied.append(commit)
7e5dd9f2 1715 else:
728b7ad8
PW
1716 if self.prepare_p4_only and i < last:
1717 print "Processing only the first commit due to option" \
1718 " --prepare-p4-only"
1719 break
7e5dd9f2
PW
1720 if i < last:
1721 quit = False
1722 while True:
6bbfd137
PW
1723 # prompt for what to do, or use the option/variable
1724 if self.conflict_behavior == "ask":
1725 print "What do you want to do?"
1726 response = raw_input("[s]kip this commit but apply"
1727 " the rest, or [q]uit? ")
1728 if not response:
1729 continue
1730 elif self.conflict_behavior == "skip":
1731 response = "s"
1732 elif self.conflict_behavior == "quit":
1733 response = "q"
1734 else:
1735 die("Unknown conflict_behavior '%s'" %
1736 self.conflict_behavior)
1737
7e5dd9f2
PW
1738 if response[0] == "s":
1739 print "Skipping this commit, but applying the rest"
1740 break
1741 if response[0] == "q":
1742 print "Quitting"
1743 quit = True
1744 break
1745 if quit:
1746 break
4f5cf76a 1747
67b0fe2e 1748 chdir(self.oldWorkingDirectory)
4f5cf76a 1749
ef739f08
PW
1750 if self.dry_run:
1751 pass
728b7ad8
PW
1752 elif self.prepare_p4_only:
1753 pass
ef739f08 1754 elif len(commits) == len(applied):
67b0fe2e 1755 print "All commits applied!"
14594f4b 1756
4c750c0d 1757 sync = P4Sync()
44e8d26c
PW
1758 if self.branch:
1759 sync.branch = self.branch
4c750c0d 1760 sync.run([])
14594f4b 1761
4c750c0d
SH
1762 rebase = P4Rebase()
1763 rebase.rebase()
4f5cf76a 1764
67b0fe2e
PW
1765 else:
1766 if len(applied) == 0:
1767 print "No commits applied."
1768 else:
1769 print "Applied only the commits marked with '*':"
1770 for c in commits:
1771 if c in applied:
1772 star = "*"
1773 else:
1774 star = " "
1775 print star, read_pipe(["git", "show", "-s",
1776 "--format=format:%h %s", c])
1777 print "You will have to do 'git p4 sync' and rebase."
1778
0d609032 1779 if gitConfigBool("git-p4.exportLabels"):
06dcd152 1780 self.exportLabels = True
06804c76
LD
1781
1782 if self.exportLabels:
1783 p4Labels = getP4Labels(self.depotPath)
1784 gitTags = getGitTags()
1785
1786 missingGitTags = gitTags - p4Labels
1787 self.exportGitTags(missingGitTags)
1788
98e023de 1789 # exit with error unless everything applied perfectly
67b0fe2e
PW
1790 if len(commits) != len(applied):
1791 sys.exit(1)
1792
b984733c
SH
1793 return True
1794
ecb7cf98
PW
1795class View(object):
1796 """Represent a p4 view ("p4 help views"), and map files in a
1797 repo according to the view."""
1798
1799 class Path(object):
1800 """A depot or client path, possibly containing wildcards.
1801 The only one supported is ... at the end, currently.
1802 Initialize with the full path, with //depot or //client."""
1803
1804 def __init__(self, path, is_depot):
1805 self.path = path
1806 self.is_depot = is_depot
1807 self.find_wildcards()
1808 # remember the prefix bit, useful for relative mappings
1809 m = re.match("(//[^/]+/)", self.path)
1810 if not m:
1811 die("Path %s does not start with //prefix/" % self.path)
1812 prefix = m.group(1)
1813 if not self.is_depot:
1814 # strip //client/ on client paths
1815 self.path = self.path[len(prefix):]
1816
1817 def find_wildcards(self):
1818 """Make sure wildcards are valid, and set up internal
1819 variables."""
1820
1821 self.ends_triple_dot = False
1822 # There are three wildcards allowed in p4 views
1823 # (see "p4 help views"). This code knows how to
1824 # handle "..." (only at the end), but cannot deal with
1825 # "%%n" or "*". Only check the depot_side, as p4 should
1826 # validate that the client_side matches too.
1827 if re.search(r'%%[1-9]', self.path):
1828 die("Can't handle %%n wildcards in view: %s" % self.path)
1829 if self.path.find("*") >= 0:
1830 die("Can't handle * wildcards in view: %s" % self.path)
1831 triple_dot_index = self.path.find("...")
1832 if triple_dot_index >= 0:
43b82bd9
PW
1833 if triple_dot_index != len(self.path) - 3:
1834 die("Can handle only single ... wildcard, at end: %s" %
ecb7cf98
PW
1835 self.path)
1836 self.ends_triple_dot = True
1837
1838 def ensure_compatible(self, other_path):
1839 """Make sure the wildcards agree."""
1840 if self.ends_triple_dot != other_path.ends_triple_dot:
1841 die("Both paths must end with ... if either does;\n" +
1842 "paths: %s %s" % (self.path, other_path.path))
1843
1844 def match_wildcards(self, test_path):
1845 """See if this test_path matches us, and fill in the value
1846 of the wildcards if so. Returns a tuple of
1847 (True|False, wildcards[]). For now, only the ... at end
1848 is supported, so at most one wildcard."""
1849 if self.ends_triple_dot:
1850 dotless = self.path[:-3]
1851 if test_path.startswith(dotless):
1852 wildcard = test_path[len(dotless):]
1853 return (True, [ wildcard ])
1854 else:
1855 if test_path == self.path:
1856 return (True, [])
1857 return (False, [])
1858
1859 def match(self, test_path):
1860 """Just return if it matches; don't bother with the wildcards."""
1861 b, _ = self.match_wildcards(test_path)
1862 return b
1863
1864 def fill_in_wildcards(self, wildcards):
1865 """Return the relative path, with the wildcards filled in
1866 if there are any."""
1867 if self.ends_triple_dot:
1868 return self.path[:-3] + wildcards[0]
1869 else:
1870 return self.path
1871
1872 class Mapping(object):
1873 def __init__(self, depot_side, client_side, overlay, exclude):
1874 # depot_side is without the trailing /... if it had one
1875 self.depot_side = View.Path(depot_side, is_depot=True)
1876 self.client_side = View.Path(client_side, is_depot=False)
1877 self.overlay = overlay # started with "+"
1878 self.exclude = exclude # started with "-"
1879 assert not (self.overlay and self.exclude)
1880 self.depot_side.ensure_compatible(self.client_side)
1881
1882 def __str__(self):
1883 c = " "
1884 if self.overlay:
1885 c = "+"
1886 if self.exclude:
1887 c = "-"
1888 return "View.Mapping: %s%s -> %s" % \
329afb8e 1889 (c, self.depot_side.path, self.client_side.path)
ecb7cf98
PW
1890
1891 def map_depot_to_client(self, depot_path):
1892 """Calculate the client path if using this mapping on the
1893 given depot path; does not consider the effect of other
1894 mappings in a view. Even excluded mappings are returned."""
1895 matches, wildcards = self.depot_side.match_wildcards(depot_path)
1896 if not matches:
1897 return ""
1898 client_path = self.client_side.fill_in_wildcards(wildcards)
1899 return client_path
1900
1901 #
1902 # View methods
1903 #
1904 def __init__(self):
1905 self.mappings = []
1906
1907 def append(self, view_line):
1908 """Parse a view line, splitting it into depot and client
1909 sides. Append to self.mappings, preserving order."""
1910
1911 # Split the view line into exactly two words. P4 enforces
1912 # structure on these lines that simplifies this quite a bit.
1913 #
1914 # Either or both words may be double-quoted.
1915 # Single quotes do not matter.
1916 # Double-quote marks cannot occur inside the words.
1917 # A + or - prefix is also inside the quotes.
1918 # There are no quotes unless they contain a space.
1919 # The line is already white-space stripped.
1920 # The two words are separated by a single space.
1921 #
1922 if view_line[0] == '"':
1923 # First word is double quoted. Find its end.
1924 close_quote_index = view_line.find('"', 1)
1925 if close_quote_index <= 0:
1926 die("No first-word closing quote found: %s" % view_line)
1927 depot_side = view_line[1:close_quote_index]
1928 # skip closing quote and space
1929 rhs_index = close_quote_index + 1 + 1
1930 else:
1931 space_index = view_line.find(" ")
1932 if space_index <= 0:
1933 die("No word-splitting space found: %s" % view_line)
1934 depot_side = view_line[0:space_index]
1935 rhs_index = space_index + 1
1936
1937 if view_line[rhs_index] == '"':
1938 # Second word is double quoted. Make sure there is a
1939 # double quote at the end too.
1940 if not view_line.endswith('"'):
1941 die("View line with rhs quote should end with one: %s" %
1942 view_line)
1943 # skip the quotes
1944 client_side = view_line[rhs_index+1:-1]
1945 else:
1946 client_side = view_line[rhs_index:]
1947
1948 # prefix + means overlay on previous mapping
1949 overlay = False
1950 if depot_side.startswith("+"):
1951 overlay = True
1952 depot_side = depot_side[1:]
1953
1954 # prefix - means exclude this path
1955 exclude = False
1956 if depot_side.startswith("-"):
1957 exclude = True
1958 depot_side = depot_side[1:]
1959
1960 m = View.Mapping(depot_side, client_side, overlay, exclude)
1961 self.mappings.append(m)
1962
1963 def map_in_client(self, depot_path):
1964 """Return the relative location in the client where this
1965 depot file should live. Returns "" if the file should
1966 not be mapped in the client."""
1967
1968 paths_filled = []
1969 client_path = ""
1970
1971 # look at later entries first
1972 for m in self.mappings[::-1]:
1973
1974 # see where will this path end up in the client
1975 p = m.map_depot_to_client(depot_path)
1976
1977 if p == "":
1978 # Depot path does not belong in client. Must remember
1979 # this, as previous items should not cause files to
1980 # exist in this path either. Remember that the list is
1981 # being walked from the end, which has higher precedence.
1982 # Overlap mappings do not exclude previous mappings.
1983 if not m.overlay:
1984 paths_filled.append(m.client_side)
1985
1986 else:
1987 # This mapping matched; no need to search any further.
1988 # But, the mapping could be rejected if the client path
6ee9a999
PW
1989 # has already been claimed by an earlier mapping (i.e.
1990 # one later in the list, which we are walking backwards).
ecb7cf98
PW
1991 already_mapped_in_client = False
1992 for f in paths_filled:
1993 # this is View.Path.match
1994 if f.match(p):
1995 already_mapped_in_client = True
1996 break
1997 if not already_mapped_in_client:
1998 # Include this file, unless it is from a line that
1999 # explicitly said to exclude it.
2000 if not m.exclude:
2001 client_path = p
2002
2003 # a match, even if rejected, always stops the search
2004 break
2005
2006 return client_path
2007
3ea2cfd4 2008class P4Sync(Command, P4UserMap):
56c09345
PW
2009 delete_actions = ( "delete", "move/delete", "purge" )
2010
b984733c
SH
2011 def __init__(self):
2012 Command.__init__(self)
3ea2cfd4 2013 P4UserMap.__init__(self)
b984733c
SH
2014 self.options = [
2015 optparse.make_option("--branch", dest="branch"),
2016 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2017 optparse.make_option("--changesfile", dest="changesFile"),
2018 optparse.make_option("--silent", dest="silent", action="store_true"),
ef48f909 2019 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
06804c76 2020 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
d2c6dd30
HWN
2021 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2022 help="Import into refs/heads/ , not refs/remotes"),
8b41a97f 2023 optparse.make_option("--max-changes", dest="maxChanges"),
86dff6b6 2024 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
3a70cdfa
TAL
2025 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2026 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2027 help="Only sync files that are included in the Perforce Client Spec")
b984733c
SH
2028 ]
2029 self.description = """Imports from Perforce into a git repository.\n
2030 example:
2031 //depot/my/project/ -- to import the current head
2032 //depot/my/project/@all -- to import everything
2033 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2034
2035 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2036
2037 self.usage += " //depot/path[@revRange]"
b984733c 2038 self.silent = False
1d7367dc
RG
2039 self.createdBranches = set()
2040 self.committedChanges = set()
569d1bd4 2041 self.branch = ""
b984733c 2042 self.detectBranches = False
cb53e1f8 2043 self.detectLabels = False
06804c76 2044 self.importLabels = False
b984733c 2045 self.changesFile = ""
01265103 2046 self.syncWithOrigin = True
a028a98e 2047 self.importIntoRemotes = True
01a9c9c5 2048 self.maxChanges = ""
8b41a97f 2049 self.keepRepoPath = False
6326aa58 2050 self.depotPaths = None
3c699645 2051 self.p4BranchesInGit = []
354081d5 2052 self.cloneExclude = []
3a70cdfa 2053 self.useClientSpec = False
a93d33ee 2054 self.useClientSpec_from_options = False
ecb7cf98 2055 self.clientSpecDirs = None
fed23693
VA
2056 self.tempBranches = []
2057 self.tempBranchLocation = "git-p4-tmp"
b984733c 2058
01265103
SH
2059 if gitConfig("git-p4.syncFromOrigin") == "false":
2060 self.syncWithOrigin = False
2061
fed23693
VA
2062 # Force a checkpoint in fast-import and wait for it to finish
2063 def checkpoint(self):
2064 self.gitStream.write("checkpoint\n\n")
2065 self.gitStream.write("progress checkpoint\n\n")
2066 out = self.gitOutput.readline()
2067 if self.verbose:
2068 print "checkpoint finished: " + out
2069
b984733c 2070 def extractFilesFromCommit(self, commit):
354081d5
TT
2071 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2072 for path in self.cloneExclude]
b984733c
SH
2073 files = []
2074 fnum = 0
2075 while commit.has_key("depotFile%s" % fnum):
2076 path = commit["depotFile%s" % fnum]
6326aa58 2077
354081d5 2078 if [p for p in self.cloneExclude
d53de8b9 2079 if p4PathStartsWith(path, p)]:
354081d5
TT
2080 found = False
2081 else:
2082 found = [p for p in self.depotPaths
d53de8b9 2083 if p4PathStartsWith(path, p)]
6326aa58 2084 if not found:
b984733c
SH
2085 fnum = fnum + 1
2086 continue
2087
2088 file = {}
2089 file["path"] = path
2090 file["rev"] = commit["rev%s" % fnum]
2091 file["action"] = commit["action%s" % fnum]
2092 file["type"] = commit["type%s" % fnum]
2093 files.append(file)
2094 fnum = fnum + 1
2095 return files
2096
6326aa58 2097 def stripRepoPath(self, path, prefixes):
21ef5df4
PW
2098 """When streaming files, this is called to map a p4 depot path
2099 to where it should go in git. The prefixes are either
2100 self.depotPaths, or self.branchPrefixes in the case of
2101 branch detection."""
2102
3952710b 2103 if self.useClientSpec:
21ef5df4
PW
2104 # branch detection moves files up a level (the branch name)
2105 # from what client spec interpretation gives
0d1696ef 2106 path = self.clientSpecDirs.map_in_client(path)
21ef5df4
PW
2107 if self.detectBranches:
2108 for b in self.knownBranches:
2109 if path.startswith(b + "/"):
2110 path = path[len(b)+1:]
2111
2112 elif self.keepRepoPath:
2113 # Preserve everything in relative path name except leading
2114 # //depot/; just look at first prefix as they all should
2115 # be in the same depot.
2116 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2117 if p4PathStartsWith(path, depot):
2118 path = path[len(depot):]
3952710b 2119
0d1696ef 2120 else:
0d1696ef
PW
2121 for p in prefixes:
2122 if p4PathStartsWith(path, p):
2123 path = path[len(p):]
21ef5df4 2124 break
8b41a97f 2125
0d1696ef 2126 path = wildcard_decode(path)
6326aa58 2127 return path
6754a299 2128
71b112d4 2129 def splitFilesIntoBranches(self, commit):
21ef5df4
PW
2130 """Look at each depotFile in the commit to figure out to what
2131 branch it belongs."""
2132
d5904674 2133 branches = {}
71b112d4
SH
2134 fnum = 0
2135 while commit.has_key("depotFile%s" % fnum):
2136 path = commit["depotFile%s" % fnum]
6326aa58 2137 found = [p for p in self.depotPaths
d53de8b9 2138 if p4PathStartsWith(path, p)]
6326aa58 2139 if not found:
71b112d4
SH
2140 fnum = fnum + 1
2141 continue
2142
2143 file = {}
2144 file["path"] = path
2145 file["rev"] = commit["rev%s" % fnum]
2146 file["action"] = commit["action%s" % fnum]
2147 file["type"] = commit["type%s" % fnum]
2148 fnum = fnum + 1
2149
21ef5df4
PW
2150 # start with the full relative path where this file would
2151 # go in a p4 client
2152 if self.useClientSpec:
2153 relPath = self.clientSpecDirs.map_in_client(path)
2154 else:
2155 relPath = self.stripRepoPath(path, self.depotPaths)
b984733c 2156
4b97ffb1 2157 for branch in self.knownBranches.keys():
21ef5df4
PW
2158 # add a trailing slash so that a commit into qt/4.2foo
2159 # doesn't end up in qt/4.2, e.g.
6754a299 2160 if relPath.startswith(branch + "/"):
d5904674
SH
2161 if branch not in branches:
2162 branches[branch] = []
71b112d4 2163 branches[branch].append(file)
6555b2cc 2164 break
b984733c
SH
2165
2166 return branches
2167
b932705b
LD
2168 # output one file from the P4 stream
2169 # - helper for streamP4Files
2170
2171 def streamOneP4File(self, file, contents):
b932705b
LD
2172 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2173 if verbose:
2174 sys.stderr.write("%s\n" % relPath)
2175
9cffb8c8
PW
2176 (type_base, type_mods) = split_p4_type(file["type"])
2177
2178 git_mode = "100644"
2179 if "x" in type_mods:
2180 git_mode = "100755"
2181 if type_base == "symlink":
2182 git_mode = "120000"
2183 # p4 print on a symlink contains "target\n"; remove the newline
b39c3612
EP
2184 data = ''.join(contents)
2185 contents = [data[:-1]]
b932705b 2186
9cffb8c8 2187 if type_base == "utf16":
55aa5714
PW
2188 # p4 delivers different text in the python output to -G
2189 # than it does when using "print -o", or normal p4 client
2190 # operations. utf16 is converted to ascii or utf8, perhaps.
2191 # But ascii text saved as -t utf16 is completely mangled.
2192 # Invoke print -o to get the real contents.
7f0e5962
PW
2193 #
2194 # On windows, the newlines will always be mangled by print, so put
2195 # them back too. This is not needed to the cygwin windows version,
2196 # just the native "NT" type.
2197 #
6de040df 2198 text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
7f0e5962
PW
2199 if p4_version_string().find("/NT") >= 0:
2200 text = text.replace("\r\n", "\n")
55aa5714
PW
2201 contents = [ text ]
2202
9f7ef0ea
PW
2203 if type_base == "apple":
2204 # Apple filetype files will be streamed as a concatenation of
2205 # its appledouble header and the contents. This is useless
2206 # on both macs and non-macs. If using "print -q -o xx", it
2207 # will create "xx" with the data, and "%xx" with the header.
2208 # This is also not very useful.
2209 #
2210 # Ideally, someday, this script can learn how to generate
2211 # appledouble files directly and import those to git, but
2212 # non-mac machines can never find a use for apple filetype.
2213 print "\nIgnoring apple filetype file %s" % file['depotFile']
2214 return
2215
55aa5714
PW
2216 # Note that we do not try to de-mangle keywords on utf16 files,
2217 # even though in theory somebody may want that.
60df071c
LD
2218 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2219 if pattern:
2220 regexp = re.compile(pattern, re.VERBOSE)
2221 text = ''.join(contents)
2222 text = regexp.sub(r'$\1$', text)
2223 contents = [ text ]
b932705b 2224
9cffb8c8 2225 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
b932705b
LD
2226
2227 # total length...
2228 length = 0
2229 for d in contents:
2230 length = length + len(d)
2231
2232 self.gitStream.write("data %d\n" % length)
2233 for d in contents:
2234 self.gitStream.write(d)
2235 self.gitStream.write("\n")
2236
2237 def streamOneP4Deletion(self, file):
2238 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2239 if verbose:
2240 sys.stderr.write("delete %s\n" % relPath)
2241 self.gitStream.write("D %s\n" % relPath)
2242
2243 # handle another chunk of streaming data
2244 def streamP4FilesCb(self, marshalled):
2245
78189bea
PW
2246 # catch p4 errors and complain
2247 err = None
2248 if "code" in marshalled:
2249 if marshalled["code"] == "error":
2250 if "data" in marshalled:
2251 err = marshalled["data"].rstrip()
2252 if err:
2253 f = None
2254 if self.stream_have_file_info:
2255 if "depotFile" in self.stream_file:
2256 f = self.stream_file["depotFile"]
2257 # force a failure in fast-import, else an empty
2258 # commit will be made
2259 self.gitStream.write("\n")
2260 self.gitStream.write("die-now\n")
2261 self.gitStream.close()
2262 # ignore errors, but make sure it exits first
2263 self.importProcess.wait()
2264 if f:
2265 die("Error from p4 print for %s: %s" % (f, err))
2266 else:
2267 die("Error from p4 print: %s" % err)
2268
c3f6163b
AG
2269 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2270 # start of a new file - output the old one first
2271 self.streamOneP4File(self.stream_file, self.stream_contents)
2272 self.stream_file = {}
2273 self.stream_contents = []
2274 self.stream_have_file_info = False
b932705b 2275
c3f6163b
AG
2276 # pick up the new file information... for the
2277 # 'data' field we need to append to our array
2278 for k in marshalled.keys():
2279 if k == 'data':
2280 self.stream_contents.append(marshalled['data'])
2281 else:
2282 self.stream_file[k] = marshalled[k]
b932705b 2283
c3f6163b 2284 self.stream_have_file_info = True
b932705b
LD
2285
2286 # Stream directly from "p4 files" into "git fast-import"
2287 def streamP4Files(self, files):
30b5940b
SH
2288 filesForCommit = []
2289 filesToRead = []
b932705b 2290 filesToDelete = []
30b5940b 2291
3a70cdfa 2292 for f in files:
ecb7cf98
PW
2293 # if using a client spec, only add the files that have
2294 # a path in the client
2295 if self.clientSpecDirs:
2296 if self.clientSpecDirs.map_in_client(f['path']) == "":
2297 continue
3a70cdfa 2298
ecb7cf98
PW
2299 filesForCommit.append(f)
2300 if f['action'] in self.delete_actions:
2301 filesToDelete.append(f)
2302 else:
2303 filesToRead.append(f)
6a49f8e2 2304
b932705b
LD
2305 # deleted files...
2306 for f in filesToDelete:
2307 self.streamOneP4Deletion(f)
1b9a4684 2308
b932705b
LD
2309 if len(filesToRead) > 0:
2310 self.stream_file = {}
2311 self.stream_contents = []
2312 self.stream_have_file_info = False
8ff45f2a 2313
c3f6163b
AG
2314 # curry self argument
2315 def streamP4FilesCbSelf(entry):
2316 self.streamP4FilesCb(entry)
6a49f8e2 2317
6de040df
LD
2318 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2319
2320 p4CmdList(["-x", "-", "print"],
2321 stdin=fileArgs,
2322 cb=streamP4FilesCbSelf)
30b5940b 2323
b932705b
LD
2324 # do the last chunk
2325 if self.stream_file.has_key('depotFile'):
2326 self.streamOneP4File(self.stream_file, self.stream_contents)
6a49f8e2 2327
affb474f
LD
2328 def make_email(self, userid):
2329 if userid in self.users:
2330 return self.users[userid]
2331 else:
2332 return "%s <a@b>" % userid
2333
06804c76
LD
2334 # Stream a p4 tag
2335 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2336 if verbose:
2337 print "writing tag %s for commit %s" % (labelName, commit)
2338 gitStream.write("tag %s\n" % labelName)
2339 gitStream.write("from %s\n" % commit)
2340
2341 if labelDetails.has_key('Owner'):
2342 owner = labelDetails["Owner"]
2343 else:
2344 owner = None
2345
2346 # Try to use the owner of the p4 label, or failing that,
2347 # the current p4 user id.
2348 if owner:
2349 email = self.make_email(owner)
2350 else:
2351 email = self.make_email(self.p4UserId())
2352 tagger = "%s %s %s" % (email, epoch, self.tz)
2353
2354 gitStream.write("tagger %s\n" % tagger)
2355
2356 print "labelDetails=",labelDetails
2357 if labelDetails.has_key('Description'):
2358 description = labelDetails['Description']
2359 else:
2360 description = 'Label from git p4'
2361
2362 gitStream.write("data %d\n" % len(description))
2363 gitStream.write(description)
2364 gitStream.write("\n")
2365
e63231e5 2366 def commit(self, details, files, branch, parent = ""):
b984733c
SH
2367 epoch = details["time"]
2368 author = details["user"]
2369
4b97ffb1
SH
2370 if self.verbose:
2371 print "commit into %s" % branch
2372
96e07dd2
HWN
2373 # start with reading files; if that fails, we should not
2374 # create a commit.
2375 new_files = []
2376 for f in files:
e63231e5 2377 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
96e07dd2
HWN
2378 new_files.append (f)
2379 else:
afa1dd9a 2380 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
96e07dd2 2381
b984733c 2382 self.gitStream.write("commit %s\n" % branch)
6a49f8e2 2383# gitStream.write("mark :%s\n" % details["change"])
b984733c
SH
2384 self.committedChanges.add(int(details["change"]))
2385 committer = ""
b607e71e
SH
2386 if author not in self.users:
2387 self.getUserMapFromPerforceServer()
affb474f 2388 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
b984733c
SH
2389
2390 self.gitStream.write("committer %s\n" % committer)
2391
2392 self.gitStream.write("data <<EOT\n")
2393 self.gitStream.write(details["desc"])
e63231e5
PW
2394 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2395 (','.join(self.branchPrefixes), details["change"]))
6581de09
SH
2396 if len(details['options']) > 0:
2397 self.gitStream.write(": options = %s" % details['options'])
2398 self.gitStream.write("]\nEOT\n\n")
b984733c
SH
2399
2400 if len(parent) > 0:
4b97ffb1
SH
2401 if self.verbose:
2402 print "parent %s" % parent
b984733c
SH
2403 self.gitStream.write("from %s\n" % parent)
2404
b932705b 2405 self.streamP4Files(new_files)
b984733c
SH
2406 self.gitStream.write("\n")
2407
1f4ba1cb
SH
2408 change = int(details["change"])
2409
9bda3a85 2410 if self.labels.has_key(change):
1f4ba1cb
SH
2411 label = self.labels[change]
2412 labelDetails = label[0]
2413 labelRevisions = label[1]
71b112d4
SH
2414 if self.verbose:
2415 print "Change %s is labelled %s" % (change, labelDetails)
1f4ba1cb 2416
6de040df 2417 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
e63231e5 2418 for p in self.branchPrefixes])
1f4ba1cb
SH
2419
2420 if len(files) == len(labelRevisions):
2421
2422 cleanedFiles = {}
2423 for info in files:
56c09345 2424 if info["action"] in self.delete_actions:
1f4ba1cb
SH
2425 continue
2426 cleanedFiles[info["depotFile"]] = info["rev"]
2427
2428 if cleanedFiles == labelRevisions:
06804c76 2429 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
1f4ba1cb
SH
2430
2431 else:
a46668fa 2432 if not self.silent:
cebdf5af
HWN
2433 print ("Tag %s does not match with change %s: files do not match."
2434 % (labelDetails["label"], change))
1f4ba1cb
SH
2435
2436 else:
a46668fa 2437 if not self.silent:
cebdf5af
HWN
2438 print ("Tag %s does not match with change %s: file count is different."
2439 % (labelDetails["label"], change))
b984733c 2440
06804c76 2441 # Build a dictionary of changelists and labels, for "detect-labels" option.
1f4ba1cb
SH
2442 def getLabels(self):
2443 self.labels = {}
2444
52a4880b 2445 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
10c3211b 2446 if len(l) > 0 and not self.silent:
183f8436 2447 print "Finding files belonging to labels in %s" % `self.depotPaths`
01ce1fe9
SH
2448
2449 for output in l:
1f4ba1cb
SH
2450 label = output["label"]
2451 revisions = {}
2452 newestChange = 0
71b112d4
SH
2453 if self.verbose:
2454 print "Querying files for label %s" % label
6de040df
LD
2455 for file in p4CmdList(["files"] +
2456 ["%s...@%s" % (p, label)
2457 for p in self.depotPaths]):
1f4ba1cb
SH
2458 revisions[file["depotFile"]] = file["rev"]
2459 change = int(file["change"])
2460 if change > newestChange:
2461 newestChange = change
2462
9bda3a85
SH
2463 self.labels[newestChange] = [output, revisions]
2464
2465 if self.verbose:
2466 print "Label changes: %s" % self.labels.keys()
1f4ba1cb 2467
06804c76
LD
2468 # Import p4 labels as git tags. A direct mapping does not
2469 # exist, so assume that if all the files are at the same revision
2470 # then we can use that, or it's something more complicated we should
2471 # just ignore.
2472 def importP4Labels(self, stream, p4Labels):
2473 if verbose:
2474 print "import p4 labels: " + ' '.join(p4Labels)
2475
2476 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
c8942a22 2477 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
06804c76
LD
2478 if len(validLabelRegexp) == 0:
2479 validLabelRegexp = defaultLabelRegexp
2480 m = re.compile(validLabelRegexp)
2481
2482 for name in p4Labels:
2483 commitFound = False
2484
2485 if not m.match(name):
2486 if verbose:
2487 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2488 continue
2489
2490 if name in ignoredP4Labels:
2491 continue
2492
2493 labelDetails = p4CmdList(['label', "-o", name])[0]
2494
2495 # get the most recent changelist for each file in this label
2496 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2497 for p in self.depotPaths])
2498
2499 if change.has_key('change'):
2500 # find the corresponding git commit; take the oldest commit
2501 changelist = int(change['change'])
2502 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2503 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2504 if len(gitCommit) == 0:
2505 print "could not find git commit for changelist %d" % changelist
2506 else:
2507 gitCommit = gitCommit.strip()
2508 commitFound = True
2509 # Convert from p4 time format
2510 try:
2511 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2512 except ValueError:
a4e9054c 2513 print "Could not convert label time %s" % labelDetails['Update']
06804c76
LD
2514 tmwhen = 1
2515
2516 when = int(time.mktime(tmwhen))
2517 self.streamTag(stream, name, labelDetails, gitCommit, when)
2518 if verbose:
2519 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2520 else:
2521 if verbose:
2522 print "Label %s has no changelists - possibly deleted?" % name
2523
2524 if not commitFound:
2525 # We can't import this label; don't try again as it will get very
2526 # expensive repeatedly fetching all the files for labels that will
2527 # never be imported. If the label is moved in the future, the
2528 # ignore will need to be removed manually.
2529 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2530
86dff6b6
HWN
2531 def guessProjectName(self):
2532 for p in self.depotPaths:
6e5295c4
SH
2533 if p.endswith("/"):
2534 p = p[:-1]
2535 p = p[p.strip().rfind("/") + 1:]
2536 if not p.endswith("/"):
2537 p += "/"
2538 return p
86dff6b6 2539
4b97ffb1 2540 def getBranchMapping(self):
6555b2cc
SH
2541 lostAndFoundBranches = set()
2542
8ace74c0
VA
2543 user = gitConfig("git-p4.branchUser")
2544 if len(user) > 0:
2545 command = "branches -u %s" % user
2546 else:
2547 command = "branches"
2548
2549 for info in p4CmdList(command):
52a4880b 2550 details = p4Cmd(["branch", "-o", info["branch"]])
4b97ffb1
SH
2551 viewIdx = 0
2552 while details.has_key("View%s" % viewIdx):
2553 paths = details["View%s" % viewIdx].split(" ")
2554 viewIdx = viewIdx + 1
2555 # require standard //depot/foo/... //depot/bar/... mapping
2556 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2557 continue
2558 source = paths[0]
2559 destination = paths[1]
6509e19c 2560 ## HACK
d53de8b9 2561 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
6509e19c
SH
2562 source = source[len(self.depotPaths[0]):-4]
2563 destination = destination[len(self.depotPaths[0]):-4]
6555b2cc 2564
1a2edf4e
SH
2565 if destination in self.knownBranches:
2566 if not self.silent:
2567 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2568 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2569 continue
2570
6555b2cc
SH
2571 self.knownBranches[destination] = source
2572
2573 lostAndFoundBranches.discard(destination)
2574
29bdbac1 2575 if source not in self.knownBranches:
6555b2cc
SH
2576 lostAndFoundBranches.add(source)
2577
7199cf13
VA
2578 # Perforce does not strictly require branches to be defined, so we also
2579 # check git config for a branch list.
2580 #
2581 # Example of branch definition in git config file:
2582 # [git-p4]
2583 # branchList=main:branchA
2584 # branchList=main:branchB
2585 # branchList=branchA:branchC
2586 configBranches = gitConfigList("git-p4.branchList")
2587 for branch in configBranches:
2588 if branch:
2589 (source, destination) = branch.split(":")
2590 self.knownBranches[destination] = source
2591
2592 lostAndFoundBranches.discard(destination)
2593
2594 if source not in self.knownBranches:
2595 lostAndFoundBranches.add(source)
2596
6555b2cc
SH
2597
2598 for branch in lostAndFoundBranches:
2599 self.knownBranches[branch] = branch
29bdbac1 2600
38f9f5ec
SH
2601 def getBranchMappingFromGitBranches(self):
2602 branches = p4BranchesInGit(self.importIntoRemotes)
2603 for branch in branches.keys():
2604 if branch == "master":
2605 branch = "main"
2606 else:
2607 branch = branch[len(self.projectName):]
2608 self.knownBranches[branch] = branch
2609
bb6e09b2
HWN
2610 def updateOptionDict(self, d):
2611 option_keys = {}
2612 if self.keepRepoPath:
2613 option_keys['keepRepoPath'] = 1
2614
2615 d["options"] = ' '.join(sorted(option_keys.keys()))
2616
2617 def readOptions(self, d):
2618 self.keepRepoPath = (d.has_key('options')
2619 and ('keepRepoPath' in d['options']))
6326aa58 2620
8134f69c
SH
2621 def gitRefForBranch(self, branch):
2622 if branch == "main":
2623 return self.refPrefix + "master"
2624
2625 if len(branch) <= 0:
2626 return branch
2627
2628 return self.refPrefix + self.projectName + branch
2629
1ca3d710
SH
2630 def gitCommitByP4Change(self, ref, change):
2631 if self.verbose:
2632 print "looking in ref " + ref + " for change %s using bisect..." % change
2633
2634 earliestCommit = ""
2635 latestCommit = parseRevision(ref)
2636
2637 while True:
2638 if self.verbose:
2639 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2640 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2641 if len(next) == 0:
2642 if self.verbose:
2643 print "argh"
2644 return ""
2645 log = extractLogMessageFromGitCommit(next)
2646 settings = extractSettingsGitLog(log)
2647 currentChange = int(settings['change'])
2648 if self.verbose:
2649 print "current change %s" % currentChange
2650
2651 if currentChange == change:
2652 if self.verbose:
2653 print "found %s" % next
2654 return next
2655
2656 if currentChange < change:
2657 earliestCommit = "^%s" % next
2658 else:
2659 latestCommit = "%s" % next
2660
2661 return ""
2662
2663 def importNewBranch(self, branch, maxChange):
2664 # make fast-import flush all changes to disk and update the refs using the checkpoint
2665 # command so that we can try to find the branch parent in the git history
2666 self.gitStream.write("checkpoint\n\n");
2667 self.gitStream.flush();
2668 branchPrefix = self.depotPaths[0] + branch + "/"
2669 range = "@1,%s" % maxChange
2670 #print "prefix" + branchPrefix
2671 changes = p4ChangesForPaths([branchPrefix], range)
2672 if len(changes) <= 0:
2673 return False
2674 firstChange = changes[0]
2675 #print "first change in branch: %s" % firstChange
2676 sourceBranch = self.knownBranches[branch]
2677 sourceDepotPath = self.depotPaths[0] + sourceBranch
2678 sourceRef = self.gitRefForBranch(sourceBranch)
2679 #print "source " + sourceBranch
2680
52a4880b 2681 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1ca3d710
SH
2682 #print "branch parent: %s" % branchParentChange
2683 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2684 if len(gitParent) > 0:
2685 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2686 #print "parent git commit: %s" % gitParent
2687
2688 self.importChanges(changes)
2689 return True
2690
fed23693
VA
2691 def searchParent(self, parent, branch, target):
2692 parentFound = False
c7d34884
PW
2693 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2694 "--no-merges", parent]):
fed23693
VA
2695 blob = blob.strip()
2696 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2697 parentFound = True
2698 if self.verbose:
2699 print "Found parent of %s in commit %s" % (branch, blob)
2700 break
2701 if parentFound:
2702 return blob
2703 else:
2704 return None
2705
e87f37ae
SH
2706 def importChanges(self, changes):
2707 cnt = 1
2708 for change in changes:
18fa13d0 2709 description = p4_describe(change)
e87f37ae
SH
2710 self.updateOptionDict(description)
2711
2712 if not self.silent:
2713 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2714 sys.stdout.flush()
2715 cnt = cnt + 1
2716
2717 try:
2718 if self.detectBranches:
2719 branches = self.splitFilesIntoBranches(description)
2720 for branch in branches.keys():
2721 ## HACK --hwn
2722 branchPrefix = self.depotPaths[0] + branch + "/"
e63231e5 2723 self.branchPrefixes = [ branchPrefix ]
e87f37ae
SH
2724
2725 parent = ""
2726
2727 filesForCommit = branches[branch]
2728
2729 if self.verbose:
2730 print "branch is %s" % branch
2731
2732 self.updatedBranches.add(branch)
2733
2734 if branch not in self.createdBranches:
2735 self.createdBranches.add(branch)
2736 parent = self.knownBranches[branch]
2737 if parent == branch:
2738 parent = ""
1ca3d710
SH
2739 else:
2740 fullBranch = self.projectName + branch
2741 if fullBranch not in self.p4BranchesInGit:
2742 if not self.silent:
2743 print("\n Importing new branch %s" % fullBranch);
2744 if self.importNewBranch(branch, change - 1):
2745 parent = ""
2746 self.p4BranchesInGit.append(fullBranch)
2747 if not self.silent:
2748 print("\n Resuming with change %s" % change);
2749
2750 if self.verbose:
2751 print "parent determined through known branches: %s" % parent
e87f37ae 2752
8134f69c
SH
2753 branch = self.gitRefForBranch(branch)
2754 parent = self.gitRefForBranch(parent)
e87f37ae
SH
2755
2756 if self.verbose:
2757 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2758
2759 if len(parent) == 0 and branch in self.initialParents:
2760 parent = self.initialParents[branch]
2761 del self.initialParents[branch]
2762
fed23693
VA
2763 blob = None
2764 if len(parent) > 0:
4f9273d2 2765 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
fed23693
VA
2766 if self.verbose:
2767 print "Creating temporary branch: " + tempBranch
e63231e5 2768 self.commit(description, filesForCommit, tempBranch)
fed23693
VA
2769 self.tempBranches.append(tempBranch)
2770 self.checkpoint()
2771 blob = self.searchParent(parent, branch, tempBranch)
2772 if blob:
e63231e5 2773 self.commit(description, filesForCommit, branch, blob)
fed23693
VA
2774 else:
2775 if self.verbose:
2776 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
e63231e5 2777 self.commit(description, filesForCommit, branch, parent)
e87f37ae
SH
2778 else:
2779 files = self.extractFilesFromCommit(description)
e63231e5 2780 self.commit(description, files, self.branch,
e87f37ae 2781 self.initialParent)
47497844 2782 # only needed once, to connect to the previous commit
e87f37ae
SH
2783 self.initialParent = ""
2784 except IOError:
2785 print self.gitError.read()
2786 sys.exit(1)
2787
c208a243
SH
2788 def importHeadRevision(self, revision):
2789 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2790
4e2e6ce4
PW
2791 details = {}
2792 details["user"] = "git perforce import user"
1494fcbb 2793 details["desc"] = ("Initial import of %s from the state at revision %s\n"
c208a243
SH
2794 % (' '.join(self.depotPaths), revision))
2795 details["change"] = revision
2796 newestRevision = 0
2797
2798 fileCnt = 0
6de040df
LD
2799 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2800
2801 for info in p4CmdList(["files"] + fileArgs):
c208a243 2802
68b28593 2803 if 'code' in info and info['code'] == 'error':
c208a243
SH
2804 sys.stderr.write("p4 returned an error: %s\n"
2805 % info['data'])
d88e707f
PW
2806 if info['data'].find("must refer to client") >= 0:
2807 sys.stderr.write("This particular p4 error is misleading.\n")
2808 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2809 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
c208a243 2810 sys.exit(1)
68b28593
PW
2811 if 'p4ExitCode' in info:
2812 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
c208a243
SH
2813 sys.exit(1)
2814
2815
2816 change = int(info["change"])
2817 if change > newestRevision:
2818 newestRevision = change
2819
56c09345 2820 if info["action"] in self.delete_actions:
c208a243
SH
2821 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2822 #fileCnt = fileCnt + 1
2823 continue
2824
2825 for prop in ["depotFile", "rev", "action", "type" ]:
2826 details["%s%s" % (prop, fileCnt)] = info[prop]
2827
2828 fileCnt = fileCnt + 1
2829
2830 details["change"] = newestRevision
4e2e6ce4 2831
9dcb9f24 2832 # Use time from top-most change so that all git p4 clones of
4e2e6ce4 2833 # the same p4 repo have the same commit SHA1s.
18fa13d0
PW
2834 res = p4_describe(newestRevision)
2835 details["time"] = res["time"]
4e2e6ce4 2836
c208a243
SH
2837 self.updateOptionDict(details)
2838 try:
e63231e5 2839 self.commit(details, self.extractFilesFromCommit(details), self.branch)
c208a243
SH
2840 except IOError:
2841 print "IO error with git fast-import. Is your git version recent enough?"
2842 print self.gitError.read()
2843
2844
b984733c 2845 def run(self, args):
6326aa58 2846 self.depotPaths = []
179caebf 2847 self.changeRange = ""
6326aa58 2848 self.previousDepotPaths = []
991a2de4 2849 self.hasOrigin = False
ce6f33c8 2850
29bdbac1
SH
2851 # map from branch depot path to parent branch
2852 self.knownBranches = {}
2853 self.initialParents = {}
2854
a028a98e
SH
2855 if self.importIntoRemotes:
2856 self.refPrefix = "refs/remotes/p4/"
2857 else:
db775559 2858 self.refPrefix = "refs/heads/p4/"
a028a98e 2859
991a2de4
PW
2860 if self.syncWithOrigin:
2861 self.hasOrigin = originP4BranchesExist()
2862 if self.hasOrigin:
2863 if not self.silent:
2864 print 'Syncing with origin first, using "git fetch origin"'
2865 system("git fetch origin")
10f880f8 2866
5a8e84cd 2867 branch_arg_given = bool(self.branch)
569d1bd4 2868 if len(self.branch) == 0:
db775559 2869 self.branch = self.refPrefix + "master"
a028a98e 2870 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
48df6fd8 2871 system("git update-ref %s refs/heads/p4" % self.branch)
55d12437 2872 system("git branch -D p4")
967f72e2 2873
a93d33ee
PW
2874 # accept either the command-line option, or the configuration variable
2875 if self.useClientSpec:
2876 # will use this after clone to set the variable
2877 self.useClientSpec_from_options = True
2878 else:
0d609032 2879 if gitConfigBool("git-p4.useclientspec"):
09fca77b
PW
2880 self.useClientSpec = True
2881 if self.useClientSpec:
543987bd 2882 self.clientSpecDirs = getClientSpec()
3a70cdfa 2883
6a49f8e2
HWN
2884 # TODO: should always look at previous commits,
2885 # merge with previous imports, if possible.
2886 if args == []:
d414c74a 2887 if self.hasOrigin:
5ca44617 2888 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3b650fc9
PW
2889
2890 # branches holds mapping from branch name to sha1
2891 branches = p4BranchesInGit(self.importIntoRemotes)
8c9e8b6e
PW
2892
2893 # restrict to just this one, disabling detect-branches
2894 if branch_arg_given:
2895 short = self.branch.split("/")[-1]
2896 if short in branches:
2897 self.p4BranchesInGit = [ short ]
2898 else:
2899 self.p4BranchesInGit = branches.keys()
abcd790f
SH
2900
2901 if len(self.p4BranchesInGit) > 1:
2902 if not self.silent:
2903 print "Importing from/into multiple branches"
2904 self.detectBranches = True
8c9e8b6e
PW
2905 for branch in branches.keys():
2906 self.initialParents[self.refPrefix + branch] = \
2907 branches[branch]
967f72e2 2908
29bdbac1
SH
2909 if self.verbose:
2910 print "branches: %s" % self.p4BranchesInGit
2911
2912 p4Change = 0
2913 for branch in self.p4BranchesInGit:
cebdf5af 2914 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
bb6e09b2
HWN
2915
2916 settings = extractSettingsGitLog(logMsg)
29bdbac1 2917
bb6e09b2
HWN
2918 self.readOptions(settings)
2919 if (settings.has_key('depot-paths')
2920 and settings.has_key ('change')):
2921 change = int(settings['change']) + 1
29bdbac1
SH
2922 p4Change = max(p4Change, change)
2923
bb6e09b2
HWN
2924 depotPaths = sorted(settings['depot-paths'])
2925 if self.previousDepotPaths == []:
6326aa58 2926 self.previousDepotPaths = depotPaths
29bdbac1 2927 else:
6326aa58
HWN
2928 paths = []
2929 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
04d277b3
VA
2930 prev_list = prev.split("/")
2931 cur_list = cur.split("/")
2932 for i in range(0, min(len(cur_list), len(prev_list))):
2933 if cur_list[i] <> prev_list[i]:
583e1707 2934 i = i - 1
6326aa58
HWN
2935 break
2936
04d277b3 2937 paths.append ("/".join(cur_list[:i + 1]))
6326aa58
HWN
2938
2939 self.previousDepotPaths = paths
29bdbac1
SH
2940
2941 if p4Change > 0:
bb6e09b2 2942 self.depotPaths = sorted(self.previousDepotPaths)
d5904674 2943 self.changeRange = "@%s,#head" % p4Change
341dc1c1 2944 if not self.silent and not self.detectBranches:
967f72e2 2945 print "Performing incremental import into %s git branch" % self.branch
569d1bd4 2946
40d69ac3
PW
2947 # accept multiple ref name abbreviations:
2948 # refs/foo/bar/branch -> use it exactly
2949 # p4/branch -> prepend refs/remotes/ or refs/heads/
2950 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
f9162f6a 2951 if not self.branch.startswith("refs/"):
40d69ac3
PW
2952 if self.importIntoRemotes:
2953 prepend = "refs/remotes/"
2954 else:
2955 prepend = "refs/heads/"
2956 if not self.branch.startswith("p4/"):
2957 prepend += "p4/"
2958 self.branch = prepend + self.branch
179caebf 2959
6326aa58 2960 if len(args) == 0 and self.depotPaths:
b984733c 2961 if not self.silent:
6326aa58 2962 print "Depot paths: %s" % ' '.join(self.depotPaths)
b984733c 2963 else:
6326aa58 2964 if self.depotPaths and self.depotPaths != args:
cebdf5af 2965 print ("previous import used depot path %s and now %s was specified. "
6326aa58
HWN
2966 "This doesn't work!" % (' '.join (self.depotPaths),
2967 ' '.join (args)))
b984733c 2968 sys.exit(1)
6326aa58 2969
bb6e09b2 2970 self.depotPaths = sorted(args)
b984733c 2971
1c49fc19 2972 revision = ""
b984733c 2973 self.users = {}
b984733c 2974
58c8bc7c
PW
2975 # Make sure no revision specifiers are used when --changesfile
2976 # is specified.
2977 bad_changesfile = False
2978 if len(self.changesFile) > 0:
2979 for p in self.depotPaths:
2980 if p.find("@") >= 0 or p.find("#") >= 0:
2981 bad_changesfile = True
2982 break
2983 if bad_changesfile:
2984 die("Option --changesfile is incompatible with revision specifiers")
2985
6326aa58
HWN
2986 newPaths = []
2987 for p in self.depotPaths:
2988 if p.find("@") != -1:
2989 atIdx = p.index("@")
2990 self.changeRange = p[atIdx:]
2991 if self.changeRange == "@all":
2992 self.changeRange = ""
6a49f8e2 2993 elif ',' not in self.changeRange:
1c49fc19 2994 revision = self.changeRange
6326aa58 2995 self.changeRange = ""
7fcff9de 2996 p = p[:atIdx]
6326aa58
HWN
2997 elif p.find("#") != -1:
2998 hashIdx = p.index("#")
1c49fc19 2999 revision = p[hashIdx:]
7fcff9de 3000 p = p[:hashIdx]
6326aa58 3001 elif self.previousDepotPaths == []:
58c8bc7c
PW
3002 # pay attention to changesfile, if given, else import
3003 # the entire p4 tree at the head revision
3004 if len(self.changesFile) == 0:
3005 revision = "#head"
6326aa58
HWN
3006
3007 p = re.sub ("\.\.\.$", "", p)
3008 if not p.endswith("/"):
3009 p += "/"
3010
3011 newPaths.append(p)
3012
3013 self.depotPaths = newPaths
3014
e63231e5
PW
3015 # --detect-branches may change this for each branch
3016 self.branchPrefixes = self.depotPaths
3017
b607e71e 3018 self.loadUserMapFromCache()
cb53e1f8
SH
3019 self.labels = {}
3020 if self.detectLabels:
3021 self.getLabels();
b984733c 3022
4b97ffb1 3023 if self.detectBranches:
df450923
SH
3024 ## FIXME - what's a P4 projectName ?
3025 self.projectName = self.guessProjectName()
3026
38f9f5ec
SH
3027 if self.hasOrigin:
3028 self.getBranchMappingFromGitBranches()
3029 else:
3030 self.getBranchMapping()
29bdbac1
SH
3031 if self.verbose:
3032 print "p4-git branches: %s" % self.p4BranchesInGit
3033 print "initial parents: %s" % self.initialParents
3034 for b in self.p4BranchesInGit:
3035 if b != "master":
6326aa58
HWN
3036
3037 ## FIXME
29bdbac1
SH
3038 b = b[len(self.projectName):]
3039 self.createdBranches.add(b)
4b97ffb1 3040
f291b4e3 3041 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
b984733c 3042
78189bea
PW
3043 self.importProcess = subprocess.Popen(["git", "fast-import"],
3044 stdin=subprocess.PIPE,
3045 stdout=subprocess.PIPE,
3046 stderr=subprocess.PIPE);
3047 self.gitOutput = self.importProcess.stdout
3048 self.gitStream = self.importProcess.stdin
3049 self.gitError = self.importProcess.stderr
b984733c 3050
1c49fc19 3051 if revision:
c208a243 3052 self.importHeadRevision(revision)
b984733c
SH
3053 else:
3054 changes = []
3055
0828ab14 3056 if len(self.changesFile) > 0:
b984733c 3057 output = open(self.changesFile).readlines()
1d7367dc 3058 changeSet = set()
b984733c
SH
3059 for line in output:
3060 changeSet.add(int(line))
3061
3062 for change in changeSet:
3063 changes.append(change)
3064
3065 changes.sort()
3066 else:
9dcb9f24
PW
3067 # catch "git p4 sync" with no new branches, in a repo that
3068 # does not have any existing p4 branches
5a8e84cd
PW
3069 if len(args) == 0:
3070 if not self.p4BranchesInGit:
3071 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3072
3073 # The default branch is master, unless --branch is used to
3074 # specify something else. Make sure it exists, or complain
3075 # nicely about how to use --branch.
3076 if not self.detectBranches:
3077 if not branch_exists(self.branch):
3078 if branch_arg_given:
3079 die("Error: branch %s does not exist." % self.branch)
3080 else:
3081 die("Error: no branch %s; perhaps specify one with --branch." %
3082 self.branch)
3083
29bdbac1 3084 if self.verbose:
86dff6b6 3085 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
6326aa58 3086 self.changeRange)
4f6432d8 3087 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
b984733c 3088
01a9c9c5 3089 if len(self.maxChanges) > 0:
7fcff9de 3090 changes = changes[:min(int(self.maxChanges), len(changes))]
01a9c9c5 3091
b984733c 3092 if len(changes) == 0:
0828ab14 3093 if not self.silent:
341dc1c1 3094 print "No changes to import!"
06804c76
LD
3095 else:
3096 if not self.silent and not self.detectBranches:
3097 print "Import destination: %s" % self.branch
3098
3099 self.updatedBranches = set()
b984733c 3100
47497844
PW
3101 if not self.detectBranches:
3102 if args:
3103 # start a new branch
3104 self.initialParent = ""
3105 else:
3106 # build on a previous revision
3107 self.initialParent = parseRevision(self.branch)
3108
06804c76 3109 self.importChanges(changes)
a9d1a27a 3110
06804c76
LD
3111 if not self.silent:
3112 print ""
3113 if len(self.updatedBranches) > 0:
3114 sys.stdout.write("Updated branches: ")
3115 for b in self.updatedBranches:
3116 sys.stdout.write("%s " % b)
3117 sys.stdout.write("\n")
341dc1c1 3118
0d609032 3119 if gitConfigBool("git-p4.importLabels"):
06dcd152 3120 self.importLabels = True
b984733c 3121
06804c76
LD
3122 if self.importLabels:
3123 p4Labels = getP4Labels(self.depotPaths)
3124 gitTags = getGitTags()
3125
3126 missingP4Labels = p4Labels - gitTags
3127 self.importP4Labels(self.gitStream, missingP4Labels)
b984733c 3128
b984733c 3129 self.gitStream.close()
78189bea 3130 if self.importProcess.wait() != 0:
29bdbac1 3131 die("fast-import failed: %s" % self.gitError.read())
b984733c
SH
3132 self.gitOutput.close()
3133 self.gitError.close()
3134
fed23693
VA
3135 # Cleanup temporary branches created during import
3136 if self.tempBranches != []:
3137 for branch in self.tempBranches:
3138 read_pipe("git update-ref -d %s" % branch)
3139 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3140
55d12437
PW
3141 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3142 # a convenient shortcut refname "p4".
3143 if self.importIntoRemotes:
3144 head_ref = self.refPrefix + "HEAD"
3145 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3146 system(["git", "symbolic-ref", head_ref, self.branch])
3147
b984733c
SH
3148 return True
3149
01ce1fe9
SH
3150class P4Rebase(Command):
3151 def __init__(self):
3152 Command.__init__(self)
06804c76
LD
3153 self.options = [
3154 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
06804c76 3155 ]
06804c76 3156 self.importLabels = False
cebdf5af
HWN
3157 self.description = ("Fetches the latest revision from perforce and "
3158 + "rebases the current work (branch) against it")
01ce1fe9
SH
3159
3160 def run(self, args):
3161 sync = P4Sync()
06804c76 3162 sync.importLabels = self.importLabels
01ce1fe9 3163 sync.run([])
d7e3868c 3164
14594f4b
SH
3165 return self.rebase()
3166
3167 def rebase(self):
36ee4ee4
SH
3168 if os.system("git update-index --refresh") != 0:
3169 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3170 if len(read_pipe("git diff-index HEAD --")) > 0:
f7e604ed 3171 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
36ee4ee4 3172
d7e3868c
SH
3173 [upstream, settings] = findUpstreamBranchPoint()
3174 if len(upstream) == 0:
3175 die("Cannot find upstream branchpoint for rebase")
3176
3177 # the branchpoint may be p4/foo~3, so strip off the parent
3178 upstream = re.sub("~[0-9]+$", "", upstream)
3179
3180 print "Rebasing the current branch onto %s" % upstream
b25b2065 3181 oldHead = read_pipe("git rev-parse HEAD").strip()
d7e3868c 3182 system("git rebase %s" % upstream)
1f52af6c 3183 system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
01ce1fe9
SH
3184 return True
3185
f9a3a4f7
SH
3186class P4Clone(P4Sync):
3187 def __init__(self):
3188 P4Sync.__init__(self)
3189 self.description = "Creates a new git repository and imports from Perforce into it"
bb6e09b2 3190 self.usage = "usage: %prog [options] //depot/path[@revRange]"
354081d5 3191 self.options += [
bb6e09b2
HWN
3192 optparse.make_option("--destination", dest="cloneDestination",
3193 action='store', default=None,
354081d5
TT
3194 help="where to leave result of the clone"),
3195 optparse.make_option("-/", dest="cloneExclude",
3196 action="append", type="string",
38200076
PW
3197 help="exclude depot path"),
3198 optparse.make_option("--bare", dest="cloneBare",
3199 action="store_true", default=False),
354081d5 3200 ]
bb6e09b2 3201 self.cloneDestination = None
f9a3a4f7 3202 self.needsGit = False
38200076 3203 self.cloneBare = False
f9a3a4f7 3204
354081d5
TT
3205 # This is required for the "append" cloneExclude action
3206 def ensure_value(self, attr, value):
3207 if not hasattr(self, attr) or getattr(self, attr) is None:
3208 setattr(self, attr, value)
3209 return getattr(self, attr)
3210
6a49f8e2
HWN
3211 def defaultDestination(self, args):
3212 ## TODO: use common prefix of args?
3213 depotPath = args[0]
3214 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3215 depotDir = re.sub("(#[^#]*)$", "", depotDir)
053d9e43 3216 depotDir = re.sub(r"\.\.\.$", "", depotDir)
6a49f8e2
HWN
3217 depotDir = re.sub(r"/$", "", depotDir)
3218 return os.path.split(depotDir)[1]
3219
f9a3a4f7
SH
3220 def run(self, args):
3221 if len(args) < 1:
3222 return False
bb6e09b2
HWN
3223
3224 if self.keepRepoPath and not self.cloneDestination:
3225 sys.stderr.write("Must specify destination for --keep-path\n")
3226 sys.exit(1)
f9a3a4f7 3227
6326aa58 3228 depotPaths = args
5e100b5c
SH
3229
3230 if not self.cloneDestination and len(depotPaths) > 1:
3231 self.cloneDestination = depotPaths[-1]
3232 depotPaths = depotPaths[:-1]
3233
354081d5 3234 self.cloneExclude = ["/"+p for p in self.cloneExclude]
6326aa58
HWN
3235 for p in depotPaths:
3236 if not p.startswith("//"):
0f487d30 3237 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
6326aa58 3238 return False
f9a3a4f7 3239
bb6e09b2 3240 if not self.cloneDestination:
98ad4faf 3241 self.cloneDestination = self.defaultDestination(args)
f9a3a4f7 3242
86dff6b6 3243 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
38200076 3244
c3bf3f13
KG
3245 if not os.path.exists(self.cloneDestination):
3246 os.makedirs(self.cloneDestination)
053fd0c1 3247 chdir(self.cloneDestination)
38200076
PW
3248
3249 init_cmd = [ "git", "init" ]
3250 if self.cloneBare:
3251 init_cmd.append("--bare")
a235e85c
BC
3252 retcode = subprocess.call(init_cmd)
3253 if retcode:
3254 raise CalledProcessError(retcode, init_cmd)
38200076 3255
6326aa58 3256 if not P4Sync.run(self, depotPaths):
f9a3a4f7 3257 return False
c595956d
PW
3258
3259 # create a master branch and check out a work tree
3260 if gitBranchExists(self.branch):
3261 system([ "git", "branch", "master", self.branch ])
3262 if not self.cloneBare:
3263 system([ "git", "checkout", "-f" ])
3264 else:
3265 print 'Not checking out any branch, use ' \
3266 '"git checkout -q -b master <branch>"'
86dff6b6 3267
a93d33ee
PW
3268 # auto-set this variable if invoked with --use-client-spec
3269 if self.useClientSpec_from_options:
3270 system("git config --bool git-p4.useclientspec true")
3271
f9a3a4f7
SH
3272 return True
3273
09d89de2
SH
3274class P4Branches(Command):
3275 def __init__(self):
3276 Command.__init__(self)
3277 self.options = [ ]
3278 self.description = ("Shows the git branches that hold imports and their "
3279 + "corresponding perforce depot paths")
3280 self.verbose = False
3281
3282 def run(self, args):
5ca44617
SH
3283 if originP4BranchesExist():
3284 createOrUpdateBranchesFromOrigin()
3285
09d89de2
SH
3286 cmdline = "git rev-parse --symbolic "
3287 cmdline += " --remotes"
3288
3289 for line in read_pipe_lines(cmdline):
3290 line = line.strip()
3291
3292 if not line.startswith('p4/') or line == "p4/HEAD":
3293 continue
3294 branch = line
3295
3296 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3297 settings = extractSettingsGitLog(log)
3298
3299 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3300 return True
3301
b984733c
SH
3302class HelpFormatter(optparse.IndentedHelpFormatter):
3303 def __init__(self):
3304 optparse.IndentedHelpFormatter.__init__(self)
3305
3306 def format_description(self, description):
3307 if description:
3308 return description + "\n"
3309 else:
3310 return ""
4f5cf76a 3311
86949eef
SH
3312def printUsage(commands):
3313 print "usage: %s <command> [options]" % sys.argv[0]
3314 print ""
3315 print "valid commands: %s" % ", ".join(commands)
3316 print ""
3317 print "Try %s <command> --help for command specific help." % sys.argv[0]
3318 print ""
3319
3320commands = {
b86f7378
HWN
3321 "debug" : P4Debug,
3322 "submit" : P4Submit,
a9834f58 3323 "commit" : P4Submit,
b86f7378
HWN
3324 "sync" : P4Sync,
3325 "rebase" : P4Rebase,
3326 "clone" : P4Clone,
09d89de2
SH
3327 "rollback" : P4RollBack,
3328 "branches" : P4Branches
86949eef
SH
3329}
3330
86949eef 3331
bb6e09b2
HWN
3332def main():
3333 if len(sys.argv[1:]) == 0:
3334 printUsage(commands.keys())
3335 sys.exit(2)
4f5cf76a 3336
bb6e09b2
HWN
3337 cmdName = sys.argv[1]
3338 try:
b86f7378
HWN
3339 klass = commands[cmdName]
3340 cmd = klass()
bb6e09b2
HWN
3341 except KeyError:
3342 print "unknown command %s" % cmdName
3343 print ""
3344 printUsage(commands.keys())
3345 sys.exit(2)
3346
3347 options = cmd.options
b86f7378 3348 cmd.gitdir = os.environ.get("GIT_DIR", None)
bb6e09b2
HWN
3349
3350 args = sys.argv[2:]
3351
b0ccc80d 3352 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
6a10b6aa
LD
3353 if cmd.needsGit:
3354 options.append(optparse.make_option("--git-dir", dest="gitdir"))
bb6e09b2 3355
6a10b6aa
LD
3356 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3357 options,
3358 description = cmd.description,
3359 formatter = HelpFormatter())
bb6e09b2 3360
6a10b6aa 3361 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
bb6e09b2
HWN
3362 global verbose
3363 verbose = cmd.verbose
3364 if cmd.needsGit:
b86f7378
HWN
3365 if cmd.gitdir == None:
3366 cmd.gitdir = os.path.abspath(".git")
3367 if not isValidGitDir(cmd.gitdir):
3368 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3369 if os.path.exists(cmd.gitdir):
bb6e09b2
HWN
3370 cdup = read_pipe("git rev-parse --show-cdup").strip()
3371 if len(cdup) > 0:
053fd0c1 3372 chdir(cdup);
e20a9e53 3373
b86f7378
HWN
3374 if not isValidGitDir(cmd.gitdir):
3375 if isValidGitDir(cmd.gitdir + "/.git"):
3376 cmd.gitdir += "/.git"
bb6e09b2 3377 else:
b86f7378 3378 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
e20a9e53 3379
b86f7378 3380 os.environ["GIT_DIR"] = cmd.gitdir
86949eef 3381
bb6e09b2
HWN
3382 if not cmd.run(args):
3383 parser.print_help()
09fca77b 3384 sys.exit(2)
4f5cf76a 3385
4f5cf76a 3386
bb6e09b2
HWN
3387if __name__ == '__main__':
3388 main()