9b120bd35d0b98172e5f5bc44e7a6af4c87b9f03
[git/git.git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
60 real_cmd = ["p4"]
61
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
64 real_cmd += ["-u",user]
65
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
68 real_cmd += ["-P", password]
69
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
72 real_cmd += ["-p", port]
73
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
76 real_cmd += ["-H", host]
77
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
80 real_cmd += ["-c", client]
81
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
89
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
94 return real_cmd
95
96 def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
106 def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
127
128 def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
138 def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
145 def write_pipe(c, stdin):
146 if verbose:
147 sys.stderr.write('Writing pipe: %s\n' % str(c))
148
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
156
157 return val
158
159 def p4_write_pipe(c, stdin):
160 real_cmd = p4_build_cmd(c)
161 return write_pipe(real_cmd, stdin)
162
163 def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
167 """
168 if verbose:
169 sys.stderr.write('Reading pipe: %s\n' % str(c))
170
171 expand = isinstance(c,basestring)
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
174 return (p.returncode, out, err)
175
176 def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
180 """
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
187 return out
188
189 def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
192 """
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
198
199 def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
202
203 def read_pipe_lines(c):
204 if verbose:
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
210 val = pipe.readlines()
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
213
214 return val
215
216 def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
218 real_cmd = p4_build_cmd(c)
219 return read_pipe_lines(real_cmd)
220
221 def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
229
230 def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
235
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
248
249 def system(cmd, ignore_error=False):
250 expand = isinstance(cmd,basestring)
251 if verbose:
252 sys.stderr.write("executing %s\n" % str(cmd))
253 retcode = subprocess.call(cmd, shell=expand)
254 if retcode and not ignore_error:
255 raise CalledProcessError(retcode, cmd)
256
257 return retcode
258
259 def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
261 real_cmd = p4_build_cmd(cmd)
262 expand = isinstance(real_cmd, basestring)
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
266
267 _p4_version_string = None
268 def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
271
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276 """
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
282
283 def p4_integrate(src, dest):
284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
285
286 def p4_sync(f, *options):
287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
288
289 def p4_add(f):
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
295
296 def p4_delete(f):
297 p4_system(["delete", wildcard_encode(f)])
298
299 def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
301
302 def p4_revert(f):
303 p4_system(["revert", wildcard_encode(f)])
304
305 def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
307
308 def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
311
312 def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
315 def p4_last_change():
316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317 return int(results[0]['change'])
318
319 def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
323
324 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328 d = ds[0]
329
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340 return d
341
342 #
343 # Canonicalize the p4 type and return a tuple of the
344 # base type, plus any modifiers. See "p4 help filetypes"
345 # for a list and explanation.
346 #
347 def split_p4_type(p4type):
348
349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
366 }
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
376
377 #
378 # return the raw p4 type of a file (text, text+ko, etc)
379 #
380 def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
382 return results[0]['headType']
383
384 #
385 # Given a type base and modifier, return a regexp matching
386 # the keywords that can be expanded in the file
387 #
388 def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
400 (:[^$\n]+)? # possibly an old expansion, followed by...
401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
406
407 #
408 # Given a file, return a regexp matching the possible
409 # RCS keywords that will be expanded, or None for files
410 # with kw expansion turned off.
411 #
412 def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
418
419 def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
422
423 p4Type = "+x"
424
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
431
432 p4_reopen(p4Type, file)
433
434 def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
436
437 result = p4_read_pipe(["opened", wildcard_encode(file)])
438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
439 if match:
440 return match.group(1)
441 else:
442 die("Could not determine file type for %s (result: '%s')" % (file, result))
443
444 # Return the set of all p4 labels
445 def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
449
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
453
454 return labels
455
456 # Return the set of all git tags
457 def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
463
464 def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
470
471 def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
473
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
476
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
487
488 If the pattern is not matched, None is returned."""
489
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
501 }
502 return None
503
504 def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
508
509 def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
511
512 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
513
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
520
521 cmd = p4_build_cmd(cmd)
522 if verbose:
523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
524
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
536 stdin_file.flush()
537 stdin_file.seek(0)
538
539 p4 = subprocess.Popen(cmd,
540 shell=expand,
541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
543
544 result = []
545 try:
546 while True:
547 entry = marshal.load(p4.stdout)
548 if skip_info:
549 if 'code' in entry and entry['code'] == 'info':
550 continue
551 if cb is not None:
552 cb(entry)
553 else:
554 result.append(entry)
555 except EOFError:
556 pass
557 exitCode = p4.wait()
558 if exitCode != 0:
559 entry = {}
560 entry["p4ExitCode"] = exitCode
561 result.append(entry)
562
563 return result
564
565 def p4Cmd(cmd):
566 list = p4CmdList(cmd)
567 result = {}
568 for entry in list:
569 result.update(entry)
570 return result;
571
572 def p4Where(depotPath):
573 if not depotPath.endswith("/"):
574 depotPath += "/"
575 depotPathLong = depotPath + "..."
576 outputList = p4CmdList(["where", depotPathLong])
577 output = None
578 for entry in outputList:
579 if "depotFile" in entry:
580 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
581 # The base path always ends with "/...".
582 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
583 output = entry
584 break
585 elif "data" in entry:
586 data = entry.get("data")
587 space = data.find(" ")
588 if data[:space] == depotPath:
589 output = entry
590 break
591 if output == None:
592 return ""
593 if output["code"] == "error":
594 return ""
595 clientPath = ""
596 if "path" in output:
597 clientPath = output.get("path")
598 elif "data" in output:
599 data = output.get("data")
600 lastSpace = data.rfind(" ")
601 clientPath = data[lastSpace + 1:]
602
603 if clientPath.endswith("..."):
604 clientPath = clientPath[:-3]
605 return clientPath
606
607 def currentGitBranch():
608 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
609
610 def isValidGitDir(path):
611 return git_dir(path) != None
612
613 def parseRevision(ref):
614 return read_pipe("git rev-parse %s" % ref).strip()
615
616 def branchExists(ref):
617 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
618 ignore_error=True)
619 return len(rev) > 0
620
621 def extractLogMessageFromGitCommit(commit):
622 logMessage = ""
623
624 ## fixme: title is first line of commit, not 1st paragraph.
625 foundTitle = False
626 for log in read_pipe_lines("git cat-file commit %s" % commit):
627 if not foundTitle:
628 if len(log) == 1:
629 foundTitle = True
630 continue
631
632 logMessage += log
633 return logMessage
634
635 def extractSettingsGitLog(log):
636 values = {}
637 for line in log.split("\n"):
638 line = line.strip()
639 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
640 if not m:
641 continue
642
643 assignments = m.group(1).split (':')
644 for a in assignments:
645 vals = a.split ('=')
646 key = vals[0].strip()
647 val = ('='.join (vals[1:])).strip()
648 if val.endswith ('\"') and val.startswith('"'):
649 val = val[1:-1]
650
651 values[key] = val
652
653 paths = values.get("depot-paths")
654 if not paths:
655 paths = values.get("depot-path")
656 if paths:
657 values['depot-paths'] = paths.split(',')
658 return values
659
660 def gitBranchExists(branch):
661 proc = subprocess.Popen(["git", "rev-parse", branch],
662 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
663 return proc.wait() == 0;
664
665 _gitConfig = {}
666
667 def gitConfig(key, typeSpecifier=None):
668 if not _gitConfig.has_key(key):
669 cmd = [ "git", "config" ]
670 if typeSpecifier:
671 cmd += [ typeSpecifier ]
672 cmd += [ key ]
673 s = read_pipe(cmd, ignore_error=True)
674 _gitConfig[key] = s.strip()
675 return _gitConfig[key]
676
677 def gitConfigBool(key):
678 """Return a bool, using git config --bool. It is True only if the
679 variable is set to true, and False if set to false or not present
680 in the config."""
681
682 if not _gitConfig.has_key(key):
683 _gitConfig[key] = gitConfig(key, '--bool') == "true"
684 return _gitConfig[key]
685
686 def gitConfigInt(key):
687 if not _gitConfig.has_key(key):
688 cmd = [ "git", "config", "--int", key ]
689 s = read_pipe(cmd, ignore_error=True)
690 v = s.strip()
691 try:
692 _gitConfig[key] = int(gitConfig(key, '--int'))
693 except ValueError:
694 _gitConfig[key] = None
695 return _gitConfig[key]
696
697 def gitConfigList(key):
698 if not _gitConfig.has_key(key):
699 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
700 _gitConfig[key] = s.strip().splitlines()
701 if _gitConfig[key] == ['']:
702 _gitConfig[key] = []
703 return _gitConfig[key]
704
705 def p4BranchesInGit(branchesAreInRemotes=True):
706 """Find all the branches whose names start with "p4/", looking
707 in remotes or heads as specified by the argument. Return
708 a dictionary of { branch: revision } for each one found.
709 The branch names are the short names, without any
710 "p4/" prefix."""
711
712 branches = {}
713
714 cmdline = "git rev-parse --symbolic "
715 if branchesAreInRemotes:
716 cmdline += "--remotes"
717 else:
718 cmdline += "--branches"
719
720 for line in read_pipe_lines(cmdline):
721 line = line.strip()
722
723 # only import to p4/
724 if not line.startswith('p4/'):
725 continue
726 # special symbolic ref to p4/master
727 if line == "p4/HEAD":
728 continue
729
730 # strip off p4/ prefix
731 branch = line[len("p4/"):]
732
733 branches[branch] = parseRevision(line)
734
735 return branches
736
737 def branch_exists(branch):
738 """Make sure that the given ref name really exists."""
739
740 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
741 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
742 out, _ = p.communicate()
743 if p.returncode:
744 return False
745 # expect exactly one line of output: the branch name
746 return out.rstrip() == branch
747
748 def findUpstreamBranchPoint(head = "HEAD"):
749 branches = p4BranchesInGit()
750 # map from depot-path to branch name
751 branchByDepotPath = {}
752 for branch in branches.keys():
753 tip = branches[branch]
754 log = extractLogMessageFromGitCommit(tip)
755 settings = extractSettingsGitLog(log)
756 if settings.has_key("depot-paths"):
757 paths = ",".join(settings["depot-paths"])
758 branchByDepotPath[paths] = "remotes/p4/" + branch
759
760 settings = None
761 parent = 0
762 while parent < 65535:
763 commit = head + "~%s" % parent
764 log = extractLogMessageFromGitCommit(commit)
765 settings = extractSettingsGitLog(log)
766 if settings.has_key("depot-paths"):
767 paths = ",".join(settings["depot-paths"])
768 if branchByDepotPath.has_key(paths):
769 return [branchByDepotPath[paths], settings]
770
771 parent = parent + 1
772
773 return ["", settings]
774
775 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
776 if not silent:
777 print ("Creating/updating branch(es) in %s based on origin branch(es)"
778 % localRefPrefix)
779
780 originPrefix = "origin/p4/"
781
782 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
783 line = line.strip()
784 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
785 continue
786
787 headName = line[len(originPrefix):]
788 remoteHead = localRefPrefix + headName
789 originHead = line
790
791 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
792 if (not original.has_key('depot-paths')
793 or not original.has_key('change')):
794 continue
795
796 update = False
797 if not gitBranchExists(remoteHead):
798 if verbose:
799 print "creating %s" % remoteHead
800 update = True
801 else:
802 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
803 if settings.has_key('change') > 0:
804 if settings['depot-paths'] == original['depot-paths']:
805 originP4Change = int(original['change'])
806 p4Change = int(settings['change'])
807 if originP4Change > p4Change:
808 print ("%s (%s) is newer than %s (%s). "
809 "Updating p4 branch from origin."
810 % (originHead, originP4Change,
811 remoteHead, p4Change))
812 update = True
813 else:
814 print ("Ignoring: %s was imported from %s while "
815 "%s was imported from %s"
816 % (originHead, ','.join(original['depot-paths']),
817 remoteHead, ','.join(settings['depot-paths'])))
818
819 if update:
820 system("git update-ref %s %s" % (remoteHead, originHead))
821
822 def originP4BranchesExist():
823 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
824
825
826 def p4ParseNumericChangeRange(parts):
827 changeStart = int(parts[0][1:])
828 if parts[1] == '#head':
829 changeEnd = p4_last_change()
830 else:
831 changeEnd = int(parts[1])
832
833 return (changeStart, changeEnd)
834
835 def chooseBlockSize(blockSize):
836 if blockSize:
837 return blockSize
838 else:
839 return defaultBlockSize
840
841 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
842 assert depotPaths
843
844 # Parse the change range into start and end. Try to find integer
845 # revision ranges as these can be broken up into blocks to avoid
846 # hitting server-side limits (maxrows, maxscanresults). But if
847 # that doesn't work, fall back to using the raw revision specifier
848 # strings, without using block mode.
849
850 if changeRange is None or changeRange == '':
851 changeStart = 1
852 changeEnd = p4_last_change()
853 block_size = chooseBlockSize(requestedBlockSize)
854 else:
855 parts = changeRange.split(',')
856 assert len(parts) == 2
857 try:
858 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
859 block_size = chooseBlockSize(requestedBlockSize)
860 except:
861 changeStart = parts[0][1:]
862 changeEnd = parts[1]
863 if requestedBlockSize:
864 die("cannot use --changes-block-size with non-numeric revisions")
865 block_size = None
866
867 changes = set()
868
869 # Retrieve changes a block at a time, to prevent running
870 # into a MaxResults/MaxScanRows error from the server.
871
872 while True:
873 cmd = ['changes']
874
875 if block_size:
876 end = min(changeEnd, changeStart + block_size)
877 revisionRange = "%d,%d" % (changeStart, end)
878 else:
879 revisionRange = "%s,%s" % (changeStart, changeEnd)
880
881 for p in depotPaths:
882 cmd += ["%s...@%s" % (p, revisionRange)]
883
884 # Insert changes in chronological order
885 for entry in reversed(p4CmdList(cmd)):
886 if entry.has_key('p4ExitCode'):
887 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
888 if not entry.has_key('change'):
889 continue
890 changes.add(int(entry['change']))
891
892 if not block_size:
893 break
894
895 if end >= changeEnd:
896 break
897
898 changeStart = end + 1
899
900 changes = sorted(changes)
901 return changes
902
903 def p4PathStartsWith(path, prefix):
904 # This method tries to remedy a potential mixed-case issue:
905 #
906 # If UserA adds //depot/DirA/file1
907 # and UserB adds //depot/dira/file2
908 #
909 # we may or may not have a problem. If you have core.ignorecase=true,
910 # we treat DirA and dira as the same directory
911 if gitConfigBool("core.ignorecase"):
912 return path.lower().startswith(prefix.lower())
913 return path.startswith(prefix)
914
915 def getClientSpec():
916 """Look at the p4 client spec, create a View() object that contains
917 all the mappings, and return it."""
918
919 specList = p4CmdList("client -o")
920 if len(specList) != 1:
921 die('Output from "client -o" is %d lines, expecting 1' %
922 len(specList))
923
924 # dictionary of all client parameters
925 entry = specList[0]
926
927 # the //client/ name
928 client_name = entry["Client"]
929
930 # just the keys that start with "View"
931 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
932
933 # hold this new View
934 view = View(client_name)
935
936 # append the lines, in order, to the view
937 for view_num in range(len(view_keys)):
938 k = "View%d" % view_num
939 if k not in view_keys:
940 die("Expected view key %s missing" % k)
941 view.append(entry[k])
942
943 return view
944
945 def getClientRoot():
946 """Grab the client directory."""
947
948 output = p4CmdList("client -o")
949 if len(output) != 1:
950 die('Output from "client -o" is %d lines, expecting 1' % len(output))
951
952 entry = output[0]
953 if "Root" not in entry:
954 die('Client has no "Root"')
955
956 return entry["Root"]
957
958 #
959 # P4 wildcards are not allowed in filenames. P4 complains
960 # if you simply add them, but you can force it with "-f", in
961 # which case it translates them into %xx encoding internally.
962 #
963 def wildcard_decode(path):
964 # Search for and fix just these four characters. Do % last so
965 # that fixing it does not inadvertently create new %-escapes.
966 # Cannot have * in a filename in windows; untested as to
967 # what p4 would do in such a case.
968 if not platform.system() == "Windows":
969 path = path.replace("%2A", "*")
970 path = path.replace("%23", "#") \
971 .replace("%40", "@") \
972 .replace("%25", "%")
973 return path
974
975 def wildcard_encode(path):
976 # do % first to avoid double-encoding the %s introduced here
977 path = path.replace("%", "%25") \
978 .replace("*", "%2A") \
979 .replace("#", "%23") \
980 .replace("@", "%40")
981 return path
982
983 def wildcard_present(path):
984 m = re.search("[*#@%]", path)
985 return m is not None
986
987 class LargeFileSystem(object):
988 """Base class for large file system support."""
989
990 def __init__(self, writeToGitStream):
991 self.largeFiles = set()
992 self.writeToGitStream = writeToGitStream
993
994 def generatePointer(self, cloneDestination, contentFile):
995 """Return the content of a pointer file that is stored in Git instead of
996 the actual content."""
997 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
998
999 def pushFile(self, localLargeFile):
1000 """Push the actual content which is not stored in the Git repository to
1001 a server."""
1002 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1003
1004 def hasLargeFileExtension(self, relPath):
1005 return reduce(
1006 lambda a, b: a or b,
1007 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1008 False
1009 )
1010
1011 def generateTempFile(self, contents):
1012 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1013 for d in contents:
1014 contentFile.write(d)
1015 contentFile.close()
1016 return contentFile.name
1017
1018 def exceedsLargeFileThreshold(self, relPath, contents):
1019 if gitConfigInt('git-p4.largeFileThreshold'):
1020 contentsSize = sum(len(d) for d in contents)
1021 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1022 return True
1023 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024 contentsSize = sum(len(d) for d in contents)
1025 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1026 return False
1027 contentTempFile = self.generateTempFile(contents)
1028 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1029 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1030 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1031 zf.close()
1032 compressedContentsSize = zf.infolist()[0].compress_size
1033 os.remove(contentTempFile)
1034 os.remove(compressedContentFile.name)
1035 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1036 return True
1037 return False
1038
1039 def addLargeFile(self, relPath):
1040 self.largeFiles.add(relPath)
1041
1042 def removeLargeFile(self, relPath):
1043 self.largeFiles.remove(relPath)
1044
1045 def isLargeFile(self, relPath):
1046 return relPath in self.largeFiles
1047
1048 def processContent(self, git_mode, relPath, contents):
1049 """Processes the content of git fast import. This method decides if a
1050 file is stored in the large file system and handles all necessary
1051 steps."""
1052 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1053 contentTempFile = self.generateTempFile(contents)
1054 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1055 if pointer_git_mode:
1056 git_mode = pointer_git_mode
1057 if localLargeFile:
1058 # Move temp file to final location in large file system
1059 largeFileDir = os.path.dirname(localLargeFile)
1060 if not os.path.isdir(largeFileDir):
1061 os.makedirs(largeFileDir)
1062 shutil.move(contentTempFile, localLargeFile)
1063 self.addLargeFile(relPath)
1064 if gitConfigBool('git-p4.largeFilePush'):
1065 self.pushFile(localLargeFile)
1066 if verbose:
1067 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1068 return (git_mode, contents)
1069
1070 class MockLFS(LargeFileSystem):
1071 """Mock large file system for testing."""
1072
1073 def generatePointer(self, contentFile):
1074 """The pointer content is the original content prefixed with "pointer-".
1075 The local filename of the large file storage is derived from the file content.
1076 """
1077 with open(contentFile, 'r') as f:
1078 content = next(f)
1079 gitMode = '100644'
1080 pointerContents = 'pointer-' + content
1081 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1082 return (gitMode, pointerContents, localLargeFile)
1083
1084 def pushFile(self, localLargeFile):
1085 """The remote filename of the large file storage is the same as the local
1086 one but in a different directory.
1087 """
1088 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1089 if not os.path.exists(remotePath):
1090 os.makedirs(remotePath)
1091 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1092
1093 class GitLFS(LargeFileSystem):
1094 """Git LFS as backend for the git-p4 large file system.
1095 See https://git-lfs.github.com/ for details."""
1096
1097 def __init__(self, *args):
1098 LargeFileSystem.__init__(self, *args)
1099 self.baseGitAttributes = []
1100
1101 def generatePointer(self, contentFile):
1102 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1103 mode and content which is stored in the Git repository instead of
1104 the actual content. Return also the new location of the actual
1105 content.
1106 """
1107 if os.path.getsize(contentFile) == 0:
1108 return (None, '', None)
1109
1110 pointerProcess = subprocess.Popen(
1111 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1112 stdout=subprocess.PIPE
1113 )
1114 pointerFile = pointerProcess.stdout.read()
1115 if pointerProcess.wait():
1116 os.remove(contentFile)
1117 die('git-lfs pointer command failed. Did you install the extension?')
1118
1119 # Git LFS removed the preamble in the output of the 'pointer' command
1120 # starting from version 1.2.0. Check for the preamble here to support
1121 # earlier versions.
1122 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1123 if pointerFile.startswith('Git LFS pointer for'):
1124 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1125
1126 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1127 localLargeFile = os.path.join(
1128 os.getcwd(),
1129 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1130 oid,
1131 )
1132 # LFS Spec states that pointer files should not have the executable bit set.
1133 gitMode = '100644'
1134 return (gitMode, pointerFile, localLargeFile)
1135
1136 def pushFile(self, localLargeFile):
1137 uploadProcess = subprocess.Popen(
1138 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1139 )
1140 if uploadProcess.wait():
1141 die('git-lfs push command failed. Did you define a remote?')
1142
1143 def generateGitAttributes(self):
1144 return (
1145 self.baseGitAttributes +
1146 [
1147 '\n',
1148 '#\n',
1149 '# Git LFS (see https://git-lfs.github.com/)\n',
1150 '#\n',
1151 ] +
1152 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1153 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1154 ] +
1155 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1156 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1157 ]
1158 )
1159
1160 def addLargeFile(self, relPath):
1161 LargeFileSystem.addLargeFile(self, relPath)
1162 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1163
1164 def removeLargeFile(self, relPath):
1165 LargeFileSystem.removeLargeFile(self, relPath)
1166 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1167
1168 def processContent(self, git_mode, relPath, contents):
1169 if relPath == '.gitattributes':
1170 self.baseGitAttributes = contents
1171 return (git_mode, self.generateGitAttributes())
1172 else:
1173 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1174
1175 class Command:
1176 def __init__(self):
1177 self.usage = "usage: %prog [options]"
1178 self.needsGit = True
1179 self.verbose = False
1180
1181 # This is required for the "append" cloneExclude action
1182 def ensure_value(self, attr, value):
1183 if not hasattr(self, attr) or getattr(self, attr) is None:
1184 setattr(self, attr, value)
1185 return getattr(self, attr)
1186
1187 class P4UserMap:
1188 def __init__(self):
1189 self.userMapFromPerforceServer = False
1190 self.myP4UserId = None
1191
1192 def p4UserId(self):
1193 if self.myP4UserId:
1194 return self.myP4UserId
1195
1196 results = p4CmdList("user -o")
1197 for r in results:
1198 if r.has_key('User'):
1199 self.myP4UserId = r['User']
1200 return r['User']
1201 die("Could not find your p4 user id")
1202
1203 def p4UserIsMe(self, p4User):
1204 # return True if the given p4 user is actually me
1205 me = self.p4UserId()
1206 if not p4User or p4User != me:
1207 return False
1208 else:
1209 return True
1210
1211 def getUserCacheFilename(self):
1212 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1213 return home + "/.gitp4-usercache.txt"
1214
1215 def getUserMapFromPerforceServer(self):
1216 if self.userMapFromPerforceServer:
1217 return
1218 self.users = {}
1219 self.emails = {}
1220
1221 for output in p4CmdList("users"):
1222 if not output.has_key("User"):
1223 continue
1224 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1225 self.emails[output["Email"]] = output["User"]
1226
1227 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1228 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1229 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1230 if mapUser and len(mapUser[0]) == 3:
1231 user = mapUser[0][0]
1232 fullname = mapUser[0][1]
1233 email = mapUser[0][2]
1234 self.users[user] = fullname + " <" + email + ">"
1235 self.emails[email] = user
1236
1237 s = ''
1238 for (key, val) in self.users.items():
1239 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1240
1241 open(self.getUserCacheFilename(), "wb").write(s)
1242 self.userMapFromPerforceServer = True
1243
1244 def loadUserMapFromCache(self):
1245 self.users = {}
1246 self.userMapFromPerforceServer = False
1247 try:
1248 cache = open(self.getUserCacheFilename(), "rb")
1249 lines = cache.readlines()
1250 cache.close()
1251 for line in lines:
1252 entry = line.strip().split("\t")
1253 self.users[entry[0]] = entry[1]
1254 except IOError:
1255 self.getUserMapFromPerforceServer()
1256
1257 class P4Debug(Command):
1258 def __init__(self):
1259 Command.__init__(self)
1260 self.options = []
1261 self.description = "A tool to debug the output of p4 -G."
1262 self.needsGit = False
1263
1264 def run(self, args):
1265 j = 0
1266 for output in p4CmdList(args):
1267 print 'Element: %d' % j
1268 j += 1
1269 print output
1270 return True
1271
1272 class P4RollBack(Command):
1273 def __init__(self):
1274 Command.__init__(self)
1275 self.options = [
1276 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1277 ]
1278 self.description = "A tool to debug the multi-branch import. Don't use :)"
1279 self.rollbackLocalBranches = False
1280
1281 def run(self, args):
1282 if len(args) != 1:
1283 return False
1284 maxChange = int(args[0])
1285
1286 if "p4ExitCode" in p4Cmd("changes -m 1"):
1287 die("Problems executing p4");
1288
1289 if self.rollbackLocalBranches:
1290 refPrefix = "refs/heads/"
1291 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1292 else:
1293 refPrefix = "refs/remotes/"
1294 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1295
1296 for line in lines:
1297 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1298 line = line.strip()
1299 ref = refPrefix + line
1300 log = extractLogMessageFromGitCommit(ref)
1301 settings = extractSettingsGitLog(log)
1302
1303 depotPaths = settings['depot-paths']
1304 change = settings['change']
1305
1306 changed = False
1307
1308 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1309 for p in depotPaths]))) == 0:
1310 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1311 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1312 continue
1313
1314 while change and int(change) > maxChange:
1315 changed = True
1316 if self.verbose:
1317 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1318 system("git update-ref %s \"%s^\"" % (ref, ref))
1319 log = extractLogMessageFromGitCommit(ref)
1320 settings = extractSettingsGitLog(log)
1321
1322
1323 depotPaths = settings['depot-paths']
1324 change = settings['change']
1325
1326 if changed:
1327 print "%s rewound to %s" % (ref, change)
1328
1329 return True
1330
1331 class P4Submit(Command, P4UserMap):
1332
1333 conflict_behavior_choices = ("ask", "skip", "quit")
1334
1335 def __init__(self):
1336 Command.__init__(self)
1337 P4UserMap.__init__(self)
1338 self.options = [
1339 optparse.make_option("--origin", dest="origin"),
1340 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1341 # preserve the user, requires relevant p4 permissions
1342 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1343 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1344 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1345 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1346 optparse.make_option("--conflict", dest="conflict_behavior",
1347 choices=self.conflict_behavior_choices),
1348 optparse.make_option("--branch", dest="branch"),
1349 optparse.make_option("--shelve", dest="shelve", action="store_true",
1350 help="Shelve instead of submit. Shelved files are reverted, "
1351 "restoring the workspace to the state before the shelve"),
1352 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1353 metavar="CHANGELIST",
1354 help="update an existing shelved changelist, implies --shelve, "
1355 "repeat in-order for multiple shelved changelists"),
1356 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1357 help="submit only the specified commit(s), one commit or xxx..xxx"),
1358 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1359 help="Disable rebase after submit is completed. Can be useful if you "
1360 "work from a local git branch that is not master")
1361 ]
1362 self.description = "Submit changes from git to the perforce depot."
1363 self.usage += " [name of git branch to submit into perforce depot]"
1364 self.origin = ""
1365 self.detectRenames = False
1366 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1367 self.dry_run = False
1368 self.shelve = False
1369 self.update_shelve = list()
1370 self.commit = ""
1371 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1372 self.prepare_p4_only = False
1373 self.conflict_behavior = None
1374 self.isWindows = (platform.system() == "Windows")
1375 self.exportLabels = False
1376 self.p4HasMoveCommand = p4_has_move_command()
1377 self.branch = None
1378
1379 if gitConfig('git-p4.largeFileSystem'):
1380 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1381
1382 def check(self):
1383 if len(p4CmdList("opened ...")) > 0:
1384 die("You have files opened with perforce! Close them before starting the sync.")
1385
1386 def separate_jobs_from_description(self, message):
1387 """Extract and return a possible Jobs field in the commit
1388 message. It goes into a separate section in the p4 change
1389 specification.
1390
1391 A jobs line starts with "Jobs:" and looks like a new field
1392 in a form. Values are white-space separated on the same
1393 line or on following lines that start with a tab.
1394
1395 This does not parse and extract the full git commit message
1396 like a p4 form. It just sees the Jobs: line as a marker
1397 to pass everything from then on directly into the p4 form,
1398 but outside the description section.
1399
1400 Return a tuple (stripped log message, jobs string)."""
1401
1402 m = re.search(r'^Jobs:', message, re.MULTILINE)
1403 if m is None:
1404 return (message, None)
1405
1406 jobtext = message[m.start():]
1407 stripped_message = message[:m.start()].rstrip()
1408 return (stripped_message, jobtext)
1409
1410 def prepareLogMessage(self, template, message, jobs):
1411 """Edits the template returned from "p4 change -o" to insert
1412 the message in the Description field, and the jobs text in
1413 the Jobs field."""
1414 result = ""
1415
1416 inDescriptionSection = False
1417
1418 for line in template.split("\n"):
1419 if line.startswith("#"):
1420 result += line + "\n"
1421 continue
1422
1423 if inDescriptionSection:
1424 if line.startswith("Files:") or line.startswith("Jobs:"):
1425 inDescriptionSection = False
1426 # insert Jobs section
1427 if jobs:
1428 result += jobs + "\n"
1429 else:
1430 continue
1431 else:
1432 if line.startswith("Description:"):
1433 inDescriptionSection = True
1434 line += "\n"
1435 for messageLine in message.split("\n"):
1436 line += "\t" + messageLine + "\n"
1437
1438 result += line + "\n"
1439
1440 return result
1441
1442 def patchRCSKeywords(self, file, pattern):
1443 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1444 (handle, outFileName) = tempfile.mkstemp(dir='.')
1445 try:
1446 outFile = os.fdopen(handle, "w+")
1447 inFile = open(file, "r")
1448 regexp = re.compile(pattern, re.VERBOSE)
1449 for line in inFile.readlines():
1450 line = regexp.sub(r'$\1$', line)
1451 outFile.write(line)
1452 inFile.close()
1453 outFile.close()
1454 # Forcibly overwrite the original file
1455 os.unlink(file)
1456 shutil.move(outFileName, file)
1457 except:
1458 # cleanup our temporary file
1459 os.unlink(outFileName)
1460 print "Failed to strip RCS keywords in %s" % file
1461 raise
1462
1463 print "Patched up RCS keywords in %s" % file
1464
1465 def p4UserForCommit(self,id):
1466 # Return the tuple (perforce user,git email) for a given git commit id
1467 self.getUserMapFromPerforceServer()
1468 gitEmail = read_pipe(["git", "log", "--max-count=1",
1469 "--format=%ae", id])
1470 gitEmail = gitEmail.strip()
1471 if not self.emails.has_key(gitEmail):
1472 return (None,gitEmail)
1473 else:
1474 return (self.emails[gitEmail],gitEmail)
1475
1476 def checkValidP4Users(self,commits):
1477 # check if any git authors cannot be mapped to p4 users
1478 for id in commits:
1479 (user,email) = self.p4UserForCommit(id)
1480 if not user:
1481 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1482 if gitConfigBool("git-p4.allowMissingP4Users"):
1483 print "%s" % msg
1484 else:
1485 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1486
1487 def lastP4Changelist(self):
1488 # Get back the last changelist number submitted in this client spec. This
1489 # then gets used to patch up the username in the change. If the same
1490 # client spec is being used by multiple processes then this might go
1491 # wrong.
1492 results = p4CmdList("client -o") # find the current client
1493 client = None
1494 for r in results:
1495 if r.has_key('Client'):
1496 client = r['Client']
1497 break
1498 if not client:
1499 die("could not get client spec")
1500 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1501 for r in results:
1502 if r.has_key('change'):
1503 return r['change']
1504 die("Could not get changelist number for last submit - cannot patch up user details")
1505
1506 def modifyChangelistUser(self, changelist, newUser):
1507 # fixup the user field of a changelist after it has been submitted.
1508 changes = p4CmdList("change -o %s" % changelist)
1509 if len(changes) != 1:
1510 die("Bad output from p4 change modifying %s to user %s" %
1511 (changelist, newUser))
1512
1513 c = changes[0]
1514 if c['User'] == newUser: return # nothing to do
1515 c['User'] = newUser
1516 input = marshal.dumps(c)
1517
1518 result = p4CmdList("change -f -i", stdin=input)
1519 for r in result:
1520 if r.has_key('code'):
1521 if r['code'] == 'error':
1522 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1523 if r.has_key('data'):
1524 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1525 return
1526 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1527
1528 def canChangeChangelists(self):
1529 # check to see if we have p4 admin or super-user permissions, either of
1530 # which are required to modify changelists.
1531 results = p4CmdList(["protects", self.depotPath])
1532 for r in results:
1533 if r.has_key('perm'):
1534 if r['perm'] == 'admin':
1535 return 1
1536 if r['perm'] == 'super':
1537 return 1
1538 return 0
1539
1540 def prepareSubmitTemplate(self, changelist=None):
1541 """Run "p4 change -o" to grab a change specification template.
1542 This does not use "p4 -G", as it is nice to keep the submission
1543 template in original order, since a human might edit it.
1544
1545 Remove lines in the Files section that show changes to files
1546 outside the depot path we're committing into."""
1547
1548 [upstream, settings] = findUpstreamBranchPoint()
1549
1550 template = """\
1551 # A Perforce Change Specification.
1552 #
1553 # Change: The change number. 'new' on a new changelist.
1554 # Date: The date this specification was last modified.
1555 # Client: The client on which the changelist was created. Read-only.
1556 # User: The user who created the changelist.
1557 # Status: Either 'pending' or 'submitted'. Read-only.
1558 # Type: Either 'public' or 'restricted'. Default is 'public'.
1559 # Description: Comments about the changelist. Required.
1560 # Jobs: What opened jobs are to be closed by this changelist.
1561 # You may delete jobs from this list. (New changelists only.)
1562 # Files: What opened files from the default changelist are to be added
1563 # to this changelist. You may delete files from this list.
1564 # (New changelists only.)
1565 """
1566 files_list = []
1567 inFilesSection = False
1568 change_entry = None
1569 args = ['change', '-o']
1570 if changelist:
1571 args.append(str(changelist))
1572 for entry in p4CmdList(args):
1573 if not entry.has_key('code'):
1574 continue
1575 if entry['code'] == 'stat':
1576 change_entry = entry
1577 break
1578 if not change_entry:
1579 die('Failed to decode output of p4 change -o')
1580 for key, value in change_entry.iteritems():
1581 if key.startswith('File'):
1582 if settings.has_key('depot-paths'):
1583 if not [p for p in settings['depot-paths']
1584 if p4PathStartsWith(value, p)]:
1585 continue
1586 else:
1587 if not p4PathStartsWith(value, self.depotPath):
1588 continue
1589 files_list.append(value)
1590 continue
1591 # Output in the order expected by prepareLogMessage
1592 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1593 if not change_entry.has_key(key):
1594 continue
1595 template += '\n'
1596 template += key + ':'
1597 if key == 'Description':
1598 template += '\n'
1599 for field_line in change_entry[key].splitlines():
1600 template += '\t'+field_line+'\n'
1601 if len(files_list) > 0:
1602 template += '\n'
1603 template += 'Files:\n'
1604 for path in files_list:
1605 template += '\t'+path+'\n'
1606 return template
1607
1608 def edit_template(self, template_file):
1609 """Invoke the editor to let the user change the submission
1610 message. Return true if okay to continue with the submit."""
1611
1612 # if configured to skip the editing part, just submit
1613 if gitConfigBool("git-p4.skipSubmitEdit"):
1614 return True
1615
1616 # look at the modification time, to check later if the user saved
1617 # the file
1618 mtime = os.stat(template_file).st_mtime
1619
1620 # invoke the editor
1621 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1622 editor = os.environ.get("P4EDITOR")
1623 else:
1624 editor = read_pipe("git var GIT_EDITOR").strip()
1625 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1626
1627 # If the file was not saved, prompt to see if this patch should
1628 # be skipped. But skip this verification step if configured so.
1629 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1630 return True
1631
1632 # modification time updated means user saved the file
1633 if os.stat(template_file).st_mtime > mtime:
1634 return True
1635
1636 while True:
1637 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1638 if response == 'y':
1639 return True
1640 if response == 'n':
1641 return False
1642
1643 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1644 # diff
1645 if os.environ.has_key("P4DIFF"):
1646 del(os.environ["P4DIFF"])
1647 diff = ""
1648 for editedFile in editedFiles:
1649 diff += p4_read_pipe(['diff', '-du',
1650 wildcard_encode(editedFile)])
1651
1652 # new file diff
1653 newdiff = ""
1654 for newFile in filesToAdd:
1655 newdiff += "==== new file ====\n"
1656 newdiff += "--- /dev/null\n"
1657 newdiff += "+++ %s\n" % newFile
1658
1659 is_link = os.path.islink(newFile)
1660 expect_link = newFile in symlinks
1661
1662 if is_link and expect_link:
1663 newdiff += "+%s\n" % os.readlink(newFile)
1664 else:
1665 f = open(newFile, "r")
1666 for line in f.readlines():
1667 newdiff += "+" + line
1668 f.close()
1669
1670 return (diff + newdiff).replace('\r\n', '\n')
1671
1672 def applyCommit(self, id):
1673 """Apply one commit, return True if it succeeded."""
1674
1675 print "Applying", read_pipe(["git", "show", "-s",
1676 "--format=format:%h %s", id])
1677
1678 (p4User, gitEmail) = self.p4UserForCommit(id)
1679
1680 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1681 filesToAdd = set()
1682 filesToChangeType = set()
1683 filesToDelete = set()
1684 editedFiles = set()
1685 pureRenameCopy = set()
1686 symlinks = set()
1687 filesToChangeExecBit = {}
1688 all_files = list()
1689
1690 for line in diff:
1691 diff = parseDiffTreeEntry(line)
1692 modifier = diff['status']
1693 path = diff['src']
1694 all_files.append(path)
1695
1696 if modifier == "M":
1697 p4_edit(path)
1698 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1699 filesToChangeExecBit[path] = diff['dst_mode']
1700 editedFiles.add(path)
1701 elif modifier == "A":
1702 filesToAdd.add(path)
1703 filesToChangeExecBit[path] = diff['dst_mode']
1704 if path in filesToDelete:
1705 filesToDelete.remove(path)
1706
1707 dst_mode = int(diff['dst_mode'], 8)
1708 if dst_mode == 0120000:
1709 symlinks.add(path)
1710
1711 elif modifier == "D":
1712 filesToDelete.add(path)
1713 if path in filesToAdd:
1714 filesToAdd.remove(path)
1715 elif modifier == "C":
1716 src, dest = diff['src'], diff['dst']
1717 p4_integrate(src, dest)
1718 pureRenameCopy.add(dest)
1719 if diff['src_sha1'] != diff['dst_sha1']:
1720 p4_edit(dest)
1721 pureRenameCopy.discard(dest)
1722 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1723 p4_edit(dest)
1724 pureRenameCopy.discard(dest)
1725 filesToChangeExecBit[dest] = diff['dst_mode']
1726 if self.isWindows:
1727 # turn off read-only attribute
1728 os.chmod(dest, stat.S_IWRITE)
1729 os.unlink(dest)
1730 editedFiles.add(dest)
1731 elif modifier == "R":
1732 src, dest = diff['src'], diff['dst']
1733 if self.p4HasMoveCommand:
1734 p4_edit(src) # src must be open before move
1735 p4_move(src, dest) # opens for (move/delete, move/add)
1736 else:
1737 p4_integrate(src, dest)
1738 if diff['src_sha1'] != diff['dst_sha1']:
1739 p4_edit(dest)
1740 else:
1741 pureRenameCopy.add(dest)
1742 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1743 if not self.p4HasMoveCommand:
1744 p4_edit(dest) # with move: already open, writable
1745 filesToChangeExecBit[dest] = diff['dst_mode']
1746 if not self.p4HasMoveCommand:
1747 if self.isWindows:
1748 os.chmod(dest, stat.S_IWRITE)
1749 os.unlink(dest)
1750 filesToDelete.add(src)
1751 editedFiles.add(dest)
1752 elif modifier == "T":
1753 filesToChangeType.add(path)
1754 else:
1755 die("unknown modifier %s for %s" % (modifier, path))
1756
1757 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1758 patchcmd = diffcmd + " | git apply "
1759 tryPatchCmd = patchcmd + "--check -"
1760 applyPatchCmd = patchcmd + "--check --apply -"
1761 patch_succeeded = True
1762
1763 if os.system(tryPatchCmd) != 0:
1764 fixed_rcs_keywords = False
1765 patch_succeeded = False
1766 print "Unfortunately applying the change failed!"
1767
1768 # Patch failed, maybe it's just RCS keyword woes. Look through
1769 # the patch to see if that's possible.
1770 if gitConfigBool("git-p4.attemptRCSCleanup"):
1771 file = None
1772 pattern = None
1773 kwfiles = {}
1774 for file in editedFiles | filesToDelete:
1775 # did this file's delta contain RCS keywords?
1776 pattern = p4_keywords_regexp_for_file(file)
1777
1778 if pattern:
1779 # this file is a possibility...look for RCS keywords.
1780 regexp = re.compile(pattern, re.VERBOSE)
1781 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1782 if regexp.search(line):
1783 if verbose:
1784 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1785 kwfiles[file] = pattern
1786 break
1787
1788 for file in kwfiles:
1789 if verbose:
1790 print "zapping %s with %s" % (line,pattern)
1791 # File is being deleted, so not open in p4. Must
1792 # disable the read-only bit on windows.
1793 if self.isWindows and file not in editedFiles:
1794 os.chmod(file, stat.S_IWRITE)
1795 self.patchRCSKeywords(file, kwfiles[file])
1796 fixed_rcs_keywords = True
1797
1798 if fixed_rcs_keywords:
1799 print "Retrying the patch with RCS keywords cleaned up"
1800 if os.system(tryPatchCmd) == 0:
1801 patch_succeeded = True
1802
1803 if not patch_succeeded:
1804 for f in editedFiles:
1805 p4_revert(f)
1806 return False
1807
1808 #
1809 # Apply the patch for real, and do add/delete/+x handling.
1810 #
1811 system(applyPatchCmd)
1812
1813 for f in filesToChangeType:
1814 p4_edit(f, "-t", "auto")
1815 for f in filesToAdd:
1816 p4_add(f)
1817 for f in filesToDelete:
1818 p4_revert(f)
1819 p4_delete(f)
1820
1821 # Set/clear executable bits
1822 for f in filesToChangeExecBit.keys():
1823 mode = filesToChangeExecBit[f]
1824 setP4ExecBit(f, mode)
1825
1826 update_shelve = 0
1827 if len(self.update_shelve) > 0:
1828 update_shelve = self.update_shelve.pop(0)
1829 p4_reopen_in_change(update_shelve, all_files)
1830
1831 #
1832 # Build p4 change description, starting with the contents
1833 # of the git commit message.
1834 #
1835 logMessage = extractLogMessageFromGitCommit(id)
1836 logMessage = logMessage.strip()
1837 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1838
1839 template = self.prepareSubmitTemplate(update_shelve)
1840 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1841
1842 if self.preserveUser:
1843 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1844
1845 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1846 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1847 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1848 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1849
1850 separatorLine = "######## everything below this line is just the diff #######\n"
1851 if not self.prepare_p4_only:
1852 submitTemplate += separatorLine
1853 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1854
1855 (handle, fileName) = tempfile.mkstemp()
1856 tmpFile = os.fdopen(handle, "w+b")
1857 if self.isWindows:
1858 submitTemplate = submitTemplate.replace("\n", "\r\n")
1859 tmpFile.write(submitTemplate)
1860 tmpFile.close()
1861
1862 if self.prepare_p4_only:
1863 #
1864 # Leave the p4 tree prepared, and the submit template around
1865 # and let the user decide what to do next
1866 #
1867 print
1868 print "P4 workspace prepared for submission."
1869 print "To submit or revert, go to client workspace"
1870 print " " + self.clientPath
1871 print
1872 print "To submit, use \"p4 submit\" to write a new description,"
1873 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1874 " \"git p4\"." % fileName
1875 print "You can delete the file \"%s\" when finished." % fileName
1876
1877 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1878 print "To preserve change ownership by user %s, you must\n" \
1879 "do \"p4 change -f <change>\" after submitting and\n" \
1880 "edit the User field."
1881 if pureRenameCopy:
1882 print "After submitting, renamed files must be re-synced."
1883 print "Invoke \"p4 sync -f\" on each of these files:"
1884 for f in pureRenameCopy:
1885 print " " + f
1886
1887 print
1888 print "To revert the changes, use \"p4 revert ...\", and delete"
1889 print "the submit template file \"%s\"" % fileName
1890 if filesToAdd:
1891 print "Since the commit adds new files, they must be deleted:"
1892 for f in filesToAdd:
1893 print " " + f
1894 print
1895 return True
1896
1897 #
1898 # Let the user edit the change description, then submit it.
1899 #
1900 submitted = False
1901
1902 try:
1903 if self.edit_template(fileName):
1904 # read the edited message and submit
1905 tmpFile = open(fileName, "rb")
1906 message = tmpFile.read()
1907 tmpFile.close()
1908 if self.isWindows:
1909 message = message.replace("\r\n", "\n")
1910 submitTemplate = message[:message.index(separatorLine)]
1911
1912 if update_shelve:
1913 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1914 elif self.shelve:
1915 p4_write_pipe(['shelve', '-i'], submitTemplate)
1916 else:
1917 p4_write_pipe(['submit', '-i'], submitTemplate)
1918 # The rename/copy happened by applying a patch that created a
1919 # new file. This leaves it writable, which confuses p4.
1920 for f in pureRenameCopy:
1921 p4_sync(f, "-f")
1922
1923 if self.preserveUser:
1924 if p4User:
1925 # Get last changelist number. Cannot easily get it from
1926 # the submit command output as the output is
1927 # unmarshalled.
1928 changelist = self.lastP4Changelist()
1929 self.modifyChangelistUser(changelist, p4User)
1930
1931 submitted = True
1932
1933 finally:
1934 # skip this patch
1935 if not submitted or self.shelve:
1936 if self.shelve:
1937 print ("Reverting shelved files.")
1938 else:
1939 print ("Submission cancelled, undoing p4 changes.")
1940 for f in editedFiles | filesToDelete:
1941 p4_revert(f)
1942 for f in filesToAdd:
1943 p4_revert(f)
1944 os.remove(f)
1945
1946 os.remove(fileName)
1947 return submitted
1948
1949 # Export git tags as p4 labels. Create a p4 label and then tag
1950 # with that.
1951 def exportGitTags(self, gitTags):
1952 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1953 if len(validLabelRegexp) == 0:
1954 validLabelRegexp = defaultLabelRegexp
1955 m = re.compile(validLabelRegexp)
1956
1957 for name in gitTags:
1958
1959 if not m.match(name):
1960 if verbose:
1961 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1962 continue
1963
1964 # Get the p4 commit this corresponds to
1965 logMessage = extractLogMessageFromGitCommit(name)
1966 values = extractSettingsGitLog(logMessage)
1967
1968 if not values.has_key('change'):
1969 # a tag pointing to something not sent to p4; ignore
1970 if verbose:
1971 print "git tag %s does not give a p4 commit" % name
1972 continue
1973 else:
1974 changelist = values['change']
1975
1976 # Get the tag details.
1977 inHeader = True
1978 isAnnotated = False
1979 body = []
1980 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1981 l = l.strip()
1982 if inHeader:
1983 if re.match(r'tag\s+', l):
1984 isAnnotated = True
1985 elif re.match(r'\s*$', l):
1986 inHeader = False
1987 continue
1988 else:
1989 body.append(l)
1990
1991 if not isAnnotated:
1992 body = ["lightweight tag imported by git p4\n"]
1993
1994 # Create the label - use the same view as the client spec we are using
1995 clientSpec = getClientSpec()
1996
1997 labelTemplate = "Label: %s\n" % name
1998 labelTemplate += "Description:\n"
1999 for b in body:
2000 labelTemplate += "\t" + b + "\n"
2001 labelTemplate += "View:\n"
2002 for depot_side in clientSpec.mappings:
2003 labelTemplate += "\t%s\n" % depot_side
2004
2005 if self.dry_run:
2006 print "Would create p4 label %s for tag" % name
2007 elif self.prepare_p4_only:
2008 print "Not creating p4 label %s for tag due to option" \
2009 " --prepare-p4-only" % name
2010 else:
2011 p4_write_pipe(["label", "-i"], labelTemplate)
2012
2013 # Use the label
2014 p4_system(["tag", "-l", name] +
2015 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2016
2017 if verbose:
2018 print "created p4 label for tag %s" % name
2019
2020 def run(self, args):
2021 if len(args) == 0:
2022 self.master = currentGitBranch()
2023 elif len(args) == 1:
2024 self.master = args[0]
2025 if not branchExists(self.master):
2026 die("Branch %s does not exist" % self.master)
2027 else:
2028 return False
2029
2030 for i in self.update_shelve:
2031 if i <= 0:
2032 sys.exit("invalid changelist %d" % i)
2033
2034 if self.master:
2035 allowSubmit = gitConfig("git-p4.allowSubmit")
2036 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2037 die("%s is not in git-p4.allowSubmit" % self.master)
2038
2039 [upstream, settings] = findUpstreamBranchPoint()
2040 self.depotPath = settings['depot-paths'][0]
2041 if len(self.origin) == 0:
2042 self.origin = upstream
2043
2044 if len(self.update_shelve) > 0:
2045 self.shelve = True
2046
2047 if self.preserveUser:
2048 if not self.canChangeChangelists():
2049 die("Cannot preserve user names without p4 super-user or admin permissions")
2050
2051 # if not set from the command line, try the config file
2052 if self.conflict_behavior is None:
2053 val = gitConfig("git-p4.conflict")
2054 if val:
2055 if val not in self.conflict_behavior_choices:
2056 die("Invalid value '%s' for config git-p4.conflict" % val)
2057 else:
2058 val = "ask"
2059 self.conflict_behavior = val
2060
2061 if self.verbose:
2062 print "Origin branch is " + self.origin
2063
2064 if len(self.depotPath) == 0:
2065 print "Internal error: cannot locate perforce depot path from existing branches"
2066 sys.exit(128)
2067
2068 self.useClientSpec = False
2069 if gitConfigBool("git-p4.useclientspec"):
2070 self.useClientSpec = True
2071 if self.useClientSpec:
2072 self.clientSpecDirs = getClientSpec()
2073
2074 # Check for the existence of P4 branches
2075 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2076
2077 if self.useClientSpec and not branchesDetected:
2078 # all files are relative to the client spec
2079 self.clientPath = getClientRoot()
2080 else:
2081 self.clientPath = p4Where(self.depotPath)
2082
2083 if self.clientPath == "":
2084 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2085
2086 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2087 self.oldWorkingDirectory = os.getcwd()
2088
2089 # ensure the clientPath exists
2090 new_client_dir = False
2091 if not os.path.exists(self.clientPath):
2092 new_client_dir = True
2093 os.makedirs(self.clientPath)
2094
2095 chdir(self.clientPath, is_client_path=True)
2096 if self.dry_run:
2097 print "Would synchronize p4 checkout in %s" % self.clientPath
2098 else:
2099 print "Synchronizing p4 checkout..."
2100 if new_client_dir:
2101 # old one was destroyed, and maybe nobody told p4
2102 p4_sync("...", "-f")
2103 else:
2104 p4_sync("...")
2105 self.check()
2106
2107 commits = []
2108 if self.master:
2109 commitish = self.master
2110 else:
2111 commitish = 'HEAD'
2112
2113 if self.commit != "":
2114 if self.commit.find("..") != -1:
2115 limits_ish = self.commit.split("..")
2116 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2117 commits.append(line.strip())
2118 commits.reverse()
2119 else:
2120 commits.append(self.commit)
2121 else:
2122 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2123 commits.append(line.strip())
2124 commits.reverse()
2125
2126 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2127 self.checkAuthorship = False
2128 else:
2129 self.checkAuthorship = True
2130
2131 if self.preserveUser:
2132 self.checkValidP4Users(commits)
2133
2134 #
2135 # Build up a set of options to be passed to diff when
2136 # submitting each commit to p4.
2137 #
2138 if self.detectRenames:
2139 # command-line -M arg
2140 self.diffOpts = "-M"
2141 else:
2142 # If not explicitly set check the config variable
2143 detectRenames = gitConfig("git-p4.detectRenames")
2144
2145 if detectRenames.lower() == "false" or detectRenames == "":
2146 self.diffOpts = ""
2147 elif detectRenames.lower() == "true":
2148 self.diffOpts = "-M"
2149 else:
2150 self.diffOpts = "-M%s" % detectRenames
2151
2152 # no command-line arg for -C or --find-copies-harder, just
2153 # config variables
2154 detectCopies = gitConfig("git-p4.detectCopies")
2155 if detectCopies.lower() == "false" or detectCopies == "":
2156 pass
2157 elif detectCopies.lower() == "true":
2158 self.diffOpts += " -C"
2159 else:
2160 self.diffOpts += " -C%s" % detectCopies
2161
2162 if gitConfigBool("git-p4.detectCopiesHarder"):
2163 self.diffOpts += " --find-copies-harder"
2164
2165 num_shelves = len(self.update_shelve)
2166 if num_shelves > 0 and num_shelves != len(commits):
2167 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2168 (len(commits), num_shelves))
2169
2170 #
2171 # Apply the commits, one at a time. On failure, ask if should
2172 # continue to try the rest of the patches, or quit.
2173 #
2174 if self.dry_run:
2175 print "Would apply"
2176 applied = []
2177 last = len(commits) - 1
2178 for i, commit in enumerate(commits):
2179 if self.dry_run:
2180 print " ", read_pipe(["git", "show", "-s",
2181 "--format=format:%h %s", commit])
2182 ok = True
2183 else:
2184 ok = self.applyCommit(commit)
2185 if ok:
2186 applied.append(commit)
2187 else:
2188 if self.prepare_p4_only and i < last:
2189 print "Processing only the first commit due to option" \
2190 " --prepare-p4-only"
2191 break
2192 if i < last:
2193 quit = False
2194 while True:
2195 # prompt for what to do, or use the option/variable
2196 if self.conflict_behavior == "ask":
2197 print "What do you want to do?"
2198 response = raw_input("[s]kip this commit but apply"
2199 " the rest, or [q]uit? ")
2200 if not response:
2201 continue
2202 elif self.conflict_behavior == "skip":
2203 response = "s"
2204 elif self.conflict_behavior == "quit":
2205 response = "q"
2206 else:
2207 die("Unknown conflict_behavior '%s'" %
2208 self.conflict_behavior)
2209
2210 if response[0] == "s":
2211 print "Skipping this commit, but applying the rest"
2212 break
2213 if response[0] == "q":
2214 print "Quitting"
2215 quit = True
2216 break
2217 if quit:
2218 break
2219
2220 chdir(self.oldWorkingDirectory)
2221 shelved_applied = "shelved" if self.shelve else "applied"
2222 if self.dry_run:
2223 pass
2224 elif self.prepare_p4_only:
2225 pass
2226 elif len(commits) == len(applied):
2227 print ("All commits {0}!".format(shelved_applied))
2228
2229 sync = P4Sync()
2230 if self.branch:
2231 sync.branch = self.branch
2232 sync.run([])
2233
2234 if self.disable_rebase is False:
2235 rebase = P4Rebase()
2236 rebase.rebase()
2237
2238 else:
2239 if len(applied) == 0:
2240 print ("No commits {0}.".format(shelved_applied))
2241 else:
2242 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2243 for c in commits:
2244 if c in applied:
2245 star = "*"
2246 else:
2247 star = " "
2248 print star, read_pipe(["git", "show", "-s",
2249 "--format=format:%h %s", c])
2250 print "You will have to do 'git p4 sync' and rebase."
2251
2252 if gitConfigBool("git-p4.exportLabels"):
2253 self.exportLabels = True
2254
2255 if self.exportLabels:
2256 p4Labels = getP4Labels(self.depotPath)
2257 gitTags = getGitTags()
2258
2259 missingGitTags = gitTags - p4Labels
2260 self.exportGitTags(missingGitTags)
2261
2262 # exit with error unless everything applied perfectly
2263 if len(commits) != len(applied):
2264 sys.exit(1)
2265
2266 return True
2267
2268 class View(object):
2269 """Represent a p4 view ("p4 help views"), and map files in a
2270 repo according to the view."""
2271
2272 def __init__(self, client_name):
2273 self.mappings = []
2274 self.client_prefix = "//%s/" % client_name
2275 # cache results of "p4 where" to lookup client file locations
2276 self.client_spec_path_cache = {}
2277
2278 def append(self, view_line):
2279 """Parse a view line, splitting it into depot and client
2280 sides. Append to self.mappings, preserving order. This
2281 is only needed for tag creation."""
2282
2283 # Split the view line into exactly two words. P4 enforces
2284 # structure on these lines that simplifies this quite a bit.
2285 #
2286 # Either or both words may be double-quoted.
2287 # Single quotes do not matter.
2288 # Double-quote marks cannot occur inside the words.
2289 # A + or - prefix is also inside the quotes.
2290 # There are no quotes unless they contain a space.
2291 # The line is already white-space stripped.
2292 # The two words are separated by a single space.
2293 #
2294 if view_line[0] == '"':
2295 # First word is double quoted. Find its end.
2296 close_quote_index = view_line.find('"', 1)
2297 if close_quote_index <= 0:
2298 die("No first-word closing quote found: %s" % view_line)
2299 depot_side = view_line[1:close_quote_index]
2300 # skip closing quote and space
2301 rhs_index = close_quote_index + 1 + 1
2302 else:
2303 space_index = view_line.find(" ")
2304 if space_index <= 0:
2305 die("No word-splitting space found: %s" % view_line)
2306 depot_side = view_line[0:space_index]
2307 rhs_index = space_index + 1
2308
2309 # prefix + means overlay on previous mapping
2310 if depot_side.startswith("+"):
2311 depot_side = depot_side[1:]
2312
2313 # prefix - means exclude this path, leave out of mappings
2314 exclude = False
2315 if depot_side.startswith("-"):
2316 exclude = True
2317 depot_side = depot_side[1:]
2318
2319 if not exclude:
2320 self.mappings.append(depot_side)
2321
2322 def convert_client_path(self, clientFile):
2323 # chop off //client/ part to make it relative
2324 if not clientFile.startswith(self.client_prefix):
2325 die("No prefix '%s' on clientFile '%s'" %
2326 (self.client_prefix, clientFile))
2327 return clientFile[len(self.client_prefix):]
2328
2329 def update_client_spec_path_cache(self, files):
2330 """ Caching file paths by "p4 where" batch query """
2331
2332 # List depot file paths exclude that already cached
2333 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2334
2335 if len(fileArgs) == 0:
2336 return # All files in cache
2337
2338 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2339 for res in where_result:
2340 if "code" in res and res["code"] == "error":
2341 # assume error is "... file(s) not in client view"
2342 continue
2343 if "clientFile" not in res:
2344 die("No clientFile in 'p4 where' output")
2345 if "unmap" in res:
2346 # it will list all of them, but only one not unmap-ped
2347 continue
2348 if gitConfigBool("core.ignorecase"):
2349 res['depotFile'] = res['depotFile'].lower()
2350 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2351
2352 # not found files or unmap files set to ""
2353 for depotFile in fileArgs:
2354 if gitConfigBool("core.ignorecase"):
2355 depotFile = depotFile.lower()
2356 if depotFile not in self.client_spec_path_cache:
2357 self.client_spec_path_cache[depotFile] = ""
2358
2359 def map_in_client(self, depot_path):
2360 """Return the relative location in the client where this
2361 depot file should live. Returns "" if the file should
2362 not be mapped in the client."""
2363
2364 if gitConfigBool("core.ignorecase"):
2365 depot_path = depot_path.lower()
2366
2367 if depot_path in self.client_spec_path_cache:
2368 return self.client_spec_path_cache[depot_path]
2369
2370 die( "Error: %s is not found in client spec path" % depot_path )
2371 return ""
2372
2373 class P4Sync(Command, P4UserMap):
2374 delete_actions = ( "delete", "move/delete", "purge" )
2375
2376 def __init__(self):
2377 Command.__init__(self)
2378 P4UserMap.__init__(self)
2379 self.options = [
2380 optparse.make_option("--branch", dest="branch"),
2381 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2382 optparse.make_option("--changesfile", dest="changesFile"),
2383 optparse.make_option("--silent", dest="silent", action="store_true"),
2384 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2385 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2386 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2387 help="Import into refs/heads/ , not refs/remotes"),
2388 optparse.make_option("--max-changes", dest="maxChanges",
2389 help="Maximum number of changes to import"),
2390 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2391 help="Internal block size to use when iteratively calling p4 changes"),
2392 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2393 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2394 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2395 help="Only sync files that are included in the Perforce Client Spec"),
2396 optparse.make_option("-/", dest="cloneExclude",
2397 action="append", type="string",
2398 help="exclude depot path"),
2399 ]
2400 self.description = """Imports from Perforce into a git repository.\n
2401 example:
2402 //depot/my/project/ -- to import the current head
2403 //depot/my/project/@all -- to import everything
2404 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2405
2406 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2407
2408 self.usage += " //depot/path[@revRange]"
2409 self.silent = False
2410 self.createdBranches = set()
2411 self.committedChanges = set()
2412 self.branch = ""
2413 self.detectBranches = False
2414 self.detectLabels = False
2415 self.importLabels = False
2416 self.changesFile = ""
2417 self.syncWithOrigin = True
2418 self.importIntoRemotes = True
2419 self.maxChanges = ""
2420 self.changes_block_size = None
2421 self.keepRepoPath = False
2422 self.depotPaths = None
2423 self.p4BranchesInGit = []
2424 self.cloneExclude = []
2425 self.useClientSpec = False
2426 self.useClientSpec_from_options = False
2427 self.clientSpecDirs = None
2428 self.tempBranches = []
2429 self.tempBranchLocation = "refs/git-p4-tmp"
2430 self.largeFileSystem = None
2431
2432 if gitConfig('git-p4.largeFileSystem'):
2433 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2434 self.largeFileSystem = largeFileSystemConstructor(
2435 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2436 )
2437
2438 if gitConfig("git-p4.syncFromOrigin") == "false":
2439 self.syncWithOrigin = False
2440
2441 # Force a checkpoint in fast-import and wait for it to finish
2442 def checkpoint(self):
2443 self.gitStream.write("checkpoint\n\n")
2444 self.gitStream.write("progress checkpoint\n\n")
2445 out = self.gitOutput.readline()
2446 if self.verbose:
2447 print "checkpoint finished: " + out
2448
2449 def extractFilesFromCommit(self, commit):
2450 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2451 for path in self.cloneExclude]
2452 files = []
2453 fnum = 0
2454 while commit.has_key("depotFile%s" % fnum):
2455 path = commit["depotFile%s" % fnum]
2456
2457 if [p for p in self.cloneExclude
2458 if p4PathStartsWith(path, p)]:
2459 found = False
2460 else:
2461 found = [p for p in self.depotPaths
2462 if p4PathStartsWith(path, p)]
2463 if not found:
2464 fnum = fnum + 1
2465 continue
2466
2467 file = {}
2468 file["path"] = path
2469 file["rev"] = commit["rev%s" % fnum]
2470 file["action"] = commit["action%s" % fnum]
2471 file["type"] = commit["type%s" % fnum]
2472 files.append(file)
2473 fnum = fnum + 1
2474 return files
2475
2476 def extractJobsFromCommit(self, commit):
2477 jobs = []
2478 jnum = 0
2479 while commit.has_key("job%s" % jnum):
2480 job = commit["job%s" % jnum]
2481 jobs.append(job)
2482 jnum = jnum + 1
2483 return jobs
2484
2485 def stripRepoPath(self, path, prefixes):
2486 """When streaming files, this is called to map a p4 depot path
2487 to where it should go in git. The prefixes are either
2488 self.depotPaths, or self.branchPrefixes in the case of
2489 branch detection."""
2490
2491 if self.useClientSpec:
2492 # branch detection moves files up a level (the branch name)
2493 # from what client spec interpretation gives
2494 path = self.clientSpecDirs.map_in_client(path)
2495 if self.detectBranches:
2496 for b in self.knownBranches:
2497 if path.startswith(b + "/"):
2498 path = path[len(b)+1:]
2499
2500 elif self.keepRepoPath:
2501 # Preserve everything in relative path name except leading
2502 # //depot/; just look at first prefix as they all should
2503 # be in the same depot.
2504 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2505 if p4PathStartsWith(path, depot):
2506 path = path[len(depot):]
2507
2508 else:
2509 for p in prefixes:
2510 if p4PathStartsWith(path, p):
2511 path = path[len(p):]
2512 break
2513
2514 path = wildcard_decode(path)
2515 return path
2516
2517 def splitFilesIntoBranches(self, commit):
2518 """Look at each depotFile in the commit to figure out to what
2519 branch it belongs."""
2520
2521 if self.clientSpecDirs:
2522 files = self.extractFilesFromCommit(commit)
2523 self.clientSpecDirs.update_client_spec_path_cache(files)
2524
2525 branches = {}
2526 fnum = 0
2527 while commit.has_key("depotFile%s" % fnum):
2528 path = commit["depotFile%s" % fnum]
2529 found = [p for p in self.depotPaths
2530 if p4PathStartsWith(path, p)]
2531 if not found:
2532 fnum = fnum + 1
2533 continue
2534
2535 file = {}
2536 file["path"] = path
2537 file["rev"] = commit["rev%s" % fnum]
2538 file["action"] = commit["action%s" % fnum]
2539 file["type"] = commit["type%s" % fnum]
2540 fnum = fnum + 1
2541
2542 # start with the full relative path where this file would
2543 # go in a p4 client
2544 if self.useClientSpec:
2545 relPath = self.clientSpecDirs.map_in_client(path)
2546 else:
2547 relPath = self.stripRepoPath(path, self.depotPaths)
2548
2549 for branch in self.knownBranches.keys():
2550 # add a trailing slash so that a commit into qt/4.2foo
2551 # doesn't end up in qt/4.2, e.g.
2552 if relPath.startswith(branch + "/"):
2553 if branch not in branches:
2554 branches[branch] = []
2555 branches[branch].append(file)
2556 break
2557
2558 return branches
2559
2560 def writeToGitStream(self, gitMode, relPath, contents):
2561 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2562 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2563 for d in contents:
2564 self.gitStream.write(d)
2565 self.gitStream.write('\n')
2566
2567 def encodeWithUTF8(self, path):
2568 try:
2569 path.decode('ascii')
2570 except:
2571 encoding = 'utf8'
2572 if gitConfig('git-p4.pathEncoding'):
2573 encoding = gitConfig('git-p4.pathEncoding')
2574 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2575 if self.verbose:
2576 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2577 return path
2578
2579 # output one file from the P4 stream
2580 # - helper for streamP4Files
2581
2582 def streamOneP4File(self, file, contents):
2583 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2584 relPath = self.encodeWithUTF8(relPath)
2585 if verbose:
2586 size = int(self.stream_file['fileSize'])
2587 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2588 sys.stdout.flush()
2589
2590 (type_base, type_mods) = split_p4_type(file["type"])
2591
2592 git_mode = "100644"
2593 if "x" in type_mods:
2594 git_mode = "100755"
2595 if type_base == "symlink":
2596 git_mode = "120000"
2597 # p4 print on a symlink sometimes contains "target\n";
2598 # if it does, remove the newline
2599 data = ''.join(contents)
2600 if not data:
2601 # Some version of p4 allowed creating a symlink that pointed
2602 # to nothing. This causes p4 errors when checking out such
2603 # a change, and errors here too. Work around it by ignoring
2604 # the bad symlink; hopefully a future change fixes it.
2605 print "\nIgnoring empty symlink in %s" % file['depotFile']
2606 return
2607 elif data[-1] == '\n':
2608 contents = [data[:-1]]
2609 else:
2610 contents = [data]
2611
2612 if type_base == "utf16":
2613 # p4 delivers different text in the python output to -G
2614 # than it does when using "print -o", or normal p4 client
2615 # operations. utf16 is converted to ascii or utf8, perhaps.
2616 # But ascii text saved as -t utf16 is completely mangled.
2617 # Invoke print -o to get the real contents.
2618 #
2619 # On windows, the newlines will always be mangled by print, so put
2620 # them back too. This is not needed to the cygwin windows version,
2621 # just the native "NT" type.
2622 #
2623 try:
2624 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2625 except Exception as e:
2626 if 'Translation of file content failed' in str(e):
2627 type_base = 'binary'
2628 else:
2629 raise e
2630 else:
2631 if p4_version_string().find('/NT') >= 0:
2632 text = text.replace('\r\n', '\n')
2633 contents = [ text ]
2634
2635 if type_base == "apple":
2636 # Apple filetype files will be streamed as a concatenation of
2637 # its appledouble header and the contents. This is useless
2638 # on both macs and non-macs. If using "print -q -o xx", it
2639 # will create "xx" with the data, and "%xx" with the header.
2640 # This is also not very useful.
2641 #
2642 # Ideally, someday, this script can learn how to generate
2643 # appledouble files directly and import those to git, but
2644 # non-mac machines can never find a use for apple filetype.
2645 print "\nIgnoring apple filetype file %s" % file['depotFile']
2646 return
2647
2648 # Note that we do not try to de-mangle keywords on utf16 files,
2649 # even though in theory somebody may want that.
2650 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2651 if pattern:
2652 regexp = re.compile(pattern, re.VERBOSE)
2653 text = ''.join(contents)
2654 text = regexp.sub(r'$\1$', text)
2655 contents = [ text ]
2656
2657 if self.largeFileSystem:
2658 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2659
2660 self.writeToGitStream(git_mode, relPath, contents)
2661
2662 def streamOneP4Deletion(self, file):
2663 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2664 relPath = self.encodeWithUTF8(relPath)
2665 if verbose:
2666 sys.stdout.write("delete %s\n" % relPath)
2667 sys.stdout.flush()
2668 self.gitStream.write("D %s\n" % relPath)
2669
2670 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2671 self.largeFileSystem.removeLargeFile(relPath)
2672
2673 # handle another chunk of streaming data
2674 def streamP4FilesCb(self, marshalled):
2675
2676 # catch p4 errors and complain
2677 err = None
2678 if "code" in marshalled:
2679 if marshalled["code"] == "error":
2680 if "data" in marshalled:
2681 err = marshalled["data"].rstrip()
2682
2683 if not err and 'fileSize' in self.stream_file:
2684 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2685 if required_bytes > 0:
2686 err = 'Not enough space left on %s! Free at least %i MB.' % (
2687 os.getcwd(), required_bytes/1024/1024
2688 )
2689
2690 if err:
2691 f = None
2692 if self.stream_have_file_info:
2693 if "depotFile" in self.stream_file:
2694 f = self.stream_file["depotFile"]
2695 # force a failure in fast-import, else an empty
2696 # commit will be made
2697 self.gitStream.write("\n")
2698 self.gitStream.write("die-now\n")
2699 self.gitStream.close()
2700 # ignore errors, but make sure it exits first
2701 self.importProcess.wait()
2702 if f:
2703 die("Error from p4 print for %s: %s" % (f, err))
2704 else:
2705 die("Error from p4 print: %s" % err)
2706
2707 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2708 # start of a new file - output the old one first
2709 self.streamOneP4File(self.stream_file, self.stream_contents)
2710 self.stream_file = {}
2711 self.stream_contents = []
2712 self.stream_have_file_info = False
2713
2714 # pick up the new file information... for the
2715 # 'data' field we need to append to our array
2716 for k in marshalled.keys():
2717 if k == 'data':
2718 if 'streamContentSize' not in self.stream_file:
2719 self.stream_file['streamContentSize'] = 0
2720 self.stream_file['streamContentSize'] += len(marshalled['data'])
2721 self.stream_contents.append(marshalled['data'])
2722 else:
2723 self.stream_file[k] = marshalled[k]
2724
2725 if (verbose and
2726 'streamContentSize' in self.stream_file and
2727 'fileSize' in self.stream_file and
2728 'depotFile' in self.stream_file):
2729 size = int(self.stream_file["fileSize"])
2730 if size > 0:
2731 progress = 100*self.stream_file['streamContentSize']/size
2732 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2733 sys.stdout.flush()
2734
2735 self.stream_have_file_info = True
2736
2737 # Stream directly from "p4 files" into "git fast-import"
2738 def streamP4Files(self, files):
2739 filesForCommit = []
2740 filesToRead = []
2741 filesToDelete = []
2742
2743 for f in files:
2744 filesForCommit.append(f)
2745 if f['action'] in self.delete_actions:
2746 filesToDelete.append(f)
2747 else:
2748 filesToRead.append(f)
2749
2750 # deleted files...
2751 for f in filesToDelete:
2752 self.streamOneP4Deletion(f)
2753
2754 if len(filesToRead) > 0:
2755 self.stream_file = {}
2756 self.stream_contents = []
2757 self.stream_have_file_info = False
2758
2759 # curry self argument
2760 def streamP4FilesCbSelf(entry):
2761 self.streamP4FilesCb(entry)
2762
2763 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2764
2765 p4CmdList(["-x", "-", "print"],
2766 stdin=fileArgs,
2767 cb=streamP4FilesCbSelf)
2768
2769 # do the last chunk
2770 if self.stream_file.has_key('depotFile'):
2771 self.streamOneP4File(self.stream_file, self.stream_contents)
2772
2773 def make_email(self, userid):
2774 if userid in self.users:
2775 return self.users[userid]
2776 else:
2777 return "%s <a@b>" % userid
2778
2779 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2780 """ Stream a p4 tag.
2781 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2782 """
2783
2784 if verbose:
2785 print "writing tag %s for commit %s" % (labelName, commit)
2786 gitStream.write("tag %s\n" % labelName)
2787 gitStream.write("from %s\n" % commit)
2788
2789 if labelDetails.has_key('Owner'):
2790 owner = labelDetails["Owner"]
2791 else:
2792 owner = None
2793
2794 # Try to use the owner of the p4 label, or failing that,
2795 # the current p4 user id.
2796 if owner:
2797 email = self.make_email(owner)
2798 else:
2799 email = self.make_email(self.p4UserId())
2800 tagger = "%s %s %s" % (email, epoch, self.tz)
2801
2802 gitStream.write("tagger %s\n" % tagger)
2803
2804 print "labelDetails=",labelDetails
2805 if labelDetails.has_key('Description'):
2806 description = labelDetails['Description']
2807 else:
2808 description = 'Label from git p4'
2809
2810 gitStream.write("data %d\n" % len(description))
2811 gitStream.write(description)
2812 gitStream.write("\n")
2813
2814 def inClientSpec(self, path):
2815 if not self.clientSpecDirs:
2816 return True
2817 inClientSpec = self.clientSpecDirs.map_in_client(path)
2818 if not inClientSpec and self.verbose:
2819 print('Ignoring file outside of client spec: {0}'.format(path))
2820 return inClientSpec
2821
2822 def hasBranchPrefix(self, path):
2823 if not self.branchPrefixes:
2824 return True
2825 hasPrefix = [p for p in self.branchPrefixes
2826 if p4PathStartsWith(path, p)]
2827 if not hasPrefix and self.verbose:
2828 print('Ignoring file outside of prefix: {0}'.format(path))
2829 return hasPrefix
2830
2831 def commit(self, details, files, branch, parent = ""):
2832 epoch = details["time"]
2833 author = details["user"]
2834 jobs = self.extractJobsFromCommit(details)
2835
2836 if self.verbose:
2837 print('commit into {0}'.format(branch))
2838
2839 if self.clientSpecDirs:
2840 self.clientSpecDirs.update_client_spec_path_cache(files)
2841
2842 files = [f for f in files
2843 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2844
2845 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2846 print('Ignoring revision {0} as it would produce an empty commit.'
2847 .format(details['change']))
2848 return
2849
2850 self.gitStream.write("commit %s\n" % branch)
2851 self.gitStream.write("mark :%s\n" % details["change"])
2852 self.committedChanges.add(int(details["change"]))
2853 committer = ""
2854 if author not in self.users:
2855 self.getUserMapFromPerforceServer()
2856 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2857
2858 self.gitStream.write("committer %s\n" % committer)
2859
2860 self.gitStream.write("data <<EOT\n")
2861 self.gitStream.write(details["desc"])
2862 if len(jobs) > 0:
2863 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2864 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2865 (','.join(self.branchPrefixes), details["change"]))
2866 if len(details['options']) > 0:
2867 self.gitStream.write(": options = %s" % details['options'])
2868 self.gitStream.write("]\nEOT\n\n")
2869
2870 if len(parent) > 0:
2871 if self.verbose:
2872 print "parent %s" % parent
2873 self.gitStream.write("from %s\n" % parent)
2874
2875 self.streamP4Files(files)
2876 self.gitStream.write("\n")
2877
2878 change = int(details["change"])
2879
2880 if self.labels.has_key(change):
2881 label = self.labels[change]
2882 labelDetails = label[0]
2883 labelRevisions = label[1]
2884 if self.verbose:
2885 print "Change %s is labelled %s" % (change, labelDetails)
2886
2887 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2888 for p in self.branchPrefixes])
2889
2890 if len(files) == len(labelRevisions):
2891
2892 cleanedFiles = {}
2893 for info in files:
2894 if info["action"] in self.delete_actions:
2895 continue
2896 cleanedFiles[info["depotFile"]] = info["rev"]
2897
2898 if cleanedFiles == labelRevisions:
2899 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2900
2901 else:
2902 if not self.silent:
2903 print ("Tag %s does not match with change %s: files do not match."
2904 % (labelDetails["label"], change))
2905
2906 else:
2907 if not self.silent:
2908 print ("Tag %s does not match with change %s: file count is different."
2909 % (labelDetails["label"], change))
2910
2911 # Build a dictionary of changelists and labels, for "detect-labels" option.
2912 def getLabels(self):
2913 self.labels = {}
2914
2915 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2916 if len(l) > 0 and not self.silent:
2917 print "Finding files belonging to labels in %s" % `self.depotPaths`
2918
2919 for output in l:
2920 label = output["label"]
2921 revisions = {}
2922 newestChange = 0
2923 if self.verbose:
2924 print "Querying files for label %s" % label
2925 for file in p4CmdList(["files"] +
2926 ["%s...@%s" % (p, label)
2927 for p in self.depotPaths]):
2928 revisions[file["depotFile"]] = file["rev"]
2929 change = int(file["change"])
2930 if change > newestChange:
2931 newestChange = change
2932
2933 self.labels[newestChange] = [output, revisions]
2934
2935 if self.verbose:
2936 print "Label changes: %s" % self.labels.keys()
2937
2938 # Import p4 labels as git tags. A direct mapping does not
2939 # exist, so assume that if all the files are at the same revision
2940 # then we can use that, or it's something more complicated we should
2941 # just ignore.
2942 def importP4Labels(self, stream, p4Labels):
2943 if verbose:
2944 print "import p4 labels: " + ' '.join(p4Labels)
2945
2946 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2947 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2948 if len(validLabelRegexp) == 0:
2949 validLabelRegexp = defaultLabelRegexp
2950 m = re.compile(validLabelRegexp)
2951
2952 for name in p4Labels:
2953 commitFound = False
2954
2955 if not m.match(name):
2956 if verbose:
2957 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2958 continue
2959
2960 if name in ignoredP4Labels:
2961 continue
2962
2963 labelDetails = p4CmdList(['label', "-o", name])[0]
2964
2965 # get the most recent changelist for each file in this label
2966 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2967 for p in self.depotPaths])
2968
2969 if change.has_key('change'):
2970 # find the corresponding git commit; take the oldest commit
2971 changelist = int(change['change'])
2972 if changelist in self.committedChanges:
2973 gitCommit = ":%d" % changelist # use a fast-import mark
2974 commitFound = True
2975 else:
2976 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2977 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2978 if len(gitCommit) == 0:
2979 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2980 else:
2981 commitFound = True
2982 gitCommit = gitCommit.strip()
2983
2984 if commitFound:
2985 # Convert from p4 time format
2986 try:
2987 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2988 except ValueError:
2989 print "Could not convert label time %s" % labelDetails['Update']
2990 tmwhen = 1
2991
2992 when = int(time.mktime(tmwhen))
2993 self.streamTag(stream, name, labelDetails, gitCommit, when)
2994 if verbose:
2995 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2996 else:
2997 if verbose:
2998 print "Label %s has no changelists - possibly deleted?" % name
2999
3000 if not commitFound:
3001 # We can't import this label; don't try again as it will get very
3002 # expensive repeatedly fetching all the files for labels that will
3003 # never be imported. If the label is moved in the future, the
3004 # ignore will need to be removed manually.
3005 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3006
3007 def guessProjectName(self):
3008 for p in self.depotPaths:
3009 if p.endswith("/"):
3010 p = p[:-1]
3011 p = p[p.strip().rfind("/") + 1:]
3012 if not p.endswith("/"):
3013 p += "/"
3014 return p
3015
3016 def getBranchMapping(self):
3017 lostAndFoundBranches = set()
3018
3019 user = gitConfig("git-p4.branchUser")
3020 if len(user) > 0:
3021 command = "branches -u %s" % user
3022 else:
3023 command = "branches"
3024
3025 for info in p4CmdList(command):
3026 details = p4Cmd(["branch", "-o", info["branch"]])
3027 viewIdx = 0
3028 while details.has_key("View%s" % viewIdx):
3029 paths = details["View%s" % viewIdx].split(" ")
3030 viewIdx = viewIdx + 1
3031 # require standard //depot/foo/... //depot/bar/... mapping
3032 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3033 continue
3034 source = paths[0]
3035 destination = paths[1]
3036 ## HACK
3037 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3038 source = source[len(self.depotPaths[0]):-4]
3039 destination = destination[len(self.depotPaths[0]):-4]
3040
3041 if destination in self.knownBranches:
3042 if not self.silent:
3043 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3044 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3045 continue
3046
3047 self.knownBranches[destination] = source
3048
3049 lostAndFoundBranches.discard(destination)
3050
3051 if source not in self.knownBranches:
3052 lostAndFoundBranches.add(source)
3053
3054 # Perforce does not strictly require branches to be defined, so we also
3055 # check git config for a branch list.
3056 #
3057 # Example of branch definition in git config file:
3058 # [git-p4]
3059 # branchList=main:branchA
3060 # branchList=main:branchB
3061 # branchList=branchA:branchC
3062 configBranches = gitConfigList("git-p4.branchList")
3063 for branch in configBranches:
3064 if branch:
3065 (source, destination) = branch.split(":")
3066 self.knownBranches[destination] = source
3067
3068 lostAndFoundBranches.discard(destination)
3069
3070 if source not in self.knownBranches:
3071 lostAndFoundBranches.add(source)
3072
3073
3074 for branch in lostAndFoundBranches:
3075 self.knownBranches[branch] = branch
3076
3077 def getBranchMappingFromGitBranches(self):
3078 branches = p4BranchesInGit(self.importIntoRemotes)
3079 for branch in branches.keys():
3080 if branch == "master":
3081 branch = "main"
3082 else:
3083 branch = branch[len(self.projectName):]
3084 self.knownBranches[branch] = branch
3085
3086 def updateOptionDict(self, d):
3087 option_keys = {}
3088 if self.keepRepoPath:
3089 option_keys['keepRepoPath'] = 1
3090
3091 d["options"] = ' '.join(sorted(option_keys.keys()))
3092
3093 def readOptions(self, d):
3094 self.keepRepoPath = (d.has_key('options')
3095 and ('keepRepoPath' in d['options']))
3096
3097 def gitRefForBranch(self, branch):
3098 if branch == "main":
3099 return self.refPrefix + "master"
3100
3101 if len(branch) <= 0:
3102 return branch
3103
3104 return self.refPrefix + self.projectName + branch
3105
3106 def gitCommitByP4Change(self, ref, change):
3107 if self.verbose:
3108 print "looking in ref " + ref + " for change %s using bisect..." % change
3109
3110 earliestCommit = ""
3111 latestCommit = parseRevision(ref)
3112
3113 while True:
3114 if self.verbose:
3115 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3116 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3117 if len(next) == 0:
3118 if self.verbose:
3119 print "argh"
3120 return ""
3121 log = extractLogMessageFromGitCommit(next)
3122 settings = extractSettingsGitLog(log)
3123 currentChange = int(settings['change'])
3124 if self.verbose:
3125 print "current change %s" % currentChange
3126
3127 if currentChange == change:
3128 if self.verbose:
3129 print "found %s" % next
3130 return next
3131
3132 if currentChange < change:
3133 earliestCommit = "^%s" % next
3134 else:
3135 latestCommit = "%s" % next
3136
3137 return ""
3138
3139 def importNewBranch(self, branch, maxChange):
3140 # make fast-import flush all changes to disk and update the refs using the checkpoint
3141 # command so that we can try to find the branch parent in the git history
3142 self.gitStream.write("checkpoint\n\n");
3143 self.gitStream.flush();
3144 branchPrefix = self.depotPaths[0] + branch + "/"
3145 range = "@1,%s" % maxChange
3146 #print "prefix" + branchPrefix
3147 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3148 if len(changes) <= 0:
3149 return False
3150 firstChange = changes[0]
3151 #print "first change in branch: %s" % firstChange
3152 sourceBranch = self.knownBranches[branch]
3153 sourceDepotPath = self.depotPaths[0] + sourceBranch
3154 sourceRef = self.gitRefForBranch(sourceBranch)
3155 #print "source " + sourceBranch
3156
3157 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3158 #print "branch parent: %s" % branchParentChange
3159 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3160 if len(gitParent) > 0:
3161 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3162 #print "parent git commit: %s" % gitParent
3163
3164 self.importChanges(changes)
3165 return True
3166
3167 def searchParent(self, parent, branch, target):
3168 parentFound = False
3169 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3170 "--no-merges", parent]):
3171 blob = blob.strip()
3172 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3173 parentFound = True
3174 if self.verbose:
3175 print "Found parent of %s in commit %s" % (branch, blob)
3176 break
3177 if parentFound:
3178 return blob
3179 else:
3180 return None
3181
3182 def importChanges(self, changes):
3183 cnt = 1
3184 for change in changes:
3185 description = p4_describe(change)
3186 self.updateOptionDict(description)
3187
3188 if not self.silent:
3189 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3190 sys.stdout.flush()
3191 cnt = cnt + 1
3192
3193 try:
3194 if self.detectBranches:
3195 branches = self.splitFilesIntoBranches(description)
3196 for branch in branches.keys():
3197 ## HACK --hwn
3198 branchPrefix = self.depotPaths[0] + branch + "/"
3199 self.branchPrefixes = [ branchPrefix ]
3200
3201 parent = ""
3202
3203 filesForCommit = branches[branch]
3204
3205 if self.verbose:
3206 print "branch is %s" % branch
3207
3208 self.updatedBranches.add(branch)
3209
3210 if branch not in self.createdBranches:
3211 self.createdBranches.add(branch)
3212 parent = self.knownBranches[branch]
3213 if parent == branch:
3214 parent = ""
3215 else:
3216 fullBranch = self.projectName + branch
3217 if fullBranch not in self.p4BranchesInGit:
3218 if not self.silent:
3219 print("\n Importing new branch %s" % fullBranch);
3220 if self.importNewBranch(branch, change - 1):
3221 parent = ""
3222 self.p4BranchesInGit.append(fullBranch)
3223 if not self.silent:
3224 print("\n Resuming with change %s" % change);
3225
3226 if self.verbose:
3227 print "parent determined through known branches: %s" % parent
3228
3229 branch = self.gitRefForBranch(branch)
3230 parent = self.gitRefForBranch(parent)
3231
3232 if self.verbose:
3233 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3234
3235 if len(parent) == 0 and branch in self.initialParents:
3236 parent = self.initialParents[branch]
3237 del self.initialParents[branch]
3238
3239 blob = None
3240 if len(parent) > 0:
3241 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3242 if self.verbose:
3243 print "Creating temporary branch: " + tempBranch
3244 self.commit(description, filesForCommit, tempBranch)
3245 self.tempBranches.append(tempBranch)
3246 self.checkpoint()
3247 blob = self.searchParent(parent, branch, tempBranch)
3248 if blob:
3249 self.commit(description, filesForCommit, branch, blob)
3250 else:
3251 if self.verbose:
3252 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3253 self.commit(description, filesForCommit, branch, parent)
3254 else:
3255 files = self.extractFilesFromCommit(description)
3256 self.commit(description, files, self.branch,
3257 self.initialParent)
3258 # only needed once, to connect to the previous commit
3259 self.initialParent = ""
3260 except IOError:
3261 print self.gitError.read()
3262 sys.exit(1)
3263
3264 def importHeadRevision(self, revision):
3265 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3266
3267 details = {}
3268 details["user"] = "git perforce import user"
3269 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3270 % (' '.join(self.depotPaths), revision))
3271 details["change"] = revision
3272 newestRevision = 0
3273
3274 fileCnt = 0
3275 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3276
3277 for info in p4CmdList(["files"] + fileArgs):
3278
3279 if 'code' in info and info['code'] == 'error':
3280 sys.stderr.write("p4 returned an error: %s\n"
3281 % info['data'])
3282 if info['data'].find("must refer to client") >= 0:
3283 sys.stderr.write("This particular p4 error is misleading.\n")
3284 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3285 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3286 sys.exit(1)
3287 if 'p4ExitCode' in info:
3288 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3289 sys.exit(1)
3290
3291
3292 change = int(info["change"])
3293 if change > newestRevision:
3294 newestRevision = change
3295
3296 if info["action"] in self.delete_actions:
3297 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3298 #fileCnt = fileCnt + 1
3299 continue
3300
3301 for prop in ["depotFile", "rev", "action", "type" ]:
3302 details["%s%s" % (prop, fileCnt)] = info[prop]
3303
3304 fileCnt = fileCnt + 1
3305
3306 details["change"] = newestRevision
3307
3308 # Use time from top-most change so that all git p4 clones of
3309 # the same p4 repo have the same commit SHA1s.
3310 res = p4_describe(newestRevision)
3311 details["time"] = res["time"]
3312
3313 self.updateOptionDict(details)
3314 try:
3315 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3316 except IOError:
3317 print "IO error with git fast-import. Is your git version recent enough?"
3318 print self.gitError.read()
3319
3320
3321 def run(self, args):
3322 self.depotPaths = []
3323 self.changeRange = ""
3324 self.previousDepotPaths = []
3325 self.hasOrigin = False
3326
3327 # map from branch depot path to parent branch
3328 self.knownBranches = {}
3329 self.initialParents = {}
3330
3331 if self.importIntoRemotes:
3332 self.refPrefix = "refs/remotes/p4/"
3333 else:
3334 self.refPrefix = "refs/heads/p4/"
3335
3336 if self.syncWithOrigin:
3337 self.hasOrigin = originP4BranchesExist()
3338 if self.hasOrigin:
3339 if not self.silent:
3340 print 'Syncing with origin first, using "git fetch origin"'
3341 system("git fetch origin")
3342
3343 branch_arg_given = bool(self.branch)
3344 if len(self.branch) == 0:
3345 self.branch = self.refPrefix + "master"
3346 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3347 system("git update-ref %s refs/heads/p4" % self.branch)
3348 system("git branch -D p4")
3349
3350 # accept either the command-line option, or the configuration variable
3351 if self.useClientSpec:
3352 # will use this after clone to set the variable
3353 self.useClientSpec_from_options = True
3354 else:
3355 if gitConfigBool("git-p4.useclientspec"):
3356 self.useClientSpec = True
3357 if self.useClientSpec:
3358 self.clientSpecDirs = getClientSpec()
3359
3360 # TODO: should always look at previous commits,
3361 # merge with previous imports, if possible.
3362 if args == []:
3363 if self.hasOrigin:
3364 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3365
3366 # branches holds mapping from branch name to sha1
3367 branches = p4BranchesInGit(self.importIntoRemotes)
3368
3369 # restrict to just this one, disabling detect-branches
3370 if branch_arg_given:
3371 short = self.branch.split("/")[-1]
3372 if short in branches:
3373 self.p4BranchesInGit = [ short ]
3374 else:
3375 self.p4BranchesInGit = branches.keys()
3376
3377 if len(self.p4BranchesInGit) > 1:
3378 if not self.silent:
3379 print "Importing from/into multiple branches"
3380 self.detectBranches = True
3381 for branch in branches.keys():
3382 self.initialParents[self.refPrefix + branch] = \
3383 branches[branch]
3384
3385 if self.verbose:
3386 print "branches: %s" % self.p4BranchesInGit
3387
3388 p4Change = 0
3389 for branch in self.p4BranchesInGit:
3390 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3391
3392 settings = extractSettingsGitLog(logMsg)
3393
3394 self.readOptions(settings)
3395 if (settings.has_key('depot-paths')
3396 and settings.has_key ('change')):
3397 change = int(settings['change']) + 1
3398 p4Change = max(p4Change, change)
3399
3400 depotPaths = sorted(settings['depot-paths'])
3401 if self.previousDepotPaths == []:
3402 self.previousDepotPaths = depotPaths
3403 else:
3404 paths = []
3405 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3406 prev_list = prev.split("/")
3407 cur_list = cur.split("/")
3408 for i in range(0, min(len(cur_list), len(prev_list))):
3409 if cur_list[i] <> prev_list[i]:
3410 i = i - 1
3411 break
3412
3413 paths.append ("/".join(cur_list[:i + 1]))
3414
3415 self.previousDepotPaths = paths
3416
3417 if p4Change > 0:
3418 self.depotPaths = sorted(self.previousDepotPaths)
3419 self.changeRange = "@%s,#head" % p4Change
3420 if not self.silent and not self.detectBranches:
3421 print "Performing incremental import into %s git branch" % self.branch
3422
3423 # accept multiple ref name abbreviations:
3424 # refs/foo/bar/branch -> use it exactly
3425 # p4/branch -> prepend refs/remotes/ or refs/heads/
3426 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3427 if not self.branch.startswith("refs/"):
3428 if self.importIntoRemotes:
3429 prepend = "refs/remotes/"
3430 else:
3431 prepend = "refs/heads/"
3432 if not self.branch.startswith("p4/"):
3433 prepend += "p4/"
3434 self.branch = prepend + self.branch
3435
3436 if len(args) == 0 and self.depotPaths:
3437 if not self.silent:
3438 print "Depot paths: %s" % ' '.join(self.depotPaths)
3439 else:
3440 if self.depotPaths and self.depotPaths != args:
3441 print ("previous import used depot path %s and now %s was specified. "
3442 "This doesn't work!" % (' '.join (self.depotPaths),
3443 ' '.join (args)))
3444 sys.exit(1)
3445
3446 self.depotPaths = sorted(args)
3447
3448 revision = ""
3449 self.users = {}
3450
3451 # Make sure no revision specifiers are used when --changesfile
3452 # is specified.
3453 bad_changesfile = False
3454 if len(self.changesFile) > 0:
3455 for p in self.depotPaths:
3456 if p.find("@") >= 0 or p.find("#") >= 0:
3457 bad_changesfile = True
3458 break
3459 if bad_changesfile:
3460 die("Option --changesfile is incompatible with revision specifiers")
3461
3462 newPaths = []
3463 for p in self.depotPaths:
3464 if p.find("@") != -1:
3465 atIdx = p.index("@")
3466 self.changeRange = p[atIdx:]
3467 if self.changeRange == "@all":
3468 self.changeRange = ""
3469 elif ',' not in self.changeRange:
3470 revision = self.changeRange
3471 self.changeRange = ""
3472 p = p[:atIdx]
3473 elif p.find("#") != -1:
3474 hashIdx = p.index("#")
3475 revision = p[hashIdx:]
3476 p = p[:hashIdx]
3477 elif self.previousDepotPaths == []:
3478 # pay attention to changesfile, if given, else import
3479 # the entire p4 tree at the head revision
3480 if len(self.changesFile) == 0:
3481 revision = "#head"
3482
3483 p = re.sub ("\.\.\.$", "", p)
3484 if not p.endswith("/"):
3485 p += "/"
3486
3487 newPaths.append(p)
3488
3489 self.depotPaths = newPaths
3490
3491 # --detect-branches may change this for each branch
3492 self.branchPrefixes = self.depotPaths
3493
3494 self.loadUserMapFromCache()
3495 self.labels = {}
3496 if self.detectLabels:
3497 self.getLabels();
3498
3499 if self.detectBranches:
3500 ## FIXME - what's a P4 projectName ?
3501 self.projectName = self.guessProjectName()
3502
3503 if self.hasOrigin:
3504 self.getBranchMappingFromGitBranches()
3505 else:
3506 self.getBranchMapping()
3507 if self.verbose:
3508 print "p4-git branches: %s" % self.p4BranchesInGit
3509 print "initial parents: %s" % self.initialParents
3510 for b in self.p4BranchesInGit:
3511 if b != "master":
3512
3513 ## FIXME
3514 b = b[len(self.projectName):]
3515 self.createdBranches.add(b)
3516
3517 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3518
3519 self.importProcess = subprocess.Popen(["git", "fast-import"],
3520 stdin=subprocess.PIPE,
3521 stdout=subprocess.PIPE,
3522 stderr=subprocess.PIPE);
3523 self.gitOutput = self.importProcess.stdout
3524 self.gitStream = self.importProcess.stdin
3525 self.gitError = self.importProcess.stderr
3526
3527 if revision:
3528 self.importHeadRevision(revision)
3529 else:
3530 changes = []
3531
3532 if len(self.changesFile) > 0:
3533 output = open(self.changesFile).readlines()
3534 changeSet = set()
3535 for line in output:
3536 changeSet.add(int(line))
3537
3538 for change in changeSet:
3539 changes.append(change)
3540
3541 changes.sort()
3542 else:
3543 # catch "git p4 sync" with no new branches, in a repo that
3544 # does not have any existing p4 branches
3545 if len(args) == 0:
3546 if not self.p4BranchesInGit:
3547 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3548
3549 # The default branch is master, unless --branch is used to
3550 # specify something else. Make sure it exists, or complain
3551 # nicely about how to use --branch.
3552 if not self.detectBranches:
3553 if not branch_exists(self.branch):
3554 if branch_arg_given:
3555 die("Error: branch %s does not exist." % self.branch)
3556 else:
3557 die("Error: no branch %s; perhaps specify one with --branch." %
3558 self.branch)
3559
3560 if self.verbose:
3561 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3562 self.changeRange)
3563 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3564
3565 if len(self.maxChanges) > 0:
3566 changes = changes[:min(int(self.maxChanges), len(changes))]
3567
3568 if len(changes) == 0:
3569 if not self.silent:
3570 print "No changes to import!"
3571 else:
3572 if not self.silent and not self.detectBranches:
3573 print "Import destination: %s" % self.branch
3574
3575 self.updatedBranches = set()
3576
3577 if not self.detectBranches:
3578 if args:
3579 # start a new branch
3580 self.initialParent = ""
3581 else:
3582 # build on a previous revision
3583 self.initialParent = parseRevision(self.branch)
3584
3585 self.importChanges(changes)
3586
3587 if not self.silent:
3588 print ""
3589 if len(self.updatedBranches) > 0:
3590 sys.stdout.write("Updated branches: ")
3591 for b in self.updatedBranches:
3592 sys.stdout.write("%s " % b)
3593 sys.stdout.write("\n")
3594
3595 if gitConfigBool("git-p4.importLabels"):
3596 self.importLabels = True
3597
3598 if self.importLabels:
3599 p4Labels = getP4Labels(self.depotPaths)
3600 gitTags = getGitTags()
3601
3602 missingP4Labels = p4Labels - gitTags
3603 self.importP4Labels(self.gitStream, missingP4Labels)
3604
3605 self.gitStream.close()