git-p4: add option to disable syncing of p4/master with p4
[git/git.git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
60 real_cmd = ["p4"]
61
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
64 real_cmd += ["-u",user]
65
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
68 real_cmd += ["-P", password]
69
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
72 real_cmd += ["-p", port]
73
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
76 real_cmd += ["-H", host]
77
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
80 real_cmd += ["-c", client]
81
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
89
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
94 return real_cmd
95
96 def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
106 def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
127
128 def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
138 def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
145 def write_pipe(c, stdin):
146 if verbose:
147 sys.stderr.write('Writing pipe: %s\n' % str(c))
148
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
156
157 return val
158
159 def p4_write_pipe(c, stdin):
160 real_cmd = p4_build_cmd(c)
161 return write_pipe(real_cmd, stdin)
162
163 def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
167 """
168 if verbose:
169 sys.stderr.write('Reading pipe: %s\n' % str(c))
170
171 expand = isinstance(c,basestring)
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
174 return (p.returncode, out, err)
175
176 def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
180 """
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
187 return out
188
189 def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
192 """
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
198
199 def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
202
203 def read_pipe_lines(c):
204 if verbose:
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
210 val = pipe.readlines()
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
213
214 return val
215
216 def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
218 real_cmd = p4_build_cmd(c)
219 return read_pipe_lines(real_cmd)
220
221 def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
229
230 def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
235
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
248
249 def system(cmd, ignore_error=False):
250 expand = isinstance(cmd,basestring)
251 if verbose:
252 sys.stderr.write("executing %s\n" % str(cmd))
253 retcode = subprocess.call(cmd, shell=expand)
254 if retcode and not ignore_error:
255 raise CalledProcessError(retcode, cmd)
256
257 return retcode
258
259 def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
261 real_cmd = p4_build_cmd(cmd)
262 expand = isinstance(real_cmd, basestring)
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
266
267 _p4_version_string = None
268 def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
271
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276 """
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
282
283 def p4_integrate(src, dest):
284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
285
286 def p4_sync(f, *options):
287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
288
289 def p4_add(f):
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
295
296 def p4_delete(f):
297 p4_system(["delete", wildcard_encode(f)])
298
299 def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
301
302 def p4_revert(f):
303 p4_system(["revert", wildcard_encode(f)])
304
305 def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
307
308 def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
311
312 def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
315 def p4_last_change():
316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317 return int(results[0]['change'])
318
319 def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
323
324 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328 d = ds[0]
329
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340 return d
341
342 #
343 # Canonicalize the p4 type and return a tuple of the
344 # base type, plus any modifiers. See "p4 help filetypes"
345 # for a list and explanation.
346 #
347 def split_p4_type(p4type):
348
349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
366 }
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
376
377 #
378 # return the raw p4 type of a file (text, text+ko, etc)
379 #
380 def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
382 return results[0]['headType']
383
384 #
385 # Given a type base and modifier, return a regexp matching
386 # the keywords that can be expanded in the file
387 #
388 def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
400 (:[^$\n]+)? # possibly an old expansion, followed by...
401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
406
407 #
408 # Given a file, return a regexp matching the possible
409 # RCS keywords that will be expanded, or None for files
410 # with kw expansion turned off.
411 #
412 def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
418
419 def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
422
423 p4Type = "+x"
424
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
431
432 p4_reopen(p4Type, file)
433
434 def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
436
437 result = p4_read_pipe(["opened", wildcard_encode(file)])
438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
439 if match:
440 return match.group(1)
441 else:
442 die("Could not determine file type for %s (result: '%s')" % (file, result))
443
444 # Return the set of all p4 labels
445 def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
449
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
453
454 return labels
455
456 # Return the set of all git tags
457 def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
463
464 def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
470
471 def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
473
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
476
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
487
488 If the pattern is not matched, None is returned."""
489
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
501 }
502 return None
503
504 def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
508
509 def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
511
512 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
513
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
520
521 cmd = p4_build_cmd(cmd)
522 if verbose:
523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
524
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
536 stdin_file.flush()
537 stdin_file.seek(0)
538
539 p4 = subprocess.Popen(cmd,
540 shell=expand,
541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
543
544 result = []
545 try:
546 while True:
547 entry = marshal.load(p4.stdout)
548 if skip_info:
549 if 'code' in entry and entry['code'] == 'info':
550 continue
551 if cb is not None:
552 cb(entry)
553 else:
554 result.append(entry)
555 except EOFError:
556 pass
557 exitCode = p4.wait()
558 if exitCode != 0:
559 entry = {}
560 entry["p4ExitCode"] = exitCode
561 result.append(entry)
562
563 return result
564
565 def p4Cmd(cmd):
566 list = p4CmdList(cmd)
567 result = {}
568 for entry in list:
569 result.update(entry)
570 return result;
571
572 def p4Where(depotPath):
573 if not depotPath.endswith("/"):
574 depotPath += "/"
575 depotPathLong = depotPath + "..."
576 outputList = p4CmdList(["where", depotPathLong])
577 output = None
578 for entry in outputList:
579 if "depotFile" in entry:
580 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
581 # The base path always ends with "/...".
582 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
583 output = entry
584 break
585 elif "data" in entry:
586 data = entry.get("data")
587 space = data.find(" ")
588 if data[:space] == depotPath:
589 output = entry
590 break
591 if output == None:
592 return ""
593 if output["code"] == "error":
594 return ""
595 clientPath = ""
596 if "path" in output:
597 clientPath = output.get("path")
598 elif "data" in output:
599 data = output.get("data")
600 lastSpace = data.rfind(" ")
601 clientPath = data[lastSpace + 1:]
602
603 if clientPath.endswith("..."):
604 clientPath = clientPath[:-3]
605 return clientPath
606
607 def currentGitBranch():
608 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
609
610 def isValidGitDir(path):
611 return git_dir(path) != None
612
613 def parseRevision(ref):
614 return read_pipe("git rev-parse %s" % ref).strip()
615
616 def branchExists(ref):
617 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
618 ignore_error=True)
619 return len(rev) > 0
620
621 def extractLogMessageFromGitCommit(commit):
622 logMessage = ""
623
624 ## fixme: title is first line of commit, not 1st paragraph.
625 foundTitle = False
626 for log in read_pipe_lines("git cat-file commit %s" % commit):
627 if not foundTitle:
628 if len(log) == 1:
629 foundTitle = True
630 continue
631
632 logMessage += log
633 return logMessage
634
635 def extractSettingsGitLog(log):
636 values = {}
637 for line in log.split("\n"):
638 line = line.strip()
639 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
640 if not m:
641 continue
642
643 assignments = m.group(1).split (':')
644 for a in assignments:
645 vals = a.split ('=')
646 key = vals[0].strip()
647 val = ('='.join (vals[1:])).strip()
648 if val.endswith ('\"') and val.startswith('"'):
649 val = val[1:-1]
650
651 values[key] = val
652
653 paths = values.get("depot-paths")
654 if not paths:
655 paths = values.get("depot-path")
656 if paths:
657 values['depot-paths'] = paths.split(',')
658 return values
659
660 def gitBranchExists(branch):
661 proc = subprocess.Popen(["git", "rev-parse", branch],
662 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
663 return proc.wait() == 0;
664
665 _gitConfig = {}
666
667 def gitConfig(key, typeSpecifier=None):
668 if not _gitConfig.has_key(key):
669 cmd = [ "git", "config" ]
670 if typeSpecifier:
671 cmd += [ typeSpecifier ]
672 cmd += [ key ]
673 s = read_pipe(cmd, ignore_error=True)
674 _gitConfig[key] = s.strip()
675 return _gitConfig[key]
676
677 def gitConfigBool(key):
678 """Return a bool, using git config --bool. It is True only if the
679 variable is set to true, and False if set to false or not present
680 in the config."""
681
682 if not _gitConfig.has_key(key):
683 _gitConfig[key] = gitConfig(key, '--bool') == "true"
684 return _gitConfig[key]
685
686 def gitConfigInt(key):
687 if not _gitConfig.has_key(key):
688 cmd = [ "git", "config", "--int", key ]
689 s = read_pipe(cmd, ignore_error=True)
690 v = s.strip()
691 try:
692 _gitConfig[key] = int(gitConfig(key, '--int'))
693 except ValueError:
694 _gitConfig[key] = None
695 return _gitConfig[key]
696
697 def gitConfigList(key):
698 if not _gitConfig.has_key(key):
699 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
700 _gitConfig[key] = s.strip().splitlines()
701 if _gitConfig[key] == ['']:
702 _gitConfig[key] = []
703 return _gitConfig[key]
704
705 def p4BranchesInGit(branchesAreInRemotes=True):
706 """Find all the branches whose names start with "p4/", looking
707 in remotes or heads as specified by the argument. Return
708 a dictionary of { branch: revision } for each one found.
709 The branch names are the short names, without any
710 "p4/" prefix."""
711
712 branches = {}
713
714 cmdline = "git rev-parse --symbolic "
715 if branchesAreInRemotes:
716 cmdline += "--remotes"
717 else:
718 cmdline += "--branches"
719
720 for line in read_pipe_lines(cmdline):
721 line = line.strip()
722
723 # only import to p4/
724 if not line.startswith('p4/'):
725 continue
726 # special symbolic ref to p4/master
727 if line == "p4/HEAD":
728 continue
729
730 # strip off p4/ prefix
731 branch = line[len("p4/"):]
732
733 branches[branch] = parseRevision(line)
734
735 return branches
736
737 def branch_exists(branch):
738 """Make sure that the given ref name really exists."""
739
740 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
741 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
742 out, _ = p.communicate()
743 if p.returncode:
744 return False
745 # expect exactly one line of output: the branch name
746 return out.rstrip() == branch
747
748 def findUpstreamBranchPoint(head = "HEAD"):
749 branches = p4BranchesInGit()
750 # map from depot-path to branch name
751 branchByDepotPath = {}
752 for branch in branches.keys():
753 tip = branches[branch]
754 log = extractLogMessageFromGitCommit(tip)
755 settings = extractSettingsGitLog(log)
756 if settings.has_key("depot-paths"):
757 paths = ",".join(settings["depot-paths"])
758 branchByDepotPath[paths] = "remotes/p4/" + branch
759
760 settings = None
761 parent = 0
762 while parent < 65535:
763 commit = head + "~%s" % parent
764 log = extractLogMessageFromGitCommit(commit)
765 settings = extractSettingsGitLog(log)
766 if settings.has_key("depot-paths"):
767 paths = ",".join(settings["depot-paths"])
768 if branchByDepotPath.has_key(paths):
769 return [branchByDepotPath[paths], settings]
770
771 parent = parent + 1
772
773 return ["", settings]
774
775 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
776 if not silent:
777 print ("Creating/updating branch(es) in %s based on origin branch(es)"
778 % localRefPrefix)
779
780 originPrefix = "origin/p4/"
781
782 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
783 line = line.strip()
784 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
785 continue
786
787 headName = line[len(originPrefix):]
788 remoteHead = localRefPrefix + headName
789 originHead = line
790
791 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
792 if (not original.has_key('depot-paths')
793 or not original.has_key('change')):
794 continue
795
796 update = False
797 if not gitBranchExists(remoteHead):
798 if verbose:
799 print "creating %s" % remoteHead
800 update = True
801 else:
802 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
803 if settings.has_key('change') > 0:
804 if settings['depot-paths'] == original['depot-paths']:
805 originP4Change = int(original['change'])
806 p4Change = int(settings['change'])
807 if originP4Change > p4Change:
808 print ("%s (%s) is newer than %s (%s). "
809 "Updating p4 branch from origin."
810 % (originHead, originP4Change,
811 remoteHead, p4Change))
812 update = True
813 else:
814 print ("Ignoring: %s was imported from %s while "
815 "%s was imported from %s"
816 % (originHead, ','.join(original['depot-paths']),
817 remoteHead, ','.join(settings['depot-paths'])))
818
819 if update:
820 system("git update-ref %s %s" % (remoteHead, originHead))
821
822 def originP4BranchesExist():
823 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
824
825
826 def p4ParseNumericChangeRange(parts):
827 changeStart = int(parts[0][1:])
828 if parts[1] == '#head':
829 changeEnd = p4_last_change()
830 else:
831 changeEnd = int(parts[1])
832
833 return (changeStart, changeEnd)
834
835 def chooseBlockSize(blockSize):
836 if blockSize:
837 return blockSize
838 else:
839 return defaultBlockSize
840
841 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
842 assert depotPaths
843
844 # Parse the change range into start and end. Try to find integer
845 # revision ranges as these can be broken up into blocks to avoid
846 # hitting server-side limits (maxrows, maxscanresults). But if
847 # that doesn't work, fall back to using the raw revision specifier
848 # strings, without using block mode.
849
850 if changeRange is None or changeRange == '':
851 changeStart = 1
852 changeEnd = p4_last_change()
853 block_size = chooseBlockSize(requestedBlockSize)
854 else:
855 parts = changeRange.split(',')
856 assert len(parts) == 2
857 try:
858 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
859 block_size = chooseBlockSize(requestedBlockSize)
860 except:
861 changeStart = parts[0][1:]
862 changeEnd = parts[1]
863 if requestedBlockSize:
864 die("cannot use --changes-block-size with non-numeric revisions")
865 block_size = None
866
867 changes = set()
868
869 # Retrieve changes a block at a time, to prevent running
870 # into a MaxResults/MaxScanRows error from the server.
871
872 while True:
873 cmd = ['changes']
874
875 if block_size:
876 end = min(changeEnd, changeStart + block_size)
877 revisionRange = "%d,%d" % (changeStart, end)
878 else:
879 revisionRange = "%s,%s" % (changeStart, changeEnd)
880
881 for p in depotPaths:
882 cmd += ["%s...@%s" % (p, revisionRange)]
883
884 # Insert changes in chronological order
885 for entry in reversed(p4CmdList(cmd)):
886 if entry.has_key('p4ExitCode'):
887 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
888 if not entry.has_key('change'):
889 continue
890 changes.add(int(entry['change']))
891
892 if not block_size:
893 break
894
895 if end >= changeEnd:
896 break
897
898 changeStart = end + 1
899
900 changes = sorted(changes)
901 return changes
902
903 def p4PathStartsWith(path, prefix):
904 # This method tries to remedy a potential mixed-case issue:
905 #
906 # If UserA adds //depot/DirA/file1
907 # and UserB adds //depot/dira/file2
908 #
909 # we may or may not have a problem. If you have core.ignorecase=true,
910 # we treat DirA and dira as the same directory
911 if gitConfigBool("core.ignorecase"):
912 return path.lower().startswith(prefix.lower())
913 return path.startswith(prefix)
914
915 def getClientSpec():
916 """Look at the p4 client spec, create a View() object that contains
917 all the mappings, and return it."""
918
919 specList = p4CmdList("client -o")
920 if len(specList) != 1:
921 die('Output from "client -o" is %d lines, expecting 1' %
922 len(specList))
923
924 # dictionary of all client parameters
925 entry = specList[0]
926
927 # the //client/ name
928 client_name = entry["Client"]
929
930 # just the keys that start with "View"
931 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
932
933 # hold this new View
934 view = View(client_name)
935
936 # append the lines, in order, to the view
937 for view_num in range(len(view_keys)):
938 k = "View%d" % view_num
939 if k not in view_keys:
940 die("Expected view key %s missing" % k)
941 view.append(entry[k])
942
943 return view
944
945 def getClientRoot():
946 """Grab the client directory."""
947
948 output = p4CmdList("client -o")
949 if len(output) != 1:
950 die('Output from "client -o" is %d lines, expecting 1' % len(output))
951
952 entry = output[0]
953 if "Root" not in entry:
954 die('Client has no "Root"')
955
956 return entry["Root"]
957
958 #
959 # P4 wildcards are not allowed in filenames. P4 complains
960 # if you simply add them, but you can force it with "-f", in
961 # which case it translates them into %xx encoding internally.
962 #
963 def wildcard_decode(path):
964 # Search for and fix just these four characters. Do % last so
965 # that fixing it does not inadvertently create new %-escapes.
966 # Cannot have * in a filename in windows; untested as to
967 # what p4 would do in such a case.
968 if not platform.system() == "Windows":
969 path = path.replace("%2A", "*")
970 path = path.replace("%23", "#") \
971 .replace("%40", "@") \
972 .replace("%25", "%")
973 return path
974
975 def wildcard_encode(path):
976 # do % first to avoid double-encoding the %s introduced here
977 path = path.replace("%", "%25") \
978 .replace("*", "%2A") \
979 .replace("#", "%23") \
980 .replace("@", "%40")
981 return path
982
983 def wildcard_present(path):
984 m = re.search("[*#@%]", path)
985 return m is not None
986
987 class LargeFileSystem(object):
988 """Base class for large file system support."""
989
990 def __init__(self, writeToGitStream):
991 self.largeFiles = set()
992 self.writeToGitStream = writeToGitStream
993
994 def generatePointer(self, cloneDestination, contentFile):
995 """Return the content of a pointer file that is stored in Git instead of
996 the actual content."""
997 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
998
999 def pushFile(self, localLargeFile):
1000 """Push the actual content which is not stored in the Git repository to
1001 a server."""
1002 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1003
1004 def hasLargeFileExtension(self, relPath):
1005 return reduce(
1006 lambda a, b: a or b,
1007 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1008 False
1009 )
1010
1011 def generateTempFile(self, contents):
1012 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1013 for d in contents:
1014 contentFile.write(d)
1015 contentFile.close()
1016 return contentFile.name
1017
1018 def exceedsLargeFileThreshold(self, relPath, contents):
1019 if gitConfigInt('git-p4.largeFileThreshold'):
1020 contentsSize = sum(len(d) for d in contents)
1021 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1022 return True
1023 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024 contentsSize = sum(len(d) for d in contents)
1025 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1026 return False
1027 contentTempFile = self.generateTempFile(contents)
1028 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1029 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1030 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1031 zf.close()
1032 compressedContentsSize = zf.infolist()[0].compress_size
1033 os.remove(contentTempFile)
1034 os.remove(compressedContentFile.name)
1035 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1036 return True
1037 return False
1038
1039 def addLargeFile(self, relPath):
1040 self.largeFiles.add(relPath)
1041
1042 def removeLargeFile(self, relPath):
1043 self.largeFiles.remove(relPath)
1044
1045 def isLargeFile(self, relPath):
1046 return relPath in self.largeFiles
1047
1048 def processContent(self, git_mode, relPath, contents):
1049 """Processes the content of git fast import. This method decides if a
1050 file is stored in the large file system and handles all necessary
1051 steps."""
1052 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1053 contentTempFile = self.generateTempFile(contents)
1054 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1055 if pointer_git_mode:
1056 git_mode = pointer_git_mode
1057 if localLargeFile:
1058 # Move temp file to final location in large file system
1059 largeFileDir = os.path.dirname(localLargeFile)
1060 if not os.path.isdir(largeFileDir):
1061 os.makedirs(largeFileDir)
1062 shutil.move(contentTempFile, localLargeFile)
1063 self.addLargeFile(relPath)
1064 if gitConfigBool('git-p4.largeFilePush'):
1065 self.pushFile(localLargeFile)
1066 if verbose:
1067 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1068 return (git_mode, contents)
1069
1070 class MockLFS(LargeFileSystem):
1071 """Mock large file system for testing."""
1072
1073 def generatePointer(self, contentFile):
1074 """The pointer content is the original content prefixed with "pointer-".
1075 The local filename of the large file storage is derived from the file content.
1076 """
1077 with open(contentFile, 'r') as f:
1078 content = next(f)
1079 gitMode = '100644'
1080 pointerContents = 'pointer-' + content
1081 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1082 return (gitMode, pointerContents, localLargeFile)
1083
1084 def pushFile(self, localLargeFile):
1085 """The remote filename of the large file storage is the same as the local
1086 one but in a different directory.
1087 """
1088 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1089 if not os.path.exists(remotePath):
1090 os.makedirs(remotePath)
1091 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1092
1093 class GitLFS(LargeFileSystem):
1094 """Git LFS as backend for the git-p4 large file system.
1095 See https://git-lfs.github.com/ for details."""
1096
1097 def __init__(self, *args):
1098 LargeFileSystem.__init__(self, *args)
1099 self.baseGitAttributes = []
1100
1101 def generatePointer(self, contentFile):
1102 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1103 mode and content which is stored in the Git repository instead of
1104 the actual content. Return also the new location of the actual
1105 content.
1106 """
1107 if os.path.getsize(contentFile) == 0:
1108 return (None, '', None)
1109
1110 pointerProcess = subprocess.Popen(
1111 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1112 stdout=subprocess.PIPE
1113 )
1114 pointerFile = pointerProcess.stdout.read()
1115 if pointerProcess.wait():
1116 os.remove(contentFile)
1117 die('git-lfs pointer command failed. Did you install the extension?')
1118
1119 # Git LFS removed the preamble in the output of the 'pointer' command
1120 # starting from version 1.2.0. Check for the preamble here to support
1121 # earlier versions.
1122 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1123 if pointerFile.startswith('Git LFS pointer for'):
1124 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1125
1126 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1127 localLargeFile = os.path.join(
1128 os.getcwd(),
1129 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1130 oid,
1131 )
1132 # LFS Spec states that pointer files should not have the executable bit set.
1133 gitMode = '100644'
1134 return (gitMode, pointerFile, localLargeFile)
1135
1136 def pushFile(self, localLargeFile):
1137 uploadProcess = subprocess.Popen(
1138 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1139 )
1140 if uploadProcess.wait():
1141 die('git-lfs push command failed. Did you define a remote?')
1142
1143 def generateGitAttributes(self):
1144 return (
1145 self.baseGitAttributes +
1146 [
1147 '\n',
1148 '#\n',
1149 '# Git LFS (see https://git-lfs.github.com/)\n',
1150 '#\n',
1151 ] +
1152 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1153 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1154 ] +
1155 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1156 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1157 ]
1158 )
1159
1160 def addLargeFile(self, relPath):
1161 LargeFileSystem.addLargeFile(self, relPath)
1162 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1163
1164 def removeLargeFile(self, relPath):
1165 LargeFileSystem.removeLargeFile(self, relPath)
1166 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1167
1168 def processContent(self, git_mode, relPath, contents):
1169 if relPath == '.gitattributes':
1170 self.baseGitAttributes = contents
1171 return (git_mode, self.generateGitAttributes())
1172 else:
1173 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1174
1175 class Command:
1176 def __init__(self):
1177 self.usage = "usage: %prog [options]"
1178 self.needsGit = True
1179 self.verbose = False
1180
1181 # This is required for the "append" cloneExclude action
1182 def ensure_value(self, attr, value):
1183 if not hasattr(self, attr) or getattr(self, attr) is None:
1184 setattr(self, attr, value)
1185 return getattr(self, attr)
1186
1187 class P4UserMap:
1188 def __init__(self):
1189 self.userMapFromPerforceServer = False
1190 self.myP4UserId = None
1191
1192 def p4UserId(self):
1193 if self.myP4UserId:
1194 return self.myP4UserId
1195
1196 results = p4CmdList("user -o")
1197 for r in results:
1198 if r.has_key('User'):
1199 self.myP4UserId = r['User']
1200 return r['User']
1201 die("Could not find your p4 user id")
1202
1203 def p4UserIsMe(self, p4User):
1204 # return True if the given p4 user is actually me
1205 me = self.p4UserId()
1206 if not p4User or p4User != me:
1207 return False
1208 else:
1209 return True
1210
1211 def getUserCacheFilename(self):
1212 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1213 return home + "/.gitp4-usercache.txt"
1214
1215 def getUserMapFromPerforceServer(self):
1216 if self.userMapFromPerforceServer:
1217 return
1218 self.users = {}
1219 self.emails = {}
1220
1221 for output in p4CmdList("users"):
1222 if not output.has_key("User"):
1223 continue
1224 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1225 self.emails[output["Email"]] = output["User"]
1226
1227 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1228 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1229 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1230 if mapUser and len(mapUser[0]) == 3:
1231 user = mapUser[0][0]
1232 fullname = mapUser[0][1]
1233 email = mapUser[0][2]
1234 self.users[user] = fullname + " <" + email + ">"
1235 self.emails[email] = user
1236
1237 s = ''
1238 for (key, val) in self.users.items():
1239 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1240
1241 open(self.getUserCacheFilename(), "wb").write(s)
1242 self.userMapFromPerforceServer = True
1243
1244 def loadUserMapFromCache(self):
1245 self.users = {}
1246 self.userMapFromPerforceServer = False
1247 try:
1248 cache = open(self.getUserCacheFilename(), "rb")
1249 lines = cache.readlines()
1250 cache.close()
1251 for line in lines:
1252 entry = line.strip().split("\t")
1253 self.users[entry[0]] = entry[1]
1254 except IOError:
1255 self.getUserMapFromPerforceServer()
1256
1257 class P4Debug(Command):
1258 def __init__(self):
1259 Command.__init__(self)
1260 self.options = []
1261 self.description = "A tool to debug the output of p4 -G."
1262 self.needsGit = False
1263
1264 def run(self, args):
1265 j = 0
1266 for output in p4CmdList(args):
1267 print 'Element: %d' % j
1268 j += 1
1269 print output
1270 return True
1271
1272 class P4RollBack(Command):
1273 def __init__(self):
1274 Command.__init__(self)
1275 self.options = [
1276 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1277 ]
1278 self.description = "A tool to debug the multi-branch import. Don't use :)"
1279 self.rollbackLocalBranches = False
1280
1281 def run(self, args):
1282 if len(args) != 1:
1283 return False
1284 maxChange = int(args[0])
1285
1286 if "p4ExitCode" in p4Cmd("changes -m 1"):
1287 die("Problems executing p4");
1288
1289 if self.rollbackLocalBranches:
1290 refPrefix = "refs/heads/"
1291 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1292 else:
1293 refPrefix = "refs/remotes/"
1294 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1295
1296 for line in lines:
1297 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1298 line = line.strip()
1299 ref = refPrefix + line
1300 log = extractLogMessageFromGitCommit(ref)
1301 settings = extractSettingsGitLog(log)
1302
1303 depotPaths = settings['depot-paths']
1304 change = settings['change']
1305
1306 changed = False
1307
1308 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1309 for p in depotPaths]))) == 0:
1310 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1311 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1312 continue
1313
1314 while change and int(change) > maxChange:
1315 changed = True
1316 if self.verbose:
1317 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1318 system("git update-ref %s \"%s^\"" % (ref, ref))
1319 log = extractLogMessageFromGitCommit(ref)
1320 settings = extractSettingsGitLog(log)
1321
1322
1323 depotPaths = settings['depot-paths']
1324 change = settings['change']
1325
1326 if changed:
1327 print "%s rewound to %s" % (ref, change)
1328
1329 return True
1330
1331 class P4Submit(Command, P4UserMap):
1332
1333 conflict_behavior_choices = ("ask", "skip", "quit")
1334
1335 def __init__(self):
1336 Command.__init__(self)
1337 P4UserMap.__init__(self)
1338 self.options = [
1339 optparse.make_option("--origin", dest="origin"),
1340 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1341 # preserve the user, requires relevant p4 permissions
1342 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1343 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1344 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1345 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1346 optparse.make_option("--conflict", dest="conflict_behavior",
1347 choices=self.conflict_behavior_choices),
1348 optparse.make_option("--branch", dest="branch"),
1349 optparse.make_option("--shelve", dest="shelve", action="store_true",
1350 help="Shelve instead of submit. Shelved files are reverted, "
1351 "restoring the workspace to the state before the shelve"),
1352 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1353 metavar="CHANGELIST",
1354 help="update an existing shelved changelist, implies --shelve, "
1355 "repeat in-order for multiple shelved changelists"),
1356 optparse.make_option("--commit", dest="commit", metavar="COMMIT",
1357 help="submit only the specified commit(s), one commit or xxx..xxx"),
1358 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",
1359 help="Disable rebase after submit is completed. Can be useful if you "
1360 "work from a local git branch that is not master"),
1361 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",
1362 help="Skip Perforce sync of p4/master after submit or shelve"),
1363 ]
1364 self.description = "Submit changes from git to the perforce depot."
1365 self.usage += " [name of git branch to submit into perforce depot]"
1366 self.origin = ""
1367 self.detectRenames = False
1368 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1369 self.dry_run = False
1370 self.shelve = False
1371 self.update_shelve = list()
1372 self.commit = ""
1373 self.disable_rebase = gitConfigBool("git-p4.disableRebase")
1374 self.disable_p4sync = gitConfigBool("git-p4.disableP4Sync")
1375 self.prepare_p4_only = False
1376 self.conflict_behavior = None
1377 self.isWindows = (platform.system() == "Windows")
1378 self.exportLabels = False
1379 self.p4HasMoveCommand = p4_has_move_command()
1380 self.branch = None
1381
1382 if gitConfig('git-p4.largeFileSystem'):
1383 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1384
1385 def check(self):
1386 if len(p4CmdList("opened ...")) > 0:
1387 die("You have files opened with perforce! Close them before starting the sync.")
1388
1389 def separate_jobs_from_description(self, message):
1390 """Extract and return a possible Jobs field in the commit
1391 message. It goes into a separate section in the p4 change
1392 specification.
1393
1394 A jobs line starts with "Jobs:" and looks like a new field
1395 in a form. Values are white-space separated on the same
1396 line or on following lines that start with a tab.
1397
1398 This does not parse and extract the full git commit message
1399 like a p4 form. It just sees the Jobs: line as a marker
1400 to pass everything from then on directly into the p4 form,
1401 but outside the description section.
1402
1403 Return a tuple (stripped log message, jobs string)."""
1404
1405 m = re.search(r'^Jobs:', message, re.MULTILINE)
1406 if m is None:
1407 return (message, None)
1408
1409 jobtext = message[m.start():]
1410 stripped_message = message[:m.start()].rstrip()
1411 return (stripped_message, jobtext)
1412
1413 def prepareLogMessage(self, template, message, jobs):
1414 """Edits the template returned from "p4 change -o" to insert
1415 the message in the Description field, and the jobs text in
1416 the Jobs field."""
1417 result = ""
1418
1419 inDescriptionSection = False
1420
1421 for line in template.split("\n"):
1422 if line.startswith("#"):
1423 result += line + "\n"
1424 continue
1425
1426 if inDescriptionSection:
1427 if line.startswith("Files:") or line.startswith("Jobs:"):
1428 inDescriptionSection = False
1429 # insert Jobs section
1430 if jobs:
1431 result += jobs + "\n"
1432 else:
1433 continue
1434 else:
1435 if line.startswith("Description:"):
1436 inDescriptionSection = True
1437 line += "\n"
1438 for messageLine in message.split("\n"):
1439 line += "\t" + messageLine + "\n"
1440
1441 result += line + "\n"
1442
1443 return result
1444
1445 def patchRCSKeywords(self, file, pattern):
1446 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1447 (handle, outFileName) = tempfile.mkstemp(dir='.')
1448 try:
1449 outFile = os.fdopen(handle, "w+")
1450 inFile = open(file, "r")
1451 regexp = re.compile(pattern, re.VERBOSE)
1452 for line in inFile.readlines():
1453 line = regexp.sub(r'$\1$', line)
1454 outFile.write(line)
1455 inFile.close()
1456 outFile.close()
1457 # Forcibly overwrite the original file
1458 os.unlink(file)
1459 shutil.move(outFileName, file)
1460 except:
1461 # cleanup our temporary file
1462 os.unlink(outFileName)
1463 print "Failed to strip RCS keywords in %s" % file
1464 raise
1465
1466 print "Patched up RCS keywords in %s" % file
1467
1468 def p4UserForCommit(self,id):
1469 # Return the tuple (perforce user,git email) for a given git commit id
1470 self.getUserMapFromPerforceServer()
1471 gitEmail = read_pipe(["git", "log", "--max-count=1",
1472 "--format=%ae", id])
1473 gitEmail = gitEmail.strip()
1474 if not self.emails.has_key(gitEmail):
1475 return (None,gitEmail)
1476 else:
1477 return (self.emails[gitEmail],gitEmail)
1478
1479 def checkValidP4Users(self,commits):
1480 # check if any git authors cannot be mapped to p4 users
1481 for id in commits:
1482 (user,email) = self.p4UserForCommit(id)
1483 if not user:
1484 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1485 if gitConfigBool("git-p4.allowMissingP4Users"):
1486 print "%s" % msg
1487 else:
1488 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1489
1490 def lastP4Changelist(self):
1491 # Get back the last changelist number submitted in this client spec. This
1492 # then gets used to patch up the username in the change. If the same
1493 # client spec is being used by multiple processes then this might go
1494 # wrong.
1495 results = p4CmdList("client -o") # find the current client
1496 client = None
1497 for r in results:
1498 if r.has_key('Client'):
1499 client = r['Client']
1500 break
1501 if not client:
1502 die("could not get client spec")
1503 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1504 for r in results:
1505 if r.has_key('change'):
1506 return r['change']
1507 die("Could not get changelist number for last submit - cannot patch up user details")
1508
1509 def modifyChangelistUser(self, changelist, newUser):
1510 # fixup the user field of a changelist after it has been submitted.
1511 changes = p4CmdList("change -o %s" % changelist)
1512 if len(changes) != 1:
1513 die("Bad output from p4 change modifying %s to user %s" %
1514 (changelist, newUser))
1515
1516 c = changes[0]
1517 if c['User'] == newUser: return # nothing to do
1518 c['User'] = newUser
1519 input = marshal.dumps(c)
1520
1521 result = p4CmdList("change -f -i", stdin=input)
1522 for r in result:
1523 if r.has_key('code'):
1524 if r['code'] == 'error':
1525 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1526 if r.has_key('data'):
1527 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1528 return
1529 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1530
1531 def canChangeChangelists(self):
1532 # check to see if we have p4 admin or super-user permissions, either of
1533 # which are required to modify changelists.
1534 results = p4CmdList(["protects", self.depotPath])
1535 for r in results:
1536 if r.has_key('perm'):
1537 if r['perm'] == 'admin':
1538 return 1
1539 if r['perm'] == 'super':
1540 return 1
1541 return 0
1542
1543 def prepareSubmitTemplate(self, changelist=None):
1544 """Run "p4 change -o" to grab a change specification template.
1545 This does not use "p4 -G", as it is nice to keep the submission
1546 template in original order, since a human might edit it.
1547
1548 Remove lines in the Files section that show changes to files
1549 outside the depot path we're committing into."""
1550
1551 [upstream, settings] = findUpstreamBranchPoint()
1552
1553 template = """\
1554 # A Perforce Change Specification.
1555 #
1556 # Change: The change number. 'new' on a new changelist.
1557 # Date: The date this specification was last modified.
1558 # Client: The client on which the changelist was created. Read-only.
1559 # User: The user who created the changelist.
1560 # Status: Either 'pending' or 'submitted'. Read-only.
1561 # Type: Either 'public' or 'restricted'. Default is 'public'.
1562 # Description: Comments about the changelist. Required.
1563 # Jobs: What opened jobs are to be closed by this changelist.
1564 # You may delete jobs from this list. (New changelists only.)
1565 # Files: What opened files from the default changelist are to be added
1566 # to this changelist. You may delete files from this list.
1567 # (New changelists only.)
1568 """
1569 files_list = []
1570 inFilesSection = False
1571 change_entry = None
1572 args = ['change', '-o']
1573 if changelist:
1574 args.append(str(changelist))
1575 for entry in p4CmdList(args):
1576 if not entry.has_key('code'):
1577 continue
1578 if entry['code'] == 'stat':
1579 change_entry = entry
1580 break
1581 if not change_entry:
1582 die('Failed to decode output of p4 change -o')
1583 for key, value in change_entry.iteritems():
1584 if key.startswith('File'):
1585 if settings.has_key('depot-paths'):
1586 if not [p for p in settings['depot-paths']
1587 if p4PathStartsWith(value, p)]:
1588 continue
1589 else:
1590 if not p4PathStartsWith(value, self.depotPath):
1591 continue
1592 files_list.append(value)
1593 continue
1594 # Output in the order expected by prepareLogMessage
1595 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1596 if not change_entry.has_key(key):
1597 continue
1598 template += '\n'
1599 template += key + ':'
1600 if key == 'Description':
1601 template += '\n'
1602 for field_line in change_entry[key].splitlines():
1603 template += '\t'+field_line+'\n'
1604 if len(files_list) > 0:
1605 template += '\n'
1606 template += 'Files:\n'
1607 for path in files_list:
1608 template += '\t'+path+'\n'
1609 return template
1610
1611 def edit_template(self, template_file):
1612 """Invoke the editor to let the user change the submission
1613 message. Return true if okay to continue with the submit."""
1614
1615 # if configured to skip the editing part, just submit
1616 if gitConfigBool("git-p4.skipSubmitEdit"):
1617 return True
1618
1619 # look at the modification time, to check later if the user saved
1620 # the file
1621 mtime = os.stat(template_file).st_mtime
1622
1623 # invoke the editor
1624 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1625 editor = os.environ.get("P4EDITOR")
1626 else:
1627 editor = read_pipe("git var GIT_EDITOR").strip()
1628 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1629
1630 # If the file was not saved, prompt to see if this patch should
1631 # be skipped. But skip this verification step if configured so.
1632 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1633 return True
1634
1635 # modification time updated means user saved the file
1636 if os.stat(template_file).st_mtime > mtime:
1637 return True
1638
1639 while True:
1640 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1641 if response == 'y':
1642 return True
1643 if response == 'n':
1644 return False
1645
1646 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1647 # diff
1648 if os.environ.has_key("P4DIFF"):
1649 del(os.environ["P4DIFF"])
1650 diff = ""
1651 for editedFile in editedFiles:
1652 diff += p4_read_pipe(['diff', '-du',
1653 wildcard_encode(editedFile)])
1654
1655 # new file diff
1656 newdiff = ""
1657 for newFile in filesToAdd:
1658 newdiff += "==== new file ====\n"
1659 newdiff += "--- /dev/null\n"
1660 newdiff += "+++ %s\n" % newFile
1661
1662 is_link = os.path.islink(newFile)
1663 expect_link = newFile in symlinks
1664
1665 if is_link and expect_link:
1666 newdiff += "+%s\n" % os.readlink(newFile)
1667 else:
1668 f = open(newFile, "r")
1669 for line in f.readlines():
1670 newdiff += "+" + line
1671 f.close()
1672
1673 return (diff + newdiff).replace('\r\n', '\n')
1674
1675 def applyCommit(self, id):
1676 """Apply one commit, return True if it succeeded."""
1677
1678 print "Applying", read_pipe(["git", "show", "-s",
1679 "--format=format:%h %s", id])
1680
1681 (p4User, gitEmail) = self.p4UserForCommit(id)
1682
1683 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1684 filesToAdd = set()
1685 filesToChangeType = set()
1686 filesToDelete = set()
1687 editedFiles = set()
1688 pureRenameCopy = set()
1689 symlinks = set()
1690 filesToChangeExecBit = {}
1691 all_files = list()
1692
1693 for line in diff:
1694 diff = parseDiffTreeEntry(line)
1695 modifier = diff['status']
1696 path = diff['src']
1697 all_files.append(path)
1698
1699 if modifier == "M":
1700 p4_edit(path)
1701 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1702 filesToChangeExecBit[path] = diff['dst_mode']
1703 editedFiles.add(path)
1704 elif modifier == "A":
1705 filesToAdd.add(path)
1706 filesToChangeExecBit[path] = diff['dst_mode']
1707 if path in filesToDelete:
1708 filesToDelete.remove(path)
1709
1710 dst_mode = int(diff['dst_mode'], 8)
1711 if dst_mode == 0120000:
1712 symlinks.add(path)
1713
1714 elif modifier == "D":
1715 filesToDelete.add(path)
1716 if path in filesToAdd:
1717 filesToAdd.remove(path)
1718 elif modifier == "C":
1719 src, dest = diff['src'], diff['dst']
1720 p4_integrate(src, dest)
1721 pureRenameCopy.add(dest)
1722 if diff['src_sha1'] != diff['dst_sha1']:
1723 p4_edit(dest)
1724 pureRenameCopy.discard(dest)
1725 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1726 p4_edit(dest)
1727 pureRenameCopy.discard(dest)
1728 filesToChangeExecBit[dest] = diff['dst_mode']
1729 if self.isWindows:
1730 # turn off read-only attribute
1731 os.chmod(dest, stat.S_IWRITE)
1732 os.unlink(dest)
1733 editedFiles.add(dest)
1734 elif modifier == "R":
1735 src, dest = diff['src'], diff['dst']
1736 if self.p4HasMoveCommand:
1737 p4_edit(src) # src must be open before move
1738 p4_move(src, dest) # opens for (move/delete, move/add)
1739 else:
1740 p4_integrate(src, dest)
1741 if diff['src_sha1'] != diff['dst_sha1']:
1742 p4_edit(dest)
1743 else:
1744 pureRenameCopy.add(dest)
1745 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1746 if not self.p4HasMoveCommand:
1747 p4_edit(dest) # with move: already open, writable
1748 filesToChangeExecBit[dest] = diff['dst_mode']
1749 if not self.p4HasMoveCommand:
1750 if self.isWindows:
1751 os.chmod(dest, stat.S_IWRITE)
1752 os.unlink(dest)
1753 filesToDelete.add(src)
1754 editedFiles.add(dest)
1755 elif modifier == "T":
1756 filesToChangeType.add(path)
1757 else:
1758 die("unknown modifier %s for %s" % (modifier, path))
1759
1760 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1761 patchcmd = diffcmd + " | git apply "
1762 tryPatchCmd = patchcmd + "--check -"
1763 applyPatchCmd = patchcmd + "--check --apply -"
1764 patch_succeeded = True
1765
1766 if os.system(tryPatchCmd) != 0:
1767 fixed_rcs_keywords = False
1768 patch_succeeded = False
1769 print "Unfortunately applying the change failed!"
1770
1771 # Patch failed, maybe it's just RCS keyword woes. Look through
1772 # the patch to see if that's possible.
1773 if gitConfigBool("git-p4.attemptRCSCleanup"):
1774 file = None
1775 pattern = None
1776 kwfiles = {}
1777 for file in editedFiles | filesToDelete:
1778 # did this file's delta contain RCS keywords?
1779 pattern = p4_keywords_regexp_for_file(file)
1780
1781 if pattern:
1782 # this file is a possibility...look for RCS keywords.
1783 regexp = re.compile(pattern, re.VERBOSE)
1784 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1785 if regexp.search(line):
1786 if verbose:
1787 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1788 kwfiles[file] = pattern
1789 break
1790
1791 for file in kwfiles:
1792 if verbose:
1793 print "zapping %s with %s" % (line,pattern)
1794 # File is being deleted, so not open in p4. Must
1795 # disable the read-only bit on windows.
1796 if self.isWindows and file not in editedFiles:
1797 os.chmod(file, stat.S_IWRITE)
1798 self.patchRCSKeywords(file, kwfiles[file])
1799 fixed_rcs_keywords = True
1800
1801 if fixed_rcs_keywords:
1802 print "Retrying the patch with RCS keywords cleaned up"
1803 if os.system(tryPatchCmd) == 0:
1804 patch_succeeded = True
1805
1806 if not patch_succeeded:
1807 for f in editedFiles:
1808 p4_revert(f)
1809 return False
1810
1811 #
1812 # Apply the patch for real, and do add/delete/+x handling.
1813 #
1814 system(applyPatchCmd)
1815
1816 for f in filesToChangeType:
1817 p4_edit(f, "-t", "auto")
1818 for f in filesToAdd:
1819 p4_add(f)
1820 for f in filesToDelete:
1821 p4_revert(f)
1822 p4_delete(f)
1823
1824 # Set/clear executable bits
1825 for f in filesToChangeExecBit.keys():
1826 mode = filesToChangeExecBit[f]
1827 setP4ExecBit(f, mode)
1828
1829 update_shelve = 0
1830 if len(self.update_shelve) > 0:
1831 update_shelve = self.update_shelve.pop(0)
1832 p4_reopen_in_change(update_shelve, all_files)
1833
1834 #
1835 # Build p4 change description, starting with the contents
1836 # of the git commit message.
1837 #
1838 logMessage = extractLogMessageFromGitCommit(id)
1839 logMessage = logMessage.strip()
1840 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1841
1842 template = self.prepareSubmitTemplate(update_shelve)
1843 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1844
1845 if self.preserveUser:
1846 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1847
1848 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1849 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1850 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1851 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1852
1853 separatorLine = "######## everything below this line is just the diff #######\n"
1854 if not self.prepare_p4_only:
1855 submitTemplate += separatorLine
1856 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1857
1858 (handle, fileName) = tempfile.mkstemp()
1859 tmpFile = os.fdopen(handle, "w+b")
1860 if self.isWindows:
1861 submitTemplate = submitTemplate.replace("\n", "\r\n")
1862 tmpFile.write(submitTemplate)
1863 tmpFile.close()
1864
1865 if self.prepare_p4_only:
1866 #
1867 # Leave the p4 tree prepared, and the submit template around
1868 # and let the user decide what to do next
1869 #
1870 print
1871 print "P4 workspace prepared for submission."
1872 print "To submit or revert, go to client workspace"
1873 print " " + self.clientPath
1874 print
1875 print "To submit, use \"p4 submit\" to write a new description,"
1876 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1877 " \"git p4\"." % fileName
1878 print "You can delete the file \"%s\" when finished." % fileName
1879
1880 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1881 print "To preserve change ownership by user %s, you must\n" \
1882 "do \"p4 change -f <change>\" after submitting and\n" \
1883 "edit the User field."
1884 if pureRenameCopy:
1885 print "After submitting, renamed files must be re-synced."
1886 print "Invoke \"p4 sync -f\" on each of these files:"
1887 for f in pureRenameCopy:
1888 print " " + f
1889
1890 print
1891 print "To revert the changes, use \"p4 revert ...\", and delete"
1892 print "the submit template file \"%s\"" % fileName
1893 if filesToAdd:
1894 print "Since the commit adds new files, they must be deleted:"
1895 for f in filesToAdd:
1896 print " " + f
1897 print
1898 return True
1899
1900 #
1901 # Let the user edit the change description, then submit it.
1902 #
1903 submitted = False
1904
1905 try:
1906 if self.edit_template(fileName):
1907 # read the edited message and submit
1908 tmpFile = open(fileName, "rb")
1909 message = tmpFile.read()
1910 tmpFile.close()
1911 if self.isWindows:
1912 message = message.replace("\r\n", "\n")
1913 submitTemplate = message[:message.index(separatorLine)]
1914
1915 if update_shelve:
1916 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1917 elif self.shelve:
1918 p4_write_pipe(['shelve', '-i'], submitTemplate)
1919 else:
1920 p4_write_pipe(['submit', '-i'], submitTemplate)
1921 # The rename/copy happened by applying a patch that created a
1922 # new file. This leaves it writable, which confuses p4.
1923 for f in pureRenameCopy:
1924 p4_sync(f, "-f")
1925
1926 if self.preserveUser:
1927 if p4User:
1928 # Get last changelist number. Cannot easily get it from
1929 # the submit command output as the output is
1930 # unmarshalled.
1931 changelist = self.lastP4Changelist()
1932 self.modifyChangelistUser(changelist, p4User)
1933
1934 submitted = True
1935
1936 finally:
1937 # skip this patch
1938 if not submitted or self.shelve:
1939 if self.shelve:
1940 print ("Reverting shelved files.")
1941 else:
1942 print ("Submission cancelled, undoing p4 changes.")
1943 for f in editedFiles | filesToDelete:
1944 p4_revert(f)
1945 for f in filesToAdd:
1946 p4_revert(f)
1947 os.remove(f)
1948
1949 os.remove(fileName)
1950 return submitted
1951
1952 # Export git tags as p4 labels. Create a p4 label and then tag
1953 # with that.
1954 def exportGitTags(self, gitTags):
1955 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1956 if len(validLabelRegexp) == 0:
1957 validLabelRegexp = defaultLabelRegexp
1958 m = re.compile(validLabelRegexp)
1959
1960 for name in gitTags:
1961
1962 if not m.match(name):
1963 if verbose:
1964 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1965 continue
1966
1967 # Get the p4 commit this corresponds to
1968 logMessage = extractLogMessageFromGitCommit(name)
1969 values = extractSettingsGitLog(logMessage)
1970
1971 if not values.has_key('change'):
1972 # a tag pointing to something not sent to p4; ignore
1973 if verbose:
1974 print "git tag %s does not give a p4 commit" % name
1975 continue
1976 else:
1977 changelist = values['change']
1978
1979 # Get the tag details.
1980 inHeader = True
1981 isAnnotated = False
1982 body = []
1983 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1984 l = l.strip()
1985 if inHeader:
1986 if re.match(r'tag\s+', l):
1987 isAnnotated = True
1988 elif re.match(r'\s*$', l):
1989 inHeader = False
1990 continue
1991 else:
1992 body.append(l)
1993
1994 if not isAnnotated:
1995 body = ["lightweight tag imported by git p4\n"]
1996
1997 # Create the label - use the same view as the client spec we are using
1998 clientSpec = getClientSpec()
1999
2000 labelTemplate = "Label: %s\n" % name
2001 labelTemplate += "Description:\n"
2002 for b in body:
2003 labelTemplate += "\t" + b + "\n"
2004 labelTemplate += "View:\n"
2005 for depot_side in clientSpec.mappings:
2006 labelTemplate += "\t%s\n" % depot_side
2007
2008 if self.dry_run:
2009 print "Would create p4 label %s for tag" % name
2010 elif self.prepare_p4_only:
2011 print "Not creating p4 label %s for tag due to option" \
2012 " --prepare-p4-only" % name
2013 else:
2014 p4_write_pipe(["label", "-i"], labelTemplate)
2015
2016 # Use the label
2017 p4_system(["tag", "-l", name] +
2018 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2019
2020 if verbose:
2021 print "created p4 label for tag %s" % name
2022
2023 def run(self, args):
2024 if len(args) == 0:
2025 self.master = currentGitBranch()
2026 elif len(args) == 1:
2027 self.master = args[0]
2028 if not branchExists(self.master):
2029 die("Branch %s does not exist" % self.master)
2030 else:
2031 return False
2032
2033 for i in self.update_shelve:
2034 if i <= 0:
2035 sys.exit("invalid changelist %d" % i)
2036
2037 if self.master:
2038 allowSubmit = gitConfig("git-p4.allowSubmit")
2039 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2040 die("%s is not in git-p4.allowSubmit" % self.master)
2041
2042 [upstream, settings] = findUpstreamBranchPoint()
2043 self.depotPath = settings['depot-paths'][0]
2044 if len(self.origin) == 0:
2045 self.origin = upstream
2046
2047 if len(self.update_shelve) > 0:
2048 self.shelve = True
2049
2050 if self.preserveUser:
2051 if not self.canChangeChangelists():
2052 die("Cannot preserve user names without p4 super-user or admin permissions")
2053
2054 # if not set from the command line, try the config file
2055 if self.conflict_behavior is None:
2056 val = gitConfig("git-p4.conflict")
2057 if val:
2058 if val not in self.conflict_behavior_choices:
2059 die("Invalid value '%s' for config git-p4.conflict" % val)
2060 else:
2061 val = "ask"
2062 self.conflict_behavior = val
2063
2064 if self.verbose:
2065 print "Origin branch is " + self.origin
2066
2067 if len(self.depotPath) == 0:
2068 print "Internal error: cannot locate perforce depot path from existing branches"
2069 sys.exit(128)
2070
2071 self.useClientSpec = False
2072 if gitConfigBool("git-p4.useclientspec"):
2073 self.useClientSpec = True
2074 if self.useClientSpec:
2075 self.clientSpecDirs = getClientSpec()
2076
2077 # Check for the existence of P4 branches
2078 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2079
2080 if self.useClientSpec and not branchesDetected:
2081 # all files are relative to the client spec
2082 self.clientPath = getClientRoot()
2083 else:
2084 self.clientPath = p4Where(self.depotPath)
2085
2086 if self.clientPath == "":
2087 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2088
2089 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2090 self.oldWorkingDirectory = os.getcwd()
2091
2092 # ensure the clientPath exists
2093 new_client_dir = False
2094 if not os.path.exists(self.clientPath):
2095 new_client_dir = True
2096 os.makedirs(self.clientPath)
2097
2098 chdir(self.clientPath, is_client_path=True)
2099 if self.dry_run:
2100 print "Would synchronize p4 checkout in %s" % self.clientPath
2101 else:
2102 print "Synchronizing p4 checkout..."
2103 if new_client_dir:
2104 # old one was destroyed, and maybe nobody told p4
2105 p4_sync("...", "-f")
2106 else:
2107 p4_sync("...")
2108 self.check()
2109
2110 commits = []
2111 if self.master:
2112 commitish = self.master
2113 else:
2114 commitish = 'HEAD'
2115
2116 if self.commit != "":
2117 if self.commit.find("..") != -1:
2118 limits_ish = self.commit.split("..")
2119 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (limits_ish[0], limits_ish[1])]):
2120 commits.append(line.strip())
2121 commits.reverse()
2122 else:
2123 commits.append(self.commit)
2124 else:
2125 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2126 commits.append(line.strip())
2127 commits.reverse()
2128
2129 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2130 self.checkAuthorship = False
2131 else:
2132 self.checkAuthorship = True
2133
2134 if self.preserveUser:
2135 self.checkValidP4Users(commits)
2136
2137 #
2138 # Build up a set of options to be passed to diff when
2139 # submitting each commit to p4.
2140 #
2141 if self.detectRenames:
2142 # command-line -M arg
2143 self.diffOpts = "-M"
2144 else:
2145 # If not explicitly set check the config variable
2146 detectRenames = gitConfig("git-p4.detectRenames")
2147
2148 if detectRenames.lower() == "false" or detectRenames == "":
2149 self.diffOpts = ""
2150 elif detectRenames.lower() == "true":
2151 self.diffOpts = "-M"
2152 else:
2153 self.diffOpts = "-M%s" % detectRenames
2154
2155 # no command-line arg for -C or --find-copies-harder, just
2156 # config variables
2157 detectCopies = gitConfig("git-p4.detectCopies")
2158 if detectCopies.lower() == "false" or detectCopies == "":
2159 pass
2160 elif detectCopies.lower() == "true":
2161 self.diffOpts += " -C"
2162 else:
2163 self.diffOpts += " -C%s" % detectCopies
2164
2165 if gitConfigBool("git-p4.detectCopiesHarder"):
2166 self.diffOpts += " --find-copies-harder"
2167
2168 num_shelves = len(self.update_shelve)
2169 if num_shelves > 0 and num_shelves != len(commits):
2170 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2171 (len(commits), num_shelves))
2172
2173 #
2174 # Apply the commits, one at a time. On failure, ask if should
2175 # continue to try the rest of the patches, or quit.
2176 #
2177 if self.dry_run:
2178 print "Would apply"
2179 applied = []
2180 last = len(commits) - 1
2181 for i, commit in enumerate(commits):
2182 if self.dry_run:
2183 print " ", read_pipe(["git", "show", "-s",
2184 "--format=format:%h %s", commit])
2185 ok = True
2186 else:
2187 ok = self.applyCommit(commit)
2188 if ok:
2189 applied.append(commit)
2190 else:
2191 if self.prepare_p4_only and i < last:
2192 print "Processing only the first commit due to option" \
2193 " --prepare-p4-only"
2194 break
2195 if i < last:
2196 quit = False
2197 while True:
2198 # prompt for what to do, or use the option/variable
2199 if self.conflict_behavior == "ask":
2200 print "What do you want to do?"
2201 response = raw_input("[s]kip this commit but apply"
2202 " the rest, or [q]uit? ")
2203 if not response:
2204 continue
2205 elif self.conflict_behavior == "skip":
2206 response = "s"
2207 elif self.conflict_behavior == "quit":
2208 response = "q"
2209 else:
2210 die("Unknown conflict_behavior '%s'" %
2211 self.conflict_behavior)
2212
2213 if response[0] == "s":
2214 print "Skipping this commit, but applying the rest"
2215 break
2216 if response[0] == "q":
2217 print "Quitting"
2218 quit = True
2219 break
2220 if quit:
2221 break
2222
2223 chdir(self.oldWorkingDirectory)
2224 shelved_applied = "shelved" if self.shelve else "applied"
2225 if self.dry_run:
2226 pass
2227 elif self.prepare_p4_only:
2228 pass
2229 elif len(commits) == len(applied):
2230 print ("All commits {0}!".format(shelved_applied))
2231
2232 sync = P4Sync()
2233 if self.branch:
2234 sync.branch = self.branch
2235 if self.disable_p4sync:
2236 sync.sync_origin_only()
2237 else:
2238 sync.run([])
2239
2240 if not self.disable_rebase:
2241 rebase = P4Rebase()
2242 rebase.rebase()
2243
2244 else:
2245 if len(applied) == 0:
2246 print ("No commits {0}.".format(shelved_applied))
2247 else:
2248 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2249 for c in commits:
2250 if c in applied:
2251 star = "*"
2252 else:
2253 star = " "
2254 print star, read_pipe(["git", "show", "-s",
2255 "--format=format:%h %s", c])
2256 print "You will have to do 'git p4 sync' and rebase."
2257
2258 if gitConfigBool("git-p4.exportLabels"):
2259 self.exportLabels = True
2260
2261 if self.exportLabels:
2262 p4Labels = getP4Labels(self.depotPath)
2263 gitTags = getGitTags()
2264
2265 missingGitTags = gitTags - p4Labels
2266 self.exportGitTags(missingGitTags)
2267
2268 # exit with error unless everything applied perfectly
2269 if len(commits) != len(applied):
2270 sys.exit(1)
2271
2272 return True
2273
2274 class View(object):
2275 """Represent a p4 view ("p4 help views"), and map files in a
2276 repo according to the view."""
2277
2278 def __init__(self, client_name):
2279 self.mappings = []
2280 self.client_prefix = "//%s/" % client_name
2281 # cache results of "p4 where" to lookup client file locations
2282 self.client_spec_path_cache = {}
2283
2284 def append(self, view_line):
2285 """Parse a view line, splitting it into depot and client
2286 sides. Append to self.mappings, preserving order. This
2287 is only needed for tag creation."""
2288
2289 # Split the view line into exactly two words. P4 enforces
2290 # structure on these lines that simplifies this quite a bit.
2291 #
2292 # Either or both words may be double-quoted.
2293 # Single quotes do not matter.
2294 # Double-quote marks cannot occur inside the words.
2295 # A + or - prefix is also inside the quotes.
2296 # There are no quotes unless they contain a space.
2297 # The line is already white-space stripped.
2298 # The two words are separated by a single space.
2299 #
2300 if view_line[0] == '"':
2301 # First word is double quoted. Find its end.
2302 close_quote_index = view_line.find('"', 1)
2303 if close_quote_index <= 0:
2304 die("No first-word closing quote found: %s" % view_line)
2305 depot_side = view_line[1:close_quote_index]
2306 # skip closing quote and space
2307 rhs_index = close_quote_index + 1 + 1
2308 else:
2309 space_index = view_line.find(" ")
2310 if space_index <= 0:
2311 die("No word-splitting space found: %s" % view_line)
2312 depot_side = view_line[0:space_index]
2313 rhs_index = space_index + 1
2314
2315 # prefix + means overlay on previous mapping
2316 if depot_side.startswith("+"):
2317 depot_side = depot_side[1:]
2318
2319 # prefix - means exclude this path, leave out of mappings
2320 exclude = False
2321 if depot_side.startswith("-"):
2322 exclude = True
2323 depot_side = depot_side[1:]
2324
2325 if not exclude:
2326 self.mappings.append(depot_side)
2327
2328 def convert_client_path(self, clientFile):
2329 # chop off //client/ part to make it relative
2330 if not clientFile.startswith(self.client_prefix):
2331 die("No prefix '%s' on clientFile '%s'" %
2332 (self.client_prefix, clientFile))
2333 return clientFile[len(self.client_prefix):]
2334
2335 def update_client_spec_path_cache(self, files):
2336 """ Caching file paths by "p4 where" batch query """
2337
2338 # List depot file paths exclude that already cached
2339 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2340
2341 if len(fileArgs) == 0:
2342 return # All files in cache
2343
2344 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2345 for res in where_result:
2346 if "code" in res and res["code"] == "error":
2347 # assume error is "... file(s) not in client view"
2348 continue
2349 if "clientFile" not in res:
2350 die("No clientFile in 'p4 where' output")
2351 if "unmap" in res:
2352 # it will list all of them, but only one not unmap-ped
2353 continue
2354 if gitConfigBool("core.ignorecase"):
2355 res['depotFile'] = res['depotFile'].lower()
2356 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2357
2358 # not found files or unmap files set to ""
2359 for depotFile in fileArgs:
2360 if gitConfigBool("core.ignorecase"):
2361 depotFile = depotFile.lower()
2362 if depotFile not in self.client_spec_path_cache:
2363 self.client_spec_path_cache[depotFile] = ""
2364
2365 def map_in_client(self, depot_path):
2366 """Return the relative location in the client where this
2367 depot file should live. Returns "" if the file should
2368 not be mapped in the client."""
2369
2370 if gitConfigBool("core.ignorecase"):
2371 depot_path = depot_path.lower()
2372
2373 if depot_path in self.client_spec_path_cache:
2374 return self.client_spec_path_cache[depot_path]
2375
2376 die( "Error: %s is not found in client spec path" % depot_path )
2377 return ""
2378
2379 class P4Sync(Command, P4UserMap):
2380 delete_actions = ( "delete", "move/delete", "purge" )
2381
2382 def __init__(self):
2383 Command.__init__(self)
2384 P4UserMap.__init__(self)
2385 self.options = [
2386 optparse.make_option("--branch", dest="branch"),
2387 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2388 optparse.make_option("--changesfile", dest="changesFile"),
2389 optparse.make_option("--silent", dest="silent", action="store_true"),
2390 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2391 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2392 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2393 help="Import into refs/heads/ , not refs/remotes"),
2394 optparse.make_option("--max-changes", dest="maxChanges",
2395 help="Maximum number of changes to import"),
2396 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2397 help="Internal block size to use when iteratively calling p4 changes"),
2398 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2399 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2400 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2401 help="Only sync files that are included in the Perforce Client Spec"),
2402 optparse.make_option("-/", dest="cloneExclude",
2403 action="append", type="string",
2404 help="exclude depot path"),
2405 ]
2406 self.description = """Imports from Perforce into a git repository.\n
2407 example:
2408 //depot/my/project/ -- to import the current head
2409 //depot/my/project/@all -- to import everything
2410 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2411
2412 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2413
2414 self.usage += " //depot/path[@revRange]"
2415 self.silent = False
2416 self.createdBranches = set()
2417 self.committedChanges = set()
2418 self.branch = ""
2419 self.detectBranches = False
2420 self.detectLabels = False
2421 self.importLabels = False
2422 self.changesFile = ""
2423 self.syncWithOrigin = True
2424 self.importIntoRemotes = True
2425 self.maxChanges = ""
2426 self.changes_block_size = None
2427 self.keepRepoPath = False
2428 self.depotPaths = None
2429 self.p4BranchesInGit = []
2430 self.cloneExclude = []
2431 self.useClientSpec = False
2432 self.useClientSpec_from_options = False
2433 self.clientSpecDirs = None
2434 self.tempBranches = []
2435 self.tempBranchLocation = "refs/git-p4-tmp"
2436 self.largeFileSystem = None
2437
2438 if gitConfig('git-p4.largeFileSystem'):
2439 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2440 self.largeFileSystem = largeFileSystemConstructor(
2441 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2442 )
2443
2444 if gitConfig("git-p4.syncFromOrigin") == "false":
2445 self.syncWithOrigin = False
2446
2447 # Force a checkpoint in fast-import and wait for it to finish
2448 def checkpoint(self):
2449 self.gitStream.write("checkpoint\n\n")
2450 self.gitStream.write("progress checkpoint\n\n")
2451 out = self.gitOutput.readline()
2452 if self.verbose:
2453 print "checkpoint finished: " + out
2454
2455 def extractFilesFromCommit(self, commit):
2456 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2457 for path in self.cloneExclude]
2458 files = []
2459 fnum = 0
2460 while commit.has_key("depotFile%s" % fnum):
2461 path = commit["depotFile%s" % fnum]
2462
2463 if [p for p in self.cloneExclude
2464 if p4PathStartsWith(path, p)]:
2465 found = False
2466 else:
2467 found = [p for p in self.depotPaths
2468 if p4PathStartsWith(path, p)]
2469 if not found:
2470 fnum = fnum + 1
2471 continue
2472
2473 file = {}
2474 file["path"] = path
2475 file["rev"] = commit["rev%s" % fnum]
2476 file["action"] = commit["action%s" % fnum]
2477 file["type"] = commit["type%s" % fnum]
2478 files.append(file)
2479 fnum = fnum + 1
2480 return files
2481
2482 def extractJobsFromCommit(self, commit):
2483 jobs = []
2484 jnum = 0
2485 while commit.has_key("job%s" % jnum):
2486 job = commit["job%s" % jnum]
2487 jobs.append(job)
2488 jnum = jnum + 1
2489 return jobs
2490
2491 def stripRepoPath(self, path, prefixes):
2492 """When streaming files, this is called to map a p4 depot path
2493 to where it should go in git. The prefixes are either
2494 self.depotPaths, or self.branchPrefixes in the case of
2495 branch detection."""
2496
2497 if self.useClientSpec:
2498 # branch detection moves files up a level (the branch name)
2499 # from what client spec interpretation gives
2500 path = self.clientSpecDirs.map_in_client(path)
2501 if self.detectBranches:
2502 for b in self.knownBranches:
2503 if path.startswith(b + "/"):
2504 path = path[len(b)+1:]
2505
2506 elif self.keepRepoPath:
2507 # Preserve everything in relative path name except leading
2508 # //depot/; just look at first prefix as they all should
2509 # be in the same depot.
2510 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2511 if p4PathStartsWith(path, depot):
2512 path = path[len(depot):]
2513
2514 else:
2515 for p in prefixes:
2516 if p4PathStartsWith(path, p):
2517 path = path[len(p):]
2518 break
2519
2520 path = wildcard_decode(path)
2521 return path
2522
2523 def splitFilesIntoBranches(self, commit):
2524 """Look at each depotFile in the commit to figure out to what
2525 branch it belongs."""
2526
2527 if self.clientSpecDirs:
2528 files = self.extractFilesFromCommit(commit)
2529 self.clientSpecDirs.update_client_spec_path_cache(files)
2530
2531 branches = {}
2532 fnum = 0
2533 while commit.has_key("depotFile%s" % fnum):
2534 path = commit["depotFile%s" % fnum]
2535 found = [p for p in self.depotPaths
2536 if p4PathStartsWith(path, p)]
2537 if not found:
2538 fnum = fnum + 1
2539 continue
2540
2541 file = {}
2542 file["path"] = path
2543 file["rev"] = commit["rev%s" % fnum]
2544 file["action"] = commit["action%s" % fnum]
2545 file["type"] = commit["type%s" % fnum]
2546 fnum = fnum + 1
2547
2548 # start with the full relative path where this file would
2549 # go in a p4 client
2550 if self.useClientSpec:
2551 relPath = self.clientSpecDirs.map_in_client(path)
2552 else:
2553 relPath = self.stripRepoPath(path, self.depotPaths)
2554
2555 for branch in self.knownBranches.keys():
2556 # add a trailing slash so that a commit into qt/4.2foo
2557 # doesn't end up in qt/4.2, e.g.
2558 if relPath.startswith(branch + "/"):
2559 if branch not in branches:
2560 branches[branch] = []
2561 branches[branch].append(file)
2562 break
2563
2564 return branches
2565
2566 def writeToGitStream(self, gitMode, relPath, contents):
2567 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2568 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2569 for d in contents:
2570 self.gitStream.write(d)
2571 self.gitStream.write('\n')
2572
2573 def encodeWithUTF8(self, path):
2574 try:
2575 path.decode('ascii')
2576 except:
2577 encoding = 'utf8'
2578 if gitConfig('git-p4.pathEncoding'):
2579 encoding = gitConfig('git-p4.pathEncoding')
2580 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2581 if self.verbose:
2582 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2583 return path
2584
2585 # output one file from the P4 stream
2586 # - helper for streamP4Files
2587
2588 def streamOneP4File(self, file, contents):
2589 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2590 relPath = self.encodeWithUTF8(relPath)
2591 if verbose:
2592 size = int(self.stream_file['fileSize'])
2593 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2594 sys.stdout.flush()
2595
2596 (type_base, type_mods) = split_p4_type(file["type"])
2597
2598 git_mode = "100644"
2599 if "x" in type_mods:
2600 git_mode = "100755"
2601 if type_base == "symlink":
2602 git_mode = "120000"
2603 # p4 print on a symlink sometimes contains "target\n";
2604 # if it does, remove the newline
2605 data = ''.join(contents)
2606 if not data:
2607 # Some version of p4 allowed creating a symlink that pointed
2608 # to nothing. This causes p4 errors when checking out such
2609 # a change, and errors here too. Work around it by ignoring
2610 # the bad symlink; hopefully a future change fixes it.
2611 print "\nIgnoring empty symlink in %s" % file['depotFile']
2612 return
2613 elif data[-1] == '\n':
2614 contents = [data[:-1]]
2615 else:
2616 contents = [data]
2617
2618 if type_base == "utf16":
2619 # p4 delivers different text in the python output to -G
2620 # than it does when using "print -o", or normal p4 client
2621 # operations. utf16 is converted to ascii or utf8, perhaps.
2622 # But ascii text saved as -t utf16 is completely mangled.
2623 # Invoke print -o to get the real contents.
2624 #
2625 # On windows, the newlines will always be mangled by print, so put
2626 # them back too. This is not needed to the cygwin windows version,
2627 # just the native "NT" type.
2628 #
2629 try:
2630 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2631 except Exception as e:
2632 if 'Translation of file content failed' in str(e):
2633 type_base = 'binary'
2634 else:
2635 raise e
2636 else:
2637 if p4_version_string().find('/NT') >= 0:
2638 text = text.replace('\r\n', '\n')
2639 contents = [ text ]
2640
2641 if type_base == "apple":
2642 # Apple filetype files will be streamed as a concatenation of
2643 # its appledouble header and the contents. This is useless
2644 # on both macs and non-macs. If using "print -q -o xx", it
2645 # will create "xx" with the data, and "%xx" with the header.
2646 # This is also not very useful.
2647 #
2648 # Ideally, someday, this script can learn how to generate
2649 # appledouble files directly and import those to git, but
2650 # non-mac machines can never find a use for apple filetype.
2651 print "\nIgnoring apple filetype file %s" % file['depotFile']
2652 return
2653
2654 # Note that we do not try to de-mangle keywords on utf16 files,
2655 # even though in theory somebody may want that.
2656 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2657 if pattern:
2658 regexp = re.compile(pattern, re.VERBOSE)
2659 text = ''.join(contents)
2660 text = regexp.sub(r'$\1$', text)
2661 contents = [ text ]
2662
2663 if self.largeFileSystem:
2664 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2665
2666 self.writeToGitStream(git_mode, relPath, contents)
2667
2668 def streamOneP4Deletion(self, file):
2669 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2670 relPath = self.encodeWithUTF8(relPath)
2671 if verbose:
2672 sys.stdout.write("delete %s\n" % relPath)
2673 sys.stdout.flush()
2674 self.gitStream.write("D %s\n" % relPath)
2675
2676 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2677 self.largeFileSystem.removeLargeFile(relPath)
2678
2679 # handle another chunk of streaming data
2680 def streamP4FilesCb(self, marshalled):
2681
2682 # catch p4 errors and complain
2683 err = None
2684 if "code" in marshalled:
2685 if marshalled["code"] == "error":
2686 if "data" in marshalled:
2687 err = marshalled["data"].rstrip()
2688
2689 if not err and 'fileSize' in self.stream_file:
2690 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2691 if required_bytes > 0:
2692 err = 'Not enough space left on %s! Free at least %i MB.' % (
2693 os.getcwd(), required_bytes/1024/1024
2694 )
2695
2696 if err:
2697 f = None
2698 if self.stream_have_file_info:
2699 if "depotFile" in self.stream_file:
2700 f = self.stream_file["depotFile"]
2701 # force a failure in fast-import, else an empty
2702 # commit will be made
2703 self.gitStream.write("\n")
2704 self.gitStream.write("die-now\n")
2705 self.gitStream.close()
2706 # ignore errors, but make sure it exits first
2707 self.importProcess.wait()
2708 if f:
2709 die("Error from p4 print for %s: %s" % (f, err))
2710 else:
2711 die("Error from p4 print: %s" % err)
2712
2713 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2714 # start of a new file - output the old one first
2715 self.streamOneP4File(self.stream_file, self.stream_contents)
2716 self.stream_file = {}
2717 self.stream_contents = []
2718 self.stream_have_file_info = False
2719
2720 # pick up the new file information... for the
2721 # 'data' field we need to append to our array
2722 for k in marshalled.keys():
2723 if k == 'data':
2724 if 'streamContentSize' not in self.stream_file:
2725 self.stream_file['streamContentSize'] = 0
2726 self.stream_file['streamContentSize'] += len(marshalled['data'])
2727 self.stream_contents.append(marshalled['data'])
2728 else:
2729 self.stream_file[k] = marshalled[k]
2730
2731 if (verbose and
2732 'streamContentSize' in self.stream_file and
2733 'fileSize' in self.stream_file and
2734 'depotFile' in self.stream_file):
2735 size = int(self.stream_file["fileSize"])
2736 if size > 0:
2737 progress = 100*self.stream_file['streamContentSize']/size
2738 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2739 sys.stdout.flush()
2740
2741 self.stream_have_file_info = True
2742
2743 # Stream directly from "p4 files" into "git fast-import"
2744 def streamP4Files(self, files):
2745 filesForCommit = []
2746 filesToRead = []
2747 filesToDelete = []
2748
2749 for f in files:
2750 filesForCommit.append(f)
2751 if f['action'] in self.delete_actions:
2752 filesToDelete.append(f)
2753 else:
2754 filesToRead.append(f)
2755
2756 # deleted files...
2757 for f in filesToDelete:
2758 self.streamOneP4Deletion(f)
2759
2760 if len(filesToRead) > 0:
2761 self.stream_file = {}
2762 self.stream_contents = []
2763 self.stream_have_file_info = False
2764
2765 # curry self argument
2766 def streamP4FilesCbSelf(entry):
2767 self.streamP4FilesCb(entry)
2768
2769 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2770
2771 p4CmdList(["-x", "-", "print"],
2772 stdin=fileArgs,
2773 cb=streamP4FilesCbSelf)
2774
2775 # do the last chunk
2776 if self.stream_file.has_key('depotFile'):
2777 self.streamOneP4File(self.stream_file, self.stream_contents)
2778
2779 def make_email(self, userid):
2780 if userid in self.users:
2781 return self.users[userid]
2782 else:
2783 return "%s <a@b>" % userid
2784
2785 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2786 """ Stream a p4 tag.
2787 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2788 """
2789
2790 if verbose:
2791 print "writing tag %s for commit %s" % (labelName, commit)
2792 gitStream.write("tag %s\n" % labelName)
2793 gitStream.write("from %s\n" % commit)
2794
2795 if labelDetails.has_key('Owner'):
2796 owner = labelDetails["Owner"]
2797 else:
2798 owner = None
2799
2800 # Try to use the owner of the p4 label, or failing that,
2801 # the current p4 user id.
2802 if owner:
2803 email = self.make_email(owner)
2804 else:
2805 email = self.make_email(self.p4UserId())
2806 tagger = "%s %s %s" % (email, epoch, self.tz)
2807
2808 gitStream.write("tagger %s\n" % tagger)
2809
2810 print "labelDetails=",labelDetails
2811 if labelDetails.has_key('Description'):
2812 description = labelDetails['Description']
2813 else:
2814 description = 'Label from git p4'
2815
2816 gitStream.write("data %d\n" % len(description))
2817 gitStream.write(description)
2818 gitStream.write("\n")
2819
2820 def inClientSpec(self, path):
2821 if not self.clientSpecDirs:
2822 return True
2823 inClientSpec = self.clientSpecDirs.map_in_client(path)
2824 if not inClientSpec and self.verbose:
2825 print('Ignoring file outside of client spec: {0}'.format(path))
2826 return inClientSpec
2827
2828 def hasBranchPrefix(self, path):
2829 if not self.branchPrefixes:
2830 return True
2831 hasPrefix = [p for p in self.branchPrefixes
2832 if p4PathStartsWith(path, p)]
2833 if not hasPrefix and self.verbose:
2834 print('Ignoring file outside of prefix: {0}'.format(path))
2835 return hasPrefix
2836
2837 def commit(self, details, files, branch, parent = ""):
2838 epoch = details["time"]
2839 author = details["user"]
2840 jobs = self.extractJobsFromCommit(details)
2841
2842 if self.verbose:
2843 print('commit into {0}'.format(branch))
2844
2845 if self.clientSpecDirs:
2846 self.clientSpecDirs.update_client_spec_path_cache(files)
2847
2848 files = [f for f in files
2849 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2850
2851 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2852 print('Ignoring revision {0} as it would produce an empty commit.'
2853 .format(details['change']))
2854 return
2855
2856 self.gitStream.write("commit %s\n" % branch)
2857 self.gitStream.write("mark :%s\n" % details["change"])
2858 self.committedChanges.add(int(details["change"]))
2859 committer = ""
2860 if author not in self.users:
2861 self.getUserMapFromPerforceServer()
2862 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2863
2864 self.gitStream.write("committer %s\n" % committer)
2865
2866 self.gitStream.write("data <<EOT\n")
2867 self.gitStream.write(details["desc"])
2868 if len(jobs) > 0:
2869 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2870 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2871 (','.join(self.branchPrefixes), details["change"]))
2872 if len(details['options']) > 0:
2873 self.gitStream.write(": options = %s" % details['options'])
2874 self.gitStream.write("]\nEOT\n\n")
2875
2876 if len(parent) > 0:
2877 if self.verbose:
2878 print "parent %s" % parent
2879 self.gitStream.write("from %s\n" % parent)
2880
2881 self.streamP4Files(files)
2882 self.gitStream.write("\n")
2883
2884 change = int(details["change"])
2885
2886 if self.labels.has_key(change):
2887 label = self.labels[change]
2888 labelDetails = label[0]
2889 labelRevisions = label[1]
2890 if self.verbose:
2891 print "Change %s is labelled %s" % (change, labelDetails)
2892
2893 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2894 for p in self.branchPrefixes])
2895
2896 if len(files) == len(labelRevisions):
2897
2898 cleanedFiles = {}
2899 for info in files:
2900 if info["action"] in self.delete_actions:
2901 continue
2902 cleanedFiles[info["depotFile"]] = info["rev"]
2903
2904 if cleanedFiles == labelRevisions:
2905 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2906
2907 else:
2908 if not self.silent:
2909 print ("Tag %s does not match with change %s: files do not match."
2910 % (labelDetails["label"], change))
2911
2912 else:
2913 if not self.silent:
2914 print ("Tag %s does not match with change %s: file count is different."
2915 % (labelDetails["label"], change))
2916
2917 # Build a dictionary of changelists and labels, for "detect-labels" option.
2918 def getLabels(self):
2919 self.labels = {}
2920
2921 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2922 if len(l) > 0 and not self.silent:
2923 print "Finding files belonging to labels in %s" % `self.depotPaths`
2924
2925 for output in l:
2926 label = output["label"]
2927 revisions = {}
2928 newestChange = 0
2929 if self.verbose:
2930 print "Querying files for label %s" % label
2931 for file in p4CmdList(["files"] +
2932 ["%s...@%s" % (p, label)
2933 for p in self.depotPaths]):
2934 revisions[file["depotFile"]] = file["rev"]
2935 change = int(file["change"])
2936 if change > newestChange:
2937 newestChange = change
2938
2939 self.labels[newestChange] = [output, revisions]
2940
2941 if self.verbose:
2942 print "Label changes: %s" % self.labels.keys()
2943
2944 # Import p4 labels as git tags. A direct mapping does not
2945 # exist, so assume that if all the files are at the same revision
2946 # then we can use that, or it's something more complicated we should
2947 # just ignore.
2948 def importP4Labels(self, stream, p4Labels):
2949 if verbose:
2950 print "import p4 labels: " + ' '.join(p4Labels)
2951
2952 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2953 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2954 if len(validLabelRegexp) == 0:
2955 validLabelRegexp = defaultLabelRegexp
2956 m = re.compile(validLabelRegexp)
2957
2958 for name in p4Labels:
2959 commitFound = False
2960
2961 if not m.match(name):
2962 if verbose:
2963 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2964 continue
2965
2966 if name in ignoredP4Labels:
2967 continue
2968
2969 labelDetails = p4CmdList(['label', "-o", name])[0]
2970
2971 # get the most recent changelist for each file in this label
2972 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2973 for p in self.depotPaths])
2974
2975 if change.has_key('change'):
2976 # find the corresponding git commit; take the oldest commit
2977 changelist = int(change['change'])
2978 if changelist in self.committedChanges:
2979 gitCommit = ":%d" % changelist # use a fast-import mark
2980 commitFound = True
2981 else:
2982 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2983 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2984 if len(gitCommit) == 0:
2985 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2986 else:
2987 commitFound = True
2988 gitCommit = gitCommit.strip()
2989
2990 if commitFound:
2991 # Convert from p4 time format
2992 try:
2993 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2994 except ValueError:
2995 print "Could not convert label time %s" % labelDetails['Update']
2996 tmwhen = 1
2997
2998 when = int(time.mktime(tmwhen))
2999 self.streamTag(stream, name, labelDetails, gitCommit, when)
3000 if verbose:
3001 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3002 else:
3003 if verbose:
3004 print "Label %s has no changelists - possibly deleted?" % name
3005
3006 if not commitFound:
3007 # We can't import this label; don't try again as it will get very
3008 # expensive repeatedly fetching all the files for labels that will
3009 # never be imported. If the label is moved in the future, the
3010 # ignore will need to be removed manually.
3011 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3012
3013 def guessProjectName(self):
3014 for p in self.depotPaths:
3015 if p.endswith("/"):
3016 p = p[:-1]
3017 p = p[p.strip().rfind("/") + 1:]
3018 if not p.endswith("/"):
3019 p += "/"
3020 return p
3021
3022 def getBranchMapping(self):
3023 lostAndFoundBranches = set()
3024
3025 user = gitConfig("git-p4.branchUser")
3026 if len(user) > 0:
3027 command = "branches -u %s" % user
3028 else:
3029 command = "branches"
3030
3031 for info in p4CmdList(command):
3032 details = p4Cmd(["branch", "-o", info["branch"]])
3033 viewIdx = 0
3034 while details.has_key("View%s" % viewIdx):
3035 paths = details["View%s" % viewIdx].split(" ")
3036 viewIdx = viewIdx + 1
3037 # require standard //depot/foo/... //depot/bar/... mapping
3038 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3039 continue
3040 source = paths[0]
3041 destination = paths[1]
3042 ## HACK
3043 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3044 source = source[len(self.depotPaths[0]):-4]
3045 destination = destination[len(self.depotPaths[0]):-4]
3046
3047 if destination in self.knownBranches:
3048 if not self.silent:
3049 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3050 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3051 continue
3052
3053 self.knownBranches[destination] = source
3054
3055 lostAndFoundBranches.discard(destination)
3056
3057 if source not in self.knownBranches:
3058 lostAndFoundBranches.add(source)
3059
3060 # Perforce does not strictly require branches to be defined, so we also
3061 # check git config for a branch list.
3062 #
3063 # Example of branch definition in git config file:
3064 # [git-p4]
3065 # branchList=main:branchA
3066 # branchList=main:branchB
3067 # branchList=branchA:branchC
3068 configBranches = gitConfigList("git-p4.branchList")
3069 for branch in configBranches:
3070 if branch:
3071 (source, destination) = branch.split(":")
3072 self.knownBranches[destination] = source
3073
3074 lostAndFoundBranches.discard(destination)
3075
3076 if source not in self.knownBranches:
3077 lostAndFoundBranches.add(source)
3078
3079
3080 for branch in lostAndFoundBranches:
3081 self.knownBranches[branch] = branch
3082
3083 def getBranchMappingFromGitBranches(self):
3084 branches = p4BranchesInGit(self.importIntoRemotes)
3085 for branch in branches.keys():
3086 if branch == "master":
3087 branch = "main"
3088 else:
3089 branch = branch[len(self.projectName):]
3090 self.knownBranches[branch] = branch
3091
3092 def updateOptionDict(self, d):
3093 option_keys = {}
3094 if self.keepRepoPath:
3095 option_keys['keepRepoPath'] = 1
3096
3097 d["options"] = ' '.join(sorted(option_keys.keys()))
3098
3099 def readOptions(self, d):
3100 self.keepRepoPath = (d.has_key('options')
3101 and ('keepRepoPath' in d['options']))
3102
3103 def gitRefForBranch(self, branch):
3104 if branch == "main":
3105 return self.refPrefix + "master"
3106
3107 if len(branch) <= 0:
3108 return branch
3109
3110 return self.refPrefix + self.projectName + branch
3111
3112 def gitCommitByP4Change(self, ref, change):
3113 if self.verbose:
3114 print "looking in ref " + ref + " for change %s using bisect..." % change
3115
3116 earliestCommit = ""
3117 latestCommit = parseRevision(ref)
3118
3119 while True:
3120 if self.verbose:
3121 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3122 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3123 if len(next) == 0:
3124 if self.verbose:
3125 print "argh"
3126 return ""
3127 log = extractLogMessageFromGitCommit(next)
3128 settings = extractSettingsGitLog(log)
3129 currentChange = int(settings['change'])
3130 if self.verbose:
3131 print "current change %s" % currentChange
3132
3133 if currentChange == change:
3134 if self.verbose:
3135 print "found %s" % next
3136 return next
3137
3138 if currentChange < change:
3139 earliestCommit = "^%s" % next
3140 else:
3141 latestCommit = "%s" % next
3142
3143 return ""
3144
3145 def importNewBranch(self, branch, maxChange):
3146 # make fast-import flush all changes to disk and update the refs using the checkpoint
3147 # command so that we can try to find the branch parent in the git history
3148 self.gitStream.write("checkpoint\n\n");
3149 self.gitStream.flush();
3150 branchPrefix = self.depotPaths[0] + branch + "/"
3151 range = "@1,%s" % maxChange
3152 #print "prefix" + branchPrefix
3153 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3154 if len(changes) <= 0:
3155 return False
3156 firstChange = changes[0]
3157 #print "first change in branch: %s" % firstChange
3158 sourceBranch = self.knownBranches[branch]
3159 sourceDepotPath = self.depotPaths[0] + sourceBranch
3160 sourceRef = self.gitRefForBranch(sourceBranch)
3161 #print "source " + sourceBranch
3162
3163 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3164 #print "branch parent: %s" % branchParentChange
3165 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3166 if len(gitParent) > 0:
3167 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3168 #print "parent git commit: %s" % gitParent
3169
3170 self.importChanges(changes)
3171 return True
3172
3173 def searchParent(self, parent, branch, target):
3174 parentFound = False
3175 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3176 "--no-merges", parent]):
3177 blob = blob.strip()
3178 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3179 parentFound = True
3180 if self.verbose:
3181 print "Found parent of %s in commit %s" % (branch, blob)
3182 break
3183 if parentFound:
3184 return blob
3185 else:
3186 return None
3187
3188 def importChanges(self, changes):
3189 cnt = 1
3190 for change in changes:
3191 description = p4_describe(change)
3192 self.updateOptionDict(description)
3193
3194 if not self.silent:
3195 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3196 sys.stdout.flush()
3197 cnt = cnt + 1
3198
3199 try:
3200 if self.detectBranches:
3201 branches = self.splitFilesIntoBranches(description)
3202 for branch in branches.keys():
3203 ## HACK --hwn
3204 branchPrefix = self.depotPaths[0] + branch + "/"
3205 self.branchPrefixes = [ branchPrefix ]
3206
3207 parent = ""
3208
3209 filesForCommit = branches[branch]
3210
3211 if self.verbose:
3212 print "branch is %s" % branch
3213
3214 self.updatedBranches.add(branch)
3215
3216 if branch not in self.createdBranches:
3217 self.createdBranches.add(branch)
3218 parent = self.knownBranches[branch]
3219 if parent == branch:
3220 parent = ""
3221 else:
3222 fullBranch = self.projectName + branch
3223 if fullBranch not in self.p4BranchesInGit:
3224 if not self.silent:
3225 print("\n Importing new branch %s" % fullBranch);
3226 if self.importNewBranch(branch, change - 1):
3227 parent = ""
3228 self.p4BranchesInGit.append(fullBranch)
3229 if not self.silent:
3230 print("\n Resuming with change %s" % change);
3231
3232 if self.verbose:
3233 print "parent determined through known branches: %s" % parent
3234
3235 branch = self.gitRefForBranch(branch)
3236 parent = self.gitRefForBranch(parent)
3237
3238 if self.verbose:
3239 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3240
3241 if len(parent) == 0 and branch in self.initialParents:
3242 parent = self.initialParents[branch]
3243 del self.initialParents[branch]
3244
3245 blob = None
3246 if len(parent) > 0:
3247 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3248 if self.verbose:
3249 print "Creating temporary branch: " + tempBranch
3250 self.commit(description, filesForCommit, tempBranch)
3251 self.tempBranches.append(tempBranch)
3252 self.checkpoint()
3253 blob = self.searchParent(parent, branch, tempBranch)
3254 if blob:
3255 self.commit(description, filesForCommit, branch, blob)
3256 else:
3257 if self.verbose:
3258 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3259 self.commit(description, filesForCommit, branch, parent)
3260 else:
3261 files = self.extractFilesFromCommit(description)
3262 self.commit(description, files, self.branch,
3263 self.initialParent)
3264 # only needed once, to connect to the previous commit
3265 self.initialParent = ""
3266 except IOError:
3267 print self.gitError.read()
3268 sys.exit(1)
3269
3270 def sync_origin_only(self):
3271 if self.syncWithOrigin:
3272 self.hasOrigin = originP4BranchesExist()
3273 if self.hasOrigin:
3274 if not self.silent:
3275 print 'Syncing with origin first, using "git fetch origin"'
3276 system("git fetch origin")
3277
3278 def importHeadRevision(self, revision):
3279 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3280
3281 details = {}
3282 details["user"] = "git perforce import user"
3283 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3284 % (' '.join(self.depotPaths), revision))
3285 details["change"] = revision
3286 newestRevision = 0
3287
3288 fileCnt = 0
3289 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3290
3291 for info in p4CmdList(["files"] + fileArgs):
3292
3293 if 'code' in info and info['code'] == 'error':
3294 sys.stderr.write("p4 returned an error: %s\n"
3295 % info['data'])
3296 if info['data'].find("must refer to client") >= 0:
3297 sys.stderr.write("This particular p4 error is misleading.\n")
3298 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3299 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3300 sys.exit(1)
3301 if 'p4ExitCode' in info:
3302 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3303 sys.exit(1)
3304
3305
3306 change = int(info["change"])
3307 if change > newestRevision:
3308 newestRevision = change
3309
3310 if info["action"] in self.delete_actions:
3311 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3312 #fileCnt = fileCnt + 1
3313 continue
3314
3315 for prop in ["depotFile", "rev", "action", "type" ]:
3316 details["%s%s" % (prop, fileCnt)] = info[prop]
3317
3318 fileCnt = fileCnt + 1
3319
3320 details["change"] = newestRevision
3321
3322 # Use time from top-most change so that all git p4 clones of
3323 # the same p4 repo have the same commit SHA1s.
3324 res = p4_describe(newestRevision)
3325 details["time"] = res["time"]
3326
3327 self.updateOptionDict(details)
3328 try:
3329 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3330 except IOError:
3331 print "IO error with git fast-import. Is your git version recent enough?"
3332 print self.gitError.read()
3333
3334
3335 def run(self, args):
3336 self.depotPaths = []
3337 self.changeRange = ""
3338 self.previousDepotPaths = []
3339 self.hasOrigin = False
3340
3341 # map from branch depot path to parent branch
3342 self.knownBranches = {}
3343 self.initialParents = {}
3344
3345 if self.importIntoRemotes:
3346 self.refPrefix = "refs/remotes/p4/"
3347 else:
3348 self.refPrefix = "refs/heads/p4/"
3349
3350 self.sync_origin_only()
3351
3352 branch_arg_given = bool(self.branch)
3353 if len(self.branch) == 0:
3354 self.branch = self.refPrefix + "master"
3355 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3356 system("git update-ref %s refs/heads/p4" % self.branch)
3357 system("git branch -D p4")
3358
3359 # accept either the command-line option, or the configuration variable
3360 if self.useClientSpec:
3361 # will use this after clone to set the variable
3362 self.useClientSpec_from_options = True
3363 else:
3364 if gitConfigBool("git-p4.useclientspec"):
3365 self.useClientSpec = True
3366 if self.useClientSpec:
3367 self.clientSpecDirs = getClientSpec()
3368
3369 # TODO: should always look at previous commits,
3370 # merge with previous imports, if possible.
3371 if args == []:
3372 if self.hasOrigin:
3373 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3374
3375 # branches holds mapping from branch name to sha1
3376 branches = p4BranchesInGit(self.importIntoRemotes)
3377
3378 # restrict to just this one, disabling detect-branches
3379 if branch_arg_given:
3380 short = self.branch.split("/")[-1]
3381 if short in branches:
3382 self.p4BranchesInGit = [ short ]
3383 else:
3384 self.p4BranchesInGit = branches.keys()
3385
3386 if len(self.p4BranchesInGit) > 1:
3387 if not self.silent:
3388 print "Importing from/into multiple branches"
3389 self.detectBranches = True
3390 for branch in branches.keys():
3391 self.initialParents[self.refPrefix + branch] = \
3392 branches[branch]
3393
3394 if self.verbose:
3395 print "branches: %s" % self.p4BranchesInGit
3396
3397 p4Change = 0
3398 for branch in self.p4BranchesInGit:
3399 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3400
3401 settings = extractSettingsGitLog(logMsg)
3402
3403 self.readOptions(settings)
3404 if (settings.has_key('depot-paths')
3405 and settings.has_key ('change')):
3406 change = int(settings['change']) + 1
3407 p4Change = max(p4Change, change)
3408
3409 depotPaths = sorted(settings['depot-paths'])
3410 if self.previousDepotPaths == []:
3411 self.previousDepotPaths = depotPaths
3412 else:
3413 paths = []
3414 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3415 prev_list = prev.split("/")
3416 cur_list = cur.split("/")
3417 for i in range(0, min(len(cur_list), len(prev_list))):
3418 if cur_list[i] <> prev_list[i]:
3419 i = i - 1
3420 break
3421
3422 paths.append ("/".join(cur_list[:i + 1]))
3423
3424 self.previousDepotPaths = paths
3425
3426 if p4Change > 0:
3427 self.depotPaths = sorted(self.previousDepotPaths)
3428 self.changeRange = "@%s,#head" % p4Change
3429 if not self.silent and not self.detectBranches:
3430 print "Performing incremental import into %s git branch" % self.branch
3431
3432 # accept multiple ref name abbreviations:
3433 # refs/foo/bar/branch -> use it exactly
3434 # p4/branch -> prepend refs/remotes/ or refs/heads/
3435 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3436 if not self.branch.startswith("refs/"):
3437 if self.importIntoRemotes:
3438 prepend = "refs/remotes/"
3439 else:
3440 prepend = "refs/heads/"
3441 if not self.branch.startswith("p4/"):
3442 prepend += "p4/"
3443 self.branch = prepend + self.branch
3444
3445 if len(args) == 0 and self.depotPaths:
3446 if not self.silent:
3447 print "Depot paths: %s" % ' '.join(self.depotPaths)
3448 else:
3449 if self.depotPaths and self.depotPaths != args:
3450 print ("previous import used depot path %s and now %s was specified. "
3451 "This doesn't work!" % (' '.join (self.depotPaths),
3452 ' '.join (args)))
3453 sys.exit(1)
3454
3455 self.depotPaths = sorted(args)
3456
3457 revision = ""
3458 self.users = {}
3459
3460 # Make sure no revision specifiers are used when --changesfile
3461 # is specified.
3462 bad_changesfile = False
3463 if len(self.changesFile) > 0:
3464 for p in self.depotPaths:
3465 if p.find("@") >= 0 or p.find("#") >= 0:
3466 bad_changesfile = True
3467 break
3468 if bad_changesfile:
3469 die("Option --changesfile is incompatible with revision specifiers")
3470
3471 newPaths = []
3472 for p in self.depotPaths:
3473 if p.find("@") != -1:
3474 atIdx = p.index("@")
3475 self.changeRange = p[atIdx:]
3476 if self.changeRange == "@all":
3477 self.changeRange = ""
3478 elif ',' not in self.changeRange:
3479 revision = self.changeRange
3480 self.changeRange = ""
3481 p = p[:atIdx]
3482 elif p.find("#") != -1:
3483 hashIdx = p.index("#")
3484 revision = p[hashIdx:]
3485 p = p[:hashIdx]
3486 elif self.previousDepotPaths == []:
3487 # pay attention to changesfile, if given, else import
3488 # the entire p4 tree at the head revision
3489 if len(self.changesFile) == 0:
3490 revision = "#head"
3491
3492 p = re.sub ("\.\.\.$", "", p)
3493 if not p.endswith("/"):
3494 p += "/"
3495
3496 newPaths.append(p)
3497
3498 self.depotPaths = newPaths
3499
3500 # --detect-branches may change this for each branch
3501 self.branchPrefixes = self.depotPaths
3502
3503 self.loadUserMapFromCache()
3504 self.labels = {}
3505 if self.detectLabels:
3506 self.getLabels();
3507
3508 if self.detectBranches:
3509 ## FIXME - what's a P4 projectName ?
3510 self.projectName = self.guessProjectName()
3511
3512 if self.hasOrigin:
3513 self.getBranchMappingFromGitBranches()
3514 else:
3515 self.getBranchMapping()
3516 if self.verbose:
3517 print "p4-git branches: %s" % self.p4BranchesInGit
3518 print "initial parents: %s" % self.initialParents
3519 for b in self.p4BranchesInGit:
3520 if b != "master":
3521
3522 ## FIXME
3523 b = b[len(self.projectName):]
3524 self.createdBranches.add(b)
3525
3526 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3527
3528 self.importProcess = subprocess.Popen(["git", "fast-import"],
3529 stdin=subprocess.PIPE,
3530 stdout=subprocess.PIPE,
3531 stderr=subprocess.PIPE);
3532 self.gitOutput = self.importProcess.stdout
3533 self.gitStream = self.importProcess.stdin
3534 self.gitError = self.importProcess.stderr
3535
3536 if revision:
3537 self.importHeadRevision(revision)
3538 else:
3539 changes = []
3540
3541 if len(self.changesFile) > 0:
3542 output = open(self.changesFile).readlines()
3543 changeSet = set()
3544 for line in output:
3545 changeSet.add(int(line))
3546
3547 for change in changeSet:
3548 changes.append(change)
3549
3550 changes.sort()
3551 else:
3552 # catch "git p4 sync" with no new branches, in a repo that
3553 # does not have any existing p4 branches
3554 if len(args) == 0:
3555 if not self.p4BranchesInGit:
3556 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3557
3558 # The default branch is master, unless --branch is used to
3559 # specify something else. Make sure it exists, or complain
3560 # nicely about how to use --branch.
3561 if not self.detectBranches:
3562 if not branch_exists(self.branch):
3563 if branch_arg_given:
3564 die("Error: branch %s does not exist." % self.branch)
3565 else:
3566 die("Error: no branch %s; perhaps specify one with --branch." %
3567 self.branch)
3568
3569 if self.verbose:
3570 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3571 self.changeRange)
3572 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3573
3574 if len(self.maxChanges) > 0:
3575 changes = changes[:min(int(self.maxChanges), len(changes))]
3576
3577 if len(changes) == 0:
3578 if not self.silent:
3579 print "No changes to import!"
3580 else:
3581 if not self.silent and not self.detectBranches:
3582 print "Import destination: %s" % self.branch
3583
3584 self.updatedBranches = set()
3585
3586 if not self.detectBranches:
3587 if args:
3588 # start a new branch
3589 self.initialParent = ""
3590 else:
3591 # build on a previous revision
3592 self.initialParent = parseRevision(self.branch)
3593
3594 self.importChanges(changes)
3595
3596 if not self.silent:
3597 print ""
3598 if len(self.updatedBranches) > 0:
3599 sys.stdout.write("Updated branches: ")
3600 for b in self.updatedBranches:
3601 sys.stdout.write("%s " % b)
3602 sys.stdout.write("\n")
3603
3604 if gitConfigBool("git-p4.importLabels"):
3605 self.importLabels = True
3606
3607 if self.importLabels:
3608 p4Labels = getP4Labels(self.depotPaths)
3609 gitTags = getGitTags()
3610
3611 missingP4Labels = p4Labels - gitTags
3612 self.importP4Labels(self.gitStream, missingP4Labels)
3613