Merge branch 'ld/p4-unshelve'
[git/git.git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
29
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
45 verbose = False
46
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
52
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
60 real_cmd = ["p4"]
61
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
64 real_cmd += ["-u",user]
65
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
68 real_cmd += ["-P", password]
69
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
72 real_cmd += ["-p", port]
73
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
76 real_cmd += ["-H", host]
77
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
80 real_cmd += ["-c", client]
81
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
89
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
94 return real_cmd
95
96 def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
106 def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
127
128 def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
138 def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
145 def write_pipe(c, stdin):
146 if verbose:
147 sys.stderr.write('Writing pipe: %s\n' % str(c))
148
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
156
157 return val
158
159 def p4_write_pipe(c, stdin):
160 real_cmd = p4_build_cmd(c)
161 return write_pipe(real_cmd, stdin)
162
163 def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
167 """
168 if verbose:
169 sys.stderr.write('Reading pipe: %s\n' % str(c))
170
171 expand = isinstance(c,basestring)
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
174 return (p.returncode, out, err)
175
176 def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
180 """
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
187 return out
188
189 def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
192 """
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
198
199 def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
202
203 def read_pipe_lines(c):
204 if verbose:
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
210 val = pipe.readlines()
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
213
214 return val
215
216 def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
218 real_cmd = p4_build_cmd(c)
219 return read_pipe_lines(real_cmd)
220
221 def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
229
230 def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
235
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
248
249 def system(cmd, ignore_error=False):
250 expand = isinstance(cmd,basestring)
251 if verbose:
252 sys.stderr.write("executing %s\n" % str(cmd))
253 retcode = subprocess.call(cmd, shell=expand)
254 if retcode and not ignore_error:
255 raise CalledProcessError(retcode, cmd)
256
257 return retcode
258
259 def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
261 real_cmd = p4_build_cmd(cmd)
262 expand = isinstance(real_cmd, basestring)
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
266
267 _p4_version_string = None
268 def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
271
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276 """
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
282
283 def p4_integrate(src, dest):
284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
285
286 def p4_sync(f, *options):
287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
288
289 def p4_add(f):
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
295
296 def p4_delete(f):
297 p4_system(["delete", wildcard_encode(f)])
298
299 def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
301
302 def p4_revert(f):
303 p4_system(["revert", wildcard_encode(f)])
304
305 def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
307
308 def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
311
312 def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
315 def p4_last_change():
316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317 return int(results[0]['change'])
318
319 def p4_describe(change, shelved=False):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
323
324 cmd = ["describe", "-s"]
325 if shelved:
326 cmd += ["-S"]
327 cmd += [str(change)]
328
329 ds = p4CmdList(cmd, skip_info=True)
330 if len(ds) != 1:
331 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
332
333 d = ds[0]
334
335 if "p4ExitCode" in d:
336 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
337 str(d)))
338 if "code" in d:
339 if d["code"] == "error":
340 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
341
342 if "time" not in d:
343 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
344
345 return d
346
347 #
348 # Canonicalize the p4 type and return a tuple of the
349 # base type, plus any modifiers. See "p4 help filetypes"
350 # for a list and explanation.
351 #
352 def split_p4_type(p4type):
353
354 p4_filetypes_historical = {
355 "ctempobj": "binary+Sw",
356 "ctext": "text+C",
357 "cxtext": "text+Cx",
358 "ktext": "text+k",
359 "kxtext": "text+kx",
360 "ltext": "text+F",
361 "tempobj": "binary+FSw",
362 "ubinary": "binary+F",
363 "uresource": "resource+F",
364 "uxbinary": "binary+Fx",
365 "xbinary": "binary+x",
366 "xltext": "text+Fx",
367 "xtempobj": "binary+Swx",
368 "xtext": "text+x",
369 "xunicode": "unicode+x",
370 "xutf16": "utf16+x",
371 }
372 if p4type in p4_filetypes_historical:
373 p4type = p4_filetypes_historical[p4type]
374 mods = ""
375 s = p4type.split("+")
376 base = s[0]
377 mods = ""
378 if len(s) > 1:
379 mods = s[1]
380 return (base, mods)
381
382 #
383 # return the raw p4 type of a file (text, text+ko, etc)
384 #
385 def p4_type(f):
386 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
387 return results[0]['headType']
388
389 #
390 # Given a type base and modifier, return a regexp matching
391 # the keywords that can be expanded in the file
392 #
393 def p4_keywords_regexp_for_type(base, type_mods):
394 if base in ("text", "unicode", "binary"):
395 kwords = None
396 if "ko" in type_mods:
397 kwords = 'Id|Header'
398 elif "k" in type_mods:
399 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
400 else:
401 return None
402 pattern = r"""
403 \$ # Starts with a dollar, followed by...
404 (%s) # one of the keywords, followed by...
405 (:[^$\n]+)? # possibly an old expansion, followed by...
406 \$ # another dollar
407 """ % kwords
408 return pattern
409 else:
410 return None
411
412 #
413 # Given a file, return a regexp matching the possible
414 # RCS keywords that will be expanded, or None for files
415 # with kw expansion turned off.
416 #
417 def p4_keywords_regexp_for_file(file):
418 if not os.path.exists(file):
419 return None
420 else:
421 (type_base, type_mods) = split_p4_type(p4_type(file))
422 return p4_keywords_regexp_for_type(type_base, type_mods)
423
424 def setP4ExecBit(file, mode):
425 # Reopens an already open file and changes the execute bit to match
426 # the execute bit setting in the passed in mode.
427
428 p4Type = "+x"
429
430 if not isModeExec(mode):
431 p4Type = getP4OpenedType(file)
432 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
433 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
434 if p4Type[-1] == "+":
435 p4Type = p4Type[0:-1]
436
437 p4_reopen(p4Type, file)
438
439 def getP4OpenedType(file):
440 # Returns the perforce file type for the given file.
441
442 result = p4_read_pipe(["opened", wildcard_encode(file)])
443 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
444 if match:
445 return match.group(1)
446 else:
447 die("Could not determine file type for %s (result: '%s')" % (file, result))
448
449 # Return the set of all p4 labels
450 def getP4Labels(depotPaths):
451 labels = set()
452 if isinstance(depotPaths,basestring):
453 depotPaths = [depotPaths]
454
455 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
456 label = l['label']
457 labels.add(label)
458
459 return labels
460
461 # Return the set of all git tags
462 def getGitTags():
463 gitTags = set()
464 for line in read_pipe_lines(["git", "tag"]):
465 tag = line.strip()
466 gitTags.add(tag)
467 return gitTags
468
469 def diffTreePattern():
470 # This is a simple generator for the diff tree regex pattern. This could be
471 # a class variable if this and parseDiffTreeEntry were a part of a class.
472 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
473 while True:
474 yield pattern
475
476 def parseDiffTreeEntry(entry):
477 """Parses a single diff tree entry into its component elements.
478
479 See git-diff-tree(1) manpage for details about the format of the diff
480 output. This method returns a dictionary with the following elements:
481
482 src_mode - The mode of the source file
483 dst_mode - The mode of the destination file
484 src_sha1 - The sha1 for the source file
485 dst_sha1 - The sha1 fr the destination file
486 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
487 status_score - The score for the status (applicable for 'C' and 'R'
488 statuses). This is None if there is no score.
489 src - The path for the source file.
490 dst - The path for the destination file. This is only present for
491 copy or renames. If it is not present, this is None.
492
493 If the pattern is not matched, None is returned."""
494
495 match = diffTreePattern().next().match(entry)
496 if match:
497 return {
498 'src_mode': match.group(1),
499 'dst_mode': match.group(2),
500 'src_sha1': match.group(3),
501 'dst_sha1': match.group(4),
502 'status': match.group(5),
503 'status_score': match.group(6),
504 'src': match.group(7),
505 'dst': match.group(10)
506 }
507 return None
508
509 def isModeExec(mode):
510 # Returns True if the given git mode represents an executable file,
511 # otherwise False.
512 return mode[-3:] == "755"
513
514 def isModeExecChanged(src_mode, dst_mode):
515 return isModeExec(src_mode) != isModeExec(dst_mode)
516
517 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
518
519 if isinstance(cmd,basestring):
520 cmd = "-G " + cmd
521 expand = True
522 else:
523 cmd = ["-G"] + cmd
524 expand = False
525
526 cmd = p4_build_cmd(cmd)
527 if verbose:
528 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
529
530 # Use a temporary file to avoid deadlocks without
531 # subprocess.communicate(), which would put another copy
532 # of stdout into memory.
533 stdin_file = None
534 if stdin is not None:
535 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
536 if isinstance(stdin,basestring):
537 stdin_file.write(stdin)
538 else:
539 for i in stdin:
540 stdin_file.write(i + '\n')
541 stdin_file.flush()
542 stdin_file.seek(0)
543
544 p4 = subprocess.Popen(cmd,
545 shell=expand,
546 stdin=stdin_file,
547 stdout=subprocess.PIPE)
548
549 result = []
550 try:
551 while True:
552 entry = marshal.load(p4.stdout)
553 if skip_info:
554 if 'code' in entry and entry['code'] == 'info':
555 continue
556 if cb is not None:
557 cb(entry)
558 else:
559 result.append(entry)
560 except EOFError:
561 pass
562 exitCode = p4.wait()
563 if exitCode != 0:
564 entry = {}
565 entry["p4ExitCode"] = exitCode
566 result.append(entry)
567
568 return result
569
570 def p4Cmd(cmd):
571 list = p4CmdList(cmd)
572 result = {}
573 for entry in list:
574 result.update(entry)
575 return result;
576
577 def p4Where(depotPath):
578 if not depotPath.endswith("/"):
579 depotPath += "/"
580 depotPathLong = depotPath + "..."
581 outputList = p4CmdList(["where", depotPathLong])
582 output = None
583 for entry in outputList:
584 if "depotFile" in entry:
585 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
586 # The base path always ends with "/...".
587 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
588 output = entry
589 break
590 elif "data" in entry:
591 data = entry.get("data")
592 space = data.find(" ")
593 if data[:space] == depotPath:
594 output = entry
595 break
596 if output == None:
597 return ""
598 if output["code"] == "error":
599 return ""
600 clientPath = ""
601 if "path" in output:
602 clientPath = output.get("path")
603 elif "data" in output:
604 data = output.get("data")
605 lastSpace = data.rfind(" ")
606 clientPath = data[lastSpace + 1:]
607
608 if clientPath.endswith("..."):
609 clientPath = clientPath[:-3]
610 return clientPath
611
612 def currentGitBranch():
613 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
614
615 def isValidGitDir(path):
616 return git_dir(path) != None
617
618 def parseRevision(ref):
619 return read_pipe("git rev-parse %s" % ref).strip()
620
621 def branchExists(ref):
622 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
623 ignore_error=True)
624 return len(rev) > 0
625
626 def extractLogMessageFromGitCommit(commit):
627 logMessage = ""
628
629 ## fixme: title is first line of commit, not 1st paragraph.
630 foundTitle = False
631 for log in read_pipe_lines("git cat-file commit %s" % commit):
632 if not foundTitle:
633 if len(log) == 1:
634 foundTitle = True
635 continue
636
637 logMessage += log
638 return logMessage
639
640 def extractSettingsGitLog(log):
641 values = {}
642 for line in log.split("\n"):
643 line = line.strip()
644 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
645 if not m:
646 continue
647
648 assignments = m.group(1).split (':')
649 for a in assignments:
650 vals = a.split ('=')
651 key = vals[0].strip()
652 val = ('='.join (vals[1:])).strip()
653 if val.endswith ('\"') and val.startswith('"'):
654 val = val[1:-1]
655
656 values[key] = val
657
658 paths = values.get("depot-paths")
659 if not paths:
660 paths = values.get("depot-path")
661 if paths:
662 values['depot-paths'] = paths.split(',')
663 return values
664
665 def gitBranchExists(branch):
666 proc = subprocess.Popen(["git", "rev-parse", branch],
667 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
668 return proc.wait() == 0;
669
670 def gitUpdateRef(ref, newvalue):
671 subprocess.check_call(["git", "update-ref", ref, newvalue])
672
673 def gitDeleteRef(ref):
674 subprocess.check_call(["git", "update-ref", "-d", ref])
675
676 _gitConfig = {}
677
678 def gitConfig(key, typeSpecifier=None):
679 if not _gitConfig.has_key(key):
680 cmd = [ "git", "config" ]
681 if typeSpecifier:
682 cmd += [ typeSpecifier ]
683 cmd += [ key ]
684 s = read_pipe(cmd, ignore_error=True)
685 _gitConfig[key] = s.strip()
686 return _gitConfig[key]
687
688 def gitConfigBool(key):
689 """Return a bool, using git config --bool. It is True only if the
690 variable is set to true, and False if set to false or not present
691 in the config."""
692
693 if not _gitConfig.has_key(key):
694 _gitConfig[key] = gitConfig(key, '--bool') == "true"
695 return _gitConfig[key]
696
697 def gitConfigInt(key):
698 if not _gitConfig.has_key(key):
699 cmd = [ "git", "config", "--int", key ]
700 s = read_pipe(cmd, ignore_error=True)
701 v = s.strip()
702 try:
703 _gitConfig[key] = int(gitConfig(key, '--int'))
704 except ValueError:
705 _gitConfig[key] = None
706 return _gitConfig[key]
707
708 def gitConfigList(key):
709 if not _gitConfig.has_key(key):
710 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
711 _gitConfig[key] = s.strip().splitlines()
712 if _gitConfig[key] == ['']:
713 _gitConfig[key] = []
714 return _gitConfig[key]
715
716 def p4BranchesInGit(branchesAreInRemotes=True):
717 """Find all the branches whose names start with "p4/", looking
718 in remotes or heads as specified by the argument. Return
719 a dictionary of { branch: revision } for each one found.
720 The branch names are the short names, without any
721 "p4/" prefix."""
722
723 branches = {}
724
725 cmdline = "git rev-parse --symbolic "
726 if branchesAreInRemotes:
727 cmdline += "--remotes"
728 else:
729 cmdline += "--branches"
730
731 for line in read_pipe_lines(cmdline):
732 line = line.strip()
733
734 # only import to p4/
735 if not line.startswith('p4/'):
736 continue
737 # special symbolic ref to p4/master
738 if line == "p4/HEAD":
739 continue
740
741 # strip off p4/ prefix
742 branch = line[len("p4/"):]
743
744 branches[branch] = parseRevision(line)
745
746 return branches
747
748 def branch_exists(branch):
749 """Make sure that the given ref name really exists."""
750
751 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
752 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
753 out, _ = p.communicate()
754 if p.returncode:
755 return False
756 # expect exactly one line of output: the branch name
757 return out.rstrip() == branch
758
759 def findUpstreamBranchPoint(head = "HEAD"):
760 branches = p4BranchesInGit()
761 # map from depot-path to branch name
762 branchByDepotPath = {}
763 for branch in branches.keys():
764 tip = branches[branch]
765 log = extractLogMessageFromGitCommit(tip)
766 settings = extractSettingsGitLog(log)
767 if settings.has_key("depot-paths"):
768 paths = ",".join(settings["depot-paths"])
769 branchByDepotPath[paths] = "remotes/p4/" + branch
770
771 settings = None
772 parent = 0
773 while parent < 65535:
774 commit = head + "~%s" % parent
775 log = extractLogMessageFromGitCommit(commit)
776 settings = extractSettingsGitLog(log)
777 if settings.has_key("depot-paths"):
778 paths = ",".join(settings["depot-paths"])
779 if branchByDepotPath.has_key(paths):
780 return [branchByDepotPath[paths], settings]
781
782 parent = parent + 1
783
784 return ["", settings]
785
786 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
787 if not silent:
788 print ("Creating/updating branch(es) in %s based on origin branch(es)"
789 % localRefPrefix)
790
791 originPrefix = "origin/p4/"
792
793 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
794 line = line.strip()
795 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
796 continue
797
798 headName = line[len(originPrefix):]
799 remoteHead = localRefPrefix + headName
800 originHead = line
801
802 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
803 if (not original.has_key('depot-paths')
804 or not original.has_key('change')):
805 continue
806
807 update = False
808 if not gitBranchExists(remoteHead):
809 if verbose:
810 print "creating %s" % remoteHead
811 update = True
812 else:
813 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
814 if settings.has_key('change') > 0:
815 if settings['depot-paths'] == original['depot-paths']:
816 originP4Change = int(original['change'])
817 p4Change = int(settings['change'])
818 if originP4Change > p4Change:
819 print ("%s (%s) is newer than %s (%s). "
820 "Updating p4 branch from origin."
821 % (originHead, originP4Change,
822 remoteHead, p4Change))
823 update = True
824 else:
825 print ("Ignoring: %s was imported from %s while "
826 "%s was imported from %s"
827 % (originHead, ','.join(original['depot-paths']),
828 remoteHead, ','.join(settings['depot-paths'])))
829
830 if update:
831 system("git update-ref %s %s" % (remoteHead, originHead))
832
833 def originP4BranchesExist():
834 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
835
836
837 def p4ParseNumericChangeRange(parts):
838 changeStart = int(parts[0][1:])
839 if parts[1] == '#head':
840 changeEnd = p4_last_change()
841 else:
842 changeEnd = int(parts[1])
843
844 return (changeStart, changeEnd)
845
846 def chooseBlockSize(blockSize):
847 if blockSize:
848 return blockSize
849 else:
850 return defaultBlockSize
851
852 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
853 assert depotPaths
854
855 # Parse the change range into start and end. Try to find integer
856 # revision ranges as these can be broken up into blocks to avoid
857 # hitting server-side limits (maxrows, maxscanresults). But if
858 # that doesn't work, fall back to using the raw revision specifier
859 # strings, without using block mode.
860
861 if changeRange is None or changeRange == '':
862 changeStart = 1
863 changeEnd = p4_last_change()
864 block_size = chooseBlockSize(requestedBlockSize)
865 else:
866 parts = changeRange.split(',')
867 assert len(parts) == 2
868 try:
869 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
870 block_size = chooseBlockSize(requestedBlockSize)
871 except:
872 changeStart = parts[0][1:]
873 changeEnd = parts[1]
874 if requestedBlockSize:
875 die("cannot use --changes-block-size with non-numeric revisions")
876 block_size = None
877
878 changes = set()
879
880 # Retrieve changes a block at a time, to prevent running
881 # into a MaxResults/MaxScanRows error from the server.
882
883 while True:
884 cmd = ['changes']
885
886 if block_size:
887 end = min(changeEnd, changeStart + block_size)
888 revisionRange = "%d,%d" % (changeStart, end)
889 else:
890 revisionRange = "%s,%s" % (changeStart, changeEnd)
891
892 for p in depotPaths:
893 cmd += ["%s...@%s" % (p, revisionRange)]
894
895 # Insert changes in chronological order
896 for entry in reversed(p4CmdList(cmd)):
897 if entry.has_key('p4ExitCode'):
898 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
899 if not entry.has_key('change'):
900 continue
901 changes.add(int(entry['change']))
902
903 if not block_size:
904 break
905
906 if end >= changeEnd:
907 break
908
909 changeStart = end + 1
910
911 changes = sorted(changes)
912 return changes
913
914 def p4PathStartsWith(path, prefix):
915 # This method tries to remedy a potential mixed-case issue:
916 #
917 # If UserA adds //depot/DirA/file1
918 # and UserB adds //depot/dira/file2
919 #
920 # we may or may not have a problem. If you have core.ignorecase=true,
921 # we treat DirA and dira as the same directory
922 if gitConfigBool("core.ignorecase"):
923 return path.lower().startswith(prefix.lower())
924 return path.startswith(prefix)
925
926 def getClientSpec():
927 """Look at the p4 client spec, create a View() object that contains
928 all the mappings, and return it."""
929
930 specList = p4CmdList("client -o")
931 if len(specList) != 1:
932 die('Output from "client -o" is %d lines, expecting 1' %
933 len(specList))
934
935 # dictionary of all client parameters
936 entry = specList[0]
937
938 # the //client/ name
939 client_name = entry["Client"]
940
941 # just the keys that start with "View"
942 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
943
944 # hold this new View
945 view = View(client_name)
946
947 # append the lines, in order, to the view
948 for view_num in range(len(view_keys)):
949 k = "View%d" % view_num
950 if k not in view_keys:
951 die("Expected view key %s missing" % k)
952 view.append(entry[k])
953
954 return view
955
956 def getClientRoot():
957 """Grab the client directory."""
958
959 output = p4CmdList("client -o")
960 if len(output) != 1:
961 die('Output from "client -o" is %d lines, expecting 1' % len(output))
962
963 entry = output[0]
964 if "Root" not in entry:
965 die('Client has no "Root"')
966
967 return entry["Root"]
968
969 #
970 # P4 wildcards are not allowed in filenames. P4 complains
971 # if you simply add them, but you can force it with "-f", in
972 # which case it translates them into %xx encoding internally.
973 #
974 def wildcard_decode(path):
975 # Search for and fix just these four characters. Do % last so
976 # that fixing it does not inadvertently create new %-escapes.
977 # Cannot have * in a filename in windows; untested as to
978 # what p4 would do in such a case.
979 if not platform.system() == "Windows":
980 path = path.replace("%2A", "*")
981 path = path.replace("%23", "#") \
982 .replace("%40", "@") \
983 .replace("%25", "%")
984 return path
985
986 def wildcard_encode(path):
987 # do % first to avoid double-encoding the %s introduced here
988 path = path.replace("%", "%25") \
989 .replace("*", "%2A") \
990 .replace("#", "%23") \
991 .replace("@", "%40")
992 return path
993
994 def wildcard_present(path):
995 m = re.search("[*#@%]", path)
996 return m is not None
997
998 class LargeFileSystem(object):
999 """Base class for large file system support."""
1000
1001 def __init__(self, writeToGitStream):
1002 self.largeFiles = set()
1003 self.writeToGitStream = writeToGitStream
1004
1005 def generatePointer(self, cloneDestination, contentFile):
1006 """Return the content of a pointer file that is stored in Git instead of
1007 the actual content."""
1008 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
1009
1010 def pushFile(self, localLargeFile):
1011 """Push the actual content which is not stored in the Git repository to
1012 a server."""
1013 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1014
1015 def hasLargeFileExtension(self, relPath):
1016 return reduce(
1017 lambda a, b: a or b,
1018 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1019 False
1020 )
1021
1022 def generateTempFile(self, contents):
1023 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1024 for d in contents:
1025 contentFile.write(d)
1026 contentFile.close()
1027 return contentFile.name
1028
1029 def exceedsLargeFileThreshold(self, relPath, contents):
1030 if gitConfigInt('git-p4.largeFileThreshold'):
1031 contentsSize = sum(len(d) for d in contents)
1032 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1033 return True
1034 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1035 contentsSize = sum(len(d) for d in contents)
1036 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1037 return False
1038 contentTempFile = self.generateTempFile(contents)
1039 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1040 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1041 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1042 zf.close()
1043 compressedContentsSize = zf.infolist()[0].compress_size
1044 os.remove(contentTempFile)
1045 os.remove(compressedContentFile.name)
1046 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1047 return True
1048 return False
1049
1050 def addLargeFile(self, relPath):
1051 self.largeFiles.add(relPath)
1052
1053 def removeLargeFile(self, relPath):
1054 self.largeFiles.remove(relPath)
1055
1056 def isLargeFile(self, relPath):
1057 return relPath in self.largeFiles
1058
1059 def processContent(self, git_mode, relPath, contents):
1060 """Processes the content of git fast import. This method decides if a
1061 file is stored in the large file system and handles all necessary
1062 steps."""
1063 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1064 contentTempFile = self.generateTempFile(contents)
1065 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1066 if pointer_git_mode:
1067 git_mode = pointer_git_mode
1068 if localLargeFile:
1069 # Move temp file to final location in large file system
1070 largeFileDir = os.path.dirname(localLargeFile)
1071 if not os.path.isdir(largeFileDir):
1072 os.makedirs(largeFileDir)
1073 shutil.move(contentTempFile, localLargeFile)
1074 self.addLargeFile(relPath)
1075 if gitConfigBool('git-p4.largeFilePush'):
1076 self.pushFile(localLargeFile)
1077 if verbose:
1078 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1079 return (git_mode, contents)
1080
1081 class MockLFS(LargeFileSystem):
1082 """Mock large file system for testing."""
1083
1084 def generatePointer(self, contentFile):
1085 """The pointer content is the original content prefixed with "pointer-".
1086 The local filename of the large file storage is derived from the file content.
1087 """
1088 with open(contentFile, 'r') as f:
1089 content = next(f)
1090 gitMode = '100644'
1091 pointerContents = 'pointer-' + content
1092 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1093 return (gitMode, pointerContents, localLargeFile)
1094
1095 def pushFile(self, localLargeFile):
1096 """The remote filename of the large file storage is the same as the local
1097 one but in a different directory.
1098 """
1099 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1100 if not os.path.exists(remotePath):
1101 os.makedirs(remotePath)
1102 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1103
1104 class GitLFS(LargeFileSystem):
1105 """Git LFS as backend for the git-p4 large file system.
1106 See https://git-lfs.github.com/ for details."""
1107
1108 def __init__(self, *args):
1109 LargeFileSystem.__init__(self, *args)
1110 self.baseGitAttributes = []
1111
1112 def generatePointer(self, contentFile):
1113 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1114 mode and content which is stored in the Git repository instead of
1115 the actual content. Return also the new location of the actual
1116 content.
1117 """
1118 if os.path.getsize(contentFile) == 0:
1119 return (None, '', None)
1120
1121 pointerProcess = subprocess.Popen(
1122 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1123 stdout=subprocess.PIPE
1124 )
1125 pointerFile = pointerProcess.stdout.read()
1126 if pointerProcess.wait():
1127 os.remove(contentFile)
1128 die('git-lfs pointer command failed. Did you install the extension?')
1129
1130 # Git LFS removed the preamble in the output of the 'pointer' command
1131 # starting from version 1.2.0. Check for the preamble here to support
1132 # earlier versions.
1133 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1134 if pointerFile.startswith('Git LFS pointer for'):
1135 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1136
1137 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1138 localLargeFile = os.path.join(
1139 os.getcwd(),
1140 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1141 oid,
1142 )
1143 # LFS Spec states that pointer files should not have the executable bit set.
1144 gitMode = '100644'
1145 return (gitMode, pointerFile, localLargeFile)
1146
1147 def pushFile(self, localLargeFile):
1148 uploadProcess = subprocess.Popen(
1149 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1150 )
1151 if uploadProcess.wait():
1152 die('git-lfs push command failed. Did you define a remote?')
1153
1154 def generateGitAttributes(self):
1155 return (
1156 self.baseGitAttributes +
1157 [
1158 '\n',
1159 '#\n',
1160 '# Git LFS (see https://git-lfs.github.com/)\n',
1161 '#\n',
1162 ] +
1163 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1164 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1165 ] +
1166 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1167 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1168 ]
1169 )
1170
1171 def addLargeFile(self, relPath):
1172 LargeFileSystem.addLargeFile(self, relPath)
1173 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1174
1175 def removeLargeFile(self, relPath):
1176 LargeFileSystem.removeLargeFile(self, relPath)
1177 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1178
1179 def processContent(self, git_mode, relPath, contents):
1180 if relPath == '.gitattributes':
1181 self.baseGitAttributes = contents
1182 return (git_mode, self.generateGitAttributes())
1183 else:
1184 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1185
1186 class Command:
1187 def __init__(self):
1188 self.usage = "usage: %prog [options]"
1189 self.needsGit = True
1190 self.verbose = False
1191
1192 # This is required for the "append" cloneExclude action
1193 def ensure_value(self, attr, value):
1194 if not hasattr(self, attr) or getattr(self, attr) is None:
1195 setattr(self, attr, value)
1196 return getattr(self, attr)
1197
1198 class P4UserMap:
1199 def __init__(self):
1200 self.userMapFromPerforceServer = False
1201 self.myP4UserId = None
1202
1203 def p4UserId(self):
1204 if self.myP4UserId:
1205 return self.myP4UserId
1206
1207 results = p4CmdList("user -o")
1208 for r in results:
1209 if r.has_key('User'):
1210 self.myP4UserId = r['User']
1211 return r['User']
1212 die("Could not find your p4 user id")
1213
1214 def p4UserIsMe(self, p4User):
1215 # return True if the given p4 user is actually me
1216 me = self.p4UserId()
1217 if not p4User or p4User != me:
1218 return False
1219 else:
1220 return True
1221
1222 def getUserCacheFilename(self):
1223 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1224 return home + "/.gitp4-usercache.txt"
1225
1226 def getUserMapFromPerforceServer(self):
1227 if self.userMapFromPerforceServer:
1228 return
1229 self.users = {}
1230 self.emails = {}
1231
1232 for output in p4CmdList("users"):
1233 if not output.has_key("User"):
1234 continue
1235 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1236 self.emails[output["Email"]] = output["User"]
1237
1238 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1239 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1240 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1241 if mapUser and len(mapUser[0]) == 3:
1242 user = mapUser[0][0]
1243 fullname = mapUser[0][1]
1244 email = mapUser[0][2]
1245 self.users[user] = fullname + " <" + email + ">"
1246 self.emails[email] = user
1247
1248 s = ''
1249 for (key, val) in self.users.items():
1250 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1251
1252 open(self.getUserCacheFilename(), "wb").write(s)
1253 self.userMapFromPerforceServer = True
1254
1255 def loadUserMapFromCache(self):
1256 self.users = {}
1257 self.userMapFromPerforceServer = False
1258 try:
1259 cache = open(self.getUserCacheFilename(), "rb")
1260 lines = cache.readlines()
1261 cache.close()
1262 for line in lines:
1263 entry = line.strip().split("\t")
1264 self.users[entry[0]] = entry[1]
1265 except IOError:
1266 self.getUserMapFromPerforceServer()
1267
1268 class P4Debug(Command):
1269 def __init__(self):
1270 Command.__init__(self)
1271 self.options = []
1272 self.description = "A tool to debug the output of p4 -G."
1273 self.needsGit = False
1274
1275 def run(self, args):
1276 j = 0
1277 for output in p4CmdList(args):
1278 print 'Element: %d' % j
1279 j += 1
1280 print output
1281 return True
1282
1283 class P4RollBack(Command):
1284 def __init__(self):
1285 Command.__init__(self)
1286 self.options = [
1287 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1288 ]
1289 self.description = "A tool to debug the multi-branch import. Don't use :)"
1290 self.rollbackLocalBranches = False
1291
1292 def run(self, args):
1293 if len(args) != 1:
1294 return False
1295 maxChange = int(args[0])
1296
1297 if "p4ExitCode" in p4Cmd("changes -m 1"):
1298 die("Problems executing p4");
1299
1300 if self.rollbackLocalBranches:
1301 refPrefix = "refs/heads/"
1302 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1303 else:
1304 refPrefix = "refs/remotes/"
1305 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1306
1307 for line in lines:
1308 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1309 line = line.strip()
1310 ref = refPrefix + line
1311 log = extractLogMessageFromGitCommit(ref)
1312 settings = extractSettingsGitLog(log)
1313
1314 depotPaths = settings['depot-paths']
1315 change = settings['change']
1316
1317 changed = False
1318
1319 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1320 for p in depotPaths]))) == 0:
1321 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1322 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1323 continue
1324
1325 while change and int(change) > maxChange:
1326 changed = True
1327 if self.verbose:
1328 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1329 system("git update-ref %s \"%s^\"" % (ref, ref))
1330 log = extractLogMessageFromGitCommit(ref)
1331 settings = extractSettingsGitLog(log)
1332
1333
1334 depotPaths = settings['depot-paths']
1335 change = settings['change']
1336
1337 if changed:
1338 print "%s rewound to %s" % (ref, change)
1339
1340 return True
1341
1342 class P4Submit(Command, P4UserMap):
1343
1344 conflict_behavior_choices = ("ask", "skip", "quit")
1345
1346 def __init__(self):
1347 Command.__init__(self)
1348 P4UserMap.__init__(self)
1349 self.options = [
1350 optparse.make_option("--origin", dest="origin"),
1351 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1352 # preserve the user, requires relevant p4 permissions
1353 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1354 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1355 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1356 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1357 optparse.make_option("--conflict", dest="conflict_behavior",
1358 choices=self.conflict_behavior_choices),
1359 optparse.make_option("--branch", dest="branch"),
1360 optparse.make_option("--shelve", dest="shelve", action="store_true",
1361 help="Shelve instead of submit. Shelved files are reverted, "
1362 "restoring the workspace to the state before the shelve"),
1363 optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1364 metavar="CHANGELIST",
1365 help="update an existing shelved changelist, implies --shelve, "
1366 "repeat in-order for multiple shelved changelists")
1367 ]
1368 self.description = "Submit changes from git to the perforce depot."
1369 self.usage += " [name of git branch to submit into perforce depot]"
1370 self.origin = ""
1371 self.detectRenames = False
1372 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1373 self.dry_run = False
1374 self.shelve = False
1375 self.update_shelve = list()
1376 self.prepare_p4_only = False
1377 self.conflict_behavior = None
1378 self.isWindows = (platform.system() == "Windows")
1379 self.exportLabels = False
1380 self.p4HasMoveCommand = p4_has_move_command()
1381 self.branch = None
1382
1383 if gitConfig('git-p4.largeFileSystem'):
1384 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1385
1386 def check(self):
1387 if len(p4CmdList("opened ...")) > 0:
1388 die("You have files opened with perforce! Close them before starting the sync.")
1389
1390 def separate_jobs_from_description(self, message):
1391 """Extract and return a possible Jobs field in the commit
1392 message. It goes into a separate section in the p4 change
1393 specification.
1394
1395 A jobs line starts with "Jobs:" and looks like a new field
1396 in a form. Values are white-space separated on the same
1397 line or on following lines that start with a tab.
1398
1399 This does not parse and extract the full git commit message
1400 like a p4 form. It just sees the Jobs: line as a marker
1401 to pass everything from then on directly into the p4 form,
1402 but outside the description section.
1403
1404 Return a tuple (stripped log message, jobs string)."""
1405
1406 m = re.search(r'^Jobs:', message, re.MULTILINE)
1407 if m is None:
1408 return (message, None)
1409
1410 jobtext = message[m.start():]
1411 stripped_message = message[:m.start()].rstrip()
1412 return (stripped_message, jobtext)
1413
1414 def prepareLogMessage(self, template, message, jobs):
1415 """Edits the template returned from "p4 change -o" to insert
1416 the message in the Description field, and the jobs text in
1417 the Jobs field."""
1418 result = ""
1419
1420 inDescriptionSection = False
1421
1422 for line in template.split("\n"):
1423 if line.startswith("#"):
1424 result += line + "\n"
1425 continue
1426
1427 if inDescriptionSection:
1428 if line.startswith("Files:") or line.startswith("Jobs:"):
1429 inDescriptionSection = False
1430 # insert Jobs section
1431 if jobs:
1432 result += jobs + "\n"
1433 else:
1434 continue
1435 else:
1436 if line.startswith("Description:"):
1437 inDescriptionSection = True
1438 line += "\n"
1439 for messageLine in message.split("\n"):
1440 line += "\t" + messageLine + "\n"
1441
1442 result += line + "\n"
1443
1444 return result
1445
1446 def patchRCSKeywords(self, file, pattern):
1447 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1448 (handle, outFileName) = tempfile.mkstemp(dir='.')
1449 try:
1450 outFile = os.fdopen(handle, "w+")
1451 inFile = open(file, "r")
1452 regexp = re.compile(pattern, re.VERBOSE)
1453 for line in inFile.readlines():
1454 line = regexp.sub(r'$\1$', line)
1455 outFile.write(line)
1456 inFile.close()
1457 outFile.close()
1458 # Forcibly overwrite the original file
1459 os.unlink(file)
1460 shutil.move(outFileName, file)
1461 except:
1462 # cleanup our temporary file
1463 os.unlink(outFileName)
1464 print "Failed to strip RCS keywords in %s" % file
1465 raise
1466
1467 print "Patched up RCS keywords in %s" % file
1468
1469 def p4UserForCommit(self,id):
1470 # Return the tuple (perforce user,git email) for a given git commit id
1471 self.getUserMapFromPerforceServer()
1472 gitEmail = read_pipe(["git", "log", "--max-count=1",
1473 "--format=%ae", id])
1474 gitEmail = gitEmail.strip()
1475 if not self.emails.has_key(gitEmail):
1476 return (None,gitEmail)
1477 else:
1478 return (self.emails[gitEmail],gitEmail)
1479
1480 def checkValidP4Users(self,commits):
1481 # check if any git authors cannot be mapped to p4 users
1482 for id in commits:
1483 (user,email) = self.p4UserForCommit(id)
1484 if not user:
1485 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1486 if gitConfigBool("git-p4.allowMissingP4Users"):
1487 print "%s" % msg
1488 else:
1489 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1490
1491 def lastP4Changelist(self):
1492 # Get back the last changelist number submitted in this client spec. This
1493 # then gets used to patch up the username in the change. If the same
1494 # client spec is being used by multiple processes then this might go
1495 # wrong.
1496 results = p4CmdList("client -o") # find the current client
1497 client = None
1498 for r in results:
1499 if r.has_key('Client'):
1500 client = r['Client']
1501 break
1502 if not client:
1503 die("could not get client spec")
1504 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1505 for r in results:
1506 if r.has_key('change'):
1507 return r['change']
1508 die("Could not get changelist number for last submit - cannot patch up user details")
1509
1510 def modifyChangelistUser(self, changelist, newUser):
1511 # fixup the user field of a changelist after it has been submitted.
1512 changes = p4CmdList("change -o %s" % changelist)
1513 if len(changes) != 1:
1514 die("Bad output from p4 change modifying %s to user %s" %
1515 (changelist, newUser))
1516
1517 c = changes[0]
1518 if c['User'] == newUser: return # nothing to do
1519 c['User'] = newUser
1520 input = marshal.dumps(c)
1521
1522 result = p4CmdList("change -f -i", stdin=input)
1523 for r in result:
1524 if r.has_key('code'):
1525 if r['code'] == 'error':
1526 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1527 if r.has_key('data'):
1528 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1529 return
1530 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1531
1532 def canChangeChangelists(self):
1533 # check to see if we have p4 admin or super-user permissions, either of
1534 # which are required to modify changelists.
1535 results = p4CmdList(["protects", self.depotPath])
1536 for r in results:
1537 if r.has_key('perm'):
1538 if r['perm'] == 'admin':
1539 return 1
1540 if r['perm'] == 'super':
1541 return 1
1542 return 0
1543
1544 def prepareSubmitTemplate(self, changelist=None):
1545 """Run "p4 change -o" to grab a change specification template.
1546 This does not use "p4 -G", as it is nice to keep the submission
1547 template in original order, since a human might edit it.
1548
1549 Remove lines in the Files section that show changes to files
1550 outside the depot path we're committing into."""
1551
1552 [upstream, settings] = findUpstreamBranchPoint()
1553
1554 template = """\
1555 # A Perforce Change Specification.
1556 #
1557 # Change: The change number. 'new' on a new changelist.
1558 # Date: The date this specification was last modified.
1559 # Client: The client on which the changelist was created. Read-only.
1560 # User: The user who created the changelist.
1561 # Status: Either 'pending' or 'submitted'. Read-only.
1562 # Type: Either 'public' or 'restricted'. Default is 'public'.
1563 # Description: Comments about the changelist. Required.
1564 # Jobs: What opened jobs are to be closed by this changelist.
1565 # You may delete jobs from this list. (New changelists only.)
1566 # Files: What opened files from the default changelist are to be added
1567 # to this changelist. You may delete files from this list.
1568 # (New changelists only.)
1569 """
1570 files_list = []
1571 inFilesSection = False
1572 change_entry = None
1573 args = ['change', '-o']
1574 if changelist:
1575 args.append(str(changelist))
1576 for entry in p4CmdList(args):
1577 if not entry.has_key('code'):
1578 continue
1579 if entry['code'] == 'stat':
1580 change_entry = entry
1581 break
1582 if not change_entry:
1583 die('Failed to decode output of p4 change -o')
1584 for key, value in change_entry.iteritems():
1585 if key.startswith('File'):
1586 if settings.has_key('depot-paths'):
1587 if not [p for p in settings['depot-paths']
1588 if p4PathStartsWith(value, p)]:
1589 continue
1590 else:
1591 if not p4PathStartsWith(value, self.depotPath):
1592 continue
1593 files_list.append(value)
1594 continue
1595 # Output in the order expected by prepareLogMessage
1596 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1597 if not change_entry.has_key(key):
1598 continue
1599 template += '\n'
1600 template += key + ':'
1601 if key == 'Description':
1602 template += '\n'
1603 for field_line in change_entry[key].splitlines():
1604 template += '\t'+field_line+'\n'
1605 if len(files_list) > 0:
1606 template += '\n'
1607 template += 'Files:\n'
1608 for path in files_list:
1609 template += '\t'+path+'\n'
1610 return template
1611
1612 def edit_template(self, template_file):
1613 """Invoke the editor to let the user change the submission
1614 message. Return true if okay to continue with the submit."""
1615
1616 # if configured to skip the editing part, just submit
1617 if gitConfigBool("git-p4.skipSubmitEdit"):
1618 return True
1619
1620 # look at the modification time, to check later if the user saved
1621 # the file
1622 mtime = os.stat(template_file).st_mtime
1623
1624 # invoke the editor
1625 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1626 editor = os.environ.get("P4EDITOR")
1627 else:
1628 editor = read_pipe("git var GIT_EDITOR").strip()
1629 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1630
1631 # If the file was not saved, prompt to see if this patch should
1632 # be skipped. But skip this verification step if configured so.
1633 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1634 return True
1635
1636 # modification time updated means user saved the file
1637 if os.stat(template_file).st_mtime > mtime:
1638 return True
1639
1640 while True:
1641 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1642 if response == 'y':
1643 return True
1644 if response == 'n':
1645 return False
1646
1647 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1648 # diff
1649 if os.environ.has_key("P4DIFF"):
1650 del(os.environ["P4DIFF"])
1651 diff = ""
1652 for editedFile in editedFiles:
1653 diff += p4_read_pipe(['diff', '-du',
1654 wildcard_encode(editedFile)])
1655
1656 # new file diff
1657 newdiff = ""
1658 for newFile in filesToAdd:
1659 newdiff += "==== new file ====\n"
1660 newdiff += "--- /dev/null\n"
1661 newdiff += "+++ %s\n" % newFile
1662
1663 is_link = os.path.islink(newFile)
1664 expect_link = newFile in symlinks
1665
1666 if is_link and expect_link:
1667 newdiff += "+%s\n" % os.readlink(newFile)
1668 else:
1669 f = open(newFile, "r")
1670 for line in f.readlines():
1671 newdiff += "+" + line
1672 f.close()
1673
1674 return (diff + newdiff).replace('\r\n', '\n')
1675
1676 def applyCommit(self, id):
1677 """Apply one commit, return True if it succeeded."""
1678
1679 print "Applying", read_pipe(["git", "show", "-s",
1680 "--format=format:%h %s", id])
1681
1682 (p4User, gitEmail) = self.p4UserForCommit(id)
1683
1684 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1685 filesToAdd = set()
1686 filesToChangeType = set()
1687 filesToDelete = set()
1688 editedFiles = set()
1689 pureRenameCopy = set()
1690 symlinks = set()
1691 filesToChangeExecBit = {}
1692 all_files = list()
1693
1694 for line in diff:
1695 diff = parseDiffTreeEntry(line)
1696 modifier = diff['status']
1697 path = diff['src']
1698 all_files.append(path)
1699
1700 if modifier == "M":
1701 p4_edit(path)
1702 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1703 filesToChangeExecBit[path] = diff['dst_mode']
1704 editedFiles.add(path)
1705 elif modifier == "A":
1706 filesToAdd.add(path)
1707 filesToChangeExecBit[path] = diff['dst_mode']
1708 if path in filesToDelete:
1709 filesToDelete.remove(path)
1710
1711 dst_mode = int(diff['dst_mode'], 8)
1712 if dst_mode == 0120000:
1713 symlinks.add(path)
1714
1715 elif modifier == "D":
1716 filesToDelete.add(path)
1717 if path in filesToAdd:
1718 filesToAdd.remove(path)
1719 elif modifier == "C":
1720 src, dest = diff['src'], diff['dst']
1721 p4_integrate(src, dest)
1722 pureRenameCopy.add(dest)
1723 if diff['src_sha1'] != diff['dst_sha1']:
1724 p4_edit(dest)
1725 pureRenameCopy.discard(dest)
1726 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1727 p4_edit(dest)
1728 pureRenameCopy.discard(dest)
1729 filesToChangeExecBit[dest] = diff['dst_mode']
1730 if self.isWindows:
1731 # turn off read-only attribute
1732 os.chmod(dest, stat.S_IWRITE)
1733 os.unlink(dest)
1734 editedFiles.add(dest)
1735 elif modifier == "R":
1736 src, dest = diff['src'], diff['dst']
1737 if self.p4HasMoveCommand:
1738 p4_edit(src) # src must be open before move
1739 p4_move(src, dest) # opens for (move/delete, move/add)
1740 else:
1741 p4_integrate(src, dest)
1742 if diff['src_sha1'] != diff['dst_sha1']:
1743 p4_edit(dest)
1744 else:
1745 pureRenameCopy.add(dest)
1746 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1747 if not self.p4HasMoveCommand:
1748 p4_edit(dest) # with move: already open, writable
1749 filesToChangeExecBit[dest] = diff['dst_mode']
1750 if not self.p4HasMoveCommand:
1751 if self.isWindows:
1752 os.chmod(dest, stat.S_IWRITE)
1753 os.unlink(dest)
1754 filesToDelete.add(src)
1755 editedFiles.add(dest)
1756 elif modifier == "T":
1757 filesToChangeType.add(path)
1758 else:
1759 die("unknown modifier %s for %s" % (modifier, path))
1760
1761 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1762 patchcmd = diffcmd + " | git apply "
1763 tryPatchCmd = patchcmd + "--check -"
1764 applyPatchCmd = patchcmd + "--check --apply -"
1765 patch_succeeded = True
1766
1767 if os.system(tryPatchCmd) != 0:
1768 fixed_rcs_keywords = False
1769 patch_succeeded = False
1770 print "Unfortunately applying the change failed!"
1771
1772 # Patch failed, maybe it's just RCS keyword woes. Look through
1773 # the patch to see if that's possible.
1774 if gitConfigBool("git-p4.attemptRCSCleanup"):
1775 file = None
1776 pattern = None
1777 kwfiles = {}
1778 for file in editedFiles | filesToDelete:
1779 # did this file's delta contain RCS keywords?
1780 pattern = p4_keywords_regexp_for_file(file)
1781
1782 if pattern:
1783 # this file is a possibility...look for RCS keywords.
1784 regexp = re.compile(pattern, re.VERBOSE)
1785 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1786 if regexp.search(line):
1787 if verbose:
1788 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1789 kwfiles[file] = pattern
1790 break
1791
1792 for file in kwfiles:
1793 if verbose:
1794 print "zapping %s with %s" % (line,pattern)
1795 # File is being deleted, so not open in p4. Must
1796 # disable the read-only bit on windows.
1797 if self.isWindows and file not in editedFiles:
1798 os.chmod(file, stat.S_IWRITE)
1799 self.patchRCSKeywords(file, kwfiles[file])
1800 fixed_rcs_keywords = True
1801
1802 if fixed_rcs_keywords:
1803 print "Retrying the patch with RCS keywords cleaned up"
1804 if os.system(tryPatchCmd) == 0:
1805 patch_succeeded = True
1806
1807 if not patch_succeeded:
1808 for f in editedFiles:
1809 p4_revert(f)
1810 return False
1811
1812 #
1813 # Apply the patch for real, and do add/delete/+x handling.
1814 #
1815 system(applyPatchCmd)
1816
1817 for f in filesToChangeType:
1818 p4_edit(f, "-t", "auto")
1819 for f in filesToAdd:
1820 p4_add(f)
1821 for f in filesToDelete:
1822 p4_revert(f)
1823 p4_delete(f)
1824
1825 # Set/clear executable bits
1826 for f in filesToChangeExecBit.keys():
1827 mode = filesToChangeExecBit[f]
1828 setP4ExecBit(f, mode)
1829
1830 update_shelve = 0
1831 if len(self.update_shelve) > 0:
1832 update_shelve = self.update_shelve.pop(0)
1833 p4_reopen_in_change(update_shelve, all_files)
1834
1835 #
1836 # Build p4 change description, starting with the contents
1837 # of the git commit message.
1838 #
1839 logMessage = extractLogMessageFromGitCommit(id)
1840 logMessage = logMessage.strip()
1841 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1842
1843 template = self.prepareSubmitTemplate(update_shelve)
1844 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1845
1846 if self.preserveUser:
1847 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1848
1849 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1850 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1851 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1852 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1853
1854 separatorLine = "######## everything below this line is just the diff #######\n"
1855 if not self.prepare_p4_only:
1856 submitTemplate += separatorLine
1857 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1858
1859 (handle, fileName) = tempfile.mkstemp()
1860 tmpFile = os.fdopen(handle, "w+b")
1861 if self.isWindows:
1862 submitTemplate = submitTemplate.replace("\n", "\r\n")
1863 tmpFile.write(submitTemplate)
1864 tmpFile.close()
1865
1866 if self.prepare_p4_only:
1867 #
1868 # Leave the p4 tree prepared, and the submit template around
1869 # and let the user decide what to do next
1870 #
1871 print
1872 print "P4 workspace prepared for submission."
1873 print "To submit or revert, go to client workspace"
1874 print " " + self.clientPath
1875 print
1876 print "To submit, use \"p4 submit\" to write a new description,"
1877 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1878 " \"git p4\"." % fileName
1879 print "You can delete the file \"%s\" when finished." % fileName
1880
1881 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1882 print "To preserve change ownership by user %s, you must\n" \
1883 "do \"p4 change -f <change>\" after submitting and\n" \
1884 "edit the User field."
1885 if pureRenameCopy:
1886 print "After submitting, renamed files must be re-synced."
1887 print "Invoke \"p4 sync -f\" on each of these files:"
1888 for f in pureRenameCopy:
1889 print " " + f
1890
1891 print
1892 print "To revert the changes, use \"p4 revert ...\", and delete"
1893 print "the submit template file \"%s\"" % fileName
1894 if filesToAdd:
1895 print "Since the commit adds new files, they must be deleted:"
1896 for f in filesToAdd:
1897 print " " + f
1898 print
1899 return True
1900
1901 #
1902 # Let the user edit the change description, then submit it.
1903 #
1904 submitted = False
1905
1906 try:
1907 if self.edit_template(fileName):
1908 # read the edited message and submit
1909 tmpFile = open(fileName, "rb")
1910 message = tmpFile.read()
1911 tmpFile.close()
1912 if self.isWindows:
1913 message = message.replace("\r\n", "\n")
1914 submitTemplate = message[:message.index(separatorLine)]
1915
1916 if update_shelve:
1917 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1918 elif self.shelve:
1919 p4_write_pipe(['shelve', '-i'], submitTemplate)
1920 else:
1921 p4_write_pipe(['submit', '-i'], submitTemplate)
1922 # The rename/copy happened by applying a patch that created a
1923 # new file. This leaves it writable, which confuses p4.
1924 for f in pureRenameCopy:
1925 p4_sync(f, "-f")
1926
1927 if self.preserveUser:
1928 if p4User:
1929 # Get last changelist number. Cannot easily get it from
1930 # the submit command output as the output is
1931 # unmarshalled.
1932 changelist = self.lastP4Changelist()
1933 self.modifyChangelistUser(changelist, p4User)
1934
1935 submitted = True
1936
1937 finally:
1938 # skip this patch
1939 if not submitted or self.shelve:
1940 if self.shelve:
1941 print ("Reverting shelved files.")
1942 else:
1943 print ("Submission cancelled, undoing p4 changes.")
1944 for f in editedFiles | filesToDelete:
1945 p4_revert(f)
1946 for f in filesToAdd:
1947 p4_revert(f)
1948 os.remove(f)
1949
1950 os.remove(fileName)
1951 return submitted
1952
1953 # Export git tags as p4 labels. Create a p4 label and then tag
1954 # with that.
1955 def exportGitTags(self, gitTags):
1956 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1957 if len(validLabelRegexp) == 0:
1958 validLabelRegexp = defaultLabelRegexp
1959 m = re.compile(validLabelRegexp)
1960
1961 for name in gitTags:
1962
1963 if not m.match(name):
1964 if verbose:
1965 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1966 continue
1967
1968 # Get the p4 commit this corresponds to
1969 logMessage = extractLogMessageFromGitCommit(name)
1970 values = extractSettingsGitLog(logMessage)
1971
1972 if not values.has_key('change'):
1973 # a tag pointing to something not sent to p4; ignore
1974 if verbose:
1975 print "git tag %s does not give a p4 commit" % name
1976 continue
1977 else:
1978 changelist = values['change']
1979
1980 # Get the tag details.
1981 inHeader = True
1982 isAnnotated = False
1983 body = []
1984 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1985 l = l.strip()
1986 if inHeader:
1987 if re.match(r'tag\s+', l):
1988 isAnnotated = True
1989 elif re.match(r'\s*$', l):
1990 inHeader = False
1991 continue
1992 else:
1993 body.append(l)
1994
1995 if not isAnnotated:
1996 body = ["lightweight tag imported by git p4\n"]
1997
1998 # Create the label - use the same view as the client spec we are using
1999 clientSpec = getClientSpec()
2000
2001 labelTemplate = "Label: %s\n" % name
2002 labelTemplate += "Description:\n"
2003 for b in body:
2004 labelTemplate += "\t" + b + "\n"
2005 labelTemplate += "View:\n"
2006 for depot_side in clientSpec.mappings:
2007 labelTemplate += "\t%s\n" % depot_side
2008
2009 if self.dry_run:
2010 print "Would create p4 label %s for tag" % name
2011 elif self.prepare_p4_only:
2012 print "Not creating p4 label %s for tag due to option" \
2013 " --prepare-p4-only" % name
2014 else:
2015 p4_write_pipe(["label", "-i"], labelTemplate)
2016
2017 # Use the label
2018 p4_system(["tag", "-l", name] +
2019 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2020
2021 if verbose:
2022 print "created p4 label for tag %s" % name
2023
2024 def run(self, args):
2025 if len(args) == 0:
2026 self.master = currentGitBranch()
2027 elif len(args) == 1:
2028 self.master = args[0]
2029 if not branchExists(self.master):
2030 die("Branch %s does not exist" % self.master)
2031 else:
2032 return False
2033
2034 for i in self.update_shelve:
2035 if i <= 0:
2036 sys.exit("invalid changelist %d" % i)
2037
2038 if self.master:
2039 allowSubmit = gitConfig("git-p4.allowSubmit")
2040 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2041 die("%s is not in git-p4.allowSubmit" % self.master)
2042
2043 [upstream, settings] = findUpstreamBranchPoint()
2044 self.depotPath = settings['depot-paths'][0]
2045 if len(self.origin) == 0:
2046 self.origin = upstream
2047
2048 if len(self.update_shelve) > 0:
2049 self.shelve = True
2050
2051 if self.preserveUser:
2052 if not self.canChangeChangelists():
2053 die("Cannot preserve user names without p4 super-user or admin permissions")
2054
2055 # if not set from the command line, try the config file
2056 if self.conflict_behavior is None:
2057 val = gitConfig("git-p4.conflict")
2058 if val:
2059 if val not in self.conflict_behavior_choices:
2060 die("Invalid value '%s' for config git-p4.conflict" % val)
2061 else:
2062 val = "ask"
2063 self.conflict_behavior = val
2064
2065 if self.verbose:
2066 print "Origin branch is " + self.origin
2067
2068 if len(self.depotPath) == 0:
2069 print "Internal error: cannot locate perforce depot path from existing branches"
2070 sys.exit(128)
2071
2072 self.useClientSpec = False
2073 if gitConfigBool("git-p4.useclientspec"):
2074 self.useClientSpec = True
2075 if self.useClientSpec:
2076 self.clientSpecDirs = getClientSpec()
2077
2078 # Check for the existence of P4 branches
2079 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2080
2081 if self.useClientSpec and not branchesDetected:
2082 # all files are relative to the client spec
2083 self.clientPath = getClientRoot()
2084 else:
2085 self.clientPath = p4Where(self.depotPath)
2086
2087 if self.clientPath == "":
2088 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2089
2090 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2091 self.oldWorkingDirectory = os.getcwd()
2092
2093 # ensure the clientPath exists
2094 new_client_dir = False
2095 if not os.path.exists(self.clientPath):
2096 new_client_dir = True
2097 os.makedirs(self.clientPath)
2098
2099 chdir(self.clientPath, is_client_path=True)
2100 if self.dry_run:
2101 print "Would synchronize p4 checkout in %s" % self.clientPath
2102 else:
2103 print "Synchronizing p4 checkout..."
2104 if new_client_dir:
2105 # old one was destroyed, and maybe nobody told p4
2106 p4_sync("...", "-f")
2107 else:
2108 p4_sync("...")
2109 self.check()
2110
2111 commits = []
2112 if self.master:
2113 committish = self.master
2114 else:
2115 committish = 'HEAD'
2116
2117 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, committish)]):
2118 commits.append(line.strip())
2119 commits.reverse()
2120
2121 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2122 self.checkAuthorship = False
2123 else:
2124 self.checkAuthorship = True
2125
2126 if self.preserveUser:
2127 self.checkValidP4Users(commits)
2128
2129 #
2130 # Build up a set of options to be passed to diff when
2131 # submitting each commit to p4.
2132 #
2133 if self.detectRenames:
2134 # command-line -M arg
2135 self.diffOpts = "-M"
2136 else:
2137 # If not explicitly set check the config variable
2138 detectRenames = gitConfig("git-p4.detectRenames")
2139
2140 if detectRenames.lower() == "false" or detectRenames == "":
2141 self.diffOpts = ""
2142 elif detectRenames.lower() == "true":
2143 self.diffOpts = "-M"
2144 else:
2145 self.diffOpts = "-M%s" % detectRenames
2146
2147 # no command-line arg for -C or --find-copies-harder, just
2148 # config variables
2149 detectCopies = gitConfig("git-p4.detectCopies")
2150 if detectCopies.lower() == "false" or detectCopies == "":
2151 pass
2152 elif detectCopies.lower() == "true":
2153 self.diffOpts += " -C"
2154 else:
2155 self.diffOpts += " -C%s" % detectCopies
2156
2157 if gitConfigBool("git-p4.detectCopiesHarder"):
2158 self.diffOpts += " --find-copies-harder"
2159
2160 num_shelves = len(self.update_shelve)
2161 if num_shelves > 0 and num_shelves != len(commits):
2162 sys.exit("number of commits (%d) must match number of shelved changelist (%d)" %
2163 (len(commits), num_shelves))
2164
2165 #
2166 # Apply the commits, one at a time. On failure, ask if should
2167 # continue to try the rest of the patches, or quit.
2168 #
2169 if self.dry_run:
2170 print "Would apply"
2171 applied = []
2172 last = len(commits) - 1
2173 for i, commit in enumerate(commits):
2174 if self.dry_run:
2175 print " ", read_pipe(["git", "show", "-s",
2176 "--format=format:%h %s", commit])
2177 ok = True
2178 else:
2179 ok = self.applyCommit(commit)
2180 if ok:
2181 applied.append(commit)
2182 else:
2183 if self.prepare_p4_only and i < last:
2184 print "Processing only the first commit due to option" \
2185 " --prepare-p4-only"
2186 break
2187 if i < last:
2188 quit = False
2189 while True:
2190 # prompt for what to do, or use the option/variable
2191 if self.conflict_behavior == "ask":
2192 print "What do you want to do?"
2193 response = raw_input("[s]kip this commit but apply"
2194 " the rest, or [q]uit? ")
2195 if not response:
2196 continue
2197 elif self.conflict_behavior == "skip":
2198 response = "s"
2199 elif self.conflict_behavior == "quit":
2200 response = "q"
2201 else:
2202 die("Unknown conflict_behavior '%s'" %
2203 self.conflict_behavior)
2204
2205 if response[0] == "s":
2206 print "Skipping this commit, but applying the rest"
2207 break
2208 if response[0] == "q":
2209 print "Quitting"
2210 quit = True
2211 break
2212 if quit:
2213 break
2214
2215 chdir(self.oldWorkingDirectory)
2216 shelved_applied = "shelved" if self.shelve else "applied"
2217 if self.dry_run:
2218 pass
2219 elif self.prepare_p4_only:
2220 pass
2221 elif len(commits) == len(applied):
2222 print ("All commits {0}!".format(shelved_applied))
2223
2224 sync = P4Sync()
2225 if self.branch:
2226 sync.branch = self.branch
2227 sync.run([])
2228
2229 rebase = P4Rebase()
2230 rebase.rebase()
2231
2232 else:
2233 if len(applied) == 0:
2234 print ("No commits {0}.".format(shelved_applied))
2235 else:
2236 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2237 for c in commits:
2238 if c in applied:
2239 star = "*"
2240 else:
2241 star = " "
2242 print star, read_pipe(["git", "show", "-s",
2243 "--format=format:%h %s", c])
2244 print "You will have to do 'git p4 sync' and rebase."
2245
2246 if gitConfigBool("git-p4.exportLabels"):
2247 self.exportLabels = True
2248
2249 if self.exportLabels:
2250 p4Labels = getP4Labels(self.depotPath)
2251 gitTags = getGitTags()
2252
2253 missingGitTags = gitTags - p4Labels
2254 self.exportGitTags(missingGitTags)
2255
2256 # exit with error unless everything applied perfectly
2257 if len(commits) != len(applied):
2258 sys.exit(1)
2259
2260 return True
2261
2262 class View(object):
2263 """Represent a p4 view ("p4 help views"), and map files in a
2264 repo according to the view."""
2265
2266 def __init__(self, client_name):
2267 self.mappings = []
2268 self.client_prefix = "//%s/" % client_name
2269 # cache results of "p4 where" to lookup client file locations
2270 self.client_spec_path_cache = {}
2271
2272 def append(self, view_line):
2273 """Parse a view line, splitting it into depot and client
2274 sides. Append to self.mappings, preserving order. This
2275 is only needed for tag creation."""
2276
2277 # Split the view line into exactly two words. P4 enforces
2278 # structure on these lines that simplifies this quite a bit.
2279 #
2280 # Either or both words may be double-quoted.
2281 # Single quotes do not matter.
2282 # Double-quote marks cannot occur inside the words.
2283 # A + or - prefix is also inside the quotes.
2284 # There are no quotes unless they contain a space.
2285 # The line is already white-space stripped.
2286 # The two words are separated by a single space.
2287 #
2288 if view_line[0] == '"':
2289 # First word is double quoted. Find its end.
2290 close_quote_index = view_line.find('"', 1)
2291 if close_quote_index <= 0:
2292 die("No first-word closing quote found: %s" % view_line)
2293 depot_side = view_line[1:close_quote_index]
2294 # skip closing quote and space
2295 rhs_index = close_quote_index + 1 + 1
2296 else:
2297 space_index = view_line.find(" ")
2298 if space_index <= 0:
2299 die("No word-splitting space found: %s" % view_line)
2300 depot_side = view_line[0:space_index]
2301 rhs_index = space_index + 1
2302
2303 # prefix + means overlay on previous mapping
2304 if depot_side.startswith("+"):
2305 depot_side = depot_side[1:]
2306
2307 # prefix - means exclude this path, leave out of mappings
2308 exclude = False
2309 if depot_side.startswith("-"):
2310 exclude = True
2311 depot_side = depot_side[1:]
2312
2313 if not exclude:
2314 self.mappings.append(depot_side)
2315
2316 def convert_client_path(self, clientFile):
2317 # chop off //client/ part to make it relative
2318 if not clientFile.startswith(self.client_prefix):
2319 die("No prefix '%s' on clientFile '%s'" %
2320 (self.client_prefix, clientFile))
2321 return clientFile[len(self.client_prefix):]
2322
2323 def update_client_spec_path_cache(self, files):
2324 """ Caching file paths by "p4 where" batch query """
2325
2326 # List depot file paths exclude that already cached
2327 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2328
2329 if len(fileArgs) == 0:
2330 return # All files in cache
2331
2332 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2333 for res in where_result:
2334 if "code" in res and res["code"] == "error":
2335 # assume error is "... file(s) not in client view"
2336 continue
2337 if "clientFile" not in res:
2338 die("No clientFile in 'p4 where' output")
2339 if "unmap" in res:
2340 # it will list all of them, but only one not unmap-ped
2341 continue
2342 if gitConfigBool("core.ignorecase"):
2343 res['depotFile'] = res['depotFile'].lower()
2344 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2345
2346 # not found files or unmap files set to ""
2347 for depotFile in fileArgs:
2348 if gitConfigBool("core.ignorecase"):
2349 depotFile = depotFile.lower()
2350 if depotFile not in self.client_spec_path_cache:
2351 self.client_spec_path_cache[depotFile] = ""
2352
2353 def map_in_client(self, depot_path):
2354 """Return the relative location in the client where this
2355 depot file should live. Returns "" if the file should
2356 not be mapped in the client."""
2357
2358 if gitConfigBool("core.ignorecase"):
2359 depot_path = depot_path.lower()
2360
2361 if depot_path in self.client_spec_path_cache:
2362 return self.client_spec_path_cache[depot_path]
2363
2364 die( "Error: %s is not found in client spec path" % depot_path )
2365 return ""
2366
2367 class P4Sync(Command, P4UserMap):
2368 delete_actions = ( "delete", "move/delete", "purge" )
2369
2370 def __init__(self):
2371 Command.__init__(self)
2372 P4UserMap.__init__(self)
2373 self.options = [
2374 optparse.make_option("--branch", dest="branch"),
2375 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2376 optparse.make_option("--changesfile", dest="changesFile"),
2377 optparse.make_option("--silent", dest="silent", action="store_true"),
2378 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2379 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2380 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2381 help="Import into refs/heads/ , not refs/remotes"),
2382 optparse.make_option("--max-changes", dest="maxChanges",
2383 help="Maximum number of changes to import"),
2384 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2385 help="Internal block size to use when iteratively calling p4 changes"),
2386 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2387 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2388 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2389 help="Only sync files that are included in the Perforce Client Spec"),
2390 optparse.make_option("-/", dest="cloneExclude",
2391 action="append", type="string",
2392 help="exclude depot path"),
2393 ]
2394 self.description = """Imports from Perforce into a git repository.\n
2395 example:
2396 //depot/my/project/ -- to import the current head
2397 //depot/my/project/@all -- to import everything
2398 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2399
2400 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2401
2402 self.usage += " //depot/path[@revRange]"
2403 self.silent = False
2404 self.createdBranches = set()
2405 self.committedChanges = set()
2406 self.branch = ""
2407 self.detectBranches = False
2408 self.detectLabels = False
2409 self.importLabels = False
2410 self.changesFile = ""
2411 self.syncWithOrigin = True
2412 self.importIntoRemotes = True
2413 self.maxChanges = ""
2414 self.changes_block_size = None
2415 self.keepRepoPath = False
2416 self.depotPaths = None
2417 self.p4BranchesInGit = []
2418 self.cloneExclude = []
2419 self.useClientSpec = False
2420 self.useClientSpec_from_options = False
2421 self.clientSpecDirs = None
2422 self.tempBranches = []
2423 self.tempBranchLocation = "refs/git-p4-tmp"
2424 self.largeFileSystem = None
2425 self.suppress_meta_comment = False
2426
2427 if gitConfig('git-p4.largeFileSystem'):
2428 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2429 self.largeFileSystem = largeFileSystemConstructor(
2430 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2431 )
2432
2433 if gitConfig("git-p4.syncFromOrigin") == "false":
2434 self.syncWithOrigin = False
2435
2436 self.depotPaths = []
2437 self.changeRange = ""
2438 self.previousDepotPaths = []
2439 self.hasOrigin = False
2440
2441 # map from branch depot path to parent branch
2442 self.knownBranches = {}
2443 self.initialParents = {}
2444
2445 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2446 self.labels = {}
2447
2448 # Force a checkpoint in fast-import and wait for it to finish
2449 def checkpoint(self):
2450 self.gitStream.write("checkpoint\n\n")
2451 self.gitStream.write("progress checkpoint\n\n")
2452 out = self.gitOutput.readline()
2453 if self.verbose:
2454 print "checkpoint finished: " + out
2455
2456 def cmp_shelved(self, path, filerev, revision):
2457 """ Determine if a path at revision #filerev is the same as the file
2458 at revision @revision for a shelved changelist. If they don't match,
2459 unshelving won't be safe (we will get other changes mixed in).
2460
2461 This is comparing the revision that the shelved changelist is *based* on, not
2462 the shelved changelist itself.
2463 """
2464 ret = p4Cmd(["diff2", "{0}#{1}".format(path, filerev), "{0}@{1}".format(path, revision)])
2465 if verbose:
2466 print("p4 diff2 path %s filerev %s revision %s => %s" % (path, filerev, revision, ret))
2467 return ret["status"] == "identical"
2468
2469 def extractFilesFromCommit(self, commit, shelved=False, shelved_cl = 0, origin_revision = 0):
2470 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2471 for path in self.cloneExclude]
2472 files = []
2473 fnum = 0
2474 while commit.has_key("depotFile%s" % fnum):
2475 path = commit["depotFile%s" % fnum]
2476
2477 if [p for p in self.cloneExclude
2478 if p4PathStartsWith(path, p)]:
2479 found = False
2480 else:
2481 found = [p for p in self.depotPaths
2482 if p4PathStartsWith(path, p)]
2483 if not found:
2484 fnum = fnum + 1
2485 continue
2486
2487 file = {}
2488 file["path"] = path
2489 file["rev"] = commit["rev%s" % fnum]
2490 file["action"] = commit["action%s" % fnum]
2491 file["type"] = commit["type%s" % fnum]
2492 if shelved:
2493 file["shelved_cl"] = int(shelved_cl)
2494
2495 # For shelved changelists, check that the revision of each file that the
2496 # shelve was based on matches the revision that we are using for the
2497 # starting point for git-fast-import (self.initialParent). Otherwise
2498 # the resulting diff will contain deltas from multiple commits.
2499
2500 if file["action"] != "add" and \
2501 not self.cmp_shelved(path, file["rev"], origin_revision):
2502 sys.exit("change {0} not based on {1} for {2}, cannot unshelve".format(
2503 commit["change"], self.initialParent, path))
2504
2505 files.append(file)
2506 fnum = fnum + 1
2507 return files
2508
2509 def extractJobsFromCommit(self, commit):
2510 jobs = []
2511 jnum = 0
2512 while commit.has_key("job%s" % jnum):
2513 job = commit["job%s" % jnum]
2514 jobs.append(job)
2515 jnum = jnum + 1
2516 return jobs
2517
2518 def stripRepoPath(self, path, prefixes):
2519 """When streaming files, this is called to map a p4 depot path
2520 to where it should go in git. The prefixes are either
2521 self.depotPaths, or self.branchPrefixes in the case of
2522 branch detection."""
2523
2524 if self.useClientSpec:
2525 # branch detection moves files up a level (the branch name)
2526 # from what client spec interpretation gives
2527 path = self.clientSpecDirs.map_in_client(path)
2528 if self.detectBranches:
2529 for b in self.knownBranches:
2530 if path.startswith(b + "/"):
2531 path = path[len(b)+1:]
2532
2533 elif self.keepRepoPath:
2534 # Preserve everything in relative path name except leading
2535 # //depot/; just look at first prefix as they all should
2536 # be in the same depot.
2537 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2538 if p4PathStartsWith(path, depot):
2539 path = path[len(depot):]
2540
2541 else:
2542 for p in prefixes:
2543 if p4PathStartsWith(path, p):
2544 path = path[len(p):]
2545 break
2546
2547 path = wildcard_decode(path)
2548 return path
2549
2550 def splitFilesIntoBranches(self, commit):
2551 """Look at each depotFile in the commit to figure out to what
2552 branch it belongs."""
2553
2554 if self.clientSpecDirs:
2555 files = self.extractFilesFromCommit(commit)
2556 self.clientSpecDirs.update_client_spec_path_cache(files)
2557
2558 branches = {}
2559 fnum = 0
2560 while commit.has_key("depotFile%s" % fnum):
2561 path = commit["depotFile%s" % fnum]
2562 found = [p for p in self.depotPaths
2563 if p4PathStartsWith(path, p)]
2564 if not found:
2565 fnum = fnum + 1
2566 continue
2567
2568 file = {}
2569 file["path"] = path
2570 file["rev"] = commit["rev%s" % fnum]
2571 file["action"] = commit["action%s" % fnum]
2572 file["type"] = commit["type%s" % fnum]
2573 fnum = fnum + 1
2574
2575 # start with the full relative path where this file would
2576 # go in a p4 client
2577 if self.useClientSpec:
2578 relPath = self.clientSpecDirs.map_in_client(path)
2579 else:
2580 relPath = self.stripRepoPath(path, self.depotPaths)
2581
2582 for branch in self.knownBranches.keys():
2583 # add a trailing slash so that a commit into qt/4.2foo
2584 # doesn't end up in qt/4.2, e.g.
2585 if relPath.startswith(branch + "/"):
2586 if branch not in branches:
2587 branches[branch] = []
2588 branches[branch].append(file)
2589 break
2590
2591 return branches
2592
2593 def writeToGitStream(self, gitMode, relPath, contents):
2594 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2595 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2596 for d in contents:
2597 self.gitStream.write(d)
2598 self.gitStream.write('\n')
2599
2600 def encodeWithUTF8(self, path):
2601 try:
2602 path.decode('ascii')
2603 except:
2604 encoding = 'utf8'
2605 if gitConfig('git-p4.pathEncoding'):
2606 encoding = gitConfig('git-p4.pathEncoding')
2607 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2608 if self.verbose:
2609 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2610 return path
2611
2612 # output one file from the P4 stream
2613 # - helper for streamP4Files
2614
2615 def streamOneP4File(self, file, contents):
2616 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2617 relPath = self.encodeWithUTF8(relPath)
2618 if verbose:
2619 size = int(self.stream_file['fileSize'])
2620 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2621 sys.stdout.flush()
2622
2623 (type_base, type_mods) = split_p4_type(file["type"])
2624
2625 git_mode = "100644"
2626 if "x" in type_mods:
2627 git_mode = "100755"
2628 if type_base == "symlink":
2629 git_mode = "120000"
2630 # p4 print on a symlink sometimes contains "target\n";
2631 # if it does, remove the newline
2632 data = ''.join(contents)
2633 if not data:
2634 # Some version of p4 allowed creating a symlink that pointed
2635 # to nothing. This causes p4 errors when checking out such
2636 # a change, and errors here too. Work around it by ignoring
2637 # the bad symlink; hopefully a future change fixes it.
2638 print "\nIgnoring empty symlink in %s" % file['depotFile']
2639 return
2640 elif data[-1] == '\n':
2641 contents = [data[:-1]]
2642 else:
2643 contents = [data]
2644
2645 if type_base == "utf16":
2646 # p4 delivers different text in the python output to -G
2647 # than it does when using "print -o", or normal p4 client
2648 # operations. utf16 is converted to ascii or utf8, perhaps.
2649 # But ascii text saved as -t utf16 is completely mangled.
2650 # Invoke print -o to get the real contents.
2651 #
2652 # On windows, the newlines will always be mangled by print, so put
2653 # them back too. This is not needed to the cygwin windows version,
2654 # just the native "NT" type.
2655 #
2656 try:
2657 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2658 except Exception as e:
2659 if 'Translation of file content failed' in str(e):
2660 type_base = 'binary'
2661 else:
2662 raise e
2663 else:
2664 if p4_version_string().find('/NT') >= 0:
2665 text = text.replace('\r\n', '\n')
2666 contents = [ text ]
2667
2668 if type_base == "apple":
2669 # Apple filetype files will be streamed as a concatenation of
2670 # its appledouble header and the contents. This is useless
2671 # on both macs and non-macs. If using "print -q -o xx", it
2672 # will create "xx" with the data, and "%xx" with the header.
2673 # This is also not very useful.
2674 #
2675 # Ideally, someday, this script can learn how to generate
2676 # appledouble files directly and import those to git, but
2677 # non-mac machines can never find a use for apple filetype.
2678 print "\nIgnoring apple filetype file %s" % file['depotFile']
2679 return
2680
2681 # Note that we do not try to de-mangle keywords on utf16 files,
2682 # even though in theory somebody may want that.
2683 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2684 if pattern:
2685 regexp = re.compile(pattern, re.VERBOSE)
2686 text = ''.join(contents)
2687 text = regexp.sub(r'$\1$', text)
2688 contents = [ text ]
2689
2690 if self.largeFileSystem:
2691 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2692
2693 self.writeToGitStream(git_mode, relPath, contents)
2694
2695 def streamOneP4Deletion(self, file):
2696 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2697 relPath = self.encodeWithUTF8(relPath)
2698 if verbose:
2699 sys.stdout.write("delete %s\n" % relPath)
2700 sys.stdout.flush()
2701 self.gitStream.write("D %s\n" % relPath)
2702
2703 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2704 self.largeFileSystem.removeLargeFile(relPath)
2705
2706 # handle another chunk of streaming data
2707 def streamP4FilesCb(self, marshalled):
2708
2709 # catch p4 errors and complain
2710 err = None
2711 if "code" in marshalled:
2712 if marshalled["code"] == "error":
2713 if "data" in marshalled:
2714 err = marshalled["data"].rstrip()
2715
2716 if not err and 'fileSize' in self.stream_file:
2717 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2718 if required_bytes > 0:
2719 err = 'Not enough space left on %s! Free at least %i MB.' % (
2720 os.getcwd(), required_bytes/1024/1024
2721 )
2722
2723 if err:
2724 f = None
2725 if self.stream_have_file_info:
2726 if "depotFile" in self.stream_file:
2727 f = self.stream_file["depotFile"]
2728 # force a failure in fast-import, else an empty
2729 # commit will be made
2730 self.gitStream.write("\n")
2731 self.gitStream.write("die-now\n")
2732 self.gitStream.close()
2733 # ignore errors, but make sure it exits first
2734 self.importProcess.wait()
2735 if f:
2736 die("Error from p4 print for %s: %s" % (f, err))
2737 else:
2738 die("Error from p4 print: %s" % err)
2739
2740 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2741 # start of a new file - output the old one first
2742 self.streamOneP4File(self.stream_file, self.stream_contents)
2743 self.stream_file = {}
2744 self.stream_contents = []
2745 self.stream_have_file_info = False
2746
2747 # pick up the new file information... for the
2748 # 'data' field we need to append to our array
2749 for k in marshalled.keys():
2750 if k == 'data':
2751 if 'streamContentSize' not in self.stream_file:
2752 self.stream_file['streamContentSize'] = 0
2753 self.stream_file['streamContentSize'] += len(marshalled['data'])
2754 self.stream_contents.append(marshalled['data'])
2755 else:
2756 self.stream_file[k] = marshalled[k]
2757
2758 if (verbose and
2759 'streamContentSize' in self.stream_file and
2760 'fileSize' in self.stream_file and
2761 'depotFile' in self.stream_file):
2762 size = int(self.stream_file["fileSize"])
2763 if size > 0:
2764 progress = 100*self.stream_file['streamContentSize']/size
2765 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2766 sys.stdout.flush()
2767
2768 self.stream_have_file_info = True
2769
2770 # Stream directly from "p4 files" into "git fast-import"
2771 def streamP4Files(self, files):
2772 filesForCommit = []
2773 filesToRead = []
2774 filesToDelete = []
2775
2776 for f in files:
2777 filesForCommit.append(f)
2778 if f['action'] in self.delete_actions:
2779 filesToDelete.append(f)
2780 else:
2781 filesToRead.append(f)
2782
2783 # deleted files...
2784 for f in filesToDelete:
2785 self.streamOneP4Deletion(f)
2786
2787 if len(filesToRead) > 0:
2788 self.stream_file = {}
2789 self.stream_contents = []
2790 self.stream_have_file_info = False
2791
2792 # curry self argument
2793 def streamP4FilesCbSelf(entry):
2794 self.streamP4FilesCb(entry)
2795
2796 fileArgs = []
2797 for f in filesToRead:
2798 if 'shelved_cl' in f:
2799 # Handle shelved CLs using the "p4 print file@=N" syntax to print
2800 # the contents
2801 fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
2802 else:
2803 fileArg = '%s#%s' % (f['path'], f['rev'])
2804
2805 fileArgs.append(fileArg)
2806
2807 p4CmdList(["-x", "-", "print"],
2808 stdin=fileArgs,
2809 cb=streamP4FilesCbSelf)
2810
2811 # do the last chunk
2812 if self.stream_file.has_key('depotFile'):
2813 self.streamOneP4File(self.stream_file, self.stream_contents)
2814
2815 def make_email(self, userid):
2816 if userid in self.users:
2817 return self.users[userid]
2818 else:
2819 return "%s <a@b>" % userid
2820
2821 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2822 """ Stream a p4 tag.
2823 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2824 """
2825
2826 if verbose:
2827 print "writing tag %s for commit %s" % (labelName, commit)
2828 gitStream.write("tag %s\n" % labelName)
2829 gitStream.write("from %s\n" % commit)
2830
2831 if labelDetails.has_key('Owner'):
2832 owner = labelDetails["Owner"]
2833 else:
2834 owner = None
2835
2836 # Try to use the owner of the p4 label, or failing that,
2837 # the current p4 user id.
2838 if owner:
2839 email = self.make_email(owner)
2840 else:
2841 email = self.make_email(self.p4UserId())
2842 tagger = "%s %s %s" % (email, epoch, self.tz)
2843
2844 gitStream.write("tagger %s\n" % tagger)
2845
2846 print "labelDetails=",labelDetails
2847 if labelDetails.has_key('Description'):
2848 description = labelDetails['Description']
2849 else:
2850 description = 'Label from git p4'
2851
2852 gitStream.write("data %d\n" % len(description))
2853 gitStream.write(description)
2854 gitStream.write("\n")
2855
2856 def inClientSpec(self, path):
2857 if not self.clientSpecDirs:
2858 return True
2859 inClientSpec = self.clientSpecDirs.map_in_client(path)
2860 if not inClientSpec and self.verbose:
2861 print('Ignoring file outside of client spec: {0}'.format(path))
2862 return inClientSpec
2863
2864 def hasBranchPrefix(self, path):
2865 if not self.branchPrefixes:
2866 return True
2867 hasPrefix = [p for p in self.branchPrefixes
2868 if p4PathStartsWith(path, p)]
2869 if not hasPrefix and self.verbose:
2870 print('Ignoring file outside of prefix: {0}'.format(path))
2871 return hasPrefix
2872
2873 def commit(self, details, files, branch, parent = ""):
2874 epoch = details["time"]
2875 author = details["user"]
2876 jobs = self.extractJobsFromCommit(details)
2877
2878 if self.verbose:
2879 print('commit into {0}'.format(branch))
2880
2881 if self.clientSpecDirs:
2882 self.clientSpecDirs.update_client_spec_path_cache(files)
2883
2884 files = [f for f in files
2885 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2886
2887 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2888 print('Ignoring revision {0} as it would produce an empty commit.'
2889 .format(details['change']))
2890 return
2891
2892 self.gitStream.write("commit %s\n" % branch)
2893 self.gitStream.write("mark :%s\n" % details["change"])
2894 self.committedChanges.add(int(details["change"]))
2895 committer = ""
2896 if author not in self.users:
2897 self.getUserMapFromPerforceServer()
2898 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2899
2900 self.gitStream.write("committer %s\n" % committer)
2901
2902 self.gitStream.write("data <<EOT\n")
2903 self.gitStream.write(details["desc"])
2904 if len(jobs) > 0:
2905 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2906
2907 if not self.suppress_meta_comment:
2908 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2909 (','.join(self.branchPrefixes), details["change"]))
2910 if len(details['options']) > 0:
2911 self.gitStream.write(": options = %s" % details['options'])
2912 self.gitStream.write("]\n")
2913
2914 self.gitStream.write("EOT\n\n")
2915
2916 if len(parent) > 0:
2917 if self.verbose:
2918 print "parent %s" % parent
2919 self.gitStream.write("from %s\n" % parent)
2920
2921 self.streamP4Files(files)
2922 self.gitStream.write("\n")
2923
2924 change = int(details["change"])
2925
2926 if self.labels.has_key(change):
2927 label = self.labels[change]
2928 labelDetails = label[0]
2929 labelRevisions = label[1]
2930 if self.verbose:
2931 print "Change %s is labelled %s" % (change, labelDetails)
2932
2933 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2934 for p in self.branchPrefixes])
2935
2936 if len(files) == len(labelRevisions):
2937
2938 cleanedFiles = {}
2939 for info in files:
2940 if info["action"] in self.delete_actions:
2941 continue
2942 cleanedFiles[info["depotFile"]] = info["rev"]
2943
2944 if cleanedFiles == labelRevisions:
2945 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2946
2947 else:
2948 if not self.silent:
2949 print ("Tag %s does not match with change %s: files do not match."
2950 % (labelDetails["label"], change))
2951
2952 else:
2953 if not self.silent:
2954 print ("Tag %s does not match with change %s: file count is different."
2955 % (labelDetails["label"], change))
2956
2957 # Build a dictionary of changelists and labels, for "detect-labels" option.
2958 def getLabels(self):
2959 self.labels = {}
2960
2961 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2962 if len(l) > 0 and not self.silent:
2963 print "Finding files belonging to labels in %s" % `self.depotPaths`
2964
2965 for output in l:
2966 label = output["label"]
2967 revisions = {}
2968 newestChange = 0
2969 if self.verbose:
2970 print "Querying files for label %s" % label
2971 for file in p4CmdList(["files"] +
2972 ["%s...@%s" % (p, label)
2973 for p in self.depotPaths]):
2974 revisions[file["depotFile"]] = file["rev"]
2975 change = int(file["change"])
2976 if change > newestChange:
2977 newestChange = change
2978
2979 self.labels[newestChange] = [output, revisions]
2980
2981 if self.verbose:
2982 print "Label changes: %s" % self.labels.keys()
2983
2984 # Import p4 labels as git tags. A direct mapping does not
2985 # exist, so assume that if all the files are at the same revision
2986 # then we can use that, or it's something more complicated we should
2987 # just ignore.
2988 def importP4Labels(self, stream, p4Labels):
2989 if verbose:
2990 print "import p4 labels: " + ' '.join(p4Labels)
2991
2992 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2993 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2994 if len(validLabelRegexp) == 0:
2995 validLabelRegexp = defaultLabelRegexp
2996 m = re.compile(validLabelRegexp)
2997
2998 for name in p4Labels:
2999 commitFound = False
3000
3001 if not m.match(name):
3002 if verbose:
3003 print "label %s does not match regexp %s" % (name,validLabelRegexp)
3004 continue
3005
3006 if name in ignoredP4Labels:
3007 continue
3008
3009 labelDetails = p4CmdList(['label', "-o", name])[0]
3010
3011 # get the most recent changelist for each file in this label
3012 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
3013 for p in self.depotPaths])
3014
3015 if change.has_key('change'):
3016 # find the corresponding git commit; take the oldest commit
3017 changelist = int(change['change'])
3018 if changelist in self.committedChanges:
3019 gitCommit = ":%d" % changelist # use a fast-import mark
3020 commitFound = True
3021 else:
3022 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
3023 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
3024 if len(gitCommit) == 0:
3025 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
3026 else:
3027 commitFound = True
3028 gitCommit = gitCommit.strip()
3029
3030 if commitFound:
3031 # Convert from p4 time format
3032 try:
3033 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
3034 except ValueError:
3035 print "Could not convert label time %s" % labelDetails['Update']
3036 tmwhen = 1
3037
3038 when = int(time.mktime(tmwhen))
3039 self.streamTag(stream, name, labelDetails, gitCommit, when)
3040 if verbose:
3041 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
3042 else:
3043 if verbose:
3044 print "Label %s has no changelists - possibly deleted?" % name
3045
3046 if not commitFound:
3047 # We can't import this label; don't try again as it will get very
3048 # expensive repeatedly fetching all the files for labels that will
3049 # never be imported. If the label is moved in the future, the
3050 # ignore will need to be removed manually.
3051 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
3052
3053 def guessProjectName(self):
3054 for p in self.depotPaths:
3055 if p.endswith("/"):
3056 p = p[:-1]
3057 p = p[p.strip().rfind("/") + 1:]
3058 if not p.endswith("/"):
3059 p += "/"
3060 return p
3061
3062 def getBranchMapping(self):
3063 lostAndFoundBranches = set()
3064
3065 user = gitConfig("git-p4.branchUser")
3066 if len(user) > 0:
3067 command = "branches -u %s" % user
3068 else:
3069 command = "branches"
3070
3071 for info in p4CmdList(command):
3072 details = p4Cmd(["branch", "-o", info["branch"]])
3073 viewIdx = 0
3074 while details.has_key("View%s" % viewIdx):
3075 paths = details["View%s" % viewIdx].split(" ")
3076 viewIdx = viewIdx + 1
3077 # require standard //depot/foo/... //depot/bar/... mapping
3078 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3079 continue
3080 source = paths[0]
3081 destination = paths[1]
3082 ## HACK
3083 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3084 source = source[len(self.depotPaths[0]):-4]
3085 destination = destination[len(self.depotPaths[0]):-4]
3086
3087 if destination in self.knownBranches:
3088 if not self.silent:
3089 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3090 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3091 continue
3092
3093 self.knownBranches[destination] = source
3094
3095 lostAndFoundBranches.discard(destination)
3096
3097 if source not in self.knownBranches:
3098 lostAndFoundBranches.add(source)
3099
3100 # Perforce does not strictly require branches to be defined, so we also
3101 # check git config for a branch list.
3102 #
3103 # Example of branch definition in git config file:
3104 # [git-p4]
3105 # branchList=main:branchA
3106 # branchList=main:branchB
3107 # branchList=branchA:branchC
3108 configBranches = gitConfigList("git-p4.branchList")
3109 for branch in configBranches:
3110 if branch:
3111 (source, destination) = branch.split(":")
3112 self.knownBranches[destination] = source
3113
3114 lostAndFoundBranches.discard(destination)
3115
3116 if source not in self.knownBranches:
3117 lostAndFoundBranches.add(source)
3118
3119
3120 for branch in lostAndFoundBranches:
3121 self.knownBranches[branch] = branch
3122
3123 def getBranchMappingFromGitBranches(self):
3124 branches = p4BranchesInGit(self.importIntoRemotes)
3125 for branch in branches.keys():
3126 if branch == "master":
3127 branch = "main"
3128 else:
3129 branch = branch[len(self.projectName):]
3130 self.knownBranches[branch] = branch
3131
3132 def updateOptionDict(self, d):
3133 option_keys = {}
3134 if self.keepRepoPath:
3135 option_keys['keepRepoPath'] = 1
3136
3137 d["options"] = ' '.join(sorted(option_keys.keys()))
3138
3139 def readOptions(self, d):
3140 self.keepRepoPath = (d.has_key('options')
3141 and ('keepRepoPath' in d['options']))
3142
3143 def gitRefForBranch(self, branch):
3144 if branch == "main":
3145 return self.refPrefix + "master"
3146
3147 if len(branch) <= 0:
3148 return branch
3149
3150 return self.refPrefix + self.projectName + branch
3151
3152 def gitCommitByP4Change(self, ref, change):
3153 if self.verbose:
3154 print "looking in ref " + ref + " for change %s using bisect..." % change
3155
3156 earliestCommit = ""
3157 latestCommit = parseRevision(ref)
3158
3159 while True:
3160 if self.verbose:
3161 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3162 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3163 if len(next) == 0:
3164 if self.verbose:
3165 print "argh"
3166 return ""
3167 log = extractLogMessageFromGitCommit(next)
3168 settings = extractSettingsGitLog(log)
3169 currentChange = int(settings['change'])
3170 if self.verbose:
3171 print "current change %s" % currentChange
3172
3173 if currentChange == change:
3174 if self.verbose:
3175 print "found %s" % next
3176 return next
3177
3178 if currentChange < change:
3179 earliestCommit = "^%s" % next
3180 else:
3181 latestCommit = "%s" % next
3182
3183 return ""
3184
3185 def importNewBranch(self, branch, maxChange):
3186 # make fast-import flush all changes to disk and update the refs using the checkpoint
3187 # command so that we can try to find the branch parent in the git history
3188 self.gitStream.write("checkpoint\n\n");
3189 self.gitStream.flush();
3190 branchPrefix = self.depotPaths[0] + branch + "/"
3191 range = "@1,%s" % maxChange
3192 #print "prefix" + branchPrefix
3193 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3194 if len(changes) <= 0:
3195 return False
3196 firstChange = changes[0]
3197 #print "first change in branch: %s" % firstChange
3198 sourceBranch = self.knownBranches[branch]
3199 sourceDepotPath = self.depotPaths[0] + sourceBranch
3200 sourceRef = self.gitRefForBranch(sourceBranch)
3201 #print "source " + sourceBranch
3202
3203 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3204 #print "branch parent: %s" % branchParentChange
3205 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3206 if len(gitParent) > 0:
3207 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3208 #print "parent git commit: %s" % gitParent
3209
3210 self.importChanges(changes)
3211 return True
3212
3213 def searchParent(self, parent, branch, target):
3214 parentFound = False
3215 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3216 "--no-merges", parent]):
3217 blob = blob.strip()
3218 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3219 parentFound = True
3220 if self.verbose:
3221 print "Found parent of %s in commit %s" % (branch, blob)
3222 break
3223 if parentFound:
3224 return blob
3225 else:
3226 return None
3227
3228 def importChanges(self, changes, shelved=False, origin_revision=0):
3229 cnt = 1
3230 for change in changes:
3231 description = p4_describe(change, shelved)
3232 self.updateOptionDict(description)
3233
3234 if not self.silent:
3235 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3236 sys.stdout.flush()
3237 cnt = cnt + 1
3238
3239 try:
3240 if self.detectBranches:
3241 branches = self.splitFilesIntoBranches(description)
3242 for branch in branches.keys():
3243 ## HACK --hwn
3244 branchPrefix = self.depotPaths[0] + branch + "/"
3245 self.branchPrefixes = [ branchPrefix ]
3246
3247 parent = ""
3248
3249 filesForCommit = branches[branch]
3250
3251 if self.verbose:
3252 print "branch is %s" % branch
3253
3254 self.updatedBranches.add(branch)
3255
3256 if branch not in self.createdBranches:
3257 self.createdBranches.add(branch)
3258 parent = self.knownBranches[branch]
3259 if parent == branch:
3260 parent = ""
3261 else:
3262 fullBranch = self.projectName + branch
3263 if fullBranch not in self.p4BranchesInGit:
3264 if not self.silent:
3265 print("\n Importing new branch %s" % fullBranch);
3266 if self.importNewBranch(branch, change - 1):
3267 parent = ""
3268 self.p4BranchesInGit.append(fullBranch)
3269 if not self.silent:
3270 print("\n Resuming with change %s" % change);
3271
3272 if self.verbose:
3273 print "parent determined through known branches: %s" % parent
3274
3275 branch = self.gitRefForBranch(branch)
3276 parent = self.gitRefForBranch(parent)
3277
3278 if self.verbose:
3279 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3280
3281 if len(parent) == 0 and branch in self.initialParents:
3282 parent = self.initialParents[branch]
3283 del self.initialParents[branch]
3284
3285 blob = None
3286 if len(parent) > 0:
3287 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3288 if self.verbose:
3289 print "Creating temporary branch: " + tempBranch
3290 self.commit(description, filesForCommit, tempBranch)
3291 self.tempBranches.append(tempBranch)
3292 self.checkpoint()
3293 blob = self.searchParent(parent, branch, tempBranch)
3294 if blob:
3295 self.commit(description, filesForCommit, branch, blob)
3296 else:
3297 if self.verbose:
3298 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3299 self.commit(description, filesForCommit, branch, parent)
3300 else:
3301 files = self.extractFilesFromCommit(description, shelved, change, origin_revision)
3302 self.commit(description, files, self.branch,
3303 self.initialParent)
3304 # only needed once, to connect to the previous commit
3305 self.initialParent = ""
3306 except IOError:
3307 print self.gitError.read()
3308 sys.exit(1)
3309
3310 def importHeadRevision(self, revision):
3311 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3312
3313 details = {}
3314 details["user"] = "git perforce import user"
3315 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3316 % (' '.join(self.depotPaths), revision))
3317 details["change"] = revision
3318 newestRevision = 0
3319
3320 fileCnt = 0
3321 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3322
3323 for info in p4CmdList(["files"] + fileArgs):
3324
3325 if 'code' in info and info['code'] == 'error':
3326 sys.stderr.write("p4 returned an error: %s\n"
3327 % info['data'])
3328 if info['data'].find("must refer to client") >= 0:
3329 sys.stderr.write("This particular p4 error is misleading.\n")
3330 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3331 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3332 sys.exit(1)
3333 if 'p4ExitCode' in info:
3334 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3335 sys.exit(1)
3336
3337
3338 change = int(info["change"])
3339 if change > newestRevision:
3340 newestRevision = change
3341
3342 if info["action"] in self.delete_actions:
3343 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3344 #fileCnt = fileCnt + 1
3345 continue
3346
3347 for prop in ["depotFile", "rev", "action", "type" ]:
3348 details["%s%s" % (prop, fileCnt)] = info[prop]
3349
3350 fileCnt = fileCnt + 1
3351
3352 details["change"] = newestRevision
3353
3354 # Use time from top-most change so that all git p4 clones of
3355 # the same p4 repo have the same commit SHA1s.
3356 res = p4_describe(newestRevision)
3357 details["time"] = res["time"]
3358
3359 self.updateOptionDict(details)
3360 try:
3361 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3362 except IOError:
3363 print "IO error with git fast-import. Is your git version recent enough?"
3364 print self.gitError.read()
3365
3366 def openStreams(self):
3367 self.importProcess = subprocess.Popen(["git", "fast-import"],
3368 stdin=subprocess.PIPE,
3369 stdout=subprocess.PIPE,
3370 stderr=subprocess.PIPE);
3371 self.gitOutput = self.importProcess.stdout
3372 self.gitStream = self.importProcess.stdin
3373 self.gitError = self.importProcess.stderr
3374
3375 def closeStreams(self):
3376 self.gitStream.close()
3377 if self.importProcess.wait() != 0:
3378 die("fast-import failed: %s" % self.gitError.read())
3379 self.gitOutput.close()
3380 self.gitError.close()
3381
3382 def run(self, args):
3383 if self.importIntoRemotes:
3384 self.refPrefix = "refs/remotes/p4/"
3385 else:
3386 self.refPrefix = "refs/heads/p4/"
3387
3388 if self.syncWithOrigin:
3389 self.hasOrigin = originP4BranchesExist()
3390 if self.hasOrigin:
3391 if not self.silent:
3392 print 'Syncing with origin first, using "git fetch origin"'
3393 system("git fetch origin")
3394
3395 branch_arg_given = bool(self.branch)
3396 if len(self.branch) == 0:
3397 self.branch = self.refPrefix + "master"
3398 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3399 system("git update-ref %s refs/heads/p4" % self.branch)
3400 system("git branch -D p4")
3401
3402 # accept either the command-line option, or the configuration variable
3403 if self.useClientSpec:
3404 # will use this after clone to set the variable
3405 self.useClientSpec_from_options = True
3406 else:
3407 if gitConfigBool("git-p4.useclientspec"):
3408 self.useClientSpec = True
3409 if self.useClientSpec:
3410 self.clientSpecDirs = getClientSpec()
3411
3412 # TODO: should always look at previous commits,
3413 # merge with previous imports, if possible.
3414 if args == []:
3415 if self.hasOrigin:
3416 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3417
3418 # branches holds mapping from branch name to sha1
3419 branches = p4BranchesInGit(self.importIntoRemotes)
3420
3421 # restrict to just this one, disabling detect-branches
3422 if branch_arg_given:
3423 short = self.branch.split("/")[-1]
3424 if short in branches:
3425 self.p4BranchesInGit = [ short ]
3426 else:
3427 self.p4BranchesInGit = branches.keys()
3428
3429 if len(self.p4BranchesInGit) > 1:
3430 if not self.silent:
3431 print "Importing from/into multiple branches"
3432 self.detectBranches = True
3433 for branch in branches.keys():
3434 self.initialParents[self.refPrefix + branch] = \
3435 branches[branch]
3436
3437 if self.verbose:
3438 print "branches: %s" % self.p4BranchesInGit
3439
3440 p4Change = 0
3441 for branch in self.p4BranchesInGit:
3442 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3443
3444 settings = extractSettingsGitLog(logMsg)
3445
3446 self.readOptions(settings)
3447 if (settings.has_key('depot-paths')
3448 and settings.has_key ('change')):
3449 change = int(settings['change']) + 1
3450 p4Change = max(p4Change, change)
3451
3452 depotPaths = sorted(settings['depot-paths'])
3453 if self.previousDepotPaths == []:
3454 self.previousDepotPaths = depotPaths
3455 else:
3456 paths = []
3457 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3458 prev_list = prev.split("/")
3459 cur_list = cur.split("/")
3460 for i in range(0, min(len(cur_list), len(prev_list))):
3461 if cur_list[i] <> prev_list[i]:
3462 i = i - 1
3463 break
3464
3465 paths.append ("/".join(cur_list[:i + 1]))
3466
3467 self.previousDepotPaths = paths
3468
3469 if p4Change > 0:
3470 self.depotPaths = sorted(self.previousDepotPaths)
3471 self.changeRange = "@%s,#head" % p4Change
3472 if not self.silent and not self.detectBranches:
3473 print "Performing incremental import into %s git branch" % self.branch
3474
3475 # accept multiple ref name abbreviations:
3476 # refs/foo/bar/branch -> use it exactly
3477 # p4/branch -> prepend refs/remotes/ or refs/heads/
3478 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3479 if not self.branch.startswith("refs/"):
3480 if self.importIntoRemotes:
3481 prepend = "refs/remotes/"
3482 else:
3483 prepend = "refs/heads/"
3484 if not self.branch.startswith("p4/"):
3485 prepend += "p4/"
3486 self.branch = prepend + self.branch
3487
3488 if len(args) == 0 and self.depotPaths:
3489 if not self.silent:
3490 print "Depot paths: %s" % ' '.join(self.depotPaths)
3491 else:
3492 if self.depotPaths and self.depotPaths != args:
3493 print ("previous import used depot path %s and now %s was specified. "
3494 "This doesn't work!" % (' '.join (self.depotPaths),
3495 ' '.join (args)))
3496 sys.exit(1)
3497
3498 self.depotPaths = sorted(args)
3499
3500 revision = ""
3501 self.users = {}
3502
3503 # Make sure no revision specifiers are used when --changesfile
3504 # is specified.
3505 bad_changesfile = False
3506 if len(self.changesFile) > 0:
3507 for p in self.depotPaths:
3508 if p.find("@") >= 0 or p.find("#") >= 0:
3509 bad_changesfile = True
3510 break
3511 if bad_changesfile:
3512 die("Option --changesfile is incompatible with revision specifiers")
3513
3514 newPaths = []
3515 for p in self.depotPaths:
3516 if p.find("@") != -1:
3517 atIdx = p.index("@")
3518 self.changeRange = p[atIdx:]
3519 if self.changeRange == "@all":
3520 self.changeRange = ""
3521 elif ',' not in self.changeRange:
3522 revision = self.changeRange
3523 self.changeRange = ""
3524 p = p[:atIdx]
3525 elif p.find("#") != -1:
3526 hashIdx = p.index("#")
3527 revision = p[hashIdx:]
3528 p = p[:hashIdx]
3529 elif self.previousDepotPaths == []:
3530 # pay attention to changesfile, if given, else import
3531 # the entire p4 tree at the head revision
3532 if len(self.changesFile) == 0:
3533 revision = "#head"
3534
3535 p = re.sub ("\.\.\.$", "", p)
3536 if not p.endswith("/"):
3537 p += "/"
3538
3539 newPaths.append(p)
3540
3541 self.depotPaths = newPaths
3542
3543 # --detect-branches may change this for each branch
3544 self.branchPrefixes = self.depotPaths
3545
3546 self.loadUserMapFromCache()
3547 self.labels = {}
3548 if self.detectLabels:
3549 self.getLabels();
3550
3551 if self.detectBranches:
3552 ## FIXME - what's a P4 projectName ?
3553 self.projectName = self.guessProjectName()
3554
3555 if self.hasOrigin:
3556 self.getBranchMappingFromGitBranches()
3557 else:
3558 self.getBranchMapping()
3559 if self.verbose:
3560 print "p4-git branches: %s" % self.p4BranchesInGit
3561 print "initial parents: %s" % self.initialParents
3562 for b in self.p4BranchesInGit:
3563 if b != "master":
3564
3565 ## FIXME
3566 b = b[len(self.projectName):]
3567 self.createdBranches.add(b)
3568
3569 self.openStreams()
3570
3571 if revision:
3572 self.importHeadRevision(revision)
3573 else:
3574 changes = []
3575
3576 if len(self.changesFile) > 0:
3577 output = open(self.changesFile).readlines()
3578 changeSet = set()
3579 for line in output:
3580 changeSet.add(int(line))
3581
3582 for change in changeSet:
3583 changes.append(change)
3584
3585 changes.sort()
3586 else:
3587 # catch "git p4 sync" with no new branches, in a repo that
3588 # does not have any existing p4 branches
3589 if len(args) == 0:
3590 if not self.p4BranchesInGit:
3591 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3592
3593 # The default branch is master, unless --branch is used to
3594 # specify something else. Make sure it exists, or complain
3595 # nicely about how to use --branch.
3596 if not self.detectBranches:
3597 if not branch_exists(self.branch):
3598 if branch_arg_given:
3599 die("Error: branch %s does not exist." % self.branch)
3600 else:
3601 die("Error: no branch %s; perhaps specify one with --branch." %
3602 self.branch)
3603
3604 if self.verbose:
3605 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3606 self.changeRange)
3607 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3608
3609 if len(self.maxChanges) > 0:
3610 changes = changes[:min(int(self.maxChanges), len(changes))]
3611
3612 if len(changes) == 0:
3613 if not self.silent:
3614 print "No changes to import!"
3615 else:
3616 if