[git/git.git] / contrib / fast-import / git-p4.py

#!/usr/bin/env python
#
# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
#
# Author: Simon Hausmann <hausmann@kde.org>
# Copyright: 2007 Simon Hausmann <hausmann@kde.org>
#            2007 Trolltech ASA
# License: MIT <http://www.opensource.org/licenses/mit-license.php>
#

import optparse, sys, os, marshal, popen2, shelve
import tempfile

gitdir = os.environ.get("GIT_DIR", "")

def p4CmdList(cmd):
    cmd = "p4 -G %s" % cmd
    pipe = os.popen(cmd, "rb")

    result = []
    try:
        while True:
            entry = marshal.load(pipe)
            result.append(entry)
    except EOFError:
        pass
    pipe.close()

    return result

def p4Cmd(cmd):
    list = p4CmdList(cmd)
    result = {}
    for entry in list:
        result.update(entry)
    return result;

def die(msg):
    sys.stderr.write(msg + "\n")
    sys.exit(1)

def currentGitBranch():
    return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1]

def isValidGitDir(path):
    if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
        return True;
    return False

def system(cmd):
    if os.system(cmd) != 0:
        die("command failed: %s" % cmd)

class P4Debug:
    def __init__(self):
        self.options = [
        ]
        self.description = "A tool to debug the output of p4 -G."

    def run(self, args):
        for output in p4CmdList(" ".join(args)):
            print output

class P4CleanTags:
    def __init__(self):
        self.options = [
#                optparse.make_option("--branch", dest="branch", default="refs/heads/master")
        ]
        self.description = "A tool to remove stale unused tags from incremental perforce imports."
    def run(self, args):
        branch = currentGitBranch()
        print "Cleaning out stale p4 import tags..."
        sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
        output = sout.read()
        try:
            tagIdx = output.index(" tags/p4/")
        except:
            print "Cannot find any p4/* tag. Nothing to do."
            sys.exit(0)

        try:
            caretIdx = output.index("^")
        except:
            caretIdx = len(output) - 1
        rev = int(output[tagIdx + 9 : caretIdx])

        allTags = os.popen("git tag -l p4/").readlines()
        for i in range(len(allTags)):
            allTags[i] = int(allTags[i][3:-1])

        allTags.sort()

        allTags.remove(rev)

        for rev in allTags:
            print os.popen("git tag -d p4/%s" % rev).read()

        print "%s tags removed." % len(allTags)

class P4Sync:
    def __init__(self):
        self.options = [
                optparse.make_option("--continue", action="store_false", dest="firstTime"),
                optparse.make_option("--origin", dest="origin"),
                optparse.make_option("--reset", action="store_true", dest="reset"),
                optparse.make_option("--master", dest="master"),
                optparse.make_option("--log-substitutions", dest="substFile"),
                optparse.make_option("--noninteractive", action="store_false"),
                optparse.make_option("--dry-run", action="store_true")
        ]
        self.description = "Submit changes from git to the perforce depot."
        self.firstTime = True
        self.reset = False
        self.interactive = True
        self.dryRun = False
        self.substFile = ""
        self.firstTime = True
        self.origin = "origin"
        self.master = ""

        self.logSubstitutions = {}
        self.logSubstitutions["<enter description here>"] = "%log%"
        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"

    def check(self):
        if len(p4CmdList("opened ...")) > 0:
            die("You have files opened with perforce! Close them before starting the sync.")

    def start(self):
        if len(self.config) > 0 and not self.reset:
            die("Cannot start sync. Previous sync config found at %s" % self.configFile)

        commits = []
        for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
            commits.append(line[:-1])
        commits.reverse()

        self.config["commits"] = commits

        print "Creating temporary p4-sync branch from %s ..." % self.origin
        system("git checkout -f -b p4-sync %s" % self.origin)

    def prepareLogMessage(self, template, message):
        result = ""

        for line in template.split("\n"):
            if line.startswith("#"):
                result += line + "\n"
                continue

            substituted = False
            for key in self.logSubstitutions.keys():
                if line.find(key) != -1:
                    value = self.logSubstitutions[key]
                    value = value.replace("%log%", message)
                    if value != "@remove@":
                        result += line.replace(key, value) + "\n"
                    substituted = True
                    break

            if not substituted:
                result += line + "\n"

        return result

    def apply(self, id):
        print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read())
        diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
        filesToAdd = set()
        filesToDelete = set()
        for line in diff:
            modifier = line[0]
            path = line[1:].strip()
            if modifier == "M":
                system("p4 edit %s" % path)
            elif modifier == "A":
                filesToAdd.add(path)
                if path in filesToDelete:
                    filesToDelete.remove(path)
            elif modifier == "D":
                filesToDelete.add(path)
                if path in filesToAdd:
                    filesToAdd.remove(path)
            else:
                die("unknown modifier %s for %s" % (modifier, path))

        system("git-diff-files --name-only -z | git-update-index --remove -z --stdin")
        system("git cherry-pick --no-commit \"%s\"" % id)

        for f in filesToAdd:
            system("p4 add %s" % f)
        for f in filesToDelete:
            system("p4 revert %s" % f)
            system("p4 delete %s" % f)

        logMessage = ""
        foundTitle = False
        for log in os.popen("git-cat-file commit %s" % id).readlines():
            if not foundTitle:
                if len(log) == 1:
                    foundTitle = 1
                continue

            if len(logMessage) > 0:
                logMessage += "\t"
            logMessage += log

        template = os.popen("p4 change -o").read()

        if self.interactive:
            submitTemplate = self.prepareLogMessage(template, logMessage)
            diff = os.popen("p4 diff -du ...").read()

            for newFile in filesToAdd:
                diff += "==== new file ====\n"
                diff += "--- /dev/null\n"
                diff += "+++ %s\n" % newFile
                f = open(newFile, "r")
                for line in f.readlines():
                    diff += "+" + line
                f.close()

            pipe = os.popen("less", "w")
            pipe.write(submitTemplate + diff)
            pipe.close()

            response = "e"
            while response == "e":
                response = raw_input("Do you want to submit this change (y/e/n)? ")
                if response == "e":
                    [handle, fileName] = tempfile.mkstemp()
                    tmpFile = os.fdopen(handle, "w+")
                    tmpFile.write(submitTemplate)
                    tmpFile.close()
                    editor = os.environ.get("EDITOR", "vi")
                    system(editor + " " + fileName)
                    tmpFile = open(fileName, "r")
                    submitTemplate = tmpFile.read()
                    tmpFile.close()
                    os.remove(fileName)

            if response == "y" or response == "yes":
               if self.dryRun:
                   print submitTemplate
                   raw_input("Press return to continue...")
               else:
                    pipe = os.popen("p4 submit -i", "w")
                    pipe.write(submitTemplate)
                    pipe.close()
            else:
                print "Not submitting!"
                self.interactive = False
        else:
            fileName = "submit.txt"
            file = open(fileName, "w+")
            file.write(self.prepareLogMessage(template, logMessage))
            file.close()
            print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)

    def run(self, args):
        if self.reset:
            self.firstTime = True

        if len(self.substFile) > 0:
            for line in open(self.substFile, "r").readlines():
                tokens = line[:-1].split("=")
                self.logSubstitutions[tokens[0]] = tokens[1]

        if len(self.master) == 0:
            self.master = currentGitBranch()
            if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
                die("Detecting current git branch failed!")

        self.check()
        self.configFile = gitdir + "/p4-git-sync.cfg"
        self.config = shelve.open(self.configFile, writeback=True)

        if self.firstTime:
            self.start()

        commits = self.config.get("commits", [])

        while len(commits) > 0:
            self.firstTime = False
            commit = commits[0]
            commits = commits[1:]
            self.config["commits"] = commits
            self.apply(commit)
            if not self.interactive:
                break

        self.config.close()

        if len(commits) == 0:
            if self.firstTime:
                print "No changes found to apply between %s and current HEAD" % self.origin
            else:
                print "All changes applied!"
                print "Deleting temporary p4-sync branch and going back to %s" % self.master
                system("git checkout %s" % self.master)
                system("git branch -D p4-sync")
                print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..."
                system("p4 edit ... >/dev/null")
                system("p4 revert ... >/dev/null")
            os.remove(self.configFile)


def printUsage(commands):
    print "usage: %s <command> [options]" % sys.argv[0]
    print ""
    print "valid commands: %s" % ", ".join(commands)
    print ""
    print "Try %s <command> --help for command specific help." % sys.argv[0]
    print ""

commands = {
    "debug" : P4Debug(),
    "clean-tags" : P4CleanTags(),
    "submit" : P4Sync()
}

if len(sys.argv[1:]) == 0:
    printUsage(commands.keys())
    sys.exit(2)

cmd = ""
cmdName = sys.argv[1]
try:
    cmd = commands[cmdName]
except KeyError:
    print "unknown command %s" % cmdName
    print ""
    printUsage(commands.keys())
    sys.exit(2)

options = cmd.options
cmd.gitdir = gitdir
options.append(optparse.make_option("--git-dir", dest="gitdir"))

parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", options,
                               description = cmd.description)

(cmd, args) = parser.parse_args(sys.argv[2:], cmd);

gitdir = cmd.gitdir
if len(gitdir) == 0:
    gitdir = ".git"

if not isValidGitDir(gitdir):
    if isValidGitDir(gitdir + "/.git"):
        gitdir += "/.git"
    else:
        die("fatal: cannot locate git repository at %s" % gitdir)

os.environ["GIT_DIR"] = gitdir

cmd.run(args)
Commit	Line	Data
86949eef SH	1	#!/usr/bin/env python
	2	#
	3	# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
	4	#
	5	# Author: Simon Hausmann <hausmann@kde.org>
83dce55a SH	6	# Copyright: 2007 Simon Hausmann <hausmann@kde.org>
83dce55a SH	7	# 2007 Trolltech ASA
86949eef SH	8	# License: MIT <http://www.opensource.org/licenses/mit-license.php>
	9	#
	10
4f5cf76a SH	11	import optparse, sys, os, marshal, popen2, shelve
	12	import tempfile
	13
	14	gitdir = os.environ.get("GIT_DIR", "")
86949eef SH	15
	16	def p4CmdList(cmd):
	17	cmd = "p4 -G %s" % cmd
	18	pipe = os.popen(cmd, "rb")
	19
	20	result = []
	21	try:
	22	while True:
	23	entry = marshal.load(pipe)
	24	result.append(entry)
	25	except EOFError:
	26	pass
	27	pipe.close()
	28
	29	return result
	30
	31	def p4Cmd(cmd):
	32	list = p4CmdList(cmd)
	33	result = {}
	34	for entry in list:
	35	result.update(entry)
	36	return result;
	37
	38	def die(msg):
	39	sys.stderr.write(msg + "\n")
	40	sys.exit(1)
	41
	42	def currentGitBranch():
	43	return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1]
	44
4f5cf76a SH	45	def isValidGitDir(path):
	46	if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
	47	return True;
	48	return False
	49
	50	def system(cmd):
	51	if os.system(cmd) != 0:
	52	die("command failed: %s" % cmd)
	53
86949eef SH	54	class P4Debug:
	55	def __init__(self):
	56	self.options = [
	57	]
c8c39116	58	self.description = "A tool to debug the output of p4 -G."
86949eef SH	59
	60	def run(self, args):
	61	for output in p4CmdList(" ".join(args)):
	62	print output
	63
	64	class P4CleanTags:
	65	def __init__(self):
	66	self.options = [
	67	# optparse.make_option("--branch", dest="branch", default="refs/heads/master")
	68	]
c8c39116	69	self.description = "A tool to remove stale unused tags from incremental perforce imports."
86949eef SH	70	def run(self, args):
	71	branch = currentGitBranch()
	72	print "Cleaning out stale p4 import tags..."
	73	sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
	74	output = sout.read()
	75	try:
	76	tagIdx = output.index(" tags/p4/")
	77	except:
	78	print "Cannot find any p4/* tag. Nothing to do."
	79	sys.exit(0)
	80
	81	try:
	82	caretIdx = output.index("^")
	83	except:
	84	caretIdx = len(output) - 1
	85	rev = int(output[tagIdx + 9 : caretIdx])
	86
	87	allTags = os.popen("git tag -l p4/").readlines()
	88	for i in range(len(allTags)):
	89	allTags[i] = int(allTags[i][3:-1])
	90
	91	allTags.sort()
	92
	93	allTags.remove(rev)
	94
	95	for rev in allTags:
	96	print os.popen("git tag -d p4/%s" % rev).read()
	97
	98	print "%s tags removed." % len(allTags)
	99
4f5cf76a SH	100	class P4Sync:
	101	def __init__(self):
	102	self.options = [
	103	optparse.make_option("--continue", action="store_false", dest="firstTime"),
	104	optparse.make_option("--origin", dest="origin"),
	105	optparse.make_option("--reset", action="store_true", dest="reset"),
	106	optparse.make_option("--master", dest="master"),
	107	optparse.make_option("--log-substitutions", dest="substFile"),
	108	optparse.make_option("--noninteractive", action="store_false"),
	109	optparse.make_option("--dry-run", action="store_true")
	110	]
	111	self.description = "Submit changes from git to the perforce depot."
	112	self.firstTime = True
	113	self.reset = False
	114	self.interactive = True
	115	self.dryRun = False
	116	self.substFile = ""
	117	self.firstTime = True
	118	self.origin = "origin"
	119	self.master = ""
	120
	121	self.logSubstitutions = {}
	122	self.logSubstitutions["<enter description here>"] = "%log%"
	123	self.logSubstitutions["\tDetails:"] = "\tDetails: %log%"
	124
	125	def check(self):
	126	if len(p4CmdList("opened ...")) > 0:
	127	die("You have files opened with perforce! Close them before starting the sync.")
	128
	129	def start(self):
	130	if len(self.config) > 0 and not self.reset:
	131	die("Cannot start sync. Previous sync config found at %s" % self.configFile)
	132
	133	commits = []
	134	for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
	135	commits.append(line[:-1])
	136	commits.reverse()
	137
	138	self.config["commits"] = commits
	139
	140	print "Creating temporary p4-sync branch from %s ..." % self.origin
	141	system("git checkout -f -b p4-sync %s" % self.origin)
	142
	143	def prepareLogMessage(self, template, message):
	144	result = ""
	145
	146	for line in template.split("\n"):
	147	if line.startswith("#"):
	148	result += line + "\n"
	149	continue
	150
	151	substituted = False
	152	for key in self.logSubstitutions.keys():
	153	if line.find(key) != -1:
	154	value = self.logSubstitutions[key]
	155	value = value.replace("%log%", message)
	156	if value != "@remove@":
	157	result += line.replace(key, value) + "\n"
	158	substituted = True
	159	break
	160
	161	if not substituted:
	162	result += line + "\n"
	163
164	return result
165
166	def apply(self, id):
167	print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read())
168	diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
169	filesToAdd = set()
170	filesToDelete = set()
171	for line in diff:
172	modifier = line[0]
173	path = line[1:].strip()
174	if modifier == "M":
175	system("p4 edit %s" % path)
176	elif modifier == "A":
177	filesToAdd.add(path)
178	if path in filesToDelete:
179	filesToDelete.remove(path)
180	elif modifier == "D":
181	filesToDelete.add(path)
182	if path in filesToAdd:
183	filesToAdd.remove(path)
184	else:
185	die("unknown modifier %s for %s" % (modifier, path))
186
187	system("git-diff-files --name-only -z \| git-update-index --remove -z --stdin")
188	system("git cherry-pick --no-commit \"%s\"" % id)
189
190	for f in filesToAdd:
191	system("p4 add %s" % f)
192	for f in filesToDelete:
193	system("p4 revert %s" % f)
194	system("p4 delete %s" % f)
195
196	logMessage = ""
197	foundTitle = False
198	for log in os.popen("git-cat-file commit %s" % id).readlines():
199	if not foundTitle:
200	if len(log) == 1:
201	foundTitle = 1
202	continue
203
204	if len(logMessage) > 0:
205	logMessage += "\t"
206	logMessage += log
207
208	template = os.popen("p4 change -o").read()
209
210	if self.interactive:
211	submitTemplate = self.prepareLogMessage(template, logMessage)
212	diff = os.popen("p4 diff -du ...").read()
213
214	for newFile in filesToAdd:
215	diff += "==== new file ====\n"
216	diff += "--- /dev/null\n"
217	diff += "+++ %s\n" % newFile
218	f = open(newFile, "r")
219	for line in f.readlines():
220	diff += "+" + line
221	f.close()
222
223	pipe = os.popen("less", "w")
224	pipe.write(submitTemplate + diff)
225	pipe.close()
226
227	response = "e"
228	while response == "e":
229	response = raw_input("Do you want to submit this change (y/e/n)? ")
230	if response == "e":
231	[handle, fileName] = tempfile.mkstemp()
232	tmpFile = os.fdopen(handle, "w+")
233	tmpFile.write(submitTemplate)
234	tmpFile.close()
235	editor = os.environ.get("EDITOR", "vi")
236	system(editor + " " + fileName)
237	tmpFile = open(fileName, "r")
238	submitTemplate = tmpFile.read()
239	tmpFile.close()
240	os.remove(fileName)
241
242	if response == "y" or response == "yes":
243	if self.dryRun:
244	print submitTemplate
245	raw_input("Press return to continue...")
246	else:
247	pipe = os.popen("p4 submit -i", "w")
248	pipe.write(submitTemplate)
249	pipe.close()
250	else:
251	print "Not submitting!"
252	self.interactive = False
253	else:
254	fileName = "submit.txt"
255	file = open(fileName, "w+")
256	file.write(self.prepareLogMessage(template, logMessage))
257	file.close()
258	print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
259
260	def run(self, args):
261	if self.reset:
262	self.firstTime = True
263
264	if len(self.substFile) > 0:
265	for line in open(self.substFile, "r").readlines():
266	tokens = line[:-1].split("=")
267	self.logSubstitutions[tokens[0]] = tokens[1]
268
269	if len(self.master) == 0:
270	self.master = currentGitBranch()
271	if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
272	die("Detecting current git branch failed!")
273
274	self.check()
275	self.configFile = gitdir + "/p4-git-sync.cfg"
276	self.config = shelve.open(self.configFile, writeback=True)
277
278	if self.firstTime:
279	self.start()
280
281	commits = self.config.get("commits", [])
282
283	while len(commits) > 0:
284	self.firstTime = False
285	commit = commits[0]
286	commits = commits[1:]
287	self.config["commits"] = commits
288	self.apply(commit)
289	if not self.interactive:
290	break
291
292	self.config.close()
293
294	if len(commits) == 0:
295	if self.firstTime:
296	print "No changes found to apply between %s and current HEAD" % self.origin
297	else:
298	print "All changes applied!"
299	print "Deleting temporary p4-sync branch and going back to %s" % self.master
300	system("git checkout %s" % self.master)
301	system("git branch -D p4-sync")
302	print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..."
303	system("p4 edit ... >/dev/null")
304	system("p4 revert ... >/dev/null")
305	os.remove(self.configFile)
306
307
86949eef SH	308	def printUsage(commands):
	309	print "usage: %s <command> [options]" % sys.argv[0]
	310	print ""
	311	print "valid commands: %s" % ", ".join(commands)
	312	print ""
	313	print "Try %s <command> --help for command specific help." % sys.argv[0]
	314	print ""
	315
	316	commands = {
	317	"debug" : P4Debug(),
4f5cf76a	318	"clean-tags" : P4CleanTags(),
05140f34	319	"submit" : P4Sync()
86949eef SH	320	}
	321
	322	if len(sys.argv[1:]) == 0:
	323	printUsage(commands.keys())
	324	sys.exit(2)
	325
	326	cmd = ""
	327	cmdName = sys.argv[1]
	328	try:
	329	cmd = commands[cmdName]
	330	except KeyError:
	331	print "unknown command %s" % cmdName
	332	print ""
	333	printUsage(commands.keys())
	334	sys.exit(2)
	335
4f5cf76a SH	336	options = cmd.options
	337	cmd.gitdir = gitdir
	338	options.append(optparse.make_option("--git-dir", dest="gitdir"))
	339
	340	parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", options,
c8c39116	341	description = cmd.description)
86949eef SH	342
	343	(cmd, args) = parser.parse_args(sys.argv[2:], cmd);
	344
4f5cf76a SH	345	gitdir = cmd.gitdir
	346	if len(gitdir) == 0:
	347	gitdir = ".git"
	348
	349	if not isValidGitDir(gitdir):
	350	if isValidGitDir(gitdir + "/.git"):
	351	gitdir += "/.git"
	352	else:
05140f34	353	die("fatal: cannot locate git repository at %s" % gitdir)
4f5cf76a SH	354
	355	os.environ["GIT_DIR"] = gitdir
	356
86949eef	357	cmd.run(args)