path: root/exporters/darcs/darcs-fast-import

#!/usr/bin/env python

"""

    darcs-fast-export - darcs backend for fast data exporters

    Copyright (c) 2008, 2009, 2010 Miklos Vajna <vmiklos@frugalware.org>
    Copyright (c) 2008 Matthias Andree <matthias.andree@gmx.de>

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2, or (at your option)
    any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

"""

import sys
import os
import re
import time
import shutil
import optparse
import subprocess

class Handler:
	def __init__(self):
		self.marks = {}
		self.files = []
		self.prevfiles = None
		self.ch = None
		self.line = None
		self.unread_line = False
		self.eof = False
		self.debug = False
		self.export_marks = []
		self.import_marks = []

	def read_next_line(self):
		if self.unread_line:
			self.unread_line = False
			return
		self.line = ""
		if self.eof:
			return
		if self.ch:
			self.line += self.ch
			self.ch = None
		buf = sys.stdin.readline()
		if not len(buf):
			self.eof = True
		else:
			self.line += buf
		if self.debug:
			print "read_next_line: '%s'" % self.line

	def read(self, length):
		buf = ""
		if self.ch:
			buf += self.ch
			self.ch = None
		buf += sys.stdin.read(length)
		if self.debug:
			print "read: '%s'" % buf
		return buf

	def skip_optional_lf(self):
		self.ch = self.read(1)
		if self.ch == "\n":
			self.ch = None

	def bug(self, s):
		raise Exception(s)

	def get_date(self, ts, tz):
		# first fix the case when tz is higher than +1200, as
		# darcs won't accept it
		if int(tz[:3]) > 12:
			ts = str(int(ts) + 60*60*24)
			tz = str(int(tz[:3])-24) + tz[3:]
                # int(ts) is seconds since epoch. Since we're trying to
                # capture both the absolute time of the commit and the
                # localtime in the timezone of the committer, we need to turn
                # the (seconds-since-epoch, committer-timezone-offset) pair
                # that we get from the git-fast-export stream format into a
                # localized-time-plus-timezone-marker string that darcs will
                # accept. Therefore, we parse the timezone-offset (which
                # looks like +0500 or +0000 or -0730 or something) and add it
                # to seconds-since-epoch before calling gmtime().
                mo = re.search(r'^([\+\-])(\d\d)(\d\d)$', tz)
                offset = 60*60*int(mo.group(2)) + 60*int(mo.group(3))
                if mo.group(1) == "-":
                        offset = -offset
                offset_time = int(ts) + offset
		s = time.strftime("%a %b %d %H:%M:%S %Y", time.gmtime(offset_time))
		items = s.split(' ')
		return " ".join(items[:-1]) + " " + tz + " " + items[-1]

	def invoke_darcs(self, cmdline):
		if os.system("darcs %s" % cmdline) != 0:
			self.bug("darcs failed")

	def invoke_add(self, path):
		self.invoke_darcs("add --boring --case-ok %s" % path)

	def handle_mark(self):
		if self.line.startswith("mark :"):
			self.mark_num = int(self.line[6:-1])
			self.read_next_line()

	def handle_data(self):
		if not self.line.startswith("data "):
			self.bug("Expected 'data n' command, found: '%s'" % self.line[:-1])
		length = int(self.line[5:-1])
		self.buf = self.read(length)
		self.skip_optional_lf()

	def handle_blob(self):
		self.read_next_line()
		self.handle_mark()
		self.handle_data()
		self.marks[self.mark_num] = self.buf

	def handle_ident(self, s):
		items = s.split(' ')
		self.ident = " ".join(items[:-2])
		self.date = self.get_date(items[-2], items[-1])

	def handle_msg(self):
		items = self.buf.split('\n')
		self.short = items[0]
		self.long = "\n".join(items[1:])

	def handle_tag(self):
		version = self.line[:-1].split(' ')[1]
		self.read_next_line()
		if self.line.startswith("from "):
			self.read_next_line()
		if self.line.startswith("tagger "):
			self.handle_ident(self.line[7:-1])
			self.read_next_line()
		self.handle_data()
		self.skip_optional_lf()
		sock = subprocess.Popen(["darcs", "tag", "--pipe"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
		buf = [self.date, self.ident, version]
		sock.stdin.write("\n".join(buf))
		sock.stdin.close()
		self.log("Tagging %s:\n%s" % (version, sock.stdout.read()))
		sock.stdout.close()
		if sock.wait() != 0:
			self.bug("darcs tag failed: '%s'" % sock.returncode)

	def handle_commit(self):
		if not self.prevfiles and self.options.import_marks:
			# first commit in an incremental continued
			# import
			for (root, dirs, files) in os.walk("."):
				for i in files:
					path = os.path.normpath(os.path.join(root, i))
					if path.startswith("_darcs") or "-darcs-backup" in path:
						continue
					self.files.append(path)
		self.prevfiles = self.files[:]
		adds = []
		symlinks = []

		self.read_next_line()
		self.handle_mark()
		if self.line.startswith("author "):
			self.handle_ident(self.line[7:-1])
			self.read_next_line()
		if self.line.startswith("committer "):
			self.handle_ident(self.line[10:-1])
			self.read_next_line()
		self.handle_data()
		self.skip_optional_lf()
		self.handle_msg()
		self.read_next_line()
		if self.line.startswith("from "):
			self.read_next_line()
		while self.line.startswith("merge "):
			self.read_next_line()
		change = False
		while len(self.line) > 0:
			if self.line.startswith("deleteall"):
				path = self.line[2:-1]
				for path in self.files:
					os.unlink(path)
				self.files = []
				change = True
			elif self.line.startswith("D "):
				path = self.line[2:-1]
				if os.path.exists(path):
					os.unlink(path)
				if path in self.files:
					self.files.remove(path)
				change = True
			elif self.line.startswith("R "):
				self.invoke_darcs("mv %s" % self.line[2:])
				change = True
			elif self.line.startswith("C "):
				src, dest = self.line[:-1].split(' ')[1:]
				shutil.copy(src.strip('"'), dest.strip('"'))
				self.invoke_add(dest)
				change = True
			elif self.line.startswith("M "):
				items = self.line.split(' ')
				path = items[3][:-1]
				dir = os.path.split(path)[0]
				if len(dir) and not os.path.exists(dir):
					os.makedirs(dir)
				if items[1] == "120000":
					if not self.options.symhack:
						print "Adding symbolic links (symlinks) is not supported by Darcs."
						sys.exit(2)
					idx = int(items[2][1:]) # TODO: handle inline symlinks
					symlinks.append((self.marks[idx], path))
					self.read_next_line()
					continue
				sock = open(path, "w")
				if items[2] != "inline":
					idx = int(items[2][1:])
					sock.write(self.marks[idx])
				else:
					self.read_next_line()
					self.handle_data()
					sock.write(self.buf)
				sock.close()
				if path not in self.prevfiles:
					adds.append(path)
				if path not in self.files:
					self.files.append(path)
				change = True
			else:
				self.unread_line = True
				break
			self.read_next_line()
			if not len(self.line):
				break

		if not change:
			# darcs does not support empty commits
			return
		for i in adds:
			self.invoke_add(i)
		args = ["darcs", "record", "--ignore-times", "-a", "--pipe"]
		buf = [self.date, self.ident]
		if not len(self.short):
			args.extend(['-m', ''])
		else:
			buf.extend([self.short, self.long])
		sock = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
		sock.stdin.write("\n".join(buf)+"\n")
		sock.stdin.close()
		self.log("Recording :%s:\n%s" % (self.mark_num, sock.stdout.read()))
		sock.stdout.close()
		if sock.wait() != 0:
			self.bug("darcs record failed: '%s'" % sock.returncode)

		for src, path in symlinks:
			# symlink does not do what we want if path is
			# already there
			if os.path.exists(path):
				# rmtree() does not work on symlinks
				if os.path.islink(path):
					os.remove(path)
				else:
					shutil.rmtree(path)
			os.symlink(src, path)
		if self.options.export_marks:
			# yeah, an xml parser would be better, but
			# should we mess with encodings just because of
			# this? i hope not
			sock = os.popen("darcs changes --last=1 --xml", "r")
			buf = sock.read()
			sock.close()
			hash = buf.split('\n')[1].split("'")[-2]
			self.export_marks.append(":%s %s" % (self.mark_num, hash))

	def handle_progress(self, s):
		print "import progress [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s.strip())
		sys.stdout.flush()

	def handle_opts(self):
		# Option Parser
		usage="%prog [options]"
		opp = optparse.OptionParser(usage=usage)
		opp.set_defaults(symhack=False)
		opp.add_option("--import-marks", metavar="IFILE",
			help="read state for incremental imports from IFILE")
		opp.add_option("--export-marks", metavar="OFILE",
			help="write state for incremental imports to OFILE")
		opp.add_option("--logfile", metavar="L",
			help="log file which contains the output of external programs invoked during the conversion")
		opp.add_option("--symhack", action="store_true", dest="symhack",
			help="Do not error out when a symlink would be created, just create it in the workdir")
		opp.add_option("--progress", metavar="P",
			help="insert progress statements after every n commit [default: 100]")
		(self.options, args) = opp.parse_args()

		if self.options.logfile:
			logfile = self.options.logfile
		else:
			logfile = "_darcs/import.log"
		self.logsock = open(os.path.abspath(logfile), "a")

		if self.options.progress:
			self.prognum = int(self.options.progress)
		else:
			self.prognum = 0
	
	def log(self, s):
		self.logsock.write("[%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s))
		self.logsock.flush()

	def handle_export_marks(self):
		if self.options.export_marks:
			sock = open(self.options.export_marks, 'w')
			sock.write("\n".join(self.export_marks))
			sock.write("\n")
			sock.close()

	def handle_import_marks(self):
		if self.options.import_marks:
			sock = open(self.options.import_marks)
			for i in sock.readlines():
				line = i.strip()
				if not len(line):
					continue
				self.import_marks.append(line.split(' ')[1])
				self.export_marks.append(line)
			sock.close()

	def handle(self):
		self.handle_opts()
		self.handle_import_marks()

		commitcount = 0
		while not self.eof:
			self.read_next_line()
			if not len(self.line[:-1]):
				pass
			elif self.line.startswith("blob"):
				self.handle_blob()
			elif self.line.startswith("commit"):
				self.handle_commit()
				commitcount += 1
				if self.prognum != 0 and commitcount % self.prognum == 0:
					self.handle_progress("%d patches" % commitcount)
			elif self.line.startswith("tag"):
				self.handle_tag()
			elif self.line.startswith("reset"):
				self.read_next_line()
				if not self.line.startswith("from "):
					self.unread_line = True
			elif self.line.startswith("checkpoint"):
				pass
			elif self.line.startswith("progress"):
				self.handle_progress(self.line[9:])
			else:
				self.bug("'%s': invalid command" % self.line[:-1])

		self.handle_export_marks()

if __name__ == "__main__":
	h = Handler()
	h.handle()