Merge fixes from Miklos for darcs fast import.

9 files changed, 197 insertions, 63 deletions

diff --git a/exporters/darcs/TODO b/exporters/darcs/TODO
index 2f199d1..c6892c8 100644
--- a/exporters/darcs/TODO
+++ b/exporters/darcs/TODO
@@ -4,3 +4,5 @@ not enabled, etc.
 parse the patches manually so we can avoid re-adding existing files manually.
 
 avoid darcs apply.
+
+import: handle evil merges (git-subtree), maybe using git log --first-parent
diff --git a/exporters/darcs/darcs-fast-export b/exporters/darcs/darcs-fast-export
index 79590a6..fa850de 100755
--- a/exporters/darcs/darcs-fast-export
+++ b/exporters/darcs/darcs-fast-export
@@ -29,6 +29,7 @@ import os
 import sys
 import gzip
 import time
+import calendar
 import shutil
 import subprocess
 import optparse
@@ -82,22 +83,12 @@ class Handler:
 			author = author[author.index('>')+2:] + ' ' + author[:author.index('>')+1]
 		return author.encode('utf-8')
 
-	def get_time_info(self, patch):
-		date = time.strptime(patch, "%a %b %d %H:%M:%S %Z %Y")
-		timestamp = int(time.mktime(date))
-		# calculate the timezone offset
-		fields=re.split('[ ]+', patch)
-		fields[4]="UTC"
-		patch_utc=" ".join(fields)
-		date_utc=time.strptime(patch_utc, "%a %b %d %H:%M:%S %Z %Y")
-		offset=int(time.timezone + time.mktime(date) - time.mktime(date_utc))
-		hours, minutes = divmod(abs(offset), 3600)
-		if offset > 0:
-			sign = "-"
-		else:
-			sign = "+"
-		zone = "%s%02d%02d" % (sign, hours, minutes // 60)
-		return timestamp, zone
+	def get_date(self, patch):
+		try:
+			date = time.strptime(patch, "%Y%m%d%H%M%S")
+		except ValueError:
+			date = time.strptime(patch[:19] + patch[-5:], '%a %b %d %H:%M:%S %Y')
+		return calendar.timegm(date)
 
 	def progress(self, s):
 		print "progress [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s)
@@ -329,8 +320,8 @@ class Handler:
 			print "mark :%s" % markcount
 			if self.options.export_marks:
 				self.export_marks.append(":%s %s" % (markcount, hash))
-			date, zone = self.get_time_info(i.attributes['local_date'].value)
-			print "committer %s %s %s" % (self.get_author(i), date, zone)
+			date = self.get_date(i.attributes['date'].value)
+			print "committer %s %s +0000" % (self.get_author(i), date)
 			print "data %d\n%s" % (len(message), message)
 			if markcount > 1:
 				print "from :%s" % (markcount-1)
@@ -354,7 +345,7 @@ class Handler:
 				tag = re.sub('[^\xe9-\xf8\w.\-]+', '_', message[4:].strip().split('\n')[0]).strip('_')
 				print "tag %s" % tag
 				print "from :%s" % markcount
-				print "tagger %s %s %s" % (self.get_author(i), date, zone)
+				print "tagger %s %s +0000" % (self.get_author(i), date)
 				print "data %d\n%s" % (len(message), message)
 			if count % self.prognum == 0:
 				self.progress("%d/%d patches" % (count, patchnum))
diff --git a/exporters/darcs/darcs-fast-import b/exporters/darcs/darcs-fast-import
index 0a1495d..c50ab90 100755
--- a/exporters/darcs/darcs-fast-import
+++ b/exporters/darcs/darcs-fast-import
@@ -4,7 +4,7 @@
 
     darcs-fast-export - darcs backend for fast data exporters
 
-    Copyright (c) 2008 Miklos Vajna <vmiklos@frugalware.org>
+    Copyright (c) 2008, 2009, 2010 Miklos Vajna <vmiklos@frugalware.org>
     Copyright (c) 2008 Matthias Andree <matthias.andree@gmx.de>
 
     This program is free software; you can redistribute it and/or modify
@@ -104,6 +104,13 @@ class Handler:
 		items = s.split(' ')
 		return " ".join(items[:-1]) + " " + tz + " " + items[-1]
 
+	def invoke_darcs(self, cmdline):
+		if os.system("darcs %s" % cmdline) != 0:
+			self.bug("darcs failed")
+
+	def invoke_add(self, path):
+		self.invoke_darcs("add --boring --case-ok %s" % path)
+
 	def handle_mark(self):
 		if self.line.startswith("mark :"):
 			self.mark_num = int(self.line[6:-1])
@@ -161,6 +168,7 @@ class Handler:
 					self.files.append(path)
 		self.prevfiles = self.files[:]
 		adds = []
+		symlinks = []
 
 		self.read_next_line()
 		self.handle_mark()
@@ -178,35 +186,47 @@ class Handler:
 			self.read_next_line()
 		while self.line.startswith("merge "):
 			self.read_next_line()
+		change = False
 		while len(self.line) > 0:
 			if self.line.startswith("deleteall"):
 				path = self.line[2:-1]
 				for path in self.files:
 					os.unlink(path)
 				self.files = []
+				change = True
 			elif self.line.startswith("D "):
 				path = self.line[2:-1]
 				if os.path.exists(path):
 					os.unlink(path)
 				if path in self.files:
 					self.files.remove(path)
+				change = True
 			elif self.line.startswith("R "):
-				os.system("darcs mv %s" % self.line[2:])
+				self.invoke_darcs("mv %s" % self.line[2:])
+				change = True
 			elif self.line.startswith("C "):
 				src, dest = self.line[:-1].split(' ')[1:]
 				shutil.copy(src.strip('"'), dest.strip('"'))
-				os.system("darcs add %s" % dest)
+				self.invoke_add(dest)
+				change = True
 			elif self.line.startswith("M "):
 				items = self.line.split(' ')
 				path = items[3][:-1]
 				dir = os.path.split(path)[0]
 				if len(dir) and not os.path.exists(dir):
 					os.makedirs(dir)
+				if items[1] == "120000":
+					if not self.options.symhack:
+						print "Adding symbolic links (symlinks) is not supported by Darcs."
+						sys.exit(2)
+					idx = int(items[2][1:]) # TODO: handle inline symlinks
+					symlinks.append((self.marks[idx], path))
+					self.read_next_line()
+					continue
 				sock = open(path, "w")
 				if items[2] != "inline":
 					idx = int(items[2][1:])
 					sock.write(self.marks[idx])
-					del self.marks[idx]
 				else:
 					self.read_next_line()
 					self.handle_data()
@@ -216,6 +236,7 @@ class Handler:
 					adds.append(path)
 				if path not in self.files:
 					self.files.append(path)
+				change = True
 			else:
 				self.unread_line = True
 				break
@@ -223,8 +244,11 @@ class Handler:
 			if not len(self.line):
 				break
 
+		if not change:
+			# darcs does not support empty commits
+			return
 		for i in adds:
-			os.system("darcs add %s" % i)
+			self.invoke_add(i)
 		sock = subprocess.Popen(["darcs", "record", "--ignore-times", "-a", "--pipe"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 		buf = [self.date, self.ident, self.short, self.long]
 		sock.stdin.write("\n".join(buf))
@@ -232,6 +256,16 @@ class Handler:
 		self.log("Recording :%s:\n%s" % (self.mark_num, sock.stdout.read()))
 		sock.stdout.close()
 
+		for src, path in symlinks:
+			# symlink does not do what we want if path is
+			# already there
+			if os.path.exists(path):
+				# rmtree() does not work on symlinks
+				if os.path.islink(path):
+					os.remove(path)
+				else:
+					shutil.rmtree(path)
+			os.symlink(src, path)
 		if self.options.export_marks:
 			# yeah, an xml parser would be better, but
 			# should we mess with encodings just because of
@@ -243,19 +277,24 @@ class Handler:
 			self.export_marks.append(":%s %s" % (self.mark_num, hash))
 
 	def handle_progress(self, s):
-		print "progress [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s.strip())
+		print "import progress [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s.strip())
 		sys.stdout.flush()
 
 	def handle_opts(self):
 		# Option Parser
 		usage="%prog [options]"
 		opp = optparse.OptionParser(usage=usage)
+		opp.set_defaults(symhack=False)
 		opp.add_option("--import-marks", metavar="IFILE",
 			help="read state for incremental imports from IFILE")
 		opp.add_option("--export-marks", metavar="OFILE",
 			help="write state for incremental imports to OFILE")
 		opp.add_option("--logfile", metavar="L",
 			help="log file which contains the output of external programs invoked during the conversion")
+		opp.add_option("--symhack", action="store_true", dest="symhack",
+			help="Do not error out when a symlink would be created, just create it in the workdir")
+		opp.add_option("--progress", metavar="P",
+			help="insert progress statements after every n commit [default: 100]")
 		(self.options, args) = opp.parse_args()
 
 		if self.options.logfile:
@@ -263,6 +302,11 @@ class Handler:
 		else:
 			logfile = "_darcs/import.log"
 		self.logsock = open(os.path.abspath(logfile), "a")
+
+		if self.options.progress:
+			self.prognum = int(self.options.progress)
+		else:
+			self.prognum = 0
 	
 	def log(self, s):
 		self.logsock.write("[%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s))
@@ -290,6 +334,7 @@ class Handler:
 		self.handle_opts()
 		self.handle_import_marks()
 
+		commitcount = 0
 		while not self.eof:
 			self.read_next_line()
 			if not len(self.line[:-1]):
@@ -298,6 +343,9 @@ class Handler:
 				self.handle_blob()
 			elif self.line.startswith("commit"):
 				self.handle_commit()
+				commitcount += 1
+				if self.prognum != 0 and commitcount % self.prognum == 0:
+					self.handle_progress("%d patches" % commitcount)
 			elif self.line.startswith("tag"):
 				self.handle_tag()
 			elif self.line.startswith("reset"):
diff --git a/exporters/darcs/darcs-fast-import.txt b/exporters/darcs/darcs-fast-import.txt
index 09c7b1e..a7f2a12 100644
--- a/exporters/darcs/darcs-fast-import.txt
+++ b/exporters/darcs/darcs-fast-import.txt
@@ -33,3 +33,25 @@ as well, via the --import-marks / --export-marks switches.
 --logfile::
 	The output of external commands are redirected to a log file. You can
 	specify the path of that file with this parameter.
+
+--symhack::
+	Enable hack for symbolic links. darcs add does not handle them
+	but in case they are just added, we can create them in the working
+	directory. This can be handy in case for example the symbolic link is in
+	a subdirectory of the project and you don't even care about that
+	subdirectory. So the hack can be useful, but be extremely careful when
+	you use it.
+
+--progress=<n>::
+	Insert progress statements after every <n> created patches. The
+	default is not to print anything as progress info is usually provided by
+	the exporter. Use this option in case the exporter does not have such a
+	switch but you still want to get some feedback.
+
+== EXIT CODES
+
+The exit code is:
+
+* 0 on success
+* 1 on unhandled exception
+* 2 in case the stream would try to let the importer create a symlink
diff --git a/exporters/darcs/git-darcs b/exporters/darcs/git-darcs
index 2272cd3..18455a2 100755
--- a/exporters/darcs/git-darcs
+++ b/exporters/darcs/git-darcs
@@ -32,20 +32,20 @@ add()
 	shift
 	if ! [ -n "$name" -a -n "$location" ]; then
 		echo "Usage: git darcs add name location [darcs-fast-export options]"
-		exit
+		return 1
 	fi
 	if git remote show |grep -q $name; then
 		echo "There is already a remote with that name"
-		exit
+		return 1
 	fi
 	if [ -n "$(git config git-darcs.$name.location)" ]; then
 		echo "There is already a darcs repo with that name"
-		exit
+		return 1
 	fi
 	repo=$location/_darcs
 	if [ ! -d $repo ] && ! wget --quiet --spider $repo; then
 		echo "Remote is not a darcs repository"
-		exit
+		return 1
 	fi
 	git config git-darcs.$name.location $location
 	echo "Darcs repo $name added. You can fetch it with 'git darcs fetch $name'"
@@ -60,7 +60,7 @@ get_location()
 	l=$(git config git-darcs.$remote.location)
 	if [ -z "$l" ]; then
 		echo "Cannot find darcs remote with name '$remote'." >&2
-		exit
+		return 1
 	fi
 	echo $l
 }
@@ -71,12 +71,12 @@ fetch()
 	shift
 	if ! [ -n "$remote" -a -z "$*" ]; then
 		echo "Usage: git darcs fetch reponame"
-		exit
+		return 1
 	fi
-	location=$(get_location $remote)
+	location=$(get_location $remote) || return $?
 	git_map=$git_dir/darcs-git/$remote-git-map
 	darcs_map=$git_dir/darcs-git/$remote-darcs-map
-	common_opts="--working $git_dir/darcs-git/repo --logfile $git_dir/darcs-git/fetch.log --git-branch=darcs/$remote"
+	common_opts="--working $git_dir/darcs-git/repo --logfile $git_dir/darcs-git/fetch.log --git-branch=refs/remotes/darcs/$remote"
         dfe_opts=$(git config git-darcs.$remote.darcs-fast-export-options)
 	pre_fetch="$(git config git-darcs.$remote.pre-fetch)"
 	if [ -n "$pre_fetch" ]; then
@@ -90,20 +90,20 @@ fetch()
 			git fast-import --export-marks=$git_map
 	elif [ -f $git_map -a -f $darcs_map ]; then
 		echo "Updating remote $remote"
-		old_rev=$(git rev-parse darcs/$remote)
+		old_rev=$(git rev-parse refs/remotes/darcs/$remote)
 		darcs-fast-export --import-marks=$darcs_map --export-marks=$darcs_map $common_opts $dfe_opts $location | \
 			git fast-import --quiet --import-marks=$git_map --export-marks=$git_map
-		new_rev=$(git rev-parse darcs/$remote)
+		new_rev=$(git rev-parse refs/remotes/darcs/$remote)
 		if [ "$old_rev" != "$new_rev" ]; then
 			echo "Fetched the following updates:"
 			git shortlog $old_rev..$new_rev
 		else
 			echo "Nothing fetched."
-			exit
+			return 0
 		fi
 	else
 		echo "One of the mapfiles is missing! Something went wrong!"
-		exit
+		return 1
 	fi
 	post_fetch="$(git config git-darcs.$remote.post-fetch)"
 	if [ -n "$post_fetch" ]; then
@@ -117,15 +117,15 @@ pull()
 	shift
 	if ! [ -n "$remote" -a -z "$*" ]; then
 		echo "Usage: git darcs pull reponame"
-		exit
+		return 1
 	fi
-	fetch $remote
+	fetch $remote || return $?
 	# see if we need to merge or rebase
 	branch=$(git symbolic-ref HEAD|sed 's|.*/||')
 	if [ "$(git config branch.$branch.rebase)" = "true" ]; then
-		git rebase darcs/$remote
+		git rebase refs/remotes/darcs/$remote
 	else
-		git merge darcs/$remote
+		git merge refs/remotes/darcs/$remote
 	fi
 }
 
@@ -135,34 +135,34 @@ push()
 	shift
 	if ! [ -n "$remote" -a -z "$*" ]; then
 		echo "Usage: git darcs push reponame"
-		exit
+		return 1
 	fi
-	location=$(get_location $remote)
-	if [ -n "$(git rev-list --left-right HEAD...darcs/$remote | sed -n '/^>/ p')" ]; then
+	location=$(get_location $remote) || return $?
+	if [ -n "$(git rev-list --left-right HEAD...refs/remotes/darcs/$remote | sed -n '/^>/ p')" ]; then
 		echo "HEAD is not a strict child of $remote, cannot push. Merge first"
-		exit
+		return 1
 	fi
-	if [ -z "$(git rev-list --left-right HEAD...darcs/$remote | sed -n '/^</ p')" ]; then
+	if [ -z "$(git rev-list --left-right HEAD...refs/remotes/darcs/$remote | sed -n '/^</ p')" ]; then
 		echo "Nothing to push. Commit something first"
-		exit
+		return 1
 	fi
 	git_map=$git_dir/darcs-git/$remote-git-map
 	darcs_map=$git_dir/darcs-git/$remote-darcs-map
 	if [ ! -f $git_map -o ! -f $darcs_map ]; then
 		echo "We do not have refmapping yet. Then how can I push?"
-		exit
+		return 1
 	fi
 	pre_push="$(git config git-darcs.$remote.pre-push)"
 	if [ -n "$pre_push" ]; then
 		$pre_push
 	fi
 	echo "Pushing the following updates:"
-	git shortlog darcs/$remote..
+	git shortlog refs/remotes/darcs/$remote..
 	git fast-export --import-marks=$git_map --export-marks=$git_map HEAD | \
 		(cd $location; darcs-fast-import --import-marks=$darcs_map --export-marks=$darcs_map \
 		--logfile $git_dir/darcs-git/push.log)
 	if [ $? == 0 ]; then
-		git update-ref darcs/$remote HEAD
+		git update-ref refs/remotes/darcs/$remote HEAD
 		post_push="$(git config git-darcs.$remote.post-push)"
 		if [ -n "$post_push" ]; then
 			$post_push
@@ -176,18 +176,18 @@ list()
 	if [ -z "$*" ]
 	then
 		git config -l | sed -n -e '/git-darcs\..*/ {s/git-darcs\.//; s/\.location=.*//p}'
-		exit
+		return 0
 	elif [ "$#" -eq 1 ]
 	then
 		case $1 in
 			-v|--verbose)
 				git config -l | sed -n -e '/git-darcs\..*/ {s/git-darcs\.//; s/\.location=/\t/p}'
-				exit
+				return 0
 				;;
 		esac
 	fi
 	echo "Usage: git darcs list [-v|--verbose]"
-	exit 1
+	return 1
 }
 
 # Find the darcs commit(s) supporting a git SHA1 prefix
@@ -198,9 +198,9 @@ find_darcs()
 	if [ -z "$sha1" -o -n "$*" ]
 	then
 		echo "Usage: git darcs find-darcs <sha1-prefix>"
-		exit 1
+		return 1
 	fi
-	for remote in $git_dir/darcs/*
+	for remote in $(git for-each-ref --format='%(refname)' refs/remotes/darcs)
 	do
 		remote=`basename $remote`
 		git_map=$git_dir/darcs-git/$remote-git-map
@@ -208,7 +208,7 @@ find_darcs()
 		if [ ! -f $git_map -o ! -f $darcs_map ]
 		then
 			echo "Missing mappings for remote $remote"
-			exit 1
+			return 1
 		fi
 		for row in `sed -n -e "/:.* $sha1.*/ s/[^ ]*/&/p" $git_map`
 		do
@@ -225,9 +225,9 @@ find_git()
 	if [ -z "$patch" -o -n "$*" ]
 	then
 		echo "Usage: git darcs find-git <patch-prefix>"
-		exit 1
+		return 1
 	fi
-	for remote in $git_dir/darcs/*
+	for remote in $(git for-each-ref --format='%(refname)' refs/remotes/darcs)
 	do
 		remote=`basename $remote`
 		git_map=$git_dir/darcs-git/$remote-git-map
@@ -235,7 +235,7 @@ find_git()
 		if [ ! -f $git_map -o ! -f $darcs_map ]
 		then
 			echo "Missing mappings for remote $remote"
-			exit 1
+			return 1
 		fi
 		for row in `sed -n -e "/:.* $patch.*/ s/[^ ]*/&/p" $darcs_map`
 		do
@@ -247,7 +247,7 @@ find_git()
 git rev-parse 2> /dev/null
 if [ $? != 0 ]; then
 	echo "Must be inside a git repository to work"
-	exit
+	exit 1
 fi
 
 git_dir=$(git rev-parse --git-dir)
@@ -270,7 +270,7 @@ case $command in
 	*)
 		echo "Usage: git darcs [COMMAND] [OPTIONS]"
 		echo "Commands: add, push, fetch, pull, list, find-darcs, find-git"
-		exit
+		exit 1
 		;;
 esac
 
diff --git a/exporters/darcs/t/lib.sh b/exporters/darcs/t/lib.sh
index dcdc0f8..095f9ef 100644
--- a/exporters/darcs/t/lib.sh
+++ b/exporters/darcs/t/lib.sh
@@ -191,6 +191,7 @@ create_git()
 	mkdir -p $1
 	cd $1
 	git init $2
+	git commit --allow-empty -m 'root commit'
 	echo A > file
 	git add file
 	git commit -a -m A
@@ -224,6 +225,23 @@ third line" | git commit -a -F -
 	git commit -a -m "add empty file"
 	rm file3
 	git commit -a -m "remove file"
+	# now add back 'file' with its old conents, so the mark gets
+	# reused
+	echo f > file
+	git add file
+	git commit -a -m "file: other -> f"
+	# this is a boring file for Darcs
+	touch foo.pyc
+	git add foo.pyc
+	git commit -a -m "boring file"
+	# replace an uppercase file to a lowercase one
+	echo SPAM > SPAM
+	git add SPAM
+	git commit -a -m SPAM
+	rm SPAM
+	echo spam > spam
+	git add spam
+	git commit -a -m "SPAM -> spam"
 	cd ..
 }
 
diff --git a/exporters/darcs/t/testimport-gitsymlink.sh b/exporters/darcs/t/testimport-gitsymlink.sh
new file mode 100644
index 0000000..100c583
--- /dev/null
+++ b/exporters/darcs/t/testimport-gitsymlink.sh
@@ -0,0 +1,45 @@
+. ./lib.sh
+
+create_git test
+cd test
+# add two dirs with the some contents, then remove the second
+# and make it a symlink to the first
+mkdir dira
+echo blabla > dira/file
+echo blablabla > dira/file2
+mkdir dirb
+touch dirb/file
+touch dirb/file2
+git add dira dirb
+git commit -a -m "add dira/dirb"
+rm -rf dirb
+ln -s dira dirb
+git add dirb
+git commit -a -m "change a dir to a symlink"
+cd ..
+
+rm -rf test.darcs
+mkdir test.darcs
+cd test.darcs
+darcs init
+cd ..
+(cd test; git fast-export --progress=2 HEAD) | (cd test.darcs; darcs-fast-import)
+# we *do* want this to fail, but with error code 2. that means that we
+# detected that symlinks are not supported and the user does not get a
+# meaningless exception
+if [ $? != 2 ]; then
+	exit 1
+fi
+
+# now try with the symhack option
+rm -rf test.darcs
+mkdir test.darcs
+cd test.darcs
+darcs init
+cd ..
+(cd test; git fast-export --progress=2 HEAD) | (cd test.darcs; darcs-fast-import --symhack)
+if [ $? != 0 ]; then
+	exit 1
+fi
+diff_importgit test
+exit $?
diff --git a/exporters/darcs/x2d b/exporters/darcs/x2d
index 61c66ef..398103d 100755
--- a/exporters/darcs/x2d
+++ b/exporters/darcs/x2d
@@ -62,6 +62,12 @@ case $format in
 		;;
 esac
 
+common_opts=""
+while [ -n "$2" ]
+do
+	common_opts="$common_opts $1"
+	shift 1
+done
 origin="$1"
 shift 1
 
@@ -77,7 +83,7 @@ fmark="$origin.darcs/_darcs/fast-import/ffi-marks"
 mkdir -p $origin.darcs
 cd $origin.darcs
 
-common_opts="--logfile $origin.darcs/_darcs/fast-import/log"
+common_opts="$common_opts --logfile $origin.darcs/_darcs/fast-import/log"
 pypath="/$(python -c 'from distutils import sysconfig; print sysconfig.get_python_lib()[1:]')/"
 
 if [ ! -f $dmark ]; then
diff --git a/exporters/darcs/x2d.txt b/exporters/darcs/x2d.txt
index eb2ec34..25ed6bb 100644
--- a/exporters/darcs/x2d.txt
+++ b/exporters/darcs/x2d.txt
@@ -6,7 +6,7 @@ x2d - convert git, bzr or hg repos to a darcs one using fast-export
 
 == SYNOPSIS
 
-x2d -f <format> <otherrepo>
+x2d -f <format> [<importoptions>] <otherrepo>
 
 == DESCRIPTION
 
@@ -24,3 +24,5 @@ importer's standard input.
 -f <format>::
 	Specify the format of the source repo. Currently supported sources are
 	git, bzr and hg. Incremental conversion is supported for all of them.
+
+The rest of the options is directly passed to darcs-fast-import.