Belatedly added filename_mapping (should have been added with 0.7.4).

Changed various files to fix --exclude-filelist problems when source
remote.


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@85 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

7 files changed, 154 insertions, 23 deletions

diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py
index 94afea3..54cf527 100644
--- a/rdiff-backup/rdiff_backup/connection.py
+++ b/rdiff-backup/rdiff_backup/connection.py
@@ -1,3 +1,4 @@
+from __future__ import generators
 execfile("rdiff.py")
 import types, os, tempfile, cPickle, shutil, traceback
 
@@ -487,3 +488,10 @@ class VirtualFile:
 
 	def close(self):
 		return self.connection.VirtualFile.closebyid(self.id)
+
+	def __iter__(self):
+		"""Iterates lines in file, like normal iter(file) behavior"""
+		while 1:
+			line = self.readline()
+			if not line: break
+			yield line
diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py
index cae6db3..bd679de 100644
--- a/rdiff-backup/rdiff_backup/selection.py
+++ b/rdiff-backup/rdiff_backup/selection.py
@@ -157,7 +157,7 @@ class Select:
 			if result is not None: return result
 		return 1
 
-	def ParseArgs(self, argtuples):
+	def ParseArgs(self, argtuples, filelists):
 		"""Create selection functions based on list of tuples
 
 		The tuples have the form (option string, additional argument)
@@ -169,6 +169,7 @@ class Select:
 		information is sent over the link.
 
 		"""
+		filelists_index = 0
 		try:
 			for opt, arg in argtuples:
 				if opt == "--exclude":
@@ -176,19 +177,22 @@ class Select:
 				elif opt == "--exclude-device-files":
 					self.add_selection_func(self.devfiles_get_sf())
 				elif opt == "--exclude-filelist":
-					self.add_selection_func(self.filelist_get_sf(arg[1],
-																 0, arg[0]))
+					self.add_selection_func(self.filelist_get_sf(
+						filelists[filelists_index], 0, arg))
+					filelists_index += 1
 				elif opt == "--exclude-regexp":
 					self.add_selection_func(self.regexp_get_sf(arg, 0))
 				elif opt == "--include":
 					self.add_selection_func(self.glob_get_sf(arg, 1))
 				elif opt == "--include-filelist":
-					self.add_selection_func(self.filelist_get_sf(arg[1],
-																 1, arg[0]))
+					self.add_selection_func(self.filelist_get_sf(
+						filelists[filelists_index], 1, arg))
+					filelists_index += 1
 				elif opt == "--include-regexp":
 					self.add_selection_func(self.regexp_get_sf(arg, 1))
 				else: assert 0, "Bad selection option %s" % opt
 		except SelectError, e: self.parse_catch_error(e)
+		assert filelists_index == len(filelists)
 
 		self.parse_last_excludes()
 		self.parse_rbdir_exclude()
diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py
index 94afea3..54cf527 100644
--- a/rdiff-backup/src/connection.py
+++ b/rdiff-backup/src/connection.py
@@ -1,3 +1,4 @@
+from __future__ import generators
 execfile("rdiff.py")
 import types, os, tempfile, cPickle, shutil, traceback
 
@@ -487,3 +488,10 @@ class VirtualFile:
 
 	def close(self):
 		return self.connection.VirtualFile.closebyid(self.id)
+
+	def __iter__(self):
+		"""Iterates lines in file, like normal iter(file) behavior"""
+		while 1:
+			line = self.readline()
+			if not line: break
+			yield line
diff --git a/rdiff-backup/src/filename_mapping.py b/rdiff-backup/src/filename_mapping.py
new file mode 100644
index 0000000..78c37d9
--- /dev/null
+++ b/rdiff-backup/src/filename_mapping.py
@@ -0,0 +1,93 @@
+execfile("selection.py")
+import re
+
+#######################################################################
+#
+# filename_mapping - used to coordinate related filenames
+#
+# For instance, some source filenames may contain characters not
+# allowed on the mirror end.  Also, if a source filename is very long
+# (say 240 characters), the extra characters added to related
+# increments may put them over the usual 255 character limit.
+#
+
+class FilenameMapping:
+	"""Contains class methods which coordinate related filenames"""
+	max_filename_length = 255
+
+	# If true, enable character quoting, and set characters making
+	# regex-style range.
+	chars_to_quote = None
+
+	# These compiled regular expressions are used in quoting and unquoting
+	chars_to_quote_regexp = None
+	unquoting_regexp = None
+
+	# Use given char to quote.  Default is set in Globals.
+	quoting_char = None
+
+	def set_init_quote_vals(cls):
+		"""Set quoting value from Globals on all conns"""
+		for conn in Globals.connections:
+			conn.FilenameMapping.set_init_quote_vals_local()
+		
+	def set_init_quote_vals_local(cls):
+		"""Set value on local connection, initialize regexps"""
+		cls.chars_to_quote = Globals.chars_to_quote
+		if len(Globals.quoting_char) != 1:
+			Log.FatalError("Expected single character for quoting char,"
+						   "got '%s' instead" % (Globals.quoting_char,))
+		cls.quoting_char = Globals.quoting_char
+		cls.init_quoting_regexps()
+
+	def init_quoting_regexps(cls):
+		"""Compile quoting regular expressions"""
+		try:
+			cls.chars_to_quote_regexp = \
+			   re.compile("[%s%s]" % (cls.chars_to_quote,
+									  cls.quoting_char), re.S)
+			cls.unquoting_regexp = \
+			   re.compile("%s[0-9]{3}" % cls.quoting_char, re.S)
+		except re.error:
+			Log.FatalError("Error '%s' when processing char quote list %s" %
+						   (re.error, cls.chars_to_quote))
+
+	def quote(cls, path):
+		"""Return quoted version of given path
+
+		Any characters quoted will be replaced by the quoting char and
+		the ascii number of the character.  For instance, "10:11:12"
+		would go to "10;05811;05812" if ":" were quoted and ";" were
+		the quoting character.
+
+		"""
+		return cls.chars_to_quote_regexp.sub(cls.quote_single, path)
+
+	def quote_single(cls, match):
+		"""Return replacement for a single character"""
+		return "%s%03d" % (cls.quoting_char, ord(match.group()))
+
+	def unquote(cls, path):
+		"""Return original version of quoted filename"""
+		return cls.unquoting_regexp.sub(cls.unquote_single, path)
+
+	def unquote_single(cls, match):
+		"""Unquote a single quoted character"""
+		assert len(match.group()) == 4
+		return chr(int(match.group()[1:]))
+
+	def get_quoted_dir_children(cls, rpath):
+		"""For rpath directory, return list of quoted children in dir"""
+		if not rpath.isdir(): return []
+		dir_pairs = [(cls.unquote(filename), filename)
+					 for filename in rpath.listdir()]
+		dir_pairs.sort() # sort by real index, not quoted part
+		child_list = []
+		for unquoted, filename in dir_pairs:
+			childrp = rpath.append(unquoted)
+			childrp.quote_path()
+			child_list.append(childrp)
+		return child_list
+
+MakeClass(FilenameMapping)
+
diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py
index 2f0b0ca..58c43d4 100644
--- a/rdiff-backup/src/globals.py
+++ b/rdiff-backup/src/globals.py
@@ -192,12 +192,19 @@ class Globals:
 		else: cls.__dict__[name] = re.compile(re_string)
 	postset_regexp_local = classmethod(postset_regexp_local)
 
-	def set_select(cls, dsrpath, tuplelist, quote_mode = None):
-		"""Initialize select object using tuplelist"""
+	def set_select(cls, dsrpath, tuplelist, quote_mode, *filelists):
+		"""Initialize select object using tuplelist
+
+		Note that each list in filelists must each be passed as
+		separate arguments, so each is recognized as a file by the
+		connection.  Otherwise we will get an error because a list
+		containing files can't be pickled.
+
+		"""
 		if dsrpath.source:
 			cls.select_source = Select(dsrpath, quote_mode)
-			cls.select_source.ParseArgs(tuplelist)
+			cls.select_source.ParseArgs(tuplelist, filelists)
 		else:
 			cls.select_mirror = Select(dsrpath, quote_mode)
-			cls.select_mirror.ParseArgs(tuplelist)
+			cls.select_mirror.ParseArgs(tuplelist, filelists)
 	set_select = classmethod(set_select)
diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py
index c130e56..72d93ce 100755
--- a/rdiff-backup/src/main.py
+++ b/rdiff-backup/src/main.py
@@ -14,6 +14,7 @@ class Main:
 		self.remote_cmd, self.remote_schema = None, None
 		self.force = None
 		self.select_opts, self.select_mirror_opts = [], []
+		self.select_files = []
 
 	def parse_cmdlineoptions(self):
 		"""Parse argument list and set global preferences"""
@@ -37,8 +38,8 @@ class Main:
 			  "restore-as-of=", "resume", "resume-window=", "server",
 			  "terminal-verbosity=", "test-server", "verbosity",
 			  "version", "windows-mode", "windows-time-format"])
-		except getopt.error:
-			self.commandline_error("Error parsing commandline options")
+		except getopt.error, e:
+			self.commandline_error("Bad commandline options: %s" % str(e))
 
 		for opt, arg in optlist:
 			if opt == "-b" or opt == "--backup-mode": self.action = "backup"
@@ -55,20 +56,24 @@ class Main:
 			elif opt == "--exclude-device-files":
 				self.select_opts.append((opt, arg))
 			elif opt == "--exclude-filelist":
-				self.select_opts.append((opt, (arg, sel_fl(arg))))
+				self.select_opts.append((opt, arg))
+				self.select_files.append(sel_fl(arg))
 			elif opt == "--exclude-filelist-stdin":
 				self.select_opts.append(("--exclude-filelist",
-										 ("standard input", sys.stdin)))
+										 "standard input"))
+				self.select_files.append(sys.stdin)
 			elif opt == "--exclude-mirror":
 				self.select_mirror_opts.append(("--exclude", arg))
 			elif opt == "--exclude-regexp": self.select_opts.append((opt, arg))
 			elif opt == "--force": self.force = 1
 			elif opt == "--include": self.select_opts.append((opt, arg))
 			elif opt == "--include-filelist":
-				self.select_opts.append((opt, (arg, sel_fl(arg))))
+				self.select_opts.append((opt, arg))
+				self.select_files.append(sel_fl(arg))
 			elif opt == "--include-filelist-stdin":
 				self.select_opts.append(("--include-filelist",
-										 ("standard input", sys.stdin)))
+										 "standard input"))
+				self.select_files.append(sys.stdin)
 			elif opt == "--include-regexp":
 				self.select_opts.append((opt, arg))
 			elif opt == "-l" or opt == "--list-increments":
@@ -227,7 +232,8 @@ rdiff-backup with the --force option if you want to mirror anyway.""" %
 
 	def backup_init_select(self, rpin, rpout):
 		"""Create Select objects on source and dest connections"""
-		rpin.conn.Globals.set_select(DSRPath(1, rpin), self.select_opts)
+		rpin.conn.Globals.set_select(DSRPath(1, rpin), self.select_opts,
+									 None, *self.select_files)
 		rpout.conn.Globals.set_select(DSRPath(None, rpout),
 									  self.select_mirror_opts, 1)
 
@@ -378,8 +384,9 @@ Try restoring from an increment file (the filenames look like
 		the restore operation isn't.
 
 		"""
-		Globals.set_select(DSRPath(1, rpin), self.select_mirror_opts) 
-		Globals.set_select(DSRPath(None, rpout), self.select_opts)
+		Globals.set_select(DSRPath(1, rpin), self.select_mirror_opts, None)
+		Globals.set_select(DSRPath(None, rpout), self.select_opts, None,
+						   *self.select_files)
 
 	def restore_get_root(self, rpin):
 		"""Return (mirror root, index) and set the data dir
diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py
index cae6db3..bd679de 100644
--- a/rdiff-backup/src/selection.py
+++ b/rdiff-backup/src/selection.py
@@ -157,7 +157,7 @@ class Select:
 			if result is not None: return result
 		return 1
 
-	def ParseArgs(self, argtuples):
+	def ParseArgs(self, argtuples, filelists):
 		"""Create selection functions based on list of tuples
 
 		The tuples have the form (option string, additional argument)
@@ -169,6 +169,7 @@ class Select:
 		information is sent over the link.
 
 		"""
+		filelists_index = 0
 		try:
 			for opt, arg in argtuples:
 				if opt == "--exclude":
@@ -176,19 +177,22 @@ class Select:
 				elif opt == "--exclude-device-files":
 					self.add_selection_func(self.devfiles_get_sf())
 				elif opt == "--exclude-filelist":
-					self.add_selection_func(self.filelist_get_sf(arg[1],
-																 0, arg[0]))
+					self.add_selection_func(self.filelist_get_sf(
+						filelists[filelists_index], 0, arg))
+					filelists_index += 1
 				elif opt == "--exclude-regexp":
 					self.add_selection_func(self.regexp_get_sf(arg, 0))
 				elif opt == "--include":
 					self.add_selection_func(self.glob_get_sf(arg, 1))
 				elif opt == "--include-filelist":
-					self.add_selection_func(self.filelist_get_sf(arg[1],
-																 1, arg[0]))
+					self.add_selection_func(self.filelist_get_sf(
+						filelists[filelists_index], 1, arg))
+					filelists_index += 1
 				elif opt == "--include-regexp":
 					self.add_selection_func(self.regexp_get_sf(arg, 1))
 				else: assert 0, "Bad selection option %s" % opt
 		except SelectError, e: self.parse_catch_error(e)
+		assert filelists_index == len(filelists)
 
 		self.parse_last_excludes()
 		self.parse_rbdir_exclude()