Various changes for 0.12.0

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@332 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

6 files changed, 141 insertions, 31 deletions

diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py
index 04ce247..5a03fb5 100644
--- a/rdiff-backup/rdiff_backup/Main.py
+++ b/rdiff-backup/rdiff_backup/Main.py
@@ -30,7 +30,7 @@ import Globals, Time, SetConnections, selection, robust, rpath, \
 action = None
 remote_cmd, remote_schema = None, None
 force = None
-select_opts, select_mirror_opts = [], []
+select_opts = []
 select_files = []
 
 def parse_cmdlineoptions(arglist):
@@ -84,8 +84,6 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--exclude-globbing-filelist":
 			select_opts.append((opt, arg))
 			select_files.append(sel_fl(arg))
-		elif opt == "--exclude-mirror":
-			select_mirror_opts.append(("--exclude", arg))
 		elif (opt == "--exclude-other-filesystems" or
 			  opt == "--exclude-regexp" or
 			  opt == "--exclude-special-files"): select_opts.append((opt, arg))
@@ -399,11 +397,23 @@ def restore_common(rpin, target, time):
 	restore_check_backup_dir(mirror_root)
 	mirror = mirror_root.new_index(index)
 	inc_rpath = datadir.append_path('increments', index)
-	restore_init_select(mirror_root, target)
+	restore_set_select(mirror_root, target)
 	restore_start_log(rpin, target, time)
 	restore.Restore(mirror, inc_rpath, target, time)
 	Log("Restore ended", 4)
 
+def restore_set_select(mirror_rp, target):
+	"""Set the selection iterator on mirror side from command line args
+
+	Here we set the selector on the mirror side, because that is where
+	we will be filtering, but the pathnames are relative to the target
+	directory.
+
+	"""
+	if select_opts: 
+		mirror_rp.conn.restore.MirrorStruct.set_mirror_select(
+			target, select_opts, *select_files)
+
 def restore_start_log(rpin, target, time):
 	"""Open restore log file, log initial message"""
 	try: Log.open_logfile(datadir.append("restore.log"))
@@ -446,19 +456,6 @@ def restore_check_backup_dir(rpin):
 		"Rerun rdiff-backup with --check-destination-dir option to revert"
 		"directory to state before unsuccessful session." % (rpin.path,))
 
-def restore_init_select(rpin, rpout):
-	"""Initialize Select
-
-	Unlike the backup selections, here they are on the local
-	connection, because the backup operation is pipelined in a way
-	the restore operation isn't.
-
-	"""
-	restore._select_mirror = selection.Select(rpin)
-	restore._select_mirror.ParseArgs(select_mirror_opts, [])
-	restore._select_mirror.parse_rbdir_exclude()
-	restore._select_source = selection.Select(rpout)
-
 def restore_get_root(rpin):
 	"""Return (mirror root, index) and set the data dir
 
diff --git a/rdiff-backup/rdiff_backup/Security.py b/rdiff-backup/rdiff_backup/Security.py
index b02d335..83ddcf2 100644
--- a/rdiff-backup/rdiff_backup/Security.py
+++ b/rdiff-backup/rdiff_backup/Security.py
@@ -148,8 +148,8 @@ def set_allowed_requests(sec_level):
 	if Globals.server:
 		allowed_requests.extend(
 			["SetConnections.init_connection_remote",
-			 "Log.setverbosity",
-			 "Log.setterm_verbosity",
+			 "log.Log.setverbosity",
+			 "log.Log.setterm_verbosity",
 			 "Time.setprevtime_local",
 			 "FilenameMapping.set_init_quote_vals_local",
 			 "Globals.postset_regexp_local",
diff --git a/rdiff-backup/rdiff_backup/backup.py b/rdiff-backup/rdiff_backup/backup.py
index 59663c6..89d7bea 100644
--- a/rdiff-backup/rdiff_backup/backup.py
+++ b/rdiff-backup/rdiff_backup/backup.py
@@ -166,9 +166,11 @@ class DestinationStruct:
 					yield iterfile.RORPIterFlushRepeat
 			else:
 				index = src_rorp and src_rorp.index or dest_rorp.index
-				cls.CCPP.flag_changed(index)
-				yield cls.get_one_sig(dest_base_rpath, index,
+				sig = cls.get_one_sig(dest_base_rpath, index,
 									  src_rorp, dest_rorp)
+				if sig:
+					cls.CCPP.flag_changed(index)
+					yield sig
 
 	def get_one_sig(cls, dest_base_rpath, index, src_rorp, dest_rorp):
 		"""Return a signature given source and destination rorps"""
@@ -180,9 +182,11 @@ class DestinationStruct:
 			dest_sig = dest_rorp.getRORPath()
 			if dest_rorp.isreg():
 				dest_rp = dest_base_rpath.new_index(index)
-				if dest_rp.isreg(): # otherwise file has changed type from reg
-					dest_sig.setfile(Rdiff.get_signature(dest_rp))
-				else: dest_sig = dest_rp.getRORPath()
+				if not dest_rp.isreg():
+					log.ErrorLog.write_if_open("UpdateError", dest_rp,
+						"File changed from regular file before signature")
+					return None
+				dest_sig.setfile(Rdiff.get_signature(dest_rp))
 		else: dest_sig = rpath.RORPath(index)
 		return dest_sig			
 
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 27c6f72..8c0b253 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -94,6 +94,7 @@ def ListAtTime(mirror_rp, inc_rp, time):
 
 class MirrorStruct:
 	"""Hold functions to be run on the mirror side"""
+	_select = None # If selection command line arguments given, use Select here
 	def set_mirror_and_rest_times(cls, restore_to_time):
 		"""Set global variabels _mirror_time and _rest_time on mirror conn"""
 		global _mirror_time, _rest_time
@@ -148,15 +149,29 @@ class MirrorStruct:
 		Usually we can use the metadata file, but if this is
 		unavailable, we may have to build it from scratch.
 
+		If the cls._select object is set, use it to filter out the
+		unwanted files from the metadata_iter.
+
 		"""
 		if rest_time is None: rest_time = _rest_time
 		metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir,
 				 rest_time, restrict_index = cls.mirror_base.index)
-		if metadata_iter: return metadata_iter
-		if require_metadata: log.Log.FatalError("Mirror metadata not found")
-		log.Log("Warning: Mirror metadata not found, "
-				"reading from directory", 2)
-		return cls.get_rorp_iter_from_rf(cls.root_rf)
+		if metadata_iter: rorp_iter = metadata_iter
+		elif require_metadata: log.Log.FatalError("Mirror metadata not found")
+		else:
+			log.Log("Warning: Mirror metadata not found, "
+					"reading from directory", 2)
+			rorp_iter = cls.get_rorp_iter_from_rf(cls.root_rf)
+
+		if cls._select:
+			rorp_iter = selection.FilterIter(cls._select, rorp_iter)
+		return rorp_iter
+
+	def set_mirror_select(cls, target_rp, select_opts, *filelists):
+		"""Initialize the mirror selection object"""
+		assert select_opts, "If no selection options, don't use selector"
+		cls._select = selection.Select(target_rp)
+		cls._select.ParseArgs(select_opts, filelists)
 
 	def get_rorp_iter_from_rf(cls, rf):
 		"""Recursively yield mirror rorps from rf"""
diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py
index d683650..9bf75d4 100644
--- a/rdiff-backup/rdiff_backup/rorpiter.py
+++ b/rdiff-backup/rdiff_backup/rorpiter.py
@@ -286,7 +286,7 @@ class IterTreeReducer:
 
 
 class ITRBranch:
-	"""Helper class for IterTreeReducer below
+	"""Helper class for IterTreeReducer above
 
 	There are five stub functions below: start_process, end_process,
 	branch_process, can_fast_process, and fast_process.  A class that
diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py
index 0fcc657..29b12e8 100644
--- a/rdiff-backup/rdiff_backup/selection.py
+++ b/rdiff-backup/rdiff_backup/selection.py
@@ -26,7 +26,7 @@ documentation on what this code does can be found on the man page.
 
 from __future__ import generators
 import re
-import FilenameMapping, robust, rpath, Globals, log
+import FilenameMapping, robust, rpath, Globals, log, rorpiter
 
 
 class SelectError(Exception):
@@ -195,6 +195,18 @@ class Select:
 				for rp in rec_func(new_rp, rec_func, sel_func):
 					yield rp
 
+	def FilterIter(self, rorp_iter):
+		"""Filter rorp_iter using Select below, removing excluded rorps"""
+		def getrpiter(rorp_iter):
+			"""Return rp iter by adding indicies of rorp_iter to self.rpath"""
+			for rorp in rorp_iter:
+				yield rpath.RPath(self.rpath.conn, self.rpath.base,
+								  rorp.index, rorp.data)
+
+		ITR = rorpiter.IterTreeReducer(FilterIterITRB, [self])
+		for rp in rp_iter: ITR(rp.index, rp)
+		ITR.Finish()
+
 	def Select(self, rp):
 		"""Run through the selection functions and return dominant val 0/1/2"""
 		for sf in self.selection_functions:
@@ -606,4 +618,86 @@ probably isn't what you meant.""" %
 		return res
 
 
+class FilterIter:
+	"""Filter rorp_iter using a Select object, removing excluded rorps"""
+	def __init__(self, select, rorp_iter):
+		"""Constructor
+
+		Input is the Select object to use and the iter of rorps to be
+		filtered.  The rorps will be converted to rps using the Select
+		base.
+
+		"""
+		self.rorp_iter = rorp_iter
+		self.base_rp = select.rpath
+		self.stored_rorps = []
+		self.ITR = rorpiter.IterTreeReducer(FilterIterITRB,
+											[select.Select, self.stored_rorps])
+		self.itr_finished = 0
+
+	def __iter__(self): return self
+
+	def next(self):
+		"""Return next object, or StopIteration"""
+		while not self.stored_rorps:
+			try: next_rorp = self.rorp_iter.next()
+			except StopIteration:
+				if self.itr_finished: raise
+				else:
+					self.ITR.Finish()
+					self.itr_finished = 1
+			else:
+				next_rp = rpath.RPath(self.base_rp.conn, self.base_rp.base,
+									  next_rorp.index, next_rorp.data)
+				self.ITR(next_rorp.index, next_rp, next_rorp)
+		return self.stored_rorps.pop(0)
+
+class FilterIterITRB(rorpiter.ITRBranch):
+	"""ITRBranch used in above FilterIter class
+
+	The reason this is necessary is because for directories sometimes
+	we don't know whether a rorp is excluded until we see what is in
+	the directory.
+
+	"""
+	def __init__(self, select, rorp_cache):
+		"""Initialize FilterIterITRB.  Called by IterTreeReducer.
+
+		select should be the relevant Select object used to test the
+		rps.  rorp_cache is the list rps should be appended to if they
+		aren't excluded.
+
+		"""
+		self.select, self.rorp_cache = select, rorp_cache
+		self.branch_excluded = None
+		self.base_queue = None # holds branch base while examining contents
+
+	def can_fast_process(self, index, next_rp, next_rorp):
+		return not next_rp.isdir()
+
+	def fast_process(self, index, next_rp, next_rorp):
+		"""For ordinary files, just append if select is positive"""
+		if self.branch_excluded: return
+		s = self.select(next_rp)
+		if s == 1:
+			if self.base_queue:
+				self.rorp_cache.append(self.base_queue)
+				self.base_queue = None
+			self.rorp_cache.append(next_rorp)
+		else: assert s == 0, "Unexpected select value %s" % (s,)
+
+	def start_process(self, index, next_rp, next_rorp):
+		s = self.select(next_rp)
+		if s == 0: self.branch_excluded = 1
+		elif s == 1: self.rorp_cache.append(next_rorp)
+		else:
+			assert s == 2, s
+			self.base_queue = next_rorp
+
+
+
+
+
+
+