Various changes for 0.12.0

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@332 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

10 files changed, 171 insertions, 53 deletions

diff --git a/rdiff-backup/CHANGELOG b/rdiff-backup/CHANGELOG
index 371640a..804c873 100644
--- a/rdiff-backup/CHANGELOG
+++ b/rdiff-backup/CHANGELOG
@@ -1,3 +1,19 @@
+New in v0.12.0 (2003/06/26)
+---------------------------
+
+Fixed (?) bug that caused crash when file changes type from regular
+file in middle of download (reported by Ty! Boyack).
+
+Failure to construct regular file in regression/restoration only
+causes warning, not fatal error.
+
+Removed --exclude-mirror option.  (Probably no one uses this, and it
+adds clutter.)
+
+--include and --exclude options should work now with restores, with
+some speed penalty.
+
+
 New in v0.11.5 (2003/06/20)
 ---------------------------
 
diff --git a/rdiff-backup/TODO b/rdiff-backup/TODO
index 38f6c40..54d091b 100644
--- a/rdiff-backup/TODO
+++ b/rdiff-backup/TODO
@@ -1,6 +1,4 @@
-Make restores tolerant of missing files
 
-Added --include/--exclude options for restoring
 
 ---------[ Medium term ]---------------------------------------
 
@@ -11,8 +9,6 @@ Look into security.py code, do some sort of security audit.
 Don't require increments.<date>.dir files to be setuid/setgid, or
 don't even have the backup files. (Andrew Bressen)
 
-Examine default settings with --windows-mode
-
 Look at Kent Borg's suggestion for restore options and digests.
 
 Add --list-files-changed-between or similar option, to list files that
@@ -24,7 +20,7 @@ Add --dry-run option (target for v1.1.x)
 
 Add # of increments option to --remove-older-than
 
-Make argument shortcut for cstream
+Make argument shortcut for cstream, or some other bandwidth limiter.
 
 Write configuration file, to make sure settings like --quoting-char,
 --windows-time-format, etc., don't change between sessions,
diff --git a/rdiff-backup/rdiff-backup.1 b/rdiff-backup/rdiff-backup.1
index 9680840..b6ecbe4 100644
--- a/rdiff-backup/rdiff-backup.1
+++ b/rdiff-backup/rdiff-backup.1
@@ -119,11 +119,6 @@ same rules as
 and
 .B --exclude.
 .TP
-.BI "--exclude-mirror " regexp
-Exclude files in the mirror area matching regexp.  This argument can
-be used multiple times.  The rdiff-backup-data directory is
-automatically excluded, so this option rarely needs to be used.
-.TP
 .B --exclude-other-filesystems
 Exclude files on file systems (identified by device number) other than
 the file system the root of the source directory is on.
@@ -235,7 +230,7 @@ Don't replicate hard links on destination side.  Note that because
 metadata is written to a separate file, hard link information will not
 be lost even if the --no-hard-links option is given (however, mirror
 files will not be linked).  If many hard-linked files are present,
-this option can drastically increase memory usage.
+this option can drastically decrease memory usage.
 .TP
 .B --null-separator
 Use nulls (\\0) instead of newlines (\\n) as line separators, which
@@ -895,11 +890,11 @@ Files whose names are close to the maximum length (e.g. 235 chars if
 the maximum is 255) may be skipped because the filenames of related
 increment files would be too long.
 .PP
-The gzip library in versions 2.2 and earlier of python have trouble
-producing files over 2GB in length.  This bug will prevent
-rdiff-backup from producing large compressed increments (snapshots or
-diffs).  A workaround is to disable compression for large
-uncompressable files.
+The gzip library in versions 2.2 and earlier of python (but fixed in
+2.3a1) has trouble producing files over 2GB in length.  This bug will
+prevent rdiff-backup from producing large compressed increments
+(snapshots or diffs).  A workaround is to disable compression for
+large uncompressable files.
 
 .SH AUTHOR
 Ben Escoto <bescoto@stanford.edu>
diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py
index 04ce247..5a03fb5 100644
--- a/rdiff-backup/rdiff_backup/Main.py
+++ b/rdiff-backup/rdiff_backup/Main.py
@@ -30,7 +30,7 @@ import Globals, Time, SetConnections, selection, robust, rpath, \
 action = None
 remote_cmd, remote_schema = None, None
 force = None
-select_opts, select_mirror_opts = [], []
+select_opts = []
 select_files = []
 
 def parse_cmdlineoptions(arglist):
@@ -84,8 +84,6 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--exclude-globbing-filelist":
 			select_opts.append((opt, arg))
 			select_files.append(sel_fl(arg))
-		elif opt == "--exclude-mirror":
-			select_mirror_opts.append(("--exclude", arg))
 		elif (opt == "--exclude-other-filesystems" or
 			  opt == "--exclude-regexp" or
 			  opt == "--exclude-special-files"): select_opts.append((opt, arg))
@@ -399,11 +397,23 @@ def restore_common(rpin, target, time):
 	restore_check_backup_dir(mirror_root)
 	mirror = mirror_root.new_index(index)
 	inc_rpath = datadir.append_path('increments', index)
-	restore_init_select(mirror_root, target)
+	restore_set_select(mirror_root, target)
 	restore_start_log(rpin, target, time)
 	restore.Restore(mirror, inc_rpath, target, time)
 	Log("Restore ended", 4)
 
+def restore_set_select(mirror_rp, target):
+	"""Set the selection iterator on mirror side from command line args
+
+	Here we set the selector on the mirror side, because that is where
+	we will be filtering, but the pathnames are relative to the target
+	directory.
+
+	"""
+	if select_opts: 
+		mirror_rp.conn.restore.MirrorStruct.set_mirror_select(
+			target, select_opts, *select_files)
+
 def restore_start_log(rpin, target, time):
 	"""Open restore log file, log initial message"""
 	try: Log.open_logfile(datadir.append("restore.log"))
@@ -446,19 +456,6 @@ def restore_check_backup_dir(rpin):
 		"Rerun rdiff-backup with --check-destination-dir option to revert"
 		"directory to state before unsuccessful session." % (rpin.path,))
 
-def restore_init_select(rpin, rpout):
-	"""Initialize Select
-
-	Unlike the backup selections, here they are on the local
-	connection, because the backup operation is pipelined in a way
-	the restore operation isn't.
-
-	"""
-	restore._select_mirror = selection.Select(rpin)
-	restore._select_mirror.ParseArgs(select_mirror_opts, [])
-	restore._select_mirror.parse_rbdir_exclude()
-	restore._select_source = selection.Select(rpout)
-
 def restore_get_root(rpin):
 	"""Return (mirror root, index) and set the data dir
 
diff --git a/rdiff-backup/rdiff_backup/Security.py b/rdiff-backup/rdiff_backup/Security.py
index b02d335..83ddcf2 100644
--- a/rdiff-backup/rdiff_backup/Security.py
+++ b/rdiff-backup/rdiff_backup/Security.py
@@ -148,8 +148,8 @@ def set_allowed_requests(sec_level):
 	if Globals.server:
 		allowed_requests.extend(
 			["SetConnections.init_connection_remote",
-			 "Log.setverbosity",
-			 "Log.setterm_verbosity",
+			 "log.Log.setverbosity",
+			 "log.Log.setterm_verbosity",
 			 "Time.setprevtime_local",
 			 "FilenameMapping.set_init_quote_vals_local",
 			 "Globals.postset_regexp_local",
diff --git a/rdiff-backup/rdiff_backup/backup.py b/rdiff-backup/rdiff_backup/backup.py
index 59663c6..89d7bea 100644
--- a/rdiff-backup/rdiff_backup/backup.py
+++ b/rdiff-backup/rdiff_backup/backup.py
@@ -166,9 +166,11 @@ class DestinationStruct:
 					yield iterfile.RORPIterFlushRepeat
 			else:
 				index = src_rorp and src_rorp.index or dest_rorp.index
-				cls.CCPP.flag_changed(index)
-				yield cls.get_one_sig(dest_base_rpath, index,
+				sig = cls.get_one_sig(dest_base_rpath, index,
 									  src_rorp, dest_rorp)
+				if sig:
+					cls.CCPP.flag_changed(index)
+					yield sig
 
 	def get_one_sig(cls, dest_base_rpath, index, src_rorp, dest_rorp):
 		"""Return a signature given source and destination rorps"""
@@ -180,9 +182,11 @@ class DestinationStruct:
 			dest_sig = dest_rorp.getRORPath()
 			if dest_rorp.isreg():
 				dest_rp = dest_base_rpath.new_index(index)
-				if dest_rp.isreg(): # otherwise file has changed type from reg
-					dest_sig.setfile(Rdiff.get_signature(dest_rp))
-				else: dest_sig = dest_rp.getRORPath()
+				if not dest_rp.isreg():
+					log.ErrorLog.write_if_open("UpdateError", dest_rp,
+						"File changed from regular file before signature")
+					return None
+				dest_sig.setfile(Rdiff.get_signature(dest_rp))
 		else: dest_sig = rpath.RORPath(index)
 		return dest_sig			
 
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 27c6f72..8c0b253 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -94,6 +94,7 @@ def ListAtTime(mirror_rp, inc_rp, time):
 
 class MirrorStruct:
 	"""Hold functions to be run on the mirror side"""
+	_select = None # If selection command line arguments given, use Select here
 	def set_mirror_and_rest_times(cls, restore_to_time):
 		"""Set global variabels _mirror_time and _rest_time on mirror conn"""
 		global _mirror_time, _rest_time
@@ -148,15 +149,29 @@ class MirrorStruct:
 		Usually we can use the metadata file, but if this is
 		unavailable, we may have to build it from scratch.
 
+		If the cls._select object is set, use it to filter out the
+		unwanted files from the metadata_iter.
+
 		"""
 		if rest_time is None: rest_time = _rest_time
 		metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir,
 				 rest_time, restrict_index = cls.mirror_base.index)
-		if metadata_iter: return metadata_iter
-		if require_metadata: log.Log.FatalError("Mirror metadata not found")
-		log.Log("Warning: Mirror metadata not found, "
-				"reading from directory", 2)
-		return cls.get_rorp_iter_from_rf(cls.root_rf)
+		if metadata_iter: rorp_iter = metadata_iter
+		elif require_metadata: log.Log.FatalError("Mirror metadata not found")
+		else:
+			log.Log("Warning: Mirror metadata not found, "
+					"reading from directory", 2)
+			rorp_iter = cls.get_rorp_iter_from_rf(cls.root_rf)
+
+		if cls._select:
+			rorp_iter = selection.FilterIter(cls._select, rorp_iter)
+		return rorp_iter
+
+	def set_mirror_select(cls, target_rp, select_opts, *filelists):
+		"""Initialize the mirror selection object"""
+		assert select_opts, "If no selection options, don't use selector"
+		cls._select = selection.Select(target_rp)
+		cls._select.ParseArgs(select_opts, filelists)
 
 	def get_rorp_iter_from_rf(cls, rf):
 		"""Recursively yield mirror rorps from rf"""
diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py
index d683650..9bf75d4 100644
--- a/rdiff-backup/rdiff_backup/rorpiter.py
+++ b/rdiff-backup/rdiff_backup/rorpiter.py
@@ -286,7 +286,7 @@ class IterTreeReducer:
 
 
 class ITRBranch:
-	"""Helper class for IterTreeReducer below
+	"""Helper class for IterTreeReducer above
 
 	There are five stub functions below: start_process, end_process,
 	branch_process, can_fast_process, and fast_process.  A class that
diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py
index 0fcc657..29b12e8 100644
--- a/rdiff-backup/rdiff_backup/selection.py
+++ b/rdiff-backup/rdiff_backup/selection.py
@@ -26,7 +26,7 @@ documentation on what this code does can be found on the man page.
 
 from __future__ import generators
 import re
-import FilenameMapping, robust, rpath, Globals, log
+import FilenameMapping, robust, rpath, Globals, log, rorpiter
 
 
 class SelectError(Exception):
@@ -195,6 +195,18 @@ class Select:
 				for rp in rec_func(new_rp, rec_func, sel_func):
 					yield rp
 
+	def FilterIter(self, rorp_iter):
+		"""Filter rorp_iter using Select below, removing excluded rorps"""
+		def getrpiter(rorp_iter):
+			"""Return rp iter by adding indicies of rorp_iter to self.rpath"""
+			for rorp in rorp_iter:
+				yield rpath.RPath(self.rpath.conn, self.rpath.base,
+								  rorp.index, rorp.data)
+
+		ITR = rorpiter.IterTreeReducer(FilterIterITRB, [self])
+		for rp in rp_iter: ITR(rp.index, rp)
+		ITR.Finish()
+
 	def Select(self, rp):
 		"""Run through the selection functions and return dominant val 0/1/2"""
 		for sf in self.selection_functions:
@@ -606,4 +618,86 @@ probably isn't what you meant.""" %
 		return res
 
 
+class FilterIter:
+	"""Filter rorp_iter using a Select object, removing excluded rorps"""
+	def __init__(self, select, rorp_iter):
+		"""Constructor
+
+		Input is the Select object to use and the iter of rorps to be
+		filtered.  The rorps will be converted to rps using the Select
+		base.
+
+		"""
+		self.rorp_iter = rorp_iter
+		self.base_rp = select.rpath
+		self.stored_rorps = []
+		self.ITR = rorpiter.IterTreeReducer(FilterIterITRB,
+											[select.Select, self.stored_rorps])
+		self.itr_finished = 0
+
+	def __iter__(self): return self
+
+	def next(self):
+		"""Return next object, or StopIteration"""
+		while not self.stored_rorps:
+			try: next_rorp = self.rorp_iter.next()
+			except StopIteration:
+				if self.itr_finished: raise
+				else:
+					self.ITR.Finish()
+					self.itr_finished = 1
+			else:
+				next_rp = rpath.RPath(self.base_rp.conn, self.base_rp.base,
+									  next_rorp.index, next_rorp.data)
+				self.ITR(next_rorp.index, next_rp, next_rorp)
+		return self.stored_rorps.pop(0)
+
+class FilterIterITRB(rorpiter.ITRBranch):
+	"""ITRBranch used in above FilterIter class
+
+	The reason this is necessary is because for directories sometimes
+	we don't know whether a rorp is excluded until we see what is in
+	the directory.
+
+	"""
+	def __init__(self, select, rorp_cache):
+		"""Initialize FilterIterITRB.  Called by IterTreeReducer.
+
+		select should be the relevant Select object used to test the
+		rps.  rorp_cache is the list rps should be appended to if they
+		aren't excluded.
+
+		"""
+		self.select, self.rorp_cache = select, rorp_cache
+		self.branch_excluded = None
+		self.base_queue = None # holds branch base while examining contents
+
+	def can_fast_process(self, index, next_rp, next_rorp):
+		return not next_rp.isdir()
+
+	def fast_process(self, index, next_rp, next_rorp):
+		"""For ordinary files, just append if select is positive"""
+		if self.branch_excluded: return
+		s = self.select(next_rp)
+		if s == 1:
+			if self.base_queue:
+				self.rorp_cache.append(self.base_queue)
+				self.base_queue = None
+			self.rorp_cache.append(next_rorp)
+		else: assert s == 0, "Unexpected select value %s" % (s,)
+
+	def start_process(self, index, next_rp, next_rorp):
+		s = self.select(next_rp)
+		if s == 0: self.branch_excluded = 1
+		elif s == 1: self.rorp_cache.append(next_rorp)
+		else:
+			assert s == 2, s
+			self.base_queue = next_rorp
+
+
+
+
+
+
+
 
diff --git a/rdiff-backup/testing/killtest.py b/rdiff-backup/testing/killtest.py
index 3eef746..2c2946b 100644
--- a/rdiff-backup/testing/killtest.py
+++ b/rdiff-backup/testing/killtest.py
@@ -127,7 +127,7 @@ class KillTest(ProcessFuncs):
 	# The following are lower and upper bounds on the amount of time
 	# rdiff-backup is expected to run.  They are used to determine how
 	# long to wait before killing the rdiff-backup process
-	time_pairs = [(0.0, 3.7), (0.0, 5.7), (0.0, 3.0), (0.0, 5.0), (0.0, 5.0)]
+	time_pairs = [(0.0, 3.7), (0.0, 3.7), (0.0, 3.0), (0.0, 5.0), (0.0, 5.0)]
 
 	def setUp(self):
 		"""Create killtest? and backup? directories if necessary"""
@@ -175,9 +175,10 @@ class KillTest(ProcessFuncs):
 
 		inc_date_pairs = map(lambda inc: (inc.getinctime(), inc), inclist)
 		inc_date_pairs.sort()
-		assert inc_date_pairs[-1][0] == curtime, \
-			   (inc_date_pairs[-1][0], curtime)
-		if len(inclist) == 2: return 1
+		if len(inclist) == 2:
+			assert inc_date_pairs[-1][0] == curtime, \
+				   (inc_date_pairs[-1][0], curtime)
+			return 1
 
 		if inc_date_pairs[-1][0] == curtime:
 			result = 0
@@ -235,7 +236,7 @@ class KillTest(ProcessFuncs):
 			elif result == -1: killed_too_soon[1] += 1
 		self.exec_rb(30000, 1, Local.kt2rp.path, Local.rpout.path)
 
-		# Now keep regressing from kt2rp, only staying there at the end
+		# Now keep regressing from kt3rp, only staying there at the end
 		for i in range(count):
 			result = cycle_once(self.time_pairs[3], 40000,
 								Local.kt3rp, Local.kt2rp)
@@ -243,7 +244,7 @@ class KillTest(ProcessFuncs):
 			elif result == -1: killed_too_soon[2] += 1
 		self.exec_rb(40000, 1, Local.kt3rp.path, Local.rpout.path)
 
-		# Now keep regressing from kt2rp, only staying there at the end
+		# Now keep regressing from kt4rp, only staying there at the end
 		for i in range(count):
 			result = cycle_once(self.time_pairs[4], 50000,
 								Local.kt4rp, Local.kt3rp)