Changed for 0.7.3 release

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@48 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

8 files changed, 128 insertions, 123 deletions

diff --git a/rdiff-backup/rdiff_backup/destructive_stepping.py b/rdiff-backup/rdiff_backup/destructive_stepping.py
index c4b1191..c2cd011 100644
--- a/rdiff-backup/rdiff_backup/destructive_stepping.py
+++ b/rdiff-backup/rdiff_backup/destructive_stepping.py
@@ -180,72 +180,4 @@ class DestructiveStepping:
 							   lambda dsrpath, x, y: dsrpath.write_changes(),
 							   initial_state)
 
-	def isexcluded(dsrp, source):
-		"""Return true if given DSRPath is excluded/ignored
-
-		If source = 1, treat as source file, otherwise treat as
-		destination file.
-
-		"""
-		return None # this code is for the test suites only, use Select instead
-		if Globals.exclude_device_files and dsrp.isdev(): return 1
-		
-		if source: exclude_regexps = Globals.exclude_regexps
-		else: exclude_regexps = Globals.exclude_mirror_regexps
-
-		for regexp in exclude_regexps:
-			if regexp.match(dsrp.path):
-				Log("Excluding %s" % dsrp.path, 6)
-				return 1
-		return None
-
-	def Iterate_from(baserp, source, starting_index = None):
-		"""Iterate dsrps from baserp, skipping any matching exclude_regexps
-
-		includes only dsrps with indicies greater than starting_index
-		if starting_index is not None.
-
-		"""
-		def helper_starting_from(dsrpath):
-			"""Like helper, but only start iterating after starting_index"""
-			if dsrpath.index > starting_index:
-				# Past starting_index, revert to normal helper
-				for dsrp in helper(dsrpath): yield dsrp
-			elif dsrpath.index == starting_index[:len(dsrpath.index)]:
-				# May encounter starting index on this branch
-				if (not DestructiveStepping.isexcluded(dsrpath, source) and
-					not DestructiveStepping.initialize(dsrpath, source)):
-					if dsrpath.isdir():
-						dir_listing = dsrpath.listdir()
-						dir_listing.sort()
-						for filename in dir_listing:
-							for dsrp in helper_starting_from(
-								dsrpath.append(filename)):
-								yield dsrp
-
-		def helper(dsrpath):
-			if (not DestructiveStepping.isexcluded(dsrpath, source) and
-				not DestructiveStepping.initialize(dsrpath, source)):
-				yield dsrpath
-				if dsrpath.isdir():
-					dir_listing = dsrpath.listdir()
-					dir_listing.sort()
-					for filename in dir_listing:
-						for dsrp in helper(dsrpath.append(filename)):
-							yield dsrp
-
-		base_dsrpath = DSRPath(baserp.conn, baserp.base,
-							   baserp.index, baserp.data)
-		if starting_index is None: return helper(base_dsrpath)
-		else: return helper_starting_from(base_dsrpath)
-
-	def Iterate_with_Finalizer(baserp, source):
-		"""Like Iterate_from, but finalize each dsrp afterwards"""
-		finalize = DestructiveStepping.Finalizer()
-		for dsrp in DestructiveStepping.Iterate_from(baserp, source):
-			yield dsrp
-			finalize(dsrp)
-		finalize.getresult()
-		
-
 MakeStatic(DestructiveStepping)
diff --git a/rdiff-backup/rdiff_backup/header.py b/rdiff-backup/rdiff_backup/header.py
index 81cbaff..38d45c8 100644
--- a/rdiff-backup/rdiff_backup/header.py
+++ b/rdiff-backup/rdiff_backup/header.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.2 released April 11, 2002
+# Version 0.7.3 released April 29, 2002
 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).
@@ -14,6 +14,6 @@
 # bugs or have any suggestions.
 
 from __future__ import nested_scopes, generators
-import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip
+import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList
 
 
diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py
index 54029c7..5a1399d 100644
--- a/rdiff-backup/rdiff_backup/highlevel.py
+++ b/rdiff-backup/rdiff_backup/highlevel.py
@@ -73,6 +73,9 @@ class HighLevel:
 		if not isinstance(target_base, DSRPath):
 			target_base = DSRPath(target_base.conn, target_base.base,
 								  target_base.index, target_base.data)
+		if not isinstance(mirror_base, DSRPath):
+			mirror_base = DSRPath(mirror_base.conn, mirror_base.base,
+								  mirror_base.index, mirror_base.data)
 		Restore.RestoreRecursive(rest_time, mirror_base, rel_index,
 								 baseinc_tup, target_base)
 
@@ -272,9 +275,9 @@ class HLDestinationStruct:
 				try: dsrp = cls.check_skip_error(error_checked, dsrp)
 				except StopIteration: break
 				SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp)
+			cls.check_skip_error(ITR.getresult, dsrp)
+			cls.check_skip_error(finalizer.getresult, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
-		ITR.getresult()
-		finalizer.getresult()
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		SaveState.checkpoint_remove()
 
@@ -288,6 +291,7 @@ class HLDestinationStruct:
 				 (exp[0] in [2,  # Means that a file is missing
 							 5,  # Reported by docv (see list)
 							 13, # Permission denied IOError
+							 20, # Means a directory changed to non-dir
 							 26] # Requested by Campbell (see list) -
                                  # happens on some NT systems
 				 ))):
diff --git a/rdiff-backup/rdiff_backup/log.py b/rdiff-backup/rdiff_backup/log.py
index 4605875..facb729 100644
--- a/rdiff-backup/rdiff_backup/log.py
+++ b/rdiff-backup/rdiff_backup/log.py
@@ -137,8 +137,8 @@ class Logger:
 
 		exc_info = sys.exc_info()
 		logging_func("Exception %s raised of class %s" %
-					 (exc_info[1], exc_info[0]), 2)
-		logging_func("".join(traceback.format_tb(exc_info[2])), 2)
+					 (exc_info[1], exc_info[0]), 3)
+		logging_func("".join(traceback.format_tb(exc_info[2])), 3)
 		
 
 Log = Logger()
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 0e0d62e..6e51b81 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -20,7 +20,6 @@ class Restore:
 		and rptarget is the rpath that will be written with the restored file.
 
 		"""
-		inclist = Restore.sortincseq(rest_time, inclist)
 		if not inclist and not (rpbase and rpbase.lstat()):
 			return # no increments were applicable
 		Log("Restoring %s with increments %s to %s" %
@@ -58,7 +57,7 @@ class Restore:
 			i = i+1
 		incpairs = incpairs[:i+1]
 
-		# Return increments in reversed order
+		# Return increments in reversed order (latest first)
 		incpairs.reverse()
 		return map(lambda pair: pair[1], incpairs)
 
@@ -106,44 +105,106 @@ class Restore:
 
 		"""
 		assert isinstance(target_base, DSRPath)
-		collated = RORPIter.CollateIterators(
-			DestructiveStepping.Iterate_from(mirror_base, None),
-			Restore.yield_inc_tuples(baseinc_tup))
+		baseinc_tup = IndexedTuple(baseinc_tup.index, (baseinc_tup[0],
+						  Restore.sortincseq(rest_time, baseinc_tup[1])))
+
+		collated = Restore.yield_collated_tuples((), mirror_base,
+										   baseinc_tup, target_base, rest_time)
 		mirror_finalizer = DestructiveStepping.Finalizer()
 		target_finalizer = DestructiveStepping.Finalizer()
 
-		for mirror, inc_tup in collated:
-			if not inc_tup:
-				inclist = []
-				target = target_base.new_index(mirror.index)
-			else:
-				inclist = inc_tup[1]
-				target = target_base.new_index(inc_tup.index)
+		for mirror, inc_tup, target in collated:
+			inclist = inc_tup and inc_tup[1] or []
 			DestructiveStepping.initialize(target, None)
 			Restore.RestoreFile(rest_time, mirror, mirror_rel_index,
 								inclist, target)
 			target_finalizer(target)
 			if mirror: mirror_finalizer(mirror)
 		target_finalizer.getresult()
-		mirror_finalizer.getresult()
+		mirror_finalizer.getresult()			
+
+	def yield_collated_tuples(index, mirrorrp, inc_tup, target, rest_time):
+		"""Iterate collated tuples starting with given args
 
-	def yield_inc_tuples(inc_tuple):
-		"""Iterate increment tuples starting with inc_tuple
+		A collated tuple is an IndexedTuple (mirrorrp, inc_tuple, target).
+		inc_tuple is itself an IndexedTuple.  target is an rpath where
+		the created file should go.
 
-		An increment tuple is an IndexedTuple (pair).  The first will
-		be the rpath of a directory, and the second is a list of all
-		the increments associated with that directory.  If there are
-		increments that do not correspond to a directory, the first
-		element will be None.  All the rpaths involved correspond to
-		files in the increment directory.
+		In this case the "mirror" directory is treated as the source,
+		and we are actually copying stuff onto what Select considers
+		the source directory.
 
 		"""
-		oldindex, rpath = inc_tuple.index, inc_tuple[0]
-		yield inc_tuple
-		if not rpath or not rpath.isdir(): return
+		select_result = Globals.select_mirror.Select(target)
+		if select_result == 0: return
+
+		inc_base = inc_tup and inc_tup[0]
+		if mirrorrp and (not Globals.select_source.Select(mirrorrp) or
+						 DestructiveStepping.initialize(mirrorrp, None)):
+			mirrorrp = None
+		collated_tuple = IndexedTuple(index, (mirrorrp, inc_tup, target))
+		if mirrorrp and mirrorrp.isdir() or inc_base and inc_base.isdir():
+			depth_tuples = Restore.yield_collated_tuples_dir(index, mirrorrp,
+									   			  inc_tup, target, rest_time)
+		else: depth_tuples = None
+
+		if select_result == 1:
+			yield collated_tuple
+			if depth_tuples:
+				for tup in depth_tuples: yield tup
+		elif select_result == 2:
+			if depth_tuples:
+				try: first = depth_tuples.next()
+				except StopIteration: return # no tuples found inside, skip
+				yield collated_tuple
+				yield first
+				for tup in depth_tuples: yield tup
+
+	def yield_collated_tuples_dir(index, mirrorrp, inc_tup, target, rest_time):
+		"""Yield collated tuples from inside given args"""
+		if not Restore.check_dir_exists(mirrorrp, inc_tup): return
+		if mirrorrp and mirrorrp.isdir():
+			dirlist = mirrorrp.listdir()
+			dirlist.sort()
+			mirror_list = map(lambda x: IndexedTuple(x, (mirrorrp.append(x),)),
+							  dirlist)
+		else: mirror_list = []
+		inc_list = Restore.get_inc_tuples(inc_tup, rest_time)
+
+		for indexed_tup in RORPIter.CollateIterators(iter(mirror_list),
+													 iter(inc_list)):
+			filename = indexed_tup.index
+			new_inc_tup = indexed_tup[1]
+			new_mirrorrp = indexed_tup[0] and indexed_tup[0][0]
+			for new_col_tup in Restore.yield_collated_tuples(
+				index + (filename,), new_mirrorrp, new_inc_tup,
+				target.append(filename), rest_time): yield new_col_tup
+
+	def check_dir_exists(mirrorrp, inc_tuple):
+		"""Return true if target should be a directory"""
+		if inc_tuple and inc_tuple[1]:
+			# Incs say dir if last (earliest) one is a dir increment
+			return inc_tuple[1][-1].getinctype() == "dir"
+		elif mirrorrp: return mirrorrp.isdir() # if no incs, copy mirror
+		else: return None
+
+	def get_inc_tuples(inc_tuple, rest_time):
+		"""Return list of inc tuples in given rpath of increment directory
+
+		An increment tuple is an IndexedTuple (pair).  The second
+		element in the pair is a list of increments with the same
+		base.  The first element is the rpath of the corresponding
+		base.  Usually this base is a directory, otherwise it is
+		ignored.  If there are increments whose corresponding base
+		doesn't exist, the first element will be None.  All the rpaths
+		involved correspond to files in the increment directory.
 
+		"""
+		if not inc_tuple: return []
+		oldindex, incdir = inc_tuple.index, inc_tuple[0]
+		if not incdir.isdir(): return []
 		inc_list_dict = {} # Index tuple lists by index
-		dirlist = rpath.listdir()
+		dirlist = incdir.listdir()		
 
 		def affirm_dict_indexed(index):
 			"""Make sure the inc_list_dict has given index"""
@@ -152,7 +213,7 @@ class Restore:
 
 		def add_to_dict(filename):
 			"""Add filename to the inc tuple dictionary"""
-			rp = rpath.append(filename)
+			rp = incdir.append(filename)
 			if rp.isincfile():
 				basename = rp.getincbase_str()
 				affirm_dict_indexed(basename)
@@ -161,20 +222,22 @@ class Restore:
 				affirm_dict_indexed(filename)
 				inc_list_dict[filename][0] = rp
 
-		def list2tuple(index):
-			"""Return inc_tuple version of dictionary entry by index"""
-			inclist = inc_list_dict[index]
-			if not inclist[1]: return None # no increments, so ignore
-			return IndexedTuple(oldindex + (index,), inclist)
+		def index2tuple(index):
+			"""Return inc_tuple version of dictionary entry by index
+
+			Also runs sortincseq to sort the increments and remove
+			irrelevant ones.  This is done here so we can avoid
+			descending into .missing directories.
+
+			"""
+			incbase, inclist = inc_list_dict[index]
+			inclist = Restore.sortincseq(rest_time, inclist)
+			if not inclist: return None # no relevant increments, so ignore
+			return IndexedTuple(index, (incbase, inclist))
 
 		for filename in dirlist: add_to_dict(filename)
 		keys = inc_list_dict.keys()
 		keys.sort()
-		for index in keys:
-			new_inc_tuple = list2tuple(index)
-			if not new_inc_tuple: continue
-			elif new_inc_tuple[0]: # corresponds to directory
-				for i in Restore.yield_inc_tuples(new_inc_tuple): yield i
-			else: yield new_inc_tuple
+		return filter(lambda x: x, map(index2tuple, keys))
 
 MakeStatic(Restore)
diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py
index 17942d3..89dc63b 100644
--- a/rdiff-backup/rdiff_backup/robust.py
+++ b/rdiff-backup/rdiff_backup/robust.py
@@ -394,6 +394,7 @@ class Resume:
 		specified.
 
 		"""
+		assert Globals.isbackup_writer
 		if Time.prevtime > later_than: return Time.prevtime # usual case
 
 		for si in cls.get_sis_covering_index(index):
@@ -416,6 +417,7 @@ class Resume:
 
 	def SetSessionInfo(cls):
 		"""Read data directory and initialize _session_info"""
+		assert Globals.isbackup_writer
 		silist = []
 		rp_quad_dict = cls.group_rps_by_time(cls.get_relevant_rps())
 		times = rp_quad_dict.keys()
diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py
index e98fa13..bad02a4 100644
--- a/rdiff-backup/rdiff_backup/rorpiter.py
+++ b/rdiff-backup/rdiff_backup/rorpiter.py
@@ -1,6 +1,6 @@
 execfile("robust.py")
 from __future__ import generators
-import tempfile
+import tempfile, UserList
 
 #######################################################################
 #
@@ -224,7 +224,7 @@ MakeStatic(RORPIter)
 
 
 
-class IndexedTuple:
+class IndexedTuple(UserList.UserList):
 	"""Like a tuple, but has .index
 
 	This is used by CollateIterator above, and can be passed to the
@@ -238,9 +238,15 @@ class IndexedTuple:
 	def __len__(self): return len(self.data)
 
 	def __getitem__(self, key):
-		"""This only works for numerical keys (faster that way)"""
+		"""This only works for numerical keys (easier this way)"""
 		return self.data[key]
 
+	def __lt__(self, other): return self.__cmp__(other) == -1
+	def __le__(self, other): return self.__cmp__(other) != 1
+	def __ne__(self, other): return not self.__eq__(other)
+	def __gt__(self, other): return self.__cmp__(other) == 1
+	def __ge__(self, other): return self.__cmp__(other) != -1
+	
 	def __cmp__(self, other):
 		assert isinstance(other, IndexedTuple)
 		if self.index < other.index: return -1
@@ -255,6 +261,4 @@ class IndexedTuple:
 		else: return None
 
 	def __str__(self):
-		assert len(self.data) == 2
-		return "(%s, %s).%s" % (str(self.data[0]), str(self.data[1]),
-								str(self.index))
+		return  "(%s).%s" % (", ".join(map(str, self.data)), self.index)
diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py
index a8f8aaa..7405491 100644
--- a/rdiff-backup/rdiff_backup/selection.py
+++ b/rdiff-backup/rdiff_backup/selection.py
@@ -150,7 +150,7 @@ class Select:
 		finalize.getresult()
 
 	def Select(self, dsrp):
-		"""Run through the selection functions and return dominant value"""
+		"""Run through the selection functions and return dominant val 0/1/2"""
 		for sf in self.selection_functions:
 			result = sf(dsrp)
 			if result is not None: return result
@@ -186,7 +186,7 @@ class Select:
 																 1, arg[0]))
 				elif opt == "--include-regexp":
 					self.add_selection_func(self.regexp_get_sf(arg, 1))
-				else: assert 0, "Bad option %s" % opt
+				else: assert 0, "Bad selection option %s" % opt
 		except SelectError, e: self.parse_catch_error(e)
 
 		self.parse_last_excludes()
@@ -254,8 +254,8 @@ probably isn't what you meant.""" %
 		i = [0] # We have to put index in list because of stupid scoping rules
 
 		def selection_function(dsrp):
-			if i[0] > len(tuple_list): return inc_default
 			while 1:
+				if i[0] >= len(tuple_list): return None
 				include, move_on = \
 						 self.filelist_pair_match(dsrp, tuple_list[i[0]])
 				if move_on:
@@ -313,9 +313,9 @@ probably isn't what you meant.""" %
 	def filelist_pair_match(self, dsrp, pair):
 		"""Matches a filelist tuple against a dsrp
 
-		Returns a pair (include, move_on, definitive).  include is
-		None if the tuple doesn't match either way, and 0/1 if the
-		tuple excludes or includes the dsrp.
+		Returns a pair (include, move_on).  include is None if the
+		tuple doesn't match either way, and 0/1 if the tuple excludes
+		or includes the dsrp.
 
 		move_on is true if the tuple cannot match a later index, and
 		so we should move on to the next tuple in the index.