Added resuming bug fixes

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@114 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

14 files changed, 112 insertions, 31 deletions

diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py
index 54cf527..a92cab8 100644
--- a/rdiff-backup/rdiff_backup/connection.py
+++ b/rdiff-backup/rdiff_backup/connection.py
@@ -338,8 +338,10 @@ class PipeConnection(LowLevelPipeConnection):
 
 	def extract_exception(self):
 		"""Return active exception"""
-		Log("Sending back exception: \n" +
-			"".join(traceback.format_tb(sys.exc_info()[2])), 2)
+		if Log.verbosity >= 5 or Log.term_verbosity >= 5:
+			Log("Sending back exception %s of type %s: \n%s" %
+				(sys.exc_info()[1], sys.exc_info()[0],
+				 "".join(traceback.format_tb(sys.exc_info()[2]))), 5)
 		return sys.exc_info()[1]
 
 	def Server(self):
diff --git a/rdiff-backup/rdiff_backup/header.py b/rdiff-backup/rdiff_backup/header.py
index 4f27dc7..1ffdfdb 100644
--- a/rdiff-backup/rdiff_backup/header.py
+++ b/rdiff-backup/rdiff_backup/header.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.5.4 released May 29, 2002
+# Version 0.7.6 released May 31, 2002
 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).
@@ -14,6 +14,6 @@
 # bugs or have any suggestions.
 
 from __future__ import nested_scopes, generators
-import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno
+import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno, signal
 
 
diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py
index ae3fa53..c0b6b61 100644
--- a/rdiff-backup/rdiff_backup/highlevel.py
+++ b/rdiff-backup/rdiff_backup/highlevel.py
@@ -207,13 +207,18 @@ class HLDestinationStruct:
 		else:
 			iitr = IncrementITR(inc_rpath)
 			iitr.override_changed()
+			Globals.ITR = iitr
+			iitr.Errors = 0
 			return iitr
 
 	def get_MirrorITR(cls, inc_rpath):
 		"""Return MirrorITR, starting from state if available"""
 		if cls._session_info and cls._session_info.ITR:
 			return cls._session_info.ITR
-		else: return MirrorITR(inc_rpath)
+		ITR = MirrorITR(inc_rpath)
+		Globals.ITR = ITR
+		ITR.Errors = 0
+		return ITR
 
 	def patch_and_finalize(cls, dest_rpath, diffs):
 		"""Apply diffs and finalize"""
@@ -244,7 +249,7 @@ class HLDestinationStruct:
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
 		finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
 		Stats.open_dir_stats_file()
-		dsrp = None
+		dsrp, finished_dsrp = None, None
 
 		try:
 			for indexed_tuple in collated:
@@ -255,9 +260,10 @@ class HLDestinationStruct:
 				ITR(dsrp.index, diff_rorp, dsrp)
 				finalizer(dsrp.index, dsrp)
 				SaveState.checkpoint(ITR, finalizer, dsrp)
+				finished_dsrp = dsrp
 			ITR.Finish()
 			finalizer.Finish()
-		except: cls.handle_last_error(dsrp, finalizer, ITR)
+		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		Stats.close_dir_stats_file()
@@ -269,7 +275,7 @@ class HLDestinationStruct:
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
 		finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
 		Stats.open_dir_stats_file()
-		dsrp = None
+		dsrp, finished_dsrp = None, None
 
 		try:
 			for indexed_tuple in collated:
@@ -281,9 +287,10 @@ class HLDestinationStruct:
 				ITR(index, diff_rorp, dsrp)
 				finalizer(index, dsrp)
 				SaveState.checkpoint(ITR, finalizer, dsrp)
+				finished_dsrp = dsrp
 			ITR.Finish()
 			finalizer.Finish()
-		except: cls.handle_last_error(dsrp, finalizer, ITR)
+		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		Stats.close_dir_stats_file()
diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py
index d7040dc..fe1a316 100644
--- a/rdiff-backup/rdiff_backup/robust.py
+++ b/rdiff-backup/rdiff_backup/robust.py
@@ -1,4 +1,4 @@
-import tempfile, errno
+import tempfile, errno, signal
 execfile("hardlink.py")
 
 #######################################################################
@@ -79,7 +79,7 @@ class RobustAction:
 
 
 class Robust:
-	"""Contains various file operations made safer using tempfiles"""
+	"""Contains various methods designed to make things safer"""
 	null_action = RobustAction(None, None, None)
 	def chain(*robust_action_list):
 		"""Return chain tying together a number of robust actions
@@ -122,9 +122,9 @@ class Robust:
 				final_vals.append(ra.final_func(init_val))
 			return final_vals
 		def error(exc, ran_init, init_val):
-			for ra, init_val in zip(ras_with_started_inits, init_return_vals):
+			for ra, init_val in zip(ras_with_started_inits, init_vals):
 				ra.error_handler(exc, 1, init_val)
-			for ra in ras_with_started_inits[len(init_return_vals):]:
+			for ra in ras_with_started_inits[len(init_vals):]:
 				ra.error_handler(exc, None, None)
 		return RobustAction(init, final, error)
 
@@ -251,6 +251,10 @@ class Robust:
 				  'ENOTDIR', 'ENAMETOOLONG', 'EINTR', 'ENOTEMPTY',
 				  'EIO', 'ETXTBSY', 'ESRCH', 'EINVAL'])):
 				Log.exception()
+				conn = Globals.backup_writer
+				if conn is not None: # increment error count
+					ITR_exists = conn.Globals.is_not_None('ITR')
+					if ITR_exists: conn.Globals.ITR.increment_stat('Errors')
 				if error_handler: return error_handler(exc, *args)
 			else:
 				Log.exception(1, 2)
@@ -265,9 +269,23 @@ class Robust:
 		dir_listing.sort()
 		return dir_listing
 
+	def signal_handler(signum, frame):
+		"""This is called when signal signum is caught"""
+		raise SignalException(signum)
+
+	def install_signal_handlers():
+		"""Install signal handlers on current connection"""
+		for signum in [signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM]:
+			signal.signal(signum, Robust.signal_handler)
+
 MakeStatic(Robust)
 
 
+class SignalException(Exception):
+	"""SignalException(signum) means signal signum has been received"""
+	pass
+
+
 class TracebackArchive:
 	"""Save last 10 caught exceptions, so they can be printed if fatal"""
 	_traceback_strings = []
diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py
index c26dfab..8b87af8 100644
--- a/rdiff-backup/rdiff_backup/selection.py
+++ b/rdiff-backup/rdiff_backup/selection.py
@@ -275,7 +275,7 @@ probably isn't what you meant.""" %
 					if include is None: continue # later line may match
 				return include
 
-		selection_function.exclude = something_excluded
+		selection_function.exclude = something_excluded or inc_default == 0
 		selection_function.name = "Filelist: " + filelist_name
 		return selection_function
 
diff --git a/rdiff-backup/rdiff_backup/statistics.py b/rdiff-backup/rdiff_backup/statistics.py
index 7dff7a3..7f99c31 100644
--- a/rdiff-backup/rdiff_backup/statistics.py
+++ b/rdiff-backup/rdiff_backup/statistics.py
@@ -17,8 +17,10 @@ class StatsObj:
 					   'ChangedFiles',
 					   'ChangedSourceSize', 'ChangedMirrorSize',
 					   'IncrementFiles', 'IncrementFileSize')
+	stat_misc_attrs = ('Errors',)
 	stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime')
-	stat_attrs = ('Filename',) + stat_time_attrs + stat_file_attrs
+	stat_attrs = (('Filename',) + stat_time_attrs +
+				  stat_misc_attrs + stat_file_attrs)
 
 	# Below, the second value in each pair is true iff the value
 	# indicates a number of bytes
@@ -50,6 +52,10 @@ class StatsObj:
 		"""Set attribute to given value"""
 		self.__dict__[attr] = value
 
+	def increment_stat(self, attr):
+		"""Add 1 to value of attribute"""
+		self.__dict__[attr] = self.get_stat(attr) + 1
+
 	def get_stats_line(self, index):
 		"""Return one line abbreviated version of full stats string"""
 		file_attrs = map(lambda attr: str(self.get_stat(attr)),
@@ -95,6 +101,8 @@ class StatsObj:
 				self.ElapsedTime = self.EndTime - self.StartTime
 			timelist.append("ElapsedTime %.2f (%s)\n" %
 				   (self.ElapsedTime, Time.inttopretty(self.ElapsedTime)))
+		if self.Errors is not None:
+			timelist.append("Errors %d\n" % self.Errors)
 		return "".join(timelist)
 
 	def get_filestats_string(self):
diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py
index 54cf527..a92cab8 100644
--- a/rdiff-backup/src/connection.py
+++ b/rdiff-backup/src/connection.py
@@ -338,8 +338,10 @@ class PipeConnection(LowLevelPipeConnection):
 
 	def extract_exception(self):
 		"""Return active exception"""
-		Log("Sending back exception: \n" +
-			"".join(traceback.format_tb(sys.exc_info()[2])), 2)
+		if Log.verbosity >= 5 or Log.term_verbosity >= 5:
+			Log("Sending back exception %s of type %s: \n%s" %
+				(sys.exc_info()[1], sys.exc_info()[0],
+				 "".join(traceback.format_tb(sys.exc_info()[2]))), 5)
 		return sys.exc_info()[1]
 
 	def Server(self):
diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py
index a2797a4..9d81ad9 100644
--- a/rdiff-backup/src/globals.py
+++ b/rdiff-backup/src/globals.py
@@ -8,7 +8,7 @@ import re, os
 class Globals:
 
 	# The current version of rdiff-backup
-	version = "0.7.5.4"
+	version = "0.7.6"
 	
 	# If this is set, use this value in seconds as the current time
 	# instead of reading it from the clock.
@@ -157,11 +157,20 @@ class Globals:
 	# replaced by the source and mirror Select objects respectively.
 	select_source, select_mirror = None, None
 
+	# On the backup writer connection, holds the main incrementing
+	# function.  Access is provided to increment error counts.
+	ITR = None
+
 	def get(cls, name):
 		"""Return the value of something in this class"""
 		return cls.__dict__[name]
 	get = classmethod(get)
 
+	def is_not_None(cls, name):
+		"""Returns true if value is not None"""
+		return cls.__dict__[name] is not None
+	is_not_None = classmethod(is_not_None)
+
 	def set(cls, name, val):
 		"""Set the value of something in this class
 
diff --git a/rdiff-backup/src/header.py b/rdiff-backup/src/header.py
index 4f27dc7..1ffdfdb 100644
--- a/rdiff-backup/src/header.py
+++ b/rdiff-backup/src/header.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.5.4 released May 29, 2002
+# Version 0.7.6 released May 31, 2002
 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).
@@ -14,6 +14,6 @@
 # bugs or have any suggestions.
 
 from __future__ import nested_scopes, generators
-import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno
+import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno, signal
 
 
diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py
index ae3fa53..c0b6b61 100644
--- a/rdiff-backup/src/highlevel.py
+++ b/rdiff-backup/src/highlevel.py
@@ -207,13 +207,18 @@ class HLDestinationStruct:
 		else:
 			iitr = IncrementITR(inc_rpath)
 			iitr.override_changed()
+			Globals.ITR = iitr
+			iitr.Errors = 0
 			return iitr
 
 	def get_MirrorITR(cls, inc_rpath):
 		"""Return MirrorITR, starting from state if available"""
 		if cls._session_info and cls._session_info.ITR:
 			return cls._session_info.ITR
-		else: return MirrorITR(inc_rpath)
+		ITR = MirrorITR(inc_rpath)
+		Globals.ITR = ITR
+		ITR.Errors = 0
+		return ITR
 
 	def patch_and_finalize(cls, dest_rpath, diffs):
 		"""Apply diffs and finalize"""
@@ -244,7 +249,7 @@ class HLDestinationStruct:
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
 		finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
 		Stats.open_dir_stats_file()
-		dsrp = None
+		dsrp, finished_dsrp = None, None
 
 		try:
 			for indexed_tuple in collated:
@@ -255,9 +260,10 @@ class HLDestinationStruct:
 				ITR(dsrp.index, diff_rorp, dsrp)
 				finalizer(dsrp.index, dsrp)
 				SaveState.checkpoint(ITR, finalizer, dsrp)
+				finished_dsrp = dsrp
 			ITR.Finish()
 			finalizer.Finish()
-		except: cls.handle_last_error(dsrp, finalizer, ITR)
+		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		Stats.close_dir_stats_file()
@@ -269,7 +275,7 @@ class HLDestinationStruct:
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
 		finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
 		Stats.open_dir_stats_file()
-		dsrp = None
+		dsrp, finished_dsrp = None, None
 
 		try:
 			for indexed_tuple in collated:
@@ -281,9 +287,10 @@ class HLDestinationStruct:
 				ITR(index, diff_rorp, dsrp)
 				finalizer(index, dsrp)
 				SaveState.checkpoint(ITR, finalizer, dsrp)
+				finished_dsrp = dsrp
 			ITR.Finish()
 			finalizer.Finish()
-		except: cls.handle_last_error(dsrp, finalizer, ITR)
+		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		Stats.close_dir_stats_file()
diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py
index 6b1db51..6359450 100755
--- a/rdiff-backup/src/main.py
+++ b/rdiff-backup/src/main.py
@@ -174,6 +174,8 @@ class Main:
 		Globals.postset_regexp('no_compression_regexp',
 							   Globals.no_compression_regexp_string)
 
+		for conn in Globals.connections: Robust.install_signal_handlers()
+
 	def take_action(self, rps):
 		"""Do whatever self.action says"""
 		if self.action == "server":
diff --git a/rdiff-backup/src/robust.py b/rdiff-backup/src/robust.py
index d7040dc..fe1a316 100644
--- a/rdiff-backup/src/robust.py
+++ b/rdiff-backup/src/robust.py
@@ -1,4 +1,4 @@
-import tempfile, errno
+import tempfile, errno, signal
 execfile("hardlink.py")
 
 #######################################################################
@@ -79,7 +79,7 @@ class RobustAction:
 
 
 class Robust:
-	"""Contains various file operations made safer using tempfiles"""
+	"""Contains various methods designed to make things safer"""
 	null_action = RobustAction(None, None, None)
 	def chain(*robust_action_list):
 		"""Return chain tying together a number of robust actions
@@ -122,9 +122,9 @@ class Robust:
 				final_vals.append(ra.final_func(init_val))
 			return final_vals
 		def error(exc, ran_init, init_val):
-			for ra, init_val in zip(ras_with_started_inits, init_return_vals):
+			for ra, init_val in zip(ras_with_started_inits, init_vals):
 				ra.error_handler(exc, 1, init_val)
-			for ra in ras_with_started_inits[len(init_return_vals):]:
+			for ra in ras_with_started_inits[len(init_vals):]:
 				ra.error_handler(exc, None, None)
 		return RobustAction(init, final, error)
 
@@ -251,6 +251,10 @@ class Robust:
 				  'ENOTDIR', 'ENAMETOOLONG', 'EINTR', 'ENOTEMPTY',
 				  'EIO', 'ETXTBSY', 'ESRCH', 'EINVAL'])):
 				Log.exception()
+				conn = Globals.backup_writer
+				if conn is not None: # increment error count
+					ITR_exists = conn.Globals.is_not_None('ITR')
+					if ITR_exists: conn.Globals.ITR.increment_stat('Errors')
 				if error_handler: return error_handler(exc, *args)
 			else:
 				Log.exception(1, 2)
@@ -265,9 +269,23 @@ class Robust:
 		dir_listing.sort()
 		return dir_listing
 
+	def signal_handler(signum, frame):
+		"""This is called when signal signum is caught"""
+		raise SignalException(signum)
+
+	def install_signal_handlers():
+		"""Install signal handlers on current connection"""
+		for signum in [signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM]:
+			signal.signal(signum, Robust.signal_handler)
+
 MakeStatic(Robust)
 
 
+class SignalException(Exception):
+	"""SignalException(signum) means signal signum has been received"""
+	pass
+
+
 class TracebackArchive:
 	"""Save last 10 caught exceptions, so they can be printed if fatal"""
 	_traceback_strings = []
diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py
index c26dfab..8b87af8 100644
--- a/rdiff-backup/src/selection.py
+++ b/rdiff-backup/src/selection.py
@@ -275,7 +275,7 @@ probably isn't what you meant.""" %
 					if include is None: continue # later line may match
 				return include
 
-		selection_function.exclude = something_excluded
+		selection_function.exclude = something_excluded or inc_default == 0
 		selection_function.name = "Filelist: " + filelist_name
 		return selection_function
 
diff --git a/rdiff-backup/src/statistics.py b/rdiff-backup/src/statistics.py
index 7dff7a3..7f99c31 100644
--- a/rdiff-backup/src/statistics.py
+++ b/rdiff-backup/src/statistics.py
@@ -17,8 +17,10 @@ class StatsObj:
 					   'ChangedFiles',
 					   'ChangedSourceSize', 'ChangedMirrorSize',
 					   'IncrementFiles', 'IncrementFileSize')
+	stat_misc_attrs = ('Errors',)
 	stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime')
-	stat_attrs = ('Filename',) + stat_time_attrs + stat_file_attrs
+	stat_attrs = (('Filename',) + stat_time_attrs +
+				  stat_misc_attrs + stat_file_attrs)
 
 	# Below, the second value in each pair is true iff the value
 	# indicates a number of bytes
@@ -50,6 +52,10 @@ class StatsObj:
 		"""Set attribute to given value"""
 		self.__dict__[attr] = value
 
+	def increment_stat(self, attr):
+		"""Add 1 to value of attribute"""
+		self.__dict__[attr] = self.get_stat(attr) + 1
+
 	def get_stats_line(self, index):
 		"""Return one line abbreviated version of full stats string"""
 		file_attrs = map(lambda attr: str(self.get_stat(attr)),
@@ -95,6 +101,8 @@ class StatsObj:
 				self.ElapsedTime = self.EndTime - self.StartTime
 			timelist.append("ElapsedTime %.2f (%s)\n" %
 				   (self.ElapsedTime, Time.inttopretty(self.ElapsedTime)))
+		if self.Errors is not None:
+			timelist.append("Errors %d\n" % self.Errors)
 		return "".join(timelist)
 
 	def get_filestats_string(self):