Added probabilistic stuff to killtest.py

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@122 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

3 files changed, 178 insertions, 54 deletions

diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py
index 6221866..04f2a8a 100644
--- a/rdiff-backup/testing/commontest.py
+++ b/rdiff-backup/testing/commontest.py
@@ -130,6 +130,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
 
 	mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir],
 												remote_schema)
+	Time.setcurtime()
 	inc = get_increment_rp(mirror_rp, time)
 	if inc:
 		_get_main().Restore(get_increment_rp(mirror_rp, time), dest_rp)
@@ -163,7 +164,8 @@ def _get_main():
 		return Globals.Main
 
 def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
-					 equality_func = None, exclude_rbdir = 1):
+					 equality_func = None, exclude_rbdir = 1,
+					 ignore_tmp_files = None):
 	"""Compare src_rp and dest_rp, which can be directories
 
 	This only compares file attributes, not the actual data.  This
@@ -179,6 +181,16 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 											   compare_hardlinks), 3)
 	src_select = Select(DSRPath(1, src_rp))
 	dest_select = Select(DSRPath(None, dest_rp))
+
+	if ignore_tmp_files:
+		# Ignoring temp files can be useful when we want to check the
+		# correctness of a backup which aborted in the middle.  In
+		# these cases it is OK to have tmp files lying around.
+		src_select.add_selection_func(src_select.regexp_get_sf(
+			".*rdiff-backup.tmp.[^/]+$", 0))
+		dest_select.add_selection_func(dest_select.regexp_get_sf(
+			".*rdiff-backup.tmp.[^/]+$", 0))
+
 	if exclude_rbdir:
 		src_select.parse_rbdir_exclude()
 		dest_select.parse_rbdir_exclude()
@@ -186,14 +198,14 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 		# include rdiff-backup-data/increments
 		src_select.add_selection_func(src_select.glob_get_tuple_sf(
 			('rdiff-backup-data', 'increments'), 1))
-		dest_select.add_selection_func(src_select.glob_get_tuple_sf(
+		dest_select.add_selection_func(dest_select.glob_get_tuple_sf(
 			('rdiff-backup-data', 'increments'), 1))
 		
 		# but exclude rdiff-backup-data
 		src_select.add_selection_func(src_select.glob_get_tuple_sf(
 			('rdiff-backup-data',), 0))
-		dest_select.add_selection_func(src_select.glob_get_tuple_sf(
-			('rdiff-backup-data',), 0))		
+		dest_select.add_selection_func(dest_select.glob_get_tuple_sf(
+			('rdiff-backup-data',), 0))
 
 	src_select.set_iter()
 	dest_select.set_iter()
@@ -221,20 +233,23 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 				if dest_rorp.index[-1].endswith('gz'): return 1
 				# Don't compare .missing increments because they don't matter
 				if dest_rorp.index[-1].endswith('.missing'): return 1
-		if src_rorp != dest_rorp: return None
-		if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
+		if compare_hardlinks:
+			if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
+		elif src_rorp == dest_rorp: return 1
 		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
 		Log("%s: %s" % (dest_rorp.index,
 						Hardlink.get_indicies(dest_rorp, None)), 3)
 		return None
 
-	if compare_hardlinks:
+	if equality_func: result = Iter.equal(dsiter1, dsiter2, 1, equality_func)
+	elif compare_hardlinks:
 		dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1)
-		dsiter2 = Hardlink.add_rorp_iter(dsiter2, None)
+		dsiter2 = Hardlink.add_rorp_iter(dsiter2, None)		
 		if exclude_rbdir:
 			result = Iter.equal(dsiter1, dsiter2, 1, hardlink_equal)
 		else: result = Iter.equal(dsiter1, dsiter2, 1, rbdir_equal)
-	elif equality_func: result = Iter.equal(dsiter1, dsiter2, 1, equality_func)
+	elif not exclude_rbdir:
+		result = Iter.equal(dsiter1, dsiter2, 1, rbdir_equal)
 	else: result = Iter.equal(dsiter1, dsiter2, 1)
 
 	for i in dsiter1: pass # make sure all files processed anyway
diff --git a/rdiff-backup/testing/killtest.py b/rdiff-backup/testing/killtest.py
index 7c157af..d0b1b40 100644
--- a/rdiff-backup/testing/killtest.py
+++ b/rdiff-backup/testing/killtest.py
@@ -1,6 +1,6 @@
 import unittest, os, signal, sys, random, time
 execfile("commontest.py")
-rbexec("restore.py")
+rbexec("main.py")
 
 """Test consistency by killing rdiff-backup as it is backing up"""
 
@@ -22,14 +22,21 @@ class Local:
 	rpout2 = get_local_rp('restoretarget2')
 	rpout3 = get_local_rp('restoretarget3')
 	rpout4 = get_local_rp('restoretarget4')
+	rpout5 = get_local_rp('restoretarget5')
 
 	back1 = get_local_rp('backup1')
 	back2 = get_local_rp('backup2')
 	back3 = get_local_rp('backup3')
 	back4 = get_local_rp('backup4')
+	back5 = get_local_rp('backup5')
 
+class TimingError(Exception):
+	"""Indicates timing error - process killed too soon or too late"""
+	pass
 
-class Kill(unittest.TestCase):
+
+class ProcessFuncs(unittest.TestCase):
+	"""Subclassed by Resume and NoResume"""
 	def delete_tmpdirs(self):
 		"""Remove any temp directories created by previous tests"""
 		assert not os.system(MiscDir + '/myrm testfiles/output* '
@@ -40,8 +47,7 @@ class Kill(unittest.TestCase):
 		"""True if there are signs of aborted backup in output/"""
 		try: dirlist = os.listdir("testfiles/output/rdiff-backup-data")
 		except OSError:
-			print "No data dir found, give process more time"
-			raise
+			raise TimingError("No data dir found, give process more time")
 		dirlist = filter(lambda f: f.startswith("last-file-incremented"),
 						 dirlist)
 		return len(dirlist) != 0
@@ -58,17 +64,19 @@ class Kill(unittest.TestCase):
 		if wait: return os.spawnvp(os.P_WAIT, 'python', arglist)
 		else: return os.spawnvp(os.P_NOWAIT, 'python', arglist)
 
-	def exec_and_kill(self, mintime, maxtime, backup_time, *args):
+	def exec_and_kill(self, mintime, maxtime, backup_time, resume, arg1, arg2):
 		"""Run rdiff-backup, then kill and run again
 
 		Kill after a time between mintime and maxtime.  First process
 		should not terminate before maxtime.
 
 		"""
-		pid = self.exec_rb(backup_time, None, *args)
+		pid = self.exec_rb(backup_time, None, arg1, arg2)
 		time.sleep(random.uniform(mintime, maxtime))
-		assert os.waitpid(pid, os.WNOHANG)[0] == 0, \
-			   "Process already quit - try lowering max time"
+		if os.waitpid(pid, os.WNOHANG)[0] != 0:
+			raise TimingError("Timing Error on %s, %s:\n"
+							  "Process already quit - try lowering max time"
+							  % (arg1, arg2))
 		os.kill(pid, self.killsignal)
 		while 1:
 			pid, exitstatus = os.waitpid(pid, os.WNOHANG)
@@ -76,73 +84,174 @@ class Kill(unittest.TestCase):
 				assert exitstatus != 0
 				break
 			time.sleep(0.2)
-		assert self.is_aborted_backup(), \
-			   "Process already finished or didn't get a chance to start"
-		os.system("ls -l %s/rdiff-backup-data" % args[1])
-		return self.exec_rb(backup_time + 5, 1, '--resume', *args)
+		if not self.is_aborted_backup():
+			raise TimingError("Timing Error on %s, %s:\n"
+							  "Process already finished or didn't "
+							  "get a chance to start" % (arg1, arg2))
+		print "---------------------- killed"
+		os.system("ls -l %s/rdiff-backup-data" % arg1)
+		if resume: self.exec_rb(backup_time + 5, 1, '--resume', arg1, arg2)
+		else: self.exec_rb(backup_time + 5000, 1, '--no-resume', arg1, arg2)
 
 	def verify_back_dirs(self):
 		"""Make sure testfiles/output/back? dirs exist"""
 		if (Local.back1.lstat() and Local.back2.lstat() and
-			Local.back3.lstat() and Local.back4.lstat()): return
+			Local.back3.lstat() and Local.back4.lstat() and
+			Local.back5.lstat()): return
 
-		os.system(MiscDir + "/myrm testfiles/backup[1-4]")
+		os.system(MiscDir + "/myrm testfiles/backup[1-5]")
 
-		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup1')
+		self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup1')
 		Local.back1.setdata()
 
-		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup2')
-		self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup2')
+		self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup2')
+		self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup2')
 		Local.back2.setdata()
 		
-		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup3')
-		self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup3')
-		self.exec_rb(30000, 1, 'testfiles/increment3', 'testfiles/backup3')
+		self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup3')
+		self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup3')
+		self.exec_rb(30000, 1, 'testfiles/increment2', 'testfiles/backup3')
 		Local.back3.setdata()
 		
-		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup4')
-		self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup4')
-		self.exec_rb(30000, 1, 'testfiles/increment3', 'testfiles/backup4')
-		self.exec_rb(40000, 1, 'testfiles/increment4', 'testfiles/backup4')
+		self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup4')
+		self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup4')
+		self.exec_rb(30000, 1, 'testfiles/increment2', 'testfiles/backup4')
+		self.exec_rb(40000, 1, 'testfiles/increment3', 'testfiles/backup4')
 		Local.back4.setdata()
 
-	def runtest(self):
+		self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup5')
+		self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup5')
+		self.exec_rb(30000, 1, 'testfiles/increment2', 'testfiles/backup5')
+		self.exec_rb(40000, 1, 'testfiles/increment3', 'testfiles/backup5')
+		self.exec_rb(50000, 1, 'testfiles/increment4', 'testfiles/backup5')
+		Local.back5.setdata()
+
+	def runtest_sequence(self, total_tests,
+						 exclude_rbdir, ignore_tmp, compare_links,
+						 stop_on_error = None):
+		timing_problems, failures = 0, 0
+		for i in range(total_tests):
+			try:
+				result = self.runtest(exclude_rbdir, ignore_tmp, compare_links)
+			except TimingError, te:
+				print te
+				timing_problems += 1
+				continue
+			if result != 1:
+				if stop_on_error: assert 0, "Compare Failure"
+				else: failures += 1
+
+		print total_tests, "tests attempted total"
+		print "%s setup problems, %s failures, %s successes" % \
+			  (timing_problems, failures,
+			   total_tests - timing_problems - failures)		
+
+
+class Resume(ProcessFuncs):
+	"""Test for graceful recovery after resumed backup"""
+	def runtest(self, exclude_rbdir, ignore_tmp_files, compare_links):
+		"""Run the actual test, returning 1 if passed and 0 otherwise"""
 		self.delete_tmpdirs()
 		self.verify_back_dirs()
 		
-		# Backing up increment1 - unfortunately, not big enough to test?
-		#self.exec_and_kill(0.6, 0.6, 10000,
-		#				   'testfiles/increment1', 'testfiles/output')
-		#assert CompareRecursive(Local.back1, Local.rpout, 1, None, None)
-		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/output')
-		time.sleep(1)
+		# Backing up increment3
+
+		# Start with increment3 because it is big and the first case
+		# is kind of special (there's no incrementing, so different
+		# code)
+		self.exec_and_kill(0.7, 1.5, 10000, 1,
+						   'testfiles/increment3', 'testfiles/output')
+		if not CompareRecursive(Local.back1, Local.rpout, compare_links,
+								None, exclude_rbdir, ignore_tmp_files):
+			return 0
+
+		# Backing up increment1
+		self.exec_and_kill(0.8, 0.8, 20000, 1,
+						   'testfiles/increment1', 'testfiles/output')
+		if not CompareRecursive(Local.back2, Local.rpout, compare_links,
+								None, exclude_rbdir, ignore_tmp_files):
+			return 0
 
 		# Backing up increment2
-		self.exec_and_kill(0.7, 1.0, 20000,
+		self.exec_and_kill(0.7, 1.0, 30000, 1,
 						   'testfiles/increment2', 'testfiles/output')
-		assert CompareRecursive(Local.back2, Local.rpout, 1, None, None)
-		time.sleep(1)
+		if not CompareRecursive(Local.back3, Local.rpout, compare_links,
+								None, exclude_rbdir, ignore_tmp_files):
+			return 0
 
 		# Backing up increment3
-		self.exec_and_kill(0.7, 2.0, 30000,
+		self.exec_and_kill(0.7, 2.0, 40000, 1,
 						   'testfiles/increment3', 'testfiles/output')
-		assert CompareRecursive(Local.back3, Local.rpout, 1, None, None)
-		time.sleep(1)
+		if not CompareRecursive(Local.back4, Local.rpout, compare_links,
+								None, exclude_rbdir, ignore_tmp_files):
+			return 0
 
 		# Backing up increment4
-		self.exec_and_kill(1.0, 5.0, 40000,
+		self.exec_and_kill(1.0, 5.0, 50000, 1,
 						   'testfiles/increment4', 'testfiles/output')
-		assert CompareRecursive(Local.back4, Local.rpout, 1, None, None)
+		if not CompareRecursive(Local.back5, Local.rpout, compare_links,
+								None, exclude_rbdir, ignore_tmp_files):
+			return 0
+		return 1
 
-	def testTERM(self):
+	def testTERM(self, total_tests = 3):
 		"""Test sending local processes a TERM signal"""
 		self.killsignal = signal.SIGTERM
-		for i in range(5):
-			self.runtest()
+		self.runtest_sequence(total_tests, None, None, 1)
 
-	def testKILL(self):
+	def testKILL(self, total_tests = 10):
 		"""Send local backup process a KILL signal"""
 		self.killsignal = signal.SIGKILL
-		self.runtest()
+		self.runtest_sequence(total_tests, None, 1, None)
+
+
+class NoResume(ProcessFuncs):
+	"""Test for consistent backup after abort and then no resume"""
+	def runtest(self, exclude_rbdir, ignore_tmp_files, compare_links):
+		self.delete_tmpdirs()
+
+		# Back up each increment to output
+		self.exec_and_kill(0.7, 1.5, 10000, 1,
+						   'testfiles/increment3', 'testfiles/output')
+		self.exec_and_kill(0.6, 0.6, 20000, 1,
+						   'testfiles/increment1', 'testfiles/output')
+		self.exec_and_kill(0.7, 1.0, 30000, 1,
+						   'testfiles/increment2', 'testfiles/output')
+		self.exec_and_kill(0.7, 2.0, 40000, 1,
+						   'testfiles/increment3', 'testfiles/output')
+		self.exec_and_kill(1.0, 5.0, 50000, 1,
+						   'testfiles/increment4', 'testfiles/output')
+
+		# Now restore each and compare
+		InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget1",
+						15000)
+		assert CompareRecursive(Local.inc3rp, Local.rpout1, compare_links,
+								None, exclude_rbdir, ignore_tmp_files)
+		InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget2",
+						25000)
+		assert CompareRecursive(Local.inc1rp, Local.rpout2, compare_links,
+								None, exclude_rbdir, ignore_tmp_files)
+		InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget3",
+						35000)
+		assert CompareRecursive(Local.inc2rp, Local.rpout3, compare_links,
+								None, exclude_rbdir, ignore_tmp_files)
+		InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget4",
+						45000)
+		assert CompareRecursive(Local.inc3rp, Local.rpout4, compare_links,
+								None, exclude_rbdir, ignore_tmp_files)
+		InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget5",
+						55000)
+		assert CompareRecursive(Local.inc4rp, Local.rpout5, compare_links,
+								None, exclude_rbdir, ignore_tmp_files)
+		return 1
+
+	def testTERM(self, total_tests = 10):
+		self.killsignal = signal.SIGTERM
+		self.runtest_sequence(total_tests, 1, None, 1)
+
+	def testKILL(self, total_tests = 20):
+		self.killsignal = signal.SIGKILL
+		self.runtest_sequence(total_tests, 1, 1, None)
+
 
 if __name__ == "__main__": unittest.main()
diff --git a/rdiff-backup/testing/regressiontest.py b/rdiff-backup/testing/regressiontest.py
index 1a486d7..80a60d0 100644
--- a/rdiff-backup/testing/regressiontest.py
+++ b/rdiff-backup/testing/regressiontest.py
@@ -13,7 +13,7 @@ testfiles
 
 Globals.set('change_source_perms', 1)
 Globals.counter = 0
-Log.setverbosity(7)
+Log.setverbosity(3)
 
 class Local:
 	"""This is just a place to put increments relative to the local