Added resuming bug fixes, kill testing for better robustness testing

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@115 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

4 files changed, 195 insertions, 7 deletions

diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py
index de897bb..6221866 100644
--- a/rdiff-backup/testing/commontest.py
+++ b/rdiff-backup/testing/commontest.py
@@ -163,7 +163,7 @@ def _get_main():
 		return Globals.Main
 
 def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
-					 equality_func = None):
+					 equality_func = None, exclude_rbdir = 1):
 	"""Compare src_rp and dest_rp, which can be directories
 
 	This only compares file attributes, not the actual data.  This
@@ -179,8 +179,22 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 											   compare_hardlinks), 3)
 	src_select = Select(DSRPath(1, src_rp))
 	dest_select = Select(DSRPath(None, dest_rp))
-	src_select.parse_rbdir_exclude()
-	dest_select.parse_rbdir_exclude()
+	if exclude_rbdir:
+		src_select.parse_rbdir_exclude()
+		dest_select.parse_rbdir_exclude()
+	else:
+		# include rdiff-backup-data/increments
+		src_select.add_selection_func(src_select.glob_get_tuple_sf(
+			('rdiff-backup-data', 'increments'), 1))
+		dest_select.add_selection_func(src_select.glob_get_tuple_sf(
+			('rdiff-backup-data', 'increments'), 1))
+		
+		# but exclude rdiff-backup-data
+		src_select.add_selection_func(src_select.glob_get_tuple_sf(
+			('rdiff-backup-data',), 0))
+		dest_select.add_selection_func(src_select.glob_get_tuple_sf(
+			('rdiff-backup-data',), 0))		
+
 	src_select.set_iter()
 	dest_select.set_iter()
 	dsiter1, dsiter2 = src_select.iterate_with_finalizer(), \
@@ -194,10 +208,32 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 						Hardlink.get_indicies(dest_rorp, None)), 3)
 		return None
 
+	def rbdir_equal(src_rorp, dest_rorp):
+		"""Like hardlink_equal, but make allowances for data directories"""
+		if not src_rorp.index and not dest_rorp.index: return 1
+		if (src_rorp.index and src_rorp.index[0] == 'rdiff-backup-data' and
+			src_rorp.index == dest_rorp.index):
+			# Don't compare dirs - they don't carry significant info
+			if dest_rorp.isdir() and src_rorp.isdir(): return 1
+			if dest_rorp.isreg() and src_rorp.isreg():
+				# Don't compare gzipped files because it is apparently
+				# non-deterministic.
+				if dest_rorp.index[-1].endswith('gz'): return 1
+				# Don't compare .missing increments because they don't matter
+				if dest_rorp.index[-1].endswith('.missing'): return 1
+		if src_rorp != dest_rorp: return None
+		if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
+		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
+		Log("%s: %s" % (dest_rorp.index,
+						Hardlink.get_indicies(dest_rorp, None)), 3)
+		return None
+
 	if compare_hardlinks:
 		dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1)
 		dsiter2 = Hardlink.add_rorp_iter(dsiter2, None)
-		result = Iter.equal(dsiter1, dsiter2, 1, hardlink_equal)
+		if exclude_rbdir:
+			result = Iter.equal(dsiter1, dsiter2, 1, hardlink_equal)
+		else: result = Iter.equal(dsiter1, dsiter2, 1, rbdir_equal)
 	elif equality_func: result = Iter.equal(dsiter1, dsiter2, 1, equality_func)
 	else: result = Iter.equal(dsiter1, dsiter2, 1)
 
diff --git a/rdiff-backup/testing/finaltest.py b/rdiff-backup/testing/finaltest.py
index 8d292de..150d5ac 100644
--- a/rdiff-backup/testing/finaltest.py
+++ b/rdiff-backup/testing/finaltest.py
@@ -113,7 +113,7 @@ class PathSetter(unittest.TestCase):
 
 		# Getting restore rps
 		inc_paths = self.getinc_paths("increments.",
-								   "testfiles/output/rdiff-backup-data")
+									  "testfiles/output/rdiff-backup-data")
 		assert len(inc_paths) == 3
 
 		# Restoring increment1
diff --git a/rdiff-backup/testing/killtest.py b/rdiff-backup/testing/killtest.py
new file mode 100644
index 0000000..7c157af
--- /dev/null
+++ b/rdiff-backup/testing/killtest.py
@@ -0,0 +1,148 @@
+import unittest, os, signal, sys, random, time
+execfile("commontest.py")
+rbexec("restore.py")
+
+"""Test consistency by killing rdiff-backup as it is backing up"""
+
+Log.setverbosity(3)
+
+class Local:
+	"""Hold some local RPaths"""
+	def get_local_rp(ext):
+		return RPath(Globals.local_connection, "testfiles/" + ext)
+
+	inc1rp = get_local_rp('increment1')
+	inc2rp = get_local_rp('increment2')
+	inc3rp = get_local_rp('increment3')
+	inc4rp = get_local_rp('increment4')
+
+	rpout = get_local_rp('output')
+	rpout_inc = get_local_rp('output_inc')
+	rpout1 = get_local_rp('restoretarget1')
+	rpout2 = get_local_rp('restoretarget2')
+	rpout3 = get_local_rp('restoretarget3')
+	rpout4 = get_local_rp('restoretarget4')
+
+	back1 = get_local_rp('backup1')
+	back2 = get_local_rp('backup2')
+	back3 = get_local_rp('backup3')
+	back4 = get_local_rp('backup4')
+
+
+class Kill(unittest.TestCase):
+	def delete_tmpdirs(self):
+		"""Remove any temp directories created by previous tests"""
+		assert not os.system(MiscDir + '/myrm testfiles/output* '
+							 'testfiles/restoretarget* testfiles/vft_out '
+							 'timbar.pyc testfiles/vft2_out')
+
+	def is_aborted_backup(self):
+		"""True if there are signs of aborted backup in output/"""
+		try: dirlist = os.listdir("testfiles/output/rdiff-backup-data")
+		except OSError:
+			print "No data dir found, give process more time"
+			raise
+		dirlist = filter(lambda f: f.startswith("last-file-incremented"),
+						 dirlist)
+		return len(dirlist) != 0
+
+	def exec_rb(self, time, wait, *args):
+		"""Run rdiff-backup return pid"""
+		arglist = ['python', '../src/rdiff-backup', '-v7']
+		if time:
+			arglist.append("--current-time")
+			arglist.append(str(time))
+		arglist.extend(args)
+
+		print "Running ", arglist
+		if wait: return os.spawnvp(os.P_WAIT, 'python', arglist)
+		else: return os.spawnvp(os.P_NOWAIT, 'python', arglist)
+
+	def exec_and_kill(self, mintime, maxtime, backup_time, *args):
+		"""Run rdiff-backup, then kill and run again
+
+		Kill after a time between mintime and maxtime.  First process
+		should not terminate before maxtime.
+
+		"""
+		pid = self.exec_rb(backup_time, None, *args)
+		time.sleep(random.uniform(mintime, maxtime))
+		assert os.waitpid(pid, os.WNOHANG)[0] == 0, \
+			   "Process already quit - try lowering max time"
+		os.kill(pid, self.killsignal)
+		while 1:
+			pid, exitstatus = os.waitpid(pid, os.WNOHANG)
+			if pid:
+				assert exitstatus != 0
+				break
+			time.sleep(0.2)
+		assert self.is_aborted_backup(), \
+			   "Process already finished or didn't get a chance to start"
+		os.system("ls -l %s/rdiff-backup-data" % args[1])
+		return self.exec_rb(backup_time + 5, 1, '--resume', *args)
+
+	def verify_back_dirs(self):
+		"""Make sure testfiles/output/back? dirs exist"""
+		if (Local.back1.lstat() and Local.back2.lstat() and
+			Local.back3.lstat() and Local.back4.lstat()): return
+
+		os.system(MiscDir + "/myrm testfiles/backup[1-4]")
+
+		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup1')
+		Local.back1.setdata()
+
+		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup2')
+		self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup2')
+		Local.back2.setdata()
+		
+		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup3')
+		self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup3')
+		self.exec_rb(30000, 1, 'testfiles/increment3', 'testfiles/backup3')
+		Local.back3.setdata()
+		
+		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup4')
+		self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup4')
+		self.exec_rb(30000, 1, 'testfiles/increment3', 'testfiles/backup4')
+		self.exec_rb(40000, 1, 'testfiles/increment4', 'testfiles/backup4')
+		Local.back4.setdata()
+
+	def runtest(self):
+		self.delete_tmpdirs()
+		self.verify_back_dirs()
+		
+		# Backing up increment1 - unfortunately, not big enough to test?
+		#self.exec_and_kill(0.6, 0.6, 10000,
+		#				   'testfiles/increment1', 'testfiles/output')
+		#assert CompareRecursive(Local.back1, Local.rpout, 1, None, None)
+		self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/output')
+		time.sleep(1)
+
+		# Backing up increment2
+		self.exec_and_kill(0.7, 1.0, 20000,
+						   'testfiles/increment2', 'testfiles/output')
+		assert CompareRecursive(Local.back2, Local.rpout, 1, None, None)
+		time.sleep(1)
+
+		# Backing up increment3
+		self.exec_and_kill(0.7, 2.0, 30000,
+						   'testfiles/increment3', 'testfiles/output')
+		assert CompareRecursive(Local.back3, Local.rpout, 1, None, None)
+		time.sleep(1)
+
+		# Backing up increment4
+		self.exec_and_kill(1.0, 5.0, 40000,
+						   'testfiles/increment4', 'testfiles/output')
+		assert CompareRecursive(Local.back4, Local.rpout, 1, None, None)
+
+	def testTERM(self):
+		"""Test sending local processes a TERM signal"""
+		self.killsignal = signal.SIGTERM
+		for i in range(5):
+			self.runtest()
+
+	def testKILL(self):
+		"""Send local backup process a KILL signal"""
+		self.killsignal = signal.SIGKILL
+		self.runtest()
+
+if __name__ == "__main__": unittest.main()
diff --git a/rdiff-backup/testing/restoretest.py b/rdiff-backup/testing/restoretest.py
index aaf84b8..e934c4a 100644
--- a/rdiff-backup/testing/restoretest.py
+++ b/rdiff-backup/testing/restoretest.py
@@ -2,7 +2,7 @@ import unittest
 
 execfile("commontest.py")
 rbexec("main.py")
-Log.setverbosity(7)
+Log.setverbosity(3)
 
 
 lc = Globals.local_connection
@@ -86,7 +86,11 @@ class RestoreTest(unittest.TestCase):
 		map(self.restoreonefiletest, ["ocaml", "mf"])
 
 	def testRestoreDir(self):
-		"""Test restoring from a real backup set"""
+		"""Test restoring from a real backup set
+
+		Run makerestoretest3 if this doesn't work.
+
+		"""
 		Myrm("testfiles/output")
 		InternalRestore(1, 1, "testfiles/restoretest3",
 						"testfiles/output", 20000)