From 9584378d91306ad9b3d8e7dbd110b7c2f472ed4b Mon Sep 17 00:00:00 2001 From: ben Date: Wed, 12 Jun 2002 04:05:41 +0000 Subject: Added probabilistic stuff to killtest.py git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@122 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/testing/commontest.py | 33 ++++-- rdiff-backup/testing/killtest.py | 197 +++++++++++++++++++++++++-------- rdiff-backup/testing/regressiontest.py | 2 +- 3 files changed, 178 insertions(+), 54 deletions(-) diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index 6221866..04f2a8a 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -130,6 +130,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time): mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir], remote_schema) + Time.setcurtime() inc = get_increment_rp(mirror_rp, time) if inc: _get_main().Restore(get_increment_rp(mirror_rp, time), dest_rp) @@ -163,7 +164,8 @@ def _get_main(): return Globals.Main def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, - equality_func = None, exclude_rbdir = 1): + equality_func = None, exclude_rbdir = 1, + ignore_tmp_files = None): """Compare src_rp and dest_rp, which can be directories This only compares file attributes, not the actual data. This @@ -179,6 +181,16 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, compare_hardlinks), 3) src_select = Select(DSRPath(1, src_rp)) dest_select = Select(DSRPath(None, dest_rp)) + + if ignore_tmp_files: + # Ignoring temp files can be useful when we want to check the + # correctness of a backup which aborted in the middle. In + # these cases it is OK to have tmp files lying around. + src_select.add_selection_func(src_select.regexp_get_sf( + ".*rdiff-backup.tmp.[^/]+$", 0)) + dest_select.add_selection_func(dest_select.regexp_get_sf( + ".*rdiff-backup.tmp.[^/]+$", 0)) + if exclude_rbdir: src_select.parse_rbdir_exclude() dest_select.parse_rbdir_exclude() @@ -186,14 +198,14 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, # include rdiff-backup-data/increments src_select.add_selection_func(src_select.glob_get_tuple_sf( ('rdiff-backup-data', 'increments'), 1)) - dest_select.add_selection_func(src_select.glob_get_tuple_sf( + dest_select.add_selection_func(dest_select.glob_get_tuple_sf( ('rdiff-backup-data', 'increments'), 1)) # but exclude rdiff-backup-data src_select.add_selection_func(src_select.glob_get_tuple_sf( ('rdiff-backup-data',), 0)) - dest_select.add_selection_func(src_select.glob_get_tuple_sf( - ('rdiff-backup-data',), 0)) + dest_select.add_selection_func(dest_select.glob_get_tuple_sf( + ('rdiff-backup-data',), 0)) src_select.set_iter() dest_select.set_iter() @@ -221,20 +233,23 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, if dest_rorp.index[-1].endswith('gz'): return 1 # Don't compare .missing increments because they don't matter if dest_rorp.index[-1].endswith('.missing'): return 1 - if src_rorp != dest_rorp: return None - if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1 + if compare_hardlinks: + if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1 + elif src_rorp == dest_rorp: return 1 Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3) Log("%s: %s" % (dest_rorp.index, Hardlink.get_indicies(dest_rorp, None)), 3) return None - if compare_hardlinks: + if equality_func: result = Iter.equal(dsiter1, dsiter2, 1, equality_func) + elif compare_hardlinks: dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1) - dsiter2 = Hardlink.add_rorp_iter(dsiter2, None) + dsiter2 = Hardlink.add_rorp_iter(dsiter2, None) if exclude_rbdir: result = Iter.equal(dsiter1, dsiter2, 1, hardlink_equal) else: result = Iter.equal(dsiter1, dsiter2, 1, rbdir_equal) - elif equality_func: result = Iter.equal(dsiter1, dsiter2, 1, equality_func) + elif not exclude_rbdir: + result = Iter.equal(dsiter1, dsiter2, 1, rbdir_equal) else: result = Iter.equal(dsiter1, dsiter2, 1) for i in dsiter1: pass # make sure all files processed anyway diff --git a/rdiff-backup/testing/killtest.py b/rdiff-backup/testing/killtest.py index 7c157af..d0b1b40 100644 --- a/rdiff-backup/testing/killtest.py +++ b/rdiff-backup/testing/killtest.py @@ -1,6 +1,6 @@ import unittest, os, signal, sys, random, time execfile("commontest.py") -rbexec("restore.py") +rbexec("main.py") """Test consistency by killing rdiff-backup as it is backing up""" @@ -22,14 +22,21 @@ class Local: rpout2 = get_local_rp('restoretarget2') rpout3 = get_local_rp('restoretarget3') rpout4 = get_local_rp('restoretarget4') + rpout5 = get_local_rp('restoretarget5') back1 = get_local_rp('backup1') back2 = get_local_rp('backup2') back3 = get_local_rp('backup3') back4 = get_local_rp('backup4') + back5 = get_local_rp('backup5') +class TimingError(Exception): + """Indicates timing error - process killed too soon or too late""" + pass -class Kill(unittest.TestCase): + +class ProcessFuncs(unittest.TestCase): + """Subclassed by Resume and NoResume""" def delete_tmpdirs(self): """Remove any temp directories created by previous tests""" assert not os.system(MiscDir + '/myrm testfiles/output* ' @@ -40,8 +47,7 @@ class Kill(unittest.TestCase): """True if there are signs of aborted backup in output/""" try: dirlist = os.listdir("testfiles/output/rdiff-backup-data") except OSError: - print "No data dir found, give process more time" - raise + raise TimingError("No data dir found, give process more time") dirlist = filter(lambda f: f.startswith("last-file-incremented"), dirlist) return len(dirlist) != 0 @@ -58,17 +64,19 @@ class Kill(unittest.TestCase): if wait: return os.spawnvp(os.P_WAIT, 'python', arglist) else: return os.spawnvp(os.P_NOWAIT, 'python', arglist) - def exec_and_kill(self, mintime, maxtime, backup_time, *args): + def exec_and_kill(self, mintime, maxtime, backup_time, resume, arg1, arg2): """Run rdiff-backup, then kill and run again Kill after a time between mintime and maxtime. First process should not terminate before maxtime. """ - pid = self.exec_rb(backup_time, None, *args) + pid = self.exec_rb(backup_time, None, arg1, arg2) time.sleep(random.uniform(mintime, maxtime)) - assert os.waitpid(pid, os.WNOHANG)[0] == 0, \ - "Process already quit - try lowering max time" + if os.waitpid(pid, os.WNOHANG)[0] != 0: + raise TimingError("Timing Error on %s, %s:\n" + "Process already quit - try lowering max time" + % (arg1, arg2)) os.kill(pid, self.killsignal) while 1: pid, exitstatus = os.waitpid(pid, os.WNOHANG) @@ -76,73 +84,174 @@ class Kill(unittest.TestCase): assert exitstatus != 0 break time.sleep(0.2) - assert self.is_aborted_backup(), \ - "Process already finished or didn't get a chance to start" - os.system("ls -l %s/rdiff-backup-data" % args[1]) - return self.exec_rb(backup_time + 5, 1, '--resume', *args) + if not self.is_aborted_backup(): + raise TimingError("Timing Error on %s, %s:\n" + "Process already finished or didn't " + "get a chance to start" % (arg1, arg2)) + print "---------------------- killed" + os.system("ls -l %s/rdiff-backup-data" % arg1) + if resume: self.exec_rb(backup_time + 5, 1, '--resume', arg1, arg2) + else: self.exec_rb(backup_time + 5000, 1, '--no-resume', arg1, arg2) def verify_back_dirs(self): """Make sure testfiles/output/back? dirs exist""" if (Local.back1.lstat() and Local.back2.lstat() and - Local.back3.lstat() and Local.back4.lstat()): return + Local.back3.lstat() and Local.back4.lstat() and + Local.back5.lstat()): return - os.system(MiscDir + "/myrm testfiles/backup[1-4]") + os.system(MiscDir + "/myrm testfiles/backup[1-5]") - self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup1') + self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup1') Local.back1.setdata() - self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup2') - self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup2') + self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup2') + self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup2') Local.back2.setdata() - self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup3') - self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup3') - self.exec_rb(30000, 1, 'testfiles/increment3', 'testfiles/backup3') + self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup3') + self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup3') + self.exec_rb(30000, 1, 'testfiles/increment2', 'testfiles/backup3') Local.back3.setdata() - self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/backup4') - self.exec_rb(20000, 1, 'testfiles/increment2', 'testfiles/backup4') - self.exec_rb(30000, 1, 'testfiles/increment3', 'testfiles/backup4') - self.exec_rb(40000, 1, 'testfiles/increment4', 'testfiles/backup4') + self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup4') + self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup4') + self.exec_rb(30000, 1, 'testfiles/increment2', 'testfiles/backup4') + self.exec_rb(40000, 1, 'testfiles/increment3', 'testfiles/backup4') Local.back4.setdata() - def runtest(self): + self.exec_rb(10000, 1, 'testfiles/increment3', 'testfiles/backup5') + self.exec_rb(20000, 1, 'testfiles/increment1', 'testfiles/backup5') + self.exec_rb(30000, 1, 'testfiles/increment2', 'testfiles/backup5') + self.exec_rb(40000, 1, 'testfiles/increment3', 'testfiles/backup5') + self.exec_rb(50000, 1, 'testfiles/increment4', 'testfiles/backup5') + Local.back5.setdata() + + def runtest_sequence(self, total_tests, + exclude_rbdir, ignore_tmp, compare_links, + stop_on_error = None): + timing_problems, failures = 0, 0 + for i in range(total_tests): + try: + result = self.runtest(exclude_rbdir, ignore_tmp, compare_links) + except TimingError, te: + print te + timing_problems += 1 + continue + if result != 1: + if stop_on_error: assert 0, "Compare Failure" + else: failures += 1 + + print total_tests, "tests attempted total" + print "%s setup problems, %s failures, %s successes" % \ + (timing_problems, failures, + total_tests - timing_problems - failures) + + +class Resume(ProcessFuncs): + """Test for graceful recovery after resumed backup""" + def runtest(self, exclude_rbdir, ignore_tmp_files, compare_links): + """Run the actual test, returning 1 if passed and 0 otherwise""" self.delete_tmpdirs() self.verify_back_dirs() - # Backing up increment1 - unfortunately, not big enough to test? - #self.exec_and_kill(0.6, 0.6, 10000, - # 'testfiles/increment1', 'testfiles/output') - #assert CompareRecursive(Local.back1, Local.rpout, 1, None, None) - self.exec_rb(10000, 1, 'testfiles/increment1', 'testfiles/output') - time.sleep(1) + # Backing up increment3 + + # Start with increment3 because it is big and the first case + # is kind of special (there's no incrementing, so different + # code) + self.exec_and_kill(0.7, 1.5, 10000, 1, + 'testfiles/increment3', 'testfiles/output') + if not CompareRecursive(Local.back1, Local.rpout, compare_links, + None, exclude_rbdir, ignore_tmp_files): + return 0 + + # Backing up increment1 + self.exec_and_kill(0.8, 0.8, 20000, 1, + 'testfiles/increment1', 'testfiles/output') + if not CompareRecursive(Local.back2, Local.rpout, compare_links, + None, exclude_rbdir, ignore_tmp_files): + return 0 # Backing up increment2 - self.exec_and_kill(0.7, 1.0, 20000, + self.exec_and_kill(0.7, 1.0, 30000, 1, 'testfiles/increment2', 'testfiles/output') - assert CompareRecursive(Local.back2, Local.rpout, 1, None, None) - time.sleep(1) + if not CompareRecursive(Local.back3, Local.rpout, compare_links, + None, exclude_rbdir, ignore_tmp_files): + return 0 # Backing up increment3 - self.exec_and_kill(0.7, 2.0, 30000, + self.exec_and_kill(0.7, 2.0, 40000, 1, 'testfiles/increment3', 'testfiles/output') - assert CompareRecursive(Local.back3, Local.rpout, 1, None, None) - time.sleep(1) + if not CompareRecursive(Local.back4, Local.rpout, compare_links, + None, exclude_rbdir, ignore_tmp_files): + return 0 # Backing up increment4 - self.exec_and_kill(1.0, 5.0, 40000, + self.exec_and_kill(1.0, 5.0, 50000, 1, 'testfiles/increment4', 'testfiles/output') - assert CompareRecursive(Local.back4, Local.rpout, 1, None, None) + if not CompareRecursive(Local.back5, Local.rpout, compare_links, + None, exclude_rbdir, ignore_tmp_files): + return 0 + return 1 - def testTERM(self): + def testTERM(self, total_tests = 3): """Test sending local processes a TERM signal""" self.killsignal = signal.SIGTERM - for i in range(5): - self.runtest() + self.runtest_sequence(total_tests, None, None, 1) - def testKILL(self): + def testKILL(self, total_tests = 10): """Send local backup process a KILL signal""" self.killsignal = signal.SIGKILL - self.runtest() + self.runtest_sequence(total_tests, None, 1, None) + + +class NoResume(ProcessFuncs): + """Test for consistent backup after abort and then no resume""" + def runtest(self, exclude_rbdir, ignore_tmp_files, compare_links): + self.delete_tmpdirs() + + # Back up each increment to output + self.exec_and_kill(0.7, 1.5, 10000, 1, + 'testfiles/increment3', 'testfiles/output') + self.exec_and_kill(0.6, 0.6, 20000, 1, + 'testfiles/increment1', 'testfiles/output') + self.exec_and_kill(0.7, 1.0, 30000, 1, + 'testfiles/increment2', 'testfiles/output') + self.exec_and_kill(0.7, 2.0, 40000, 1, + 'testfiles/increment3', 'testfiles/output') + self.exec_and_kill(1.0, 5.0, 50000, 1, + 'testfiles/increment4', 'testfiles/output') + + # Now restore each and compare + InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget1", + 15000) + assert CompareRecursive(Local.inc3rp, Local.rpout1, compare_links, + None, exclude_rbdir, ignore_tmp_files) + InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget2", + 25000) + assert CompareRecursive(Local.inc1rp, Local.rpout2, compare_links, + None, exclude_rbdir, ignore_tmp_files) + InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget3", + 35000) + assert CompareRecursive(Local.inc2rp, Local.rpout3, compare_links, + None, exclude_rbdir, ignore_tmp_files) + InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget4", + 45000) + assert CompareRecursive(Local.inc3rp, Local.rpout4, compare_links, + None, exclude_rbdir, ignore_tmp_files) + InternalRestore(1, 1, "testfiles/output", "testfiles/restoretarget5", + 55000) + assert CompareRecursive(Local.inc4rp, Local.rpout5, compare_links, + None, exclude_rbdir, ignore_tmp_files) + return 1 + + def testTERM(self, total_tests = 10): + self.killsignal = signal.SIGTERM + self.runtest_sequence(total_tests, 1, None, 1) + + def testKILL(self, total_tests = 20): + self.killsignal = signal.SIGKILL + self.runtest_sequence(total_tests, 1, 1, None) + if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/regressiontest.py b/rdiff-backup/testing/regressiontest.py index 1a486d7..80a60d0 100644 --- a/rdiff-backup/testing/regressiontest.py +++ b/rdiff-backup/testing/regressiontest.py @@ -13,7 +13,7 @@ testfiles Globals.set('change_source_perms', 1) Globals.counter = 0 -Log.setverbosity(7) +Log.setverbosity(3) class Local: """This is just a place to put increments relative to the local -- cgit v1.2.1