From b53bfd4d41252426cb050ef896676034d92e3ef7 Mon Sep 17 00:00:00 2001 From: bescoto Date: Tue, 31 Dec 2002 08:46:22 +0000 Subject: Various changes for v0.11.1 (see CHANGELOG) git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@256 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/testing/benchmark.py | 141 ++++++++++++++++++ rdiff-backup/testing/commontest.py | 13 +- rdiff-backup/testing/finaltest.py | 73 ++-------- rdiff-backup/testing/hardlinktest.py | 113 +++++++++------ rdiff-backup/testing/incrementtest.py | 98 ++----------- rdiff-backup/testing/metadatatest.py | 3 +- rdiff-backup/testing/rdifftest.py | 37 ++--- rdiff-backup/testing/regressiontest.py | 154 ++++++-------------- rdiff-backup/testing/restoretest.py | 255 +++++++++++++++++++-------------- rdiff-backup/testing/robusttest.py | 60 +++----- rdiff-backup/testing/roottest.py | 68 ++++++++- rdiff-backup/testing/rorpitertest.py | 19 --- rdiff-backup/testing/selectiontest.py | 26 ++-- rdiff-backup/testing/statisticstest.py | 93 +++++++++++- rdiff-backup/testing/timetest.py | 2 +- 15 files changed, 628 insertions(+), 527 deletions(-) create mode 100644 rdiff-backup/testing/benchmark.py (limited to 'rdiff-backup/testing') diff --git a/rdiff-backup/testing/benchmark.py b/rdiff-backup/testing/benchmark.py new file mode 100644 index 0000000..ccd9f0b --- /dev/null +++ b/rdiff-backup/testing/benchmark.py @@ -0,0 +1,141 @@ +import sys, time +from commontest import * +from rdiff_backup import rpath, Globals + +"""benchmark.py + +When possible, use 'rdiff-backup' from the shell, which allows using +different versions of rdiff-backup by altering the PYTHONPATH. We +just use clock time, so this isn't exact at all. + +""" + +output_local = 1 +output_desc = "testfiles/output" +new_pythonpath = None + +def run_cmd(cmd): + """Run the given cmd, return the amount of time it took""" + if new_pythonpath: full_cmd = "PYTHONPATH=%s %s" % (new_pythonpath, cmd) + else: full_cmd = cmd + print "Running command '%s'" % (full_cmd,) + t = time.time() + assert not os.system(full_cmd) + return time.time() - t + +def create_many_files(dirname, s, count = 1000): + """Create many short files in the dirname directory + + There will be count files in the directory, and each file will + contain the string s. + + """ + Myrm("testfiles/many_out") + dir_rp = rpath.RPath(Globals.local_connection, dirname) + dir_rp.mkdir() + for i in xrange(count): + rp = dir_rp.append(str(i)) + fp = rp.open("wb") + fp.write(s) + assert not fp.close() + +def create_nested(dirname, s, depth, branch_factor = 10): + """Create many short files in branching directory""" + def write(rp): + fp = rp.open("wb") + fp.write(s) + assert not fp.close() + + def helper(rp, depth): + rp.mkdir() + sub_rps = map(lambda i: rp.append(str(i)), range(branch_factor)) + if depth == 1: map(write, sub_rps) + else: map(lambda rp: helper(rp, depth-1), sub_rps) + + Myrm("testfiles/nested_out") + helper(rpath.RPath(Globals.local_connection, dirname), depth) + +def benchmark(backup_cmd, restore_cmd, desc, update_func = None): + """Print benchmark using backup_cmd and restore_cmd + + If update_func is given, run it and then do backup a third time. + + """ + print "Initially backing up %s: %ss" % (desc, run_cmd(backup_cmd)) + print "Updating %s, no change: %ss" % (desc, run_cmd(backup_cmd)) + + if update_func: + update_func() + print "Updating %s, all changed: %ss" % (desc, run_cmd(backup_cmd)) + + Myrm("testfiles/rest_out") + print "Restoring %s to empty dir: %ss" % (desc, run_cmd(restore_cmd)) + print "Restoring %s to unchanged dir: %ss" % (desc, run_cmd(restore_cmd)) + +def many_files(): + """Time backup and restore of 2000 files""" + count = 2000 + create_many_files("testfiles/many_out", "a", count) + backup_cmd = "rdiff-backup testfiles/many_out " + output_desc + restore_cmd = "rdiff-backup --force -r now %s testfiles/rest_out" % \ + (output_desc,) + update_func = lambda: create_many_files("testfiles/many_out", "e", count) + benchmark(backup_cmd, restore_cmd, "2000 1-byte files", update_func) + +def many_files_rsync(): + """Test rsync benchmark""" + count = 2000 + create_many_files("testfiles/many_out", "a", count) + rsync_command = ("rsync -e ssh -aH --delete testfiles/many_out " + + output_desc) + print "Initial rsync: %ss" % (run_cmd(rsync_command),) + print "rsync update: %ss" % (run_cmd(rsync_command),) + + create_many_files("testfiles/many_out", "e", count) + print "Update changed rsync: %ss" % (run_cmd(rsync_command),) + +def nested_files(): + """Time backup and restore of 10000 nested files""" + depth = 4 + create_nested("testfiles/nested_out", "a", depth) + backup_cmd = "rdiff-backup testfiles/nested_out " + output_desc + restore_cmd = "rdiff-backup --force -r now %s testfiles/rest_out" % \ + (output_desc,) + update_func = lambda: create_nested("testfiles/nested_out", "e", depth) + benchmark(backup_cmd, restore_cmd, "10000 1-byte nested files", + update_func) + +def nested_files_rsync(): + """Test rsync on nested files""" + depth = 4 + create_nested("testfiles/nested_out", "a", depth) + rsync_command = ("rsync -e ssh -aH --delete testfiles/nested_out " + + output_desc) + print "Initial rsync: %ss" % (run_cmd(rsync_command),) + print "rsync update: %ss" % (run_cmd(rsync_command),) + + create_nested("testfiles/nested_out", "e", depth) + print "Update changed rsync: %ss" % (run_cmd(rsync_command),) + +if len(sys.argv) < 2 or len(sys.argv) > 3: + print "Syntax: benchmark.py benchmark_func [output_description]" + print + print "Where output_description defaults to 'testfiles/output'." + print "Currently benchmark_func includes:" + print "'many_files', 'many_files_rsync', and, 'nested_files'." + sys.exit(1) + +if len(sys.argv) == 3: + output_desc = sys.argv[2] + if ":" in output_desc: output_local = None + +if output_local: + assert not rpath.RPath(Globals.local_connection, output_desc).lstat(), \ + "Outfile file %s exists, try deleting it first" % (output_desc,) + +if os.environ.has_key('BENCHMARKPYPATH'): + new_pythonpath = os.environ['BENCHMARKPYPATH'] + +function_name = sys.argv[1] +print "Running ", function_name +eval(sys.argv[1])() diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index 19b2c60..dae825a 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -3,7 +3,7 @@ import os, sys from rdiff_backup.log import Log from rdiff_backup.rpath import RPath from rdiff_backup import Globals, Hardlink, SetConnections, Main, \ - selection, highlevel, lazy, Time, rpath + selection, lazy, Time, rpath SourceDir = "../src" AbsCurdir = os.getcwd() # Absolute path name of current directory @@ -13,6 +13,9 @@ __no_execute__ = 1 # Keeps the actual rdiff-backup program from running def Myrm(dirstring): """Run myrm on given directory string""" + root_rp = rpath.RPath(Globals.local_connection, dirstring) + for rp in selection.Select(root_rp).set_iter(): + if rp.isdir(): rp.chmod(0700) # otherwise may not be able to remove assert not os.system("rm -rf %s" % (dirstring,)) def Make(): @@ -21,6 +24,13 @@ def Make(): os.system("python ./Make") os.chdir(AbsCurdir) +def MakeOutputDir(): + """Initialize the testfiles/output directory""" + Myrm("testfiles/output") + rp = rpath.RPath(Globals.local_connection, "testfiles/output") + rp.mkdir() + return rp + def rdiff_backup(source_local, dest_local, src_dir, dest_dir, current_time = None, extra_options = ""): """Run rdiff-backup with the given options @@ -121,6 +131,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time): the testing directory and will be modified for remote trials. """ + Main.force = 1 remote_schema = '%s' #_reset_connections() if not mirror_local: diff --git a/rdiff-backup/testing/finaltest.py b/rdiff-backup/testing/finaltest.py index ba1128e..ae48462 100644 --- a/rdiff-backup/testing/finaltest.py +++ b/rdiff-backup/testing/finaltest.py @@ -1,11 +1,11 @@ import unittest, os, re, sys, time from commontest import * -from rdiff_backup import Globals, log, rpath +from rdiff_backup import Globals, log, rpath, robust """Regression tests""" Globals.exclude_mirror_regexps = [re.compile(".*/rdiff-backup-data")] -log.Log.setverbosity(7) +log.Log.setverbosity(3) lc = Globals.local_connection @@ -43,7 +43,7 @@ class PathSetter(unittest.TestCase): def reset_schema(self): self.rb_schema = SourceDir + \ - "/../rdiff-backup -v3 --remote-schema './chdir-wrapper2 %s' " + "/../rdiff-backup -v7 --remote-schema './chdir-wrapper2 %s' " def refresh(self, *rp_list): """Reread data for the given rps""" @@ -168,8 +168,9 @@ class PathSetter(unittest.TestCase): def getinc_paths(self, basename, directory): """Return increment.______.dir paths""" - incfiles = filter(lambda s: s.startswith(basename), - os.listdir(directory)) + dirrp = rpath.RPath(Globals.local_connection, directory) + incfiles = [filename for filename in robust.listrp(dirrp) + if filename.startswith(basename)] incfiles.sort() incrps = map(lambda f: rpath.RPath(lc, directory+"/"+f), incfiles) return map(lambda x: x.path, filter(rpath.RPath.isincfile, incrps)) @@ -196,26 +197,15 @@ class Final(PathSetter): self.set_connections(None, None, "test2/tmp", "../../") self.runtest() -# def testMirroringLocal(self): -# """Run mirroring only everything remote""" -# self.delete_tmpdirs() -# self.set_connections(None, None, None, None) -# self.exec_rb_extra_args(10000, "-m", -# "testfiles/various_file_types", -# "testfiles/output") -# assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None) - -# def testMirroringRemote(self): -# """Run mirroring only everything remote""" -# self.delete_tmpdirs() -# self.set_connections("test1/", "../", "test2/tmp/", "../../") -# self.exec_rb_extra_args(10000, "-m", -# "testfiles/various_file_types", -# "testfiles/output") -# assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None) + def testProcLocal(self): + """Test initial backup of /proc locally""" + Myrm("testfiles/procoutput") + self.set_connections(None, None, None, None) + self.exec_rb(None, '../../../../../../proc', 'testfiles/procoutput') def testProcRemote(self): """Test mirroring proc""" + Myrm("testfiles/procoutput") self.set_connections(None, None, "test2/tmp/", "../../") self.exec_rb(None, '../../../../../../proc', 'testfiles/procoutput') @@ -352,44 +342,5 @@ testfiles/increment2/changed_dir""") self.assertRaises(OSError, os.lstat, 'testfiles/restoretarget1/executable2') - -class FinalCorrupt(PathSetter): - def testBackupOverlay(self): - """Test backing up onto a directory already backed up for that time - - This will test to see if rdiff-backup will ignore files who - already have an increment where it wants to put something. - Just make sure rdiff-backup doesn't exit with an error. - - """ - self.delete_tmpdirs() - assert not os.system("cp -a testfiles/corruptbackup testfiles/output") - self.set_connections(None, None, None, None) - self.exec_rb(None, 'testfiles/corruptbackup_source', - 'testfiles/output') - - def testBackupOverlayRemote(self): - """Like above but destination is remote""" - self.delete_tmpdirs() - assert not os.system("cp -a testfiles/corruptbackup testfiles/output") - self.set_connections(None, None, "test1/", '../') - self.exec_rb(None, 'testfiles/corruptbackup_source', - 'testfiles/output') - - def testCheckpointData(self): - """Destination directory has bad checkpoint data, no sym""" - self.delete_tmpdirs() - assert not os.system("cp -a testfiles/corrupt_dest1 testfiles/output") - self.set_connections(None, None, None, None) - self.exec_rb(None, 'testfiles/various_file_types', 'testfiles/output') - - def testCheckpointData2(self): - """Destination directory has bad checkpoint data, with sym""" - self.delete_tmpdirs() - assert not os.system("cp -a testfiles/corrupt_dest2 testfiles/output") - self.set_connections(None, None, None, None) - self.exec_rb(None, 'testfiles/various_file_types', 'testfiles/output') - - if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/hardlinktest.py b/rdiff-backup/testing/hardlinktest.py index 2b70ed4..a675e8a 100644 --- a/rdiff-backup/testing/hardlinktest.py +++ b/rdiff-backup/testing/hardlinktest.py @@ -1,5 +1,4 @@ - -import os, unittest +import os, unittest, time from commontest import * from rdiff_backup import Globals, Hardlink, selection, rpath @@ -30,48 +29,6 @@ class HardlinkTest(unittest.TestCase): assert not CompareRecursive(self.hardlink_dir1, self.hardlink_dir2, compare_hardlinks = 1) - def testCheckpointing(self): - """Test saving and recovering of various dictionaries""" - d1 = {1:1} - d2 = {2:2} - d3 = {3:3} - d4 = {} - - Hardlink._src_inode_indicies = d1 - Hardlink._src_index_indicies = d2 - Hardlink._dest_inode_indicies = d3 - Hardlink._dest_index_indicies = d4 - - self.reset_output() - Time.setcurtime(12345) - Globals.isbackup_writer = 1 - Hardlink.final_checkpoint(self.outputrp) - - reset_hardlink_dicts() - assert Hardlink.retrieve_checkpoint(self.outputrp, 12345) - assert Hardlink._src_inode_indicies == d1, \ - Hardlink._src_inode_indicies - assert Hardlink._src_index_indicies == d2, \ - Hardlink._src_index_indicies - assert Hardlink._dest_inode_indicies == d3, \ - Hardlink._dest_inode_indicies - assert Hardlink._dest_index_indicies == d4, \ - Hardlink._dest_index_indicies - - def testFinalwrite(self): - """Test writing of the final database""" - Globals.isbackup_writer = 1 - Time.setcurtime(123456) - Globals.rbdir = self.outputrp - finald = Hardlink._src_index_indicies = {'hello':'world'} - - self.reset_output() - Hardlink.final_writedata() - - Hardlink._src_index_indicies = None - assert Hardlink.retrieve_final(123456) - assert Hardlink._src_index_indicies == finald - def testBuildingDict(self): """See if the partial inode dictionary is correct""" Globals.preserve_hardlinks = 1 @@ -143,6 +100,74 @@ class HardlinkTest(unittest.TestCase): BackupRestoreSeries(None, None, dirlist, compare_hardlinks=1) BackupRestoreSeries(1, 1, dirlist, compare_hardlinks=1) + def testInnerRestore(self): + """Restore part of a dir, see if hard links preserved""" + MakeOutputDir() + output = rpath.RPath(Globals.local_connection, + "testfiles/output") + + # Now set up directories out_hardlink1 and out_hardlink2 + hlout1 = rpath.RPath(Globals.local_connection, + "testfiles/out_hardlink1") + if hlout1.lstat(): hlout1.delete() + hlout1.mkdir() + hlout1_sub = hlout1.append("subdir") + hlout1_sub.mkdir() + hl1_1 = hlout1_sub.append("hardlink1") + hl1_2 = hlout1_sub.append("hardlink2") + hl1_3 = hlout1_sub.append("hardlink3") + hl1_4 = hlout1_sub.append("hardlink4") + # 1 and 2 are hard linked, as are 3 and 4 + hl1_1.touch() + hl1_2.hardlink(hl1_1.path) + hl1_3.touch() + hl1_4.hardlink(hl1_3.path) + + hlout2 = rpath.RPath(Globals.local_connection, + "testfiles/out_hardlink2") + if hlout2.lstat(): hlout2.delete() + assert not os.system("cp -a testfiles/out_hardlink1 " + "testfiles/out_hardlink2") + hlout2_sub = hlout2.append("subdir") + hl2_1 = hlout2_sub.append("hardlink1") + hl2_2 = hlout2_sub.append("hardlink2") + hl2_3 = hlout2_sub.append("hardlink3") + hl2_4 = hlout2_sub.append("hardlink4") + # Now 2 and 3 are hard linked, also 1 and 4 + rpath.copy_with_attribs(hl1_1, hl2_1) + rpath.copy_with_attribs(hl1_2, hl2_2) + hl2_3.delete() + hl2_3.hardlink(hl2_2.path) + hl2_4.delete() + hl2_4.hardlink(hl2_1.path) + rpath.copy_attribs(hlout1_sub, hlout2_sub) + + InternalBackup(1, 1, hlout1.path, output.path) + time.sleep(1) + InternalBackup(1, 1, hlout2.path, output.path) + + out2 = rpath.RPath(Globals.local_connection, "testfiles/out2") + hlout1 = out2.append("hardlink1") + hlout2 = out2.append("hardlink2") + hlout3 = out2.append("hardlink3") + hlout4 = out2.append("hardlink4") + + if out2.lstat(): out2.delete() + InternalRestore(1, 1, "testfiles/output/subdir", "testfiles/out2", 1) + out2.setdata() + for rp in [hlout1, hlout2, hlout3, hlout4]: rp.setdata() + assert hlout1.getinode() == hlout2.getinode() + assert hlout3.getinode() == hlout4.getinode() + assert hlout1.getinode() != hlout3.getinode() + + if out2.lstat(): out2.delete() + InternalRestore(1, 1, "testfiles/output/subdir", "testfiles/out2", + int(time.time())) + out2.setdata() + for rp in [hlout1, hlout2, hlout3, hlout4]: rp.setdata() + assert hlout1.getinode() == hlout4.getinode() + assert hlout2.getinode() == hlout3.getinode() + assert hlout1.getinode() != hlout2.getinode() if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/incrementtest.py b/rdiff-backup/testing/incrementtest.py index 0aa52ea..165c55f 100644 --- a/rdiff-backup/testing/incrementtest.py +++ b/rdiff-backup/testing/incrementtest.py @@ -1,7 +1,6 @@ import unittest, os, re, time from commontest import * -from rdiff_backup import log, rpath, restore, increment, Time, \ - Rdiff, statistics +from rdiff_backup import log, rpath, increment, Time, Rdiff, statistics lc = Globals.local_connection Globals.change_source_perms = 1 @@ -21,14 +20,14 @@ dir = getrp(".") sym = getrp("symbolic_link") nothing = getrp("nothing") -target = rpath.RPath(lc, "testfiles/out") -out2 = rpath.RPath(lc, "testfiles/out2") -out_gz = rpath.RPath(lc, "testfiles/out.gz") +target = rpath.RPath(lc, "testfiles/output/out") +out2 = rpath.RPath(lc, "testfiles/output/out2") +out_gz = rpath.RPath(lc, "testfiles/output/out.gz") Time.setprevtime(999424113) prevtimestr = "2001-09-02T02:48:33-07:00" -t_pref = "testfiles/out.2001-09-02T02:48:33-07:00" -t_diff = "testfiles/out.2001-09-02T02:48:33-07:00.diff" +t_pref = "testfiles/output/out.2001-09-02T02:48:33-07:00" +t_diff = "testfiles/output/out.2001-09-02T02:48:33-07:00.diff" Globals.no_compression_regexp = \ re.compile(Globals.no_compression_regexp_string, re.I) @@ -37,11 +36,12 @@ class inctest(unittest.TestCase): """Test the incrementRP function""" def setUp(self): Globals.set('isbackup_writer',1) + MakeOutputDir() def check_time(self, rp): """Make sure that rp is an inc file, and time is Time.prevtime""" assert rp.isincfile(), rp - t = Time.stringtotime(rp.getinctime()) + t = rp.getinctime() assert t == Time.prevtime, (t, Time.prevtime) def testreg(self): @@ -114,7 +114,7 @@ class inctest(unittest.TestCase): rp = increment.Increment(rf, rf2, target) self.check_time(rp) assert rpath.cmp_attribs(rp, rf2) - Rdiff.patch_action(rf, rp, out2).execute() + Rdiff.patch_local(rf, rp, out2) assert rpath.cmp(rf2, out2) rp.delete() out2.delete() @@ -125,7 +125,7 @@ class inctest(unittest.TestCase): rp = increment.Increment(rf, rf2, target) self.check_time(rp) assert rpath.cmp_attribs(rp, rf2) - Rdiff.patch_action(rf, rp, out2, delta_compressed = 1).execute() + Rdiff.patch_local(rf, rp, out2, delta_compressed = 1) assert rpath.cmp(rf2, out2) rp.delete() out2.delete() @@ -139,86 +139,10 @@ class inctest(unittest.TestCase): rp = increment.Increment(rf, out_gz, target) self.check_time(rp) assert rpath.cmp_attribs(rp, out_gz) - Rdiff.patch_action(rf, rp, out2).execute() + Rdiff.patch_local(rf, rp, out2) assert rpath.cmp(out_gz, out2) rp.delete() out2.delete() out_gz.delete() -class inctest2(unittest.TestCase): - """Like inctest but contains more elaborate tests""" - def stats_check_initial(self, s): - """Make sure stats object s compatible with initial mirroring - - A lot of the off by one stuff is because the root directory - exists in the below examples. - - """ - assert s.MirrorFiles == 1 or s.MirrorFiles == 0 - assert s.MirrorFileSize < 20000 - assert s.NewFiles <= s.SourceFiles <= s.NewFiles + 1 - assert s.NewFileSize <= s.SourceFileSize <= s.NewFileSize + 20000 - assert s.ChangedFiles == 1 or s.ChangedFiles == 0 - assert s.ChangedSourceSize < 20000 - assert s.ChangedMirrorSize < 20000 - assert s.DeletedFiles == s.DeletedFileSize == 0 - assert s.IncrementFileSize == 0 - - def testStatistics(self): - """Test the writing of statistics - - The file sizes are approximate because the size of directories - could change with different file systems... - - """ - Globals.compression = 1 - Myrm("testfiles/output") - InternalBackup(1, 1, "testfiles/stattest1", "testfiles/output") - InternalBackup(1, 1, "testfiles/stattest2", "testfiles/output", - time.time()+1) - - rbdir = rpath.RPath(Globals.local_connection, - "testfiles/output/rdiff-backup-data") - - #incs = Restore.get_inclist(rbdir.append("subdir"). - # append("directory_statistics")) - #assert len(incs) == 2 - #s1 = StatsObj().read_stats_from_rp(incs[0]) # initial mirror stats - #assert s1.SourceFiles == 2 - #assert 400000 < s1.SourceFileSize < 420000 - #self.stats_check_initial(s1) - - #subdir_stats = StatsObj().read_stats_from_rp(incs[1]) # increment stats - #assert subdir_stats.SourceFiles == 2 - #assert 400000 < subdir_stats.SourceFileSize < 420000 - #assert subdir_stats.MirrorFiles == 2 - #assert 400000 < subdir_stats.MirrorFileSize < 420000 - #assert subdir_stats.NewFiles == subdir_stats.NewFileSize == 0 - #assert subdir_stats.DeletedFiles == subdir_stats.DeletedFileSize == 0 - #assert subdir_stats.ChangedFiles == 2 - #assert 400000 < subdir_stats.ChangedSourceSize < 420000 - #assert 400000 < subdir_stats.ChangedMirrorSize < 420000 - #assert 10 < subdir_stats.IncrementFileSize < 20000 - - incs = restore.get_inclist(rbdir.append("session_statistics")) - assert len(incs) == 2 - s2 = statistics.StatsObj().read_stats_from_rp(incs[0]) - assert s2.SourceFiles == 7 - assert 700000 < s2.SourceFileSize < 750000 - self.stats_check_initial(s2) - - root_stats = statistics.StatsObj().read_stats_from_rp(incs[1]) - assert root_stats.SourceFiles == 7, root_stats.SourceFiles - assert 550000 < root_stats.SourceFileSize < 570000 - assert root_stats.MirrorFiles == 7 - assert 700000 < root_stats.MirrorFileSize < 750000 - assert root_stats.NewFiles == 1 - assert root_stats.NewFileSize == 0 - assert root_stats.DeletedFiles == 1 - assert root_stats.DeletedFileSize == 200000 - assert 3 <= root_stats.ChangedFiles <= 4, root_stats.ChangedFiles - assert 450000 < root_stats.ChangedSourceSize < 470000 - assert 400000 < root_stats.ChangedMirrorSize < 420000 - assert 10 < root_stats.IncrementFileSize < 30000 - if __name__ == '__main__': unittest.main() diff --git a/rdiff-backup/testing/metadatatest.py b/rdiff-backup/testing/metadatatest.py index 570dd79..7b6a91a 100644 --- a/rdiff-backup/testing/metadatatest.py +++ b/rdiff-backup/testing/metadatatest.py @@ -1,7 +1,6 @@ import unittest, os, cStringIO, time from rdiff_backup.metadata import * -from rdiff_backup import rpath, connection, Globals, selection, \ - destructive_stepping +from rdiff_backup import rpath, connection, Globals, selection tempdir = rpath.RPath(Globals.local_connection, "testfiles/output") diff --git a/rdiff-backup/testing/rdifftest.py b/rdiff-backup/testing/rdifftest.py index 999f1ac..6079f1a 100644 --- a/rdiff-backup/testing/rdifftest.py +++ b/rdiff-backup/testing/rdifftest.py @@ -50,7 +50,7 @@ class RdiffTest(unittest.TestCase): self.delta.write_from_fileobj(Rdiff.get_delta_sigrp(self.signature, self.new)) assert self.delta.lstat() - Rdiff.patch_action(self.basis, self.delta, self.output).execute() + Rdiff.patch_local(self.basis, self.delta, self.output) assert rpath.cmp(self.new, self.output) map(rpath.RPath.delete, rplist) @@ -74,14 +74,14 @@ class RdiffTest(unittest.TestCase): os.system("mv %s %s" % (self.delta.path + ".gz", self.delta.path)) self.delta.setdata() - Rdiff.patch_action(self.basis, self.delta, self.output, - delta_compressed = 1).execute() + Rdiff.patch_local(self.basis, self.delta, self.output, + delta_compressed = 1) assert rpath.cmp(self.new, self.output) map(rpath.RPath.delete, rplist) def testWriteDelta(self): """Test write delta feature of rdiff""" - self.delta.delete() + if self.delta.lstat(): self.delta.delete() rplist = [self.basis, self.new, self.delta, self.output] MakeRandomFile(self.basis.path) MakeRandomFile(self.new.path) @@ -90,7 +90,7 @@ class RdiffTest(unittest.TestCase): Rdiff.write_delta(self.basis, self.new, self.delta) assert self.delta.lstat() - Rdiff.patch_action(self.basis, self.delta, self.output).execute() + Rdiff.patch_local(self.basis, self.delta, self.output) assert rpath.cmp(self.new, self.output) map(rpath.RPath.delete, rplist) @@ -109,7 +109,7 @@ class RdiffTest(unittest.TestCase): os.system("gunzip " + delta_gz.path) delta_gz.setdata() self.delta.setdata() - Rdiff.patch_action(self.basis, self.delta, self.output).execute() + Rdiff.patch_local(self.basis, self.delta, self.output) assert rpath.cmp(self.new, self.output) map(rpath.RPath.delete, rplist) @@ -128,7 +128,7 @@ class RdiffTest(unittest.TestCase): self.delta.write_from_fileobj(Rdiff.get_delta_sigrp(self.signature, self.new)) assert self.delta.lstat() - Rdiff.patch_action(self.basis, self.delta).execute() + Rdiff.patch_local(self.basis, self.delta) assert rpath.cmp(self.basis, self.new) map(rpath.RPath.delete, rplist) @@ -141,31 +141,10 @@ class RdiffTest(unittest.TestCase): MakeRandomFile(self.basis.path) MakeRandomFile(self.new.path) map(rpath.RPath.setdata, rplist) - Rdiff.copy_action(self.basis, self.new).execute() + Rdiff.copy_local(self.basis, self.new) assert rpath.cmp(self.basis, self.new) map(rpath.RPath.delete, rplist) - def testPatchWithAttribs(self): - """Using rdiff to copy two files with attributes""" - rplist = [self.basis, self.new, self.delta] - for rp in rplist: - if rp.lstat(): rp.delete() - - MakeRandomFile(self.basis.path) - MakeRandomFile(self.new.path) - self.new.chmod(0401) - map(rpath.RPath.setdata, rplist) - Rdiff.write_delta(self.basis, self.new, self.delta) - rpath.copy_attribs(self.new, self.delta) - assert self.delta.getperms() == 0401 - - assert not self.basis == self.new - Rdiff.patch_with_attribs_action(self.basis, self.delta).execute() - if not self.basis == self.new: - print self.basis, self.new - assert 0 - map(rpath.RPath.delete, rplist) - if __name__ == '__main__': unittest.main() diff --git a/rdiff-backup/testing/regressiontest.py b/rdiff-backup/testing/regressiontest.py index 57b57ab..5c55986 100644 --- a/rdiff-backup/testing/regressiontest.py +++ b/rdiff-backup/testing/regressiontest.py @@ -1,6 +1,6 @@ import unittest, os from commontest import * -from rdiff_backup import Globals, SetConnections, log, rpath +from rdiff_backup import Globals, SetConnections, log, rpath, backup """Regression tests @@ -12,14 +12,14 @@ testfiles Globals.set('change_source_perms', 1) Globals.counter = 0 -log.Log.setverbosity(7) +log.Log.setverbosity(3) + +def get_local_rp(extension): + return rpath.RPath(Globals.local_connection, "testfiles/" + extension) class Local: """This is just a place to put increments relative to the local connection""" - def get_local_rp(extension): - return rpath.RPath(Globals.local_connection, "testfiles/" + extension) - inc1rp = get_local_rp('increment1') inc2rp = get_local_rp('increment2') inc3rp = get_local_rp('increment3') @@ -152,98 +152,52 @@ class IncrementTest1(unittest.TestCase): """Increment/Restore when both directories are remote""" BackupRestoreSeries(None, None, self.dirlist) + def testNoWrite(self): + """Test backup/restore on dirs without write permissions""" + def write_string(rp, s = ""): + """Write string s to file""" + fp = rp.open("wb") + fp.write(s) + assert not fp.close() -class IncrementTest2(PathSetter): - def OldtestRecoveryLocal(self): - """Test to see if rdiff-backup can continue with bad increment""" - assert not os.system("rm -rf testfiles/recovery_out_backup") - self.setPathnames(None, None, None, None) - Time.setprevtime(1006136450) - Time.setcurtime() - Globals.add_regexp('.*rdiff-backup-data', 1) - os.system('cp -a testfiles/recovery_out testfiles/recovery_out_backup') - recovery_in = self.get_src_rp('testfiles/recovery') - recovery_out = self.get_dest_rp('testfiles/recovery_out_backup') - recovery_inc = self.get_dest_rp('testfiles/recovery_out_backup/' - 'rdiff-backup-data/increments') - highlevel.Mirror_and_increment(recovery_in, recovery_out, recovery_inc) - # Should probably check integrity of increments, but for now - # allow if it doesn't during the Mirror_and_increment - - def OldtestRecoveryRemote(self): - """Test Recovery with both connections remote""" - assert not os.system('rm -rf testfiles/recovery_out_backup') - self.setPathnames('test1', '../', 'test2/tmp', '../../') - Time.setprevtime(1006136450) - Time.setcurtime() - Globals.add_regexp('.*rdiff-backup-data', 1) - os.system('cp -a testfiles/recovery_out testfiles/recovery_out_backup') - recovery_in = self.get_src_rp('testfiles/recovery') - recovery_out = self.get_dest_rp('testfiles/recovery_out_backup') - recovery_inc = self.get_dest_rp('testfiles/recovery_out_backup/' - 'rdiff-backup-data/increments') - highlevel.Mirror_and_increment(recovery_in, recovery_out, recovery_inc) - # Should probably check integrity of increments, but for now - # allow if it doesn't during the Mirror_and_increment - - def runtest(self): - """After setting connections, etc., run actual test using this""" - Time.setcurtime() - - Main.backup_set_select(Local.inc1rp) - highlevel.Mirror(self.inc1rp, self.rpout) - assert CompareRecursive(Local.inc1rp, Local.rpout) - - Time.setcurtime() - Time.setprevtime(999500000) - Main.backup_set_select(self.inc2rp) - highlevel.Mirror_and_increment(self.inc2rp, self.rpout, self.rpout_inc) - assert CompareRecursive(Local.inc2rp, Local.rpout) + def make_subdirs(): + """Make testfiles/no_write_out and testfiles/no_write_out2""" + nw_out1 = get_local_rp("no_write_out") + nw_out1.mkdir() - Time.setcurtime() - Time.setprevtime(999510000) - Main.backup_set_select(self.inc3rp) - highlevel.Mirror_and_increment(self.inc3rp, self.rpout, self.rpout_inc) - assert CompareRecursive(Local.inc3rp, Local.rpout) + nw_out1_1 = get_local_rp("no_write_out/1") + write_string(nw_out1_1) + nw_out1_1.chmod(0) - Time.setcurtime() - Time.setprevtime(999520000) - Main.backup_set_select(self.inc4rp) - highlevel.Mirror_and_increment(self.inc4rp, self.rpout, self.rpout_inc) - assert CompareRecursive(Local.inc4rp, Local.rpout) - + nw_out1_2 = get_local_rp("no_write_out/2") + write_string(nw_out1_2, 'e') + nw_out1_1.chmod(0400) - print "Restoring to self.inc4" - highlevel.Restore(999530000, self.rpout, self.get_inctup(), - self.rpout4) - assert CompareRecursive(Local.inc4rp, Local.rpout4) + nw1_sub = get_local_rp("no_write_out/subdir") + nw1_sub.mkdir() - print "Restoring to self.inc3" - highlevel.Restore(999520000, self.rpout, self.get_inctup(), - self.rpout3) - assert CompareRecursive(Local.inc3rp, Local.rpout3) + nw_out1_sub1 = get_local_rp("no_write_out/subdir/1") + write_string(nw_out1_sub1, 'f') + nw1_sub.chmod(0500) + nw_out1.chmod(0500) - print "Restoring to self.inc2" - highlevel.Restore(999510000, self.rpout, self.get_inctup(), - self.rpout2) - assert CompareRecursive(Local.inc2rp, Local.rpout2) + nw_out2 = get_local_rp("no_write_out2") + nw_out2.mkdir() - print "Restoring to self.inc1" - highlevel.Restore(999500000, self.rpout, self.get_inctup(), - self.rpout1) - assert CompareRecursive(Local.inc1rp, Local.rpout1) + nw_out2_1 = get_local_rp("no_write_out2/1") + write_string(nw_out2_1, 'g') - def get_inctup(self): - """Return inc tuples as expected by Restore.RestoreRecursive + nw_out2_2 = get_local_rp("no_write_out2/2") + write_string(nw_out2_2, 'aeu') + nw_out1.chmod(0500) - Assumes output increment directory is - testfiles/output_inc._____. - - """ - filenames = filter(lambda x: x.startswith("output_inc."), - Local.prefix.listdir()) - rplist = map(lambda x: Local.prefix.append(x), filenames) - return IndexedTuple((), (Local.prefix.append("output_inc"), rplist)) + Myrm("testfiles/no_write_out") + Myrm("testfiles/no_write_out2") + Myrm("testfiles/output") + make_subdirs() + BackupRestoreSeries(1, 1, ['testfiles/no_write_out', + 'testfiles/no_write_out2', + 'testfiles/empty']) class MirrorTest(PathSetter): @@ -317,7 +271,7 @@ class MirrorTest(PathSetter): Globals.change_ownership = 1 self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # add uid/gid info - highlevel.Mirror(self.rootfiles, self.rootfiles_out) + backup.Mirror(self.rootfiles, self.rootfiles_out) assert CompareRecursive(Local.rootfiles, Local.rootfiles_out) Globals.change_ownership = None self.refresh(self.rootfiles, self.rootfiles_out, @@ -330,29 +284,13 @@ class MirrorTest(PathSetter): conn.Globals.set('change_ownership', 1) self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # add uid/gid info - highlevel.Mirror(self.rootfiles, self.rootfiles_out) + backup.Mirror(self.rootfiles, self.rootfiles_out) assert CompareRecursive(Local.rootfiles, Local.rootfiles_out) for coon in Globals.connections: conn.Globals.set('change_ownership', None) self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # remove that info - def testRoot2Local(self): - """Make sure we can backup a directory we don't own""" - self.setPathnames(None, None, None, None) - Globals.change_ownership = Globals.change_source_perms = None - self.refresh(self.rootfiles2, self.rootfiles_out2, - Local.rootfiles2, Local.rootfiles_out2) # add uid/gid info - self.Mirror(self.rootfiles2, self.rootfiles_out2) - assert CompareRecursive(Local.rootfiles2, Local.rootfiles_out2) - self.refresh(self.rootfiles2, self.rootfiles_out2, - Local.rootfiles2, Local.rootfiles_out2) # remove that info - self.Mirror(self.rootfiles21, self.rootfiles_out2) - assert CompareRecursive(Local.rootfiles21, Local.rootfiles_out2) - self.refresh(self.rootfiles21, self.rootfiles_out2, - Local.rootfiles21, Local.rootfiles_out2) # remove that info - Globals.change_source_perms = 1 - def deleteoutput(self): assert not os.system("rm -rf testfiles/output*") self.rbdir = self.rpout.append('rdiff-backup-data') @@ -395,12 +333,12 @@ class MirrorTest(PathSetter): assert CompareRecursive(Local.inc2rp, Local.rpout) def Mirror(self, rpin, rpout): - """Like highlevel.Mirror, but run misc_setup first""" + """Like backup.Mirror, but setup first, cleanup later""" Main.force = 1 Main.misc_setup([rpin, rpout]) Main.backup_set_select(rpin) Main.backup_init_dirs(rpin, rpout) - highlevel.Mirror(rpin, rpout) + backup.Mirror(rpin, rpout) Log.close_logfile() Hardlink.clear_dictionaries() diff --git a/rdiff-backup/testing/restoretest.py b/rdiff-backup/testing/restoretest.py index 9cb6ebb..df99300 100644 --- a/rdiff-backup/testing/restoretest.py +++ b/rdiff-backup/testing/restoretest.py @@ -1,120 +1,163 @@ import unittest from commontest import * -from rdiff_backup import log, restore, Globals, rpath +from rdiff_backup import log, restore, Globals, rpath, TempFile Log.setverbosity(3) - - lc = Globals.local_connection +tempdir = rpath.RPath(Globals.local_connection, "testfiles/output") +restore_base_rp = rpath.RPath(Globals.local_connection, + "testfiles/restoretest") +restore_base_filenames = restore_base_rp.listdir() +mirror_time = 1041109438 # just some late time + +class RestoreFileComparer: + """Holds a file to be restored and tests against it + + Each object has a restore file and a dictionary of times -> + rpaths. When the restore file is restored to one of the given + times, the resulting file should be the same as the related rpath. + + """ + def __init__(self, rf): + self.rf = rf + self.time_rp_dict = {} + + def add_rpath(self, rp, t): + """Add rp, which represents what rf should be at given time t""" + assert not self.time_rp_dict.has_key(t) + self.time_rp_dict[t] = rp + + def compare_at_time(self, t): + """Restore file, make sure it is the same at time t""" + log.Log("Checking result at time %s" % (t,), 7) + tf = TempFile.new(tempdir.append("foo")) + restore._mirror_time = mirror_time + restore._rest_time = t + self.rf.set_relevant_incs() + out_rorpath = self.rf.get_attribs().getRORPath() + correct_result = self.time_rp_dict[t] + + if out_rorpath.isreg(): + out_rorpath.setfile(self.rf.get_restore_fp()) + rpath.copy_with_attribs(out_rorpath, tf) + assert tf.equal_verbose(correct_result, check_index = 0), \ + "%s, %s" % (tf, correct_result) + if tf.isreg(): + assert rpath.cmpfileobj(tf.open("rb"), correct_result.open("rb")) + if tf.lstat(): tf.delete() + + def compare_all(self): + """Check restore results for all available times""" + for t in self.time_rp_dict.keys(): self.compare_at_time(t) + class RestoreTest(unittest.TestCase): """Test Restore class""" - prefix = "testfiles/restoretest/" - def maketesttuples(self, basename): - """Make testing tuples from available files starting with prefix - - tuples is a sorted (oldest to newest) list of pairs (rp1, rp2) - where rp1 is an increment file and rp2 is the same but without - the final extension. incs is a list of all increment files. - - """ - dirlist = os.listdir(self.prefix) - dirlist.sort() - baselist = filter(lambda f: f.startswith(basename), dirlist) - rps = map(lambda f: rpath.RPath(lc, self.prefix+f), baselist) - incs = filter(lambda rp: rp.isincfile(), rps) - tuples = map(lambda rp: (rp, rpath.RPath(lc, "%s.%s" % - (rp.getincbase().path, - rp.getinctime()))), - incs) - return tuples, incs - - def restoreonefiletest(self, basename): - tuples, incs = self.maketesttuples(basename) - rpbase = rpath.RPath(lc, self.prefix + basename) - rptarget = rpath.RPath(lc, "testfiles/outfile") - Hardlink.initialize_dictionaries() - - for pair in tuples: - print "Processing file " + pair[0].path - if rptarget.lstat(): rptarget.delete() - rest_time = Time.stringtotime(pair[0].getinctime()) - rid = restore.RestoreIncrementData((), rpbase, incs) - rid.sortincseq(rest_time, 10000000000) # pick some really late time - rcd = restore.RestoreCombinedData(rid, rpbase, rptarget) - rcd.RestoreFile() - #sorted_incs = Restore.sortincseq(rest_time, incs) - #Restore.RestoreFile(rest_time, rpbase, (), sorted_incs, rptarget) - rptarget.setdata() - if not rptarget.lstat(): assert not pair[1].lstat() - elif not pair[1].lstat(): assert not rptarget.lstat() - else: - assert rpath.cmp(rptarget, pair[1]), \ - "%s %s" % (rptarget.path, pair[1].path) - assert rpath.cmp_attribs(rptarget, pair[1]), \ - "%s %s" % (rptarget.path, pair[1].path) - rptarget.delete() - - def testsortincseq(self): - """Test the Restore.sortincseq function - - This test just makes sure that it comes up with the right - number of increments for each base name - given a list of - increments, we should eventually get sorted sequences that - end in each one (each one will be the last increment once). - - """ - for basename in ['ocaml', 'mf']: - tuples, unused = self.maketesttuples(basename) - incs = [tuple[0] for tuple in tuples] - - # Now we need a time newer than any inc - mirror_time = Time.stringtotime(incs[-1].getinctime()) + 10000 - - for inc, incbase in tuples: - assert inc.isincfile() - inctime = Time.stringtotime(inc.getinctime()) - rid1 = restore.RestoreIncrementData(basename, incbase, incs) - rid1.sortincseq(inctime, mirror_time) - assert rid1.inc_list, rid1.inc_list - # oldest increment should be exactly inctime - ridtime = Time.stringtotime(rid1.inc_list[-1].getinctime()) - assert ridtime == inctime, (ridtime, inctime) - - - def testRestorefiles(self): - """Testing restoration of files one at a time""" - map(self.restoreonefiletest, ["ocaml", "mf"]) - - def testRestoreDir(self): - """Test restoring from a real backup set - - Run makerestoretest3 if this doesn't work. - - """ - Myrm("testfiles/output") - InternalRestore(1, 1, "testfiles/restoretest3", - "testfiles/output", 20000) - - src_rp = rpath.RPath(Globals.local_connection, "testfiles/increment2") - restore_rp = rpath.RPath(Globals.local_connection, "testfiles/output") - assert CompareRecursive(src_rp, restore_rp) - - def testRestoreCorrupt(self): - """Test restoring a partially corrupt archive - - The problem here is that a directory is missing from what is - to be restored, but because the previous backup was aborted in - the middle, some of the files in that directory weren't marked - as .missing. + def get_rfcs(self): + """Return available RestoreFileCompararer objects""" + base_rf = restore.RestoreFile(restore_base_rp, restore_base_rp, []) + rfs = base_rf.yield_sub_rfs() + rfcs = [] + for rf in rfs: + if rf.mirror_rp.dirsplit()[1] in ["dir"]: + log.Log("skipping 'dir'", 5) + continue + + rfc = RestoreFileComparer(rf) + for inc in rf.inc_list: + test_time = inc.getinctime() + rfc.add_rpath(self.get_correct(rf.mirror_rp, test_time), + test_time) + rfc.add_rpath(rf.mirror_rp, mirror_time) + rfcs.append(rfc) + return rfcs + + def get_correct(self, mirror_rp, test_time): + """Return correct version with base mirror_rp at time test_time""" + assert -1 < test_time < 2000000000, test_time + dirname, basename = mirror_rp.dirsplit() + for filename in restore_base_filenames: + comps = filename.split(".") + base = ".".join(comps[:-1]) + t = Time.stringtotime(comps[-1]) + if t == test_time and basename == base: + return restore_base_rp.append(filename) + # Correct rp must be empty + return restore_base_rp.append("%s.%s" % + (basename, Time.timetostring(test_time))) + + def testRestoreSingle(self): + """Test restoring files one at at a time""" + MakeOutputDir() + for rfc in self.get_rfcs(): + if rfc.rf.inc_rp.isincfile(): continue + log.Log("Comparing %s" % (rfc.rf.inc_rp.path,), 5) + rfc.compare_all() + + def testBothLocal(self): + """Test directory restore everything local""" + self.restore_dir_test(1,1) + + def testMirrorRemote(self): + """Test directory restore mirror is remote""" + self.restore_dir_test(0, 1) + + def testDestRemote(self): + """Test directory restore destination is remote""" + self.restore_dir_test(1, 0) + + def testBothRemote(self): + """Test directory restore everything is remote""" + self.restore_dir_test(0, 0) + + def restore_dir_test(self, mirror_local, dest_local): + """Run whole dir tests + + If any of the above tests don't work, try rerunning + makerestoretest3. """ Myrm("testfiles/output") - InternalRestore(1, 1, "testfiles/restoretest4", "testfiles/output", - 10000) - assert os.lstat("testfiles/output") - self.assertRaises(OSError, os.lstat, "testfiles/output/tmp") - self.assertRaises(OSError, os.lstat, "testfiles/output/rdiff-backup") + target_rp = rpath.RPath(Globals.local_connection, "testfiles/output") + mirror_rp = rpath.RPath(Globals.local_connection, + "testfiles/restoretest3") + inc1_rp = rpath.RPath(Globals.local_connection, + "testfiles/increment1") + inc2_rp = rpath.RPath(Globals.local_connection, + "testfiles/increment2") + inc3_rp = rpath.RPath(Globals.local_connection, + "testfiles/increment3") + inc4_rp = rpath.RPath(Globals.local_connection, + "testfiles/increment4") + + InternalRestore(mirror_local, dest_local, "testfiles/restoretest3", + "testfiles/output", 45000) + assert CompareRecursive(inc4_rp, target_rp) + InternalRestore(mirror_local, dest_local, "testfiles/restoretest3", + "testfiles/output", 35000) + assert CompareRecursive(inc3_rp, target_rp, compare_hardlinks = 0) + InternalRestore(mirror_local, dest_local, "testfiles/restoretest3", + "testfiles/output", 25000) + assert CompareRecursive(inc2_rp, target_rp, compare_hardlinks = 0) + InternalRestore(mirror_local, dest_local, "testfiles/restoretest3", + "testfiles/output", 5000) + assert CompareRecursive(inc1_rp, target_rp, compare_hardlinks = 0) + +# def testRestoreCorrupt(self): +# """Test restoring a partially corrupt archive +# +# The problem here is that a directory is missing from what is +# to be restored, but because the previous backup was aborted in +# the middle, some of the files in that directory weren't marked +# as .missing. +# +# """ +# Myrm("testfiles/output") +# InternalRestore(1, 1, "testfiles/restoretest4", "testfiles/output", +# 10000) +# assert os.lstat("testfiles/output") +# self.assertRaises(OSError, os.lstat, "testfiles/output/tmp") +# self.assertRaises(OSError, os.lstat, "testfiles/output/rdiff-backup") def testRestoreNoincs(self): """Test restoring a directory with no increments, just mirror""" diff --git a/rdiff-backup/testing/robusttest.py b/rdiff-backup/testing/robusttest.py index 8c6d51c..6b9e356 100644 --- a/rdiff-backup/testing/robusttest.py +++ b/rdiff-backup/testing/robusttest.py @@ -1,32 +1,11 @@ + import os, unittest from commontest import * from rdiff_backup import rpath, robust, TempFile, Globals - - -class TestRobustAction(unittest.TestCase): - """Test some robust actions""" - def testCopyWithAttribs(self): - """Test copy with attribs action""" - rpin = rpath.RPath(Globals.local_connection, "./testfiles/robust/in") - fp = open("./testfiles/robust/in", "wb") - fp.write("hello there") - fp.close() - os.chmod("./testfiles/robust/in", 0604) - rpin.setdata() - assert rpin.isreg() and rpin.getperms() % 01000 == 0604 - - rpout = rpath.RPath(Globals.local_connection, "./testfiles/robust/out") - robust.copy_with_attribs_action(rpin, rpout).execute() - if not rpout == rpin: - print rpout, rpin - assert 0 - - rpout.delete() - rpin.delete() class TempFileTest(unittest.TestCase): - """Test creation and management of tempfiles""" + """Test creation and management of tempfiles in TempFile module""" rp_base = rpath.RPath(Globals.local_connection, "./testfiles/robust/testfile_base") def testBasic(self): @@ -61,26 +40,19 @@ class TempFileTest(unittest.TestCase): assert destination.lstat() destination.delete() - -class SaveStateTest(unittest.TestCase): - """Test SaveState class""" - data_dir = rpath.RPath(Globals.local_connection, "testfiles/robust") - def testSymlinking(self): - """Test recording last file with symlink""" - last_rorp = rpath.RORPath(('usr', 'local', 'bin', 'ls')) - Globals.rbdir = self.data_dir - Time.setcurtime() - SetConnections.BackupInitConnections(Globals.local_connection, - Globals.local_connection) - robust.SaveState.init_filenames() - robust.SaveState.record_last_file_action(last_rorp).execute() - - sym_rp = rpath.RPath(Globals.local_connection, - "testfiles/robust/last-file-incremented.%s.data" % - Time.curtimestr) - assert sym_rp.issym() - assert sym_rp.readlink() == "increments/usr/local/bin/ls" - sym_rp.delete() - +class RobustTest(unittest.TestCase): + """Test robust module""" + def test_check_common_error(self): + """Test capturing errors""" + def cause_catchable_error(a): + os.lstat("aoenuthaoeu/aosutnhcg.4fpr,38p") + def cause_uncatchable_error(): + ansoethusaotneuhsaotneuhsaontehuaou + result = robust.check_common_error(None, cause_catchable_error, [1]) + assert result is None, result + try: robust.check_common_error(None, cause_uncatchable_error) + except NameError: pass + else: assert 0, "Key error not raised" + if __name__ == '__main__': unittest.main() diff --git a/rdiff-backup/testing/roottest.py b/rdiff-backup/testing/roottest.py index fbeaaa1..81292b2 100644 --- a/rdiff-backup/testing/roottest.py +++ b/rdiff-backup/testing/roottest.py @@ -1,7 +1,6 @@ import unittest, os from commontest import * -from rdiff_backup.log import * -from rdiff_backup import Globals +from rdiff_backup import Globals, log """Root tests @@ -11,7 +10,11 @@ that are meant to be run as root. Globals.set('change_source_perms', None) Globals.counter = 0 -Log.setverbosity(4) +log.Log.setverbosity(4) + +def Run(cmd): + print "Running: ", cmd + assert not os.system(cmd) class RootTest(unittest.TestCase): dirlist1 = ["testfiles/root", "testfiles/various_file_types", "testfiles/increment4"] @@ -21,8 +24,63 @@ class RootTest(unittest.TestCase): def testLocal2(self): BackupRestoreSeries(1, 1, self.dirlist2) def testRemote(self): BackupRestoreSeries(None, None, self.dirlist1) - def tearDown(self): - os.system(MiscDir + "/myrm testfiles/output testfiles/rest_out") +class NonRoot(unittest.TestCase): + """Test backing up as non-root user + + Test backing up a directory with files of different userids and + with device files in it, as a non-root user. When restoring as + root, everything should be restored normally. + + """ + user = 'ben' + def make_root_dir(self): + """Make directory createable only by root""" + rp = rpath.RPath(Globals.local_connection, "testfiles/root_out") + if rp.lstat(): Myrm(rp.path) + rp.mkdir() + rp1 = rp.append("1") + rp1.touch() + rp2 = rp.append("2") + rp2.touch() + rp2.chown(1, 1) + rp3 = rp.append("3") + rp3.touch() + rp3.chown(2, 2) + rp4 = rp.append("dev") + rp4.makedev('c', 4, 28) + return rp + + def test_non_root(self): + """Main non-root -> root test""" + Myrm("testfiles/output") + input_rp = self.make_root_dir() + Globals.change_ownership = 1 + output_rp = rpath.RPath(Globals.local_connection, "testfiles/output") + restore_rp = rpath.RPath(Globals.local_connection, + "testfiles/rest_out") + empty_rp = rpath.RPath(Globals.local_connection, "testfiles/empty") + + backup_cmd = "rdiff-backup %s %s" % (input_rp.path, output_rp.path) + Run("su %s -c '%s'" % (self.user, backup_cmd)) + + Myrm("testfiles/rest_out") + restore_cmd = "rdiff-backup -r now %s %s" % (output_rp.path, + restore_rp.path,) + Run(restore_cmd) + assert CompareRecursive(input_rp, restore_rp) + + backup_cmd = "rdiff-backup %s %s" % (empty_rp.path, output_rp.path) + Run("su %s -c '%s'" % (self.user, backup_cmd)) + + Myrm("testfiles/rest_out") + Run(restore_cmd) + assert CompareRecursive(empty_rp, restore_rp) + Myrm("testfiles/rest_out") + restore_cmd = "rdiff-backup -r 1 %s %s" % (output_rp.path, + restore_rp.path,) + Run(restore_cmd) + assert CompareRecursive(input_rp, restore_rp) + if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/rorpitertest.py b/rdiff-backup/testing/rorpitertest.py index ec786c2..f43a085 100644 --- a/rdiff-backup/testing/rorpitertest.py +++ b/rdiff-backup/testing/rorpitertest.py @@ -52,25 +52,6 @@ class RORPIterTest(unittest.TestCase): iter([]))) - def testCombinedPatching(self): - """Combined signature, patch, and diff operations""" - if self.output.lstat(): - Myrm(self.output.path) - self.output.setdata() - - def turninto(final_rp): - sigfile = rorpiter.ToFile(rorpiter.GetSignatureIter(self.output)) - diff_file = rorpiter.ToFile(rorpiter.GetDiffIter( - rorpiter.FromFile(sigfile), rorpiter.IterateRPaths(final_rp))) - rorpiter.PatchIter(self.output, rorpiter.FromFile(diff_file)) - - turninto(self.inc1rp) - rpath.copy_attribs(self.inc1rp, self.output) # Update time - assert self.compare_no_times(self.inc1rp, self.output) - turninto(self.inc2rp) - rpath.copy_attribs(self.inc2rp, self.output) - assert self.compare_no_times(self.inc2rp, self.output) - def compare_no_times(self, src_rp, dest_rp): """Compare but disregard directories attributes""" def equal(src_rorp, dest_rorp): diff --git a/rdiff-backup/testing/selectiontest.py b/rdiff-backup/testing/selectiontest.py index 8fa970d..2e0cd78 100644 --- a/rdiff-backup/testing/selectiontest.py +++ b/rdiff-backup/testing/selectiontest.py @@ -417,19 +417,19 @@ testfiles/select**/2 ("--exclude", "/")], [(), ("home",)]) - def testParseStartingFrom(self): - """Test parse, this time starting from inside""" - self.root = rpath.RPath(Globals.local_connection, "testfiles/select") - self.Select = Select(self.root) - self.Select.ParseArgs([("--include", "testfiles/select/1/1"), - ("--exclude", "**")], []) - self.Select.set_iter(('1', '1')) - assert lazy.Iter.equal(lazy.Iter.map(lambda dsrp: dsrp.index, - self.Select), - iter([("1", '1', '1'), - ('1', '1', '2'), - ('1', '1', '3')]), - verbose = 1) +# def testParseStartingFrom(self): +# """Test parse, this time starting from inside""" +# self.root = rpath.RPath(Globals.local_connection, "testfiles/select") +# self.Select = Select(self.root) +# self.Select.ParseArgs([("--include", "testfiles/select/1/1"), +# ("--exclude", "**")], []) +# self.Select.set_iter(('1', '1')) +# assert lazy.Iter.equal(lazy.Iter.map(lambda dsrp: dsrp.index, +# self.Select), +# iter([("1", '1', '1'), +# ('1', '1', '2'), +# ('1', '1', '3')]), +# verbose = 1) if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/statisticstest.py b/rdiff-backup/testing/statisticstest.py index 7e8f02c..85a1b68 100644 --- a/rdiff-backup/testing/statisticstest.py +++ b/rdiff-backup/testing/statisticstest.py @@ -1,6 +1,6 @@ -import unittest +import unittest, time from commontest import * -from rdiff_backup import statistics, rpath +from rdiff_backup import statistics, rpath, restore class StatsObjTest(unittest.TestCase): """Test StatsObj class""" @@ -29,7 +29,7 @@ class StatsObjTest(unittest.TestCase): self.set_obj(s) assert s.get_stat('SourceFiles') == 1 - s1 = statistics.ITRB() + s1 = statistics.StatFileObj() assert s1.get_stat('SourceFiles') == 0 def test_get_stats_string(self): @@ -40,10 +40,12 @@ class StatsObjTest(unittest.TestCase): self.set_obj(s) stats_string = s.get_stats_string() - assert stats_string == \ -"""StartTime 11.00 (Wed Dec 31 16:00:11 1969) -EndTime 12.00 (Wed Dec 31 16:00:12 1969) -ElapsedTime 1.00 (1 second) + ss_list = stats_string.split("\n") + tail = "\n".join(ss_list[2:]) # Time varies by time zone, don't check +#"""StartTime 11.00 (Wed Dec 31 16:00:11 1969) +#EndTime 12.00 (Wed Dec 31 16:00:12 1969)" + assert tail == \ +"""ElapsedTime 1.00 (1 second) SourceFiles 1 SourceFileSize 2 (2 bytes) MirrorFiles 13 @@ -143,4 +145,81 @@ TotalDestinationSizeChange 7 (7 bytes) assert s3.SourceFiles == 75 +class IncStatTest(unittest.TestCase): + """Test statistics as produced by actual backup""" + def stats_check_initial(self, s): + """Make sure stats object s compatible with initial mirroring + + A lot of the off by one stuff is because the root directory + exists in the below examples. + + """ + assert s.MirrorFiles == 1 or s.MirrorFiles == 0 + assert s.MirrorFileSize < 20000 + assert s.NewFiles <= s.SourceFiles <= s.NewFiles + 1 + assert s.NewFileSize <= s.SourceFileSize <= s.NewFileSize + 20000 + assert s.ChangedFiles == 1 or s.ChangedFiles == 0 + assert s.ChangedSourceSize < 20000 + assert s.ChangedMirrorSize < 20000 + assert s.DeletedFiles == s.DeletedFileSize == 0 + assert s.IncrementFileSize == 0 + + def testStatistics(self): + """Test the writing of statistics + + The file sizes are approximate because the size of directories + could change with different file systems... + + """ + Globals.compression = 1 + Myrm("testfiles/output") + InternalBackup(1, 1, "testfiles/stattest1", "testfiles/output") + InternalBackup(1, 1, "testfiles/stattest2", "testfiles/output", + time.time()+1) + + rbdir = rpath.RPath(Globals.local_connection, + "testfiles/output/rdiff-backup-data") + + #incs = Restore.get_inclist(rbdir.append("subdir"). + # append("directory_statistics")) + #assert len(incs) == 2 + #s1 = StatsObj().read_stats_from_rp(incs[0]) # initial mirror stats + #assert s1.SourceFiles == 2 + #assert 400000 < s1.SourceFileSize < 420000 + #self.stats_check_initial(s1) + + #subdir_stats = StatsObj().read_stats_from_rp(incs[1]) # increment stats + #assert subdir_stats.SourceFiles == 2 + #assert 400000 < subdir_stats.SourceFileSize < 420000 + #assert subdir_stats.MirrorFiles == 2 + #assert 400000 < subdir_stats.MirrorFileSize < 420000 + #assert subdir_stats.NewFiles == subdir_stats.NewFileSize == 0 + #assert subdir_stats.DeletedFiles == subdir_stats.DeletedFileSize == 0 + #assert subdir_stats.ChangedFiles == 2 + #assert 400000 < subdir_stats.ChangedSourceSize < 420000 + #assert 400000 < subdir_stats.ChangedMirrorSize < 420000 + #assert 10 < subdir_stats.IncrementFileSize < 20000 + + incs = restore.get_inclist(rbdir.append("session_statistics")) + assert len(incs) == 2 + s2 = statistics.StatsObj().read_stats_from_rp(incs[0]) + assert s2.SourceFiles == 7 + assert 700000 <= s2.SourceFileSize < 750000 + self.stats_check_initial(s2) + + root_stats = statistics.StatsObj().read_stats_from_rp(incs[1]) + assert root_stats.SourceFiles == 7, root_stats.SourceFiles + assert 550000 <= root_stats.SourceFileSize < 570000 + assert root_stats.MirrorFiles == 7 + assert 700000 <= root_stats.MirrorFileSize < 750000 + assert root_stats.NewFiles == 1 + assert root_stats.NewFileSize == 0 + assert root_stats.DeletedFiles == 1 + assert root_stats.DeletedFileSize == 200000 + assert 3 <= root_stats.ChangedFiles <= 4, root_stats.ChangedFiles + assert 450000 <= root_stats.ChangedSourceSize < 470000 + assert 400000 <= root_stats.ChangedMirrorSize < 420000, \ + root_stats.ChangedMirrorSize + assert 10 < root_stats.IncrementFileSize < 30000 + if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/timetest.py b/rdiff-backup/testing/timetest.py index 97286a8..367a4f9 100644 --- a/rdiff-backup/testing/timetest.py +++ b/rdiff-backup/testing/timetest.py @@ -6,7 +6,7 @@ class TimeTest(unittest.TestCase): def testConversion(self): """test timetostring and stringtotime""" Time.setcurtime() - assert type(Time.curtime) is types.FloatType + assert type(Time.curtime) is types.FloatType or types.LongType assert type(Time.curtimestr) is types.StringType assert (Time.cmp(int(Time.curtime), Time.curtimestr) == 0 or Time.cmp(int(Time.curtime) + 1, Time.curtimestr) == 0) -- cgit v1.2.1