From 5ccca65dec1ef00d3551493958af5ee27e7df212 Mon Sep 17 00:00:00 2001 From: bescoto Date: Sun, 12 Oct 2003 05:24:25 +0000 Subject: New hard link system should use less memory git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@468 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/testing/commontest.py | 129 +++++++++++++++++------------------ rdiff-backup/testing/hardlinktest.py | 58 +++------------- 2 files changed, 72 insertions(+), 115 deletions(-) (limited to 'rdiff-backup/testing') diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index e80d699..9d1ee81 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -3,7 +3,7 @@ import os, sys, code from rdiff_backup.log import Log from rdiff_backup.rpath import RPath from rdiff_backup import Globals, Hardlink, SetConnections, Main, \ - selection, lazy, Time, rpath, eas_acls + selection, lazy, Time, rpath, eas_acls, rorpiter RBBin = "../rdiff-backup" SourceDir = "../rdiff_backup" @@ -175,63 +175,78 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, specified. """ - if compare_hardlinks: reset_hardlink_dicts() - src_rp.setdata() - dest_rp.setdata() - - Log("Comparing %s and %s, hardlinks %s, eas %s, acls %s" % - (src_rp.path, dest_rp.path, compare_hardlinks, - compare_eas, compare_acls), 3) - src_select = selection.Select(src_rp) - dest_select = selection.Select(dest_rp) - - if ignore_tmp_files: - # Ignoring temp files can be useful when we want to check the - # correctness of a backup which aborted in the middle. In - # these cases it is OK to have tmp files lying around. - src_select.add_selection_func(src_select.regexp_get_sf( - ".*rdiff-backup.tmp.[^/]+$", 0)) - dest_select.add_selection_func(dest_select.regexp_get_sf( - ".*rdiff-backup.tmp.[^/]+$", 0)) - - if exclude_rbdir: - src_select.parse_rbdir_exclude() - dest_select.parse_rbdir_exclude() - else: - # include rdiff-backup-data/increments - src_select.add_selection_func(src_select.glob_get_tuple_sf( - ('rdiff-backup-data', 'increments'), 1)) - dest_select.add_selection_func(dest_select.glob_get_tuple_sf( - ('rdiff-backup-data', 'increments'), 1)) - - # but exclude rdiff-backup-data - src_select.add_selection_func(src_select.glob_get_tuple_sf( - ('rdiff-backup-data',), 0)) - dest_select.add_selection_func(dest_select.glob_get_tuple_sf( - ('rdiff-backup-data',), 0)) - - dsiter1, dsiter2 = src_select.set_iter(), dest_select.set_iter() - - def hardlink_equal(src_rorp, dest_rorp): + def get_selection_functions(): + """Return generators of files in source, dest""" + src_rp.setdata() + dest_rp.setdata() + src_select = selection.Select(src_rp) + dest_select = selection.Select(dest_rp) + + if ignore_tmp_files: + # Ignoring temp files can be useful when we want to check the + # correctness of a backup which aborted in the middle. In + # these cases it is OK to have tmp files lying around. + src_select.add_selection_func(src_select.regexp_get_sf( + ".*rdiff-backup.tmp.[^/]+$", 0)) + dest_select.add_selection_func(dest_select.regexp_get_sf( + ".*rdiff-backup.tmp.[^/]+$", 0)) + + if exclude_rbdir: # Exclude rdiff-backup-data directory + src_select.parse_rbdir_exclude() + dest_select.parse_rbdir_exclude() + + return src_select.set_iter(), dest_select.set_iter() + + def preprocess(src_rorp, dest_rorp): + """Initially process src and dest_rorp""" + if compare_hardlinks and src_rorp: + Hardlink.add_rorp(src_rorp, dest_rorp) + + def postprocess(src_rorp, dest_rorp): + """After comparison, process src_rorp and dest_rorp""" + if compare_hardlinks and src_rorp: + Hardlink.del_rorp(src_rorp) + + def equality_func(src_rorp, dest_rorp): + """Combined eq func returns true iff two files compare same""" + if not src_rorp: + Log("Source rorp missing: " + str(dest_rorp), 3) + return 0 + if not dest_rorp: + Log("Dest rorp missing: " + str(src_rorp), 3) + return 0 if not src_rorp.equal_verbose(dest_rorp, compare_ownership = compare_ownership): - return None - if not Hardlink.rorp_eq(src_rorp, dest_rorp): + return 0 + if compare_hardlinks and not Hardlink.rorp_eq(src_rorp, dest_rorp): + Log("Hardlink compare failure", 3) Log("%s: %s" % (src_rorp.index, - Hardlink.get_indicies(src_rorp, 1)), 3) + Hardlink.get_inode_key(src_rorp)), 3) Log("%s: %s" % (dest_rorp.index, - Hardlink.get_indicies(dest_rorp, None)), 3) - return None + Hardlink.get_inode_key(dest_rorp)), 3) + return 0 if compare_eas and not eas_acls.ea_compare_rps(src_rorp, dest_rorp): Log("Different EAs in files %s and %s" % (src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3) - return None + return 0 if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp): Log("Different ACLs in files %s and %s" % (src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3) - return None + return 0 return 1 + Log("Comparing %s and %s, hardlinks %s, eas %s, acls %s" % + (src_rp.path, dest_rp.path, compare_hardlinks, + compare_eas, compare_acls), 3) + if compare_hardlinks: reset_hardlink_dicts() + src_iter, dest_iter = get_selection_functions() + for src_rorp, dest_rorp in rorpiter.Collate2Iters(src_iter, dest_iter): + preprocess(src_rorp, dest_rorp) + if not equality_func(src_rorp, dest_rorp): return 0 + postprocess(src_rorp, dest_rorp) + return 1 + + def rbdir_equal(src_rorp, dest_rorp): """Like hardlink_equal, but make allowances for data directories""" if not src_rorp.index and not dest_rorp.index: return 1 @@ -263,30 +278,10 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1, Hardlink.get_indicies(dest_rorp, None)), 3) return None - if equality_func: result = lazy.Iter.equal(dsiter1, dsiter2, - 1, equality_func) - elif compare_hardlinks: - dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1) - dsiter2 = Hardlink.add_rorp_iter(dsiter2, None) - if exclude_rbdir: - result = lazy.Iter.equal(dsiter1, dsiter2, 1, hardlink_equal) - else: result = lazy.Iter.equal(dsiter1, dsiter2, 1, rbdir_equal) - elif not exclude_rbdir: - result = lazy.Iter.equal(dsiter1, dsiter2, 1, rbdir_equal) - else: result = lazy.Iter.equal(dsiter1, dsiter2, 1, - lambda x, y: x.equal_verbose(y, compare_ownership = compare_ownership)) - - for i in dsiter1: pass # make sure all files processed anyway - for i in dsiter2: pass - return result def reset_hardlink_dicts(): """Clear the hardlink dictionaries""" - Hardlink._src_inode_indicies = {} - Hardlink._src_index_indicies = {} - Hardlink._dest_inode_indicies = {} - Hardlink._dest_index_indicies = {} - Hardlink._restore_index_path = {} + Hardlink._inode_index = {} def BackupRestoreSeries(source_local, dest_local, list_of_dirnames, compare_hardlinks = 1, diff --git a/rdiff-backup/testing/hardlinktest.py b/rdiff-backup/testing/hardlinktest.py index 22dc31d..c3827f6 100644 --- a/rdiff-backup/testing/hardlinktest.py +++ b/rdiff-backup/testing/hardlinktest.py @@ -2,7 +2,7 @@ import os, unittest, time from commontest import * from rdiff_backup import Globals, Hardlink, selection, rpath -Log.setverbosity(3) +Log.setverbosity(6) class HardlinkTest(unittest.TestCase): """Test cases for Hard links""" @@ -34,62 +34,24 @@ class HardlinkTest(unittest.TestCase): Globals.preserve_hardlinks = 1 reset_hardlink_dicts() for dsrp in selection.Select(self.hardlink_dir3).set_iter(): - Hardlink.add_rorp(dsrp, 1) + Hardlink.add_rorp(dsrp) - assert len(Hardlink._src_inode_indicies.keys()) == 3, \ - Hardlink._src_inode_indicies - assert len(Hardlink._src_index_indicies.keys()) == 3, \ - Hardlink._src_index_indicies - vals1 = Hardlink._src_inode_indicies.values() - vals2 = Hardlink._src_index_indicies.values() - vals1.sort() - vals2.sort() - assert vals1 == vals2 - - def testBuildingDict2(self): - """Same as testBuildingDict but test destination building""" - Globals.preserve_hardlinks = 1 - reset_hardlink_dicts() - for dsrp in selection.Select(self.hardlink_dir3).set_iter(): - Hardlink.add_rorp(dsrp, None) - - assert len(Hardlink._dest_inode_indicies.keys()) == 3, \ - Hardlink._dest_inode_indicies - assert len(Hardlink._dest_index_indicies.keys()) == 3, \ - Hardlink._dest_index_indicies - vals1 = Hardlink._dest_inode_indicies.values() - vals2 = Hardlink._dest_index_indicies.values() - vals1.sort() - vals2.sort() - assert vals1 == vals2 + assert len(Hardlink._inode_index.keys()) == 3, \ + Hardlink._inode_index def testCompletedDict(self): """See if the hardlink dictionaries are built correctly""" reset_hardlink_dicts() for dsrp in selection.Select(self.hardlink_dir1).set_iter(): - Hardlink.add_rorp(dsrp, 1) - assert Hardlink._src_inode_indicies == {}, \ - Hardlink._src_inode_indicies - - hll1 = [('file1',), ('file2',), ('file3',)] - hll2 = [('file4',), ('file5',), ('file6',)] - dict = {} - for index in hll1: dict[index] = hll1 - for index in hll2: dict[index] = hll2 - assert Hardlink._src_index_indicies == dict + Hardlink.add_rorp(dsrp) + Hardlink.del_rorp(dsrp) + assert Hardlink._inode_index == {}, Hardlink._inode_index reset_hardlink_dicts() for dsrp in selection.Select(self.hardlink_dir2).set_iter(): - Hardlink.add_rorp(dsrp, 1) - assert Hardlink._src_inode_indicies == {}, \ - Hardlink._src_inode_indicies - - hll1 = [('file1',), ('file3',), ('file4',)] - hll2 = [('file2',), ('file5',), ('file6',)] - dict = {} - for index in hll1: dict[index] = hll1 - for index in hll2: dict[index] = hll2 - assert Hardlink._src_index_indicies == dict + Hardlink.add_rorp(dsrp) + Hardlink.del_rorp(dsrp) + assert Hardlink._inode_index == {}, Hardlink._inode_index def testSeries(self): """Test hardlink system by backing up and restoring a few dirs""" -- cgit v1.2.1