From 73d24ff28e1561fb000edce42965b439bc04f53b Mon Sep 17 00:00:00 2001 From: ben Date: Sat, 11 May 2002 21:33:34 +0000 Subject: Various final bug fixes for 0.7.4 git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@79 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/header.py | 2 +- rdiff-backup/rdiff_backup/increment.py | 63 +++++++++++++++++++++++++++-- rdiff-backup/rdiff_backup/restore.py | 2 +- rdiff-backup/rdiff_backup/rpath.py | 4 +- rdiff-backup/src/header.py | 2 +- rdiff-backup/src/increment.py | 63 +++++++++++++++++++++++++++-- rdiff-backup/src/restore.py | 2 +- rdiff-backup/src/rpath.py | 4 +- rdiff-backup/testing/commontest.py | 15 ++++--- rdiff-backup/testing/incrementtest.py | 74 +++++++++++++++++++++++++++++++++- rdiff-backup/testing/restoretest.py | 5 --- rdiff-backup/testing/robusttest.py | 2 +- rdiff-backup/testing/roottest.py | 2 +- 13 files changed, 209 insertions(+), 31 deletions(-) diff --git a/rdiff-backup/rdiff_backup/header.py b/rdiff-backup/rdiff_backup/header.py index 38d45c8..020a63f 100644 --- a/rdiff-backup/rdiff_backup/header.py +++ b/rdiff-backup/rdiff_backup/header.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.3 released April 29, 2002 +# Version 0.7.4 released May 11, 2002 # Copyright (C) 2001, 2002 Ben Escoto # # This program is licensed under the GNU General Public License (GPL). diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index b28b315..4e96e59 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -9,6 +9,12 @@ execfile("filename_mapping.py") class Inc: """Class containing increment functions""" + # This is a hack. _inc_file holds the dsrp of the latest + # increment file created, to be used in IncrementITR for + # statistics purposes. It should be given directly to the ITR + # object but there didn't seem to be a good way to pass it out. + _inc_file = None + def Increment_action(new, mirror, incpref): """Main file incrementing function, returns RobustAction @@ -95,7 +101,9 @@ class Inc: while 1: inctime = Resume.FindTime(rp.index, inctime) incrp = get_newinc(Time.timetostring(inctime)) - if not incrp.lstat(): return incrp + if not incrp.lstat(): break + Inc._inc_file = incrp + return incrp MakeStatic(Inc) @@ -134,6 +142,7 @@ class IncrementITR(IterTreeReducer): dsrp is the local file to be incremented """ + self.init_statistics(diff_rorp, dsrp) incpref = self.inc_rpath.new_index(index) if Globals.quoting_enabled: incpref.quote_path() if dsrp.isdir(): @@ -141,6 +150,26 @@ class IncrementITR(IterTreeReducer): self.setvals(diff_rorp, dsrp, incpref) else: self.init_non_dir(dsrp, diff_rorp, incpref) + def init_statistics(self, diff_rorp, dsrp): + """Set initial values for various statistics + + These refer to the old mirror or to new increment files. Note + that changed_file_size could be bigger than total_file_size. + The other statistic, increment_file_size, is set later when we + have that information. + + """ + if dsrp.lstat(): + self.total_files = 1 + self.total_file_size = dsrp.getsize() + else: self.total_files = self.total_file_size = 0 + if diff_rorp: + self.changed_files = 1 + if dsrp.lstat(): self.changed_file_size = dsrp.getsize() + else: self.changed_file_size = 0 + else: self.changed_files = self.changed_file_size = 0 + self.increment_file_size = 0 + def override_changed(self): """Set changed flag to true @@ -195,6 +224,9 @@ class IncrementITR(IterTreeReducer): Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref), RORPIter.patchonce_action(None, dsrp, diff_rorp)] ).execute() + + self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat() + and Inc._inc_file.getsize()) or 0) self.changed = 1 def end_process(self): @@ -213,9 +245,32 @@ class IncrementITR(IterTreeReducer): if diff_rorp: RORPIter.patchonce_action(None, dsrp, diff_rorp).execute() + self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat() + and Inc._inc_file.getsize()) or 0) + self.write_statistics() + + def write_statistics(self): + """Write the accumulated totals into file in inc directory""" + if not self.incpref.isdir(): return # only write for directories + statrp = Inc.get_inc_ext(self.incpref.append("directory_statistics"), + "data") + tf = TempFileManager.new(statrp) + def init_thunk(): + fp = tf.open("w") + fp.write("TotalFiles %d\n" % self.total_files) + fp.write("TotalFileSize %d\n" % self.total_file_size) + fp.write("ChangedFiles %d\n" % self.changed_files) + fp.write("ChangedFileSize %d\n" % self.changed_file_size) + fp.write("IncrementFileSize %d\n" % self.increment_file_size) + fp.close() + Robust.make_tf_robustaction(init_thunk, (tf,), (statrp,)).execute() + def branch_process(self, subinstance): - """Update the has_changed flag if change in branch""" + """Update statistics, and the has_changed flag if change in branch""" if subinstance.changed: self.changed = 1 - - + self.total_files += subinstance.total_files + self.total_file_size += subinstance.total_file_size + self.changed_files += subinstance.changed_files + self.changed_file_size += subinstance.changed_file_size + self.increment_file_size += subinstance.increment_file_size diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 0faa9b2..5026716 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -213,7 +213,7 @@ class Restore: """Add filename to the inc tuple dictionary""" rp = rid.inc_rpath.append(filename) if Globals.quoting_enabled: rp.quote_path() - if rp.isincfile(): + if rp.isincfile() and rp.getinctype() != 'data': basename = rp.getincbase_str() affirm_dict_indexed(basename) rid_dict[basename].inc_list.append(rp) diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 748cab4..5ed36f5 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -456,7 +456,6 @@ class RPath(RORPath): if stat.S_ISREG(mode): type = 'reg' - data['size'] = statblock[stat.ST_SIZE] elif stat.S_ISDIR(mode): type = 'dir' elif stat.S_ISCHR(mode): type = 'dev' @@ -471,6 +470,7 @@ class RPath(RORPath): elif stat.S_ISSOCK(mode): type = 'sock' else: raise RPathException("Unknown type for %s" % self.path) data['type'] = type + data['size'] = statblock[stat.ST_SIZE] data['perms'] = stat.S_IMODE(mode) data['uid'] = statblock[stat.ST_UID] data['gid'] = statblock[stat.ST_GID] @@ -754,7 +754,7 @@ class RPath(RORPath): if type == 'c': datatype = 'chr' elif type == 'b': datatype = 'blk' else: raise RPathException - self.data = {'type': datatype, 'devnums': (type, major, minor)} + self.setdata() def getRORPath(self, include_contents = None): """Return read only version of self""" diff --git a/rdiff-backup/src/header.py b/rdiff-backup/src/header.py index 38d45c8..020a63f 100644 --- a/rdiff-backup/src/header.py +++ b/rdiff-backup/src/header.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.3 released April 29, 2002 +# Version 0.7.4 released May 11, 2002 # Copyright (C) 2001, 2002 Ben Escoto # # This program is licensed under the GNU General Public License (GPL). diff --git a/rdiff-backup/src/increment.py b/rdiff-backup/src/increment.py index b28b315..4e96e59 100644 --- a/rdiff-backup/src/increment.py +++ b/rdiff-backup/src/increment.py @@ -9,6 +9,12 @@ execfile("filename_mapping.py") class Inc: """Class containing increment functions""" + # This is a hack. _inc_file holds the dsrp of the latest + # increment file created, to be used in IncrementITR for + # statistics purposes. It should be given directly to the ITR + # object but there didn't seem to be a good way to pass it out. + _inc_file = None + def Increment_action(new, mirror, incpref): """Main file incrementing function, returns RobustAction @@ -95,7 +101,9 @@ class Inc: while 1: inctime = Resume.FindTime(rp.index, inctime) incrp = get_newinc(Time.timetostring(inctime)) - if not incrp.lstat(): return incrp + if not incrp.lstat(): break + Inc._inc_file = incrp + return incrp MakeStatic(Inc) @@ -134,6 +142,7 @@ class IncrementITR(IterTreeReducer): dsrp is the local file to be incremented """ + self.init_statistics(diff_rorp, dsrp) incpref = self.inc_rpath.new_index(index) if Globals.quoting_enabled: incpref.quote_path() if dsrp.isdir(): @@ -141,6 +150,26 @@ class IncrementITR(IterTreeReducer): self.setvals(diff_rorp, dsrp, incpref) else: self.init_non_dir(dsrp, diff_rorp, incpref) + def init_statistics(self, diff_rorp, dsrp): + """Set initial values for various statistics + + These refer to the old mirror or to new increment files. Note + that changed_file_size could be bigger than total_file_size. + The other statistic, increment_file_size, is set later when we + have that information. + + """ + if dsrp.lstat(): + self.total_files = 1 + self.total_file_size = dsrp.getsize() + else: self.total_files = self.total_file_size = 0 + if diff_rorp: + self.changed_files = 1 + if dsrp.lstat(): self.changed_file_size = dsrp.getsize() + else: self.changed_file_size = 0 + else: self.changed_files = self.changed_file_size = 0 + self.increment_file_size = 0 + def override_changed(self): """Set changed flag to true @@ -195,6 +224,9 @@ class IncrementITR(IterTreeReducer): Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref), RORPIter.patchonce_action(None, dsrp, diff_rorp)] ).execute() + + self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat() + and Inc._inc_file.getsize()) or 0) self.changed = 1 def end_process(self): @@ -213,9 +245,32 @@ class IncrementITR(IterTreeReducer): if diff_rorp: RORPIter.patchonce_action(None, dsrp, diff_rorp).execute() + self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat() + and Inc._inc_file.getsize()) or 0) + self.write_statistics() + + def write_statistics(self): + """Write the accumulated totals into file in inc directory""" + if not self.incpref.isdir(): return # only write for directories + statrp = Inc.get_inc_ext(self.incpref.append("directory_statistics"), + "data") + tf = TempFileManager.new(statrp) + def init_thunk(): + fp = tf.open("w") + fp.write("TotalFiles %d\n" % self.total_files) + fp.write("TotalFileSize %d\n" % self.total_file_size) + fp.write("ChangedFiles %d\n" % self.changed_files) + fp.write("ChangedFileSize %d\n" % self.changed_file_size) + fp.write("IncrementFileSize %d\n" % self.increment_file_size) + fp.close() + Robust.make_tf_robustaction(init_thunk, (tf,), (statrp,)).execute() + def branch_process(self, subinstance): - """Update the has_changed flag if change in branch""" + """Update statistics, and the has_changed flag if change in branch""" if subinstance.changed: self.changed = 1 - - + self.total_files += subinstance.total_files + self.total_file_size += subinstance.total_file_size + self.changed_files += subinstance.changed_files + self.changed_file_size += subinstance.changed_file_size + self.increment_file_size += subinstance.increment_file_size diff --git a/rdiff-backup/src/restore.py b/rdiff-backup/src/restore.py index 0faa9b2..5026716 100644 --- a/rdiff-backup/src/restore.py +++ b/rdiff-backup/src/restore.py @@ -213,7 +213,7 @@ class Restore: """Add filename to the inc tuple dictionary""" rp = rid.inc_rpath.append(filename) if Globals.quoting_enabled: rp.quote_path() - if rp.isincfile(): + if rp.isincfile() and rp.getinctype() != 'data': basename = rp.getincbase_str() affirm_dict_indexed(basename) rid_dict[basename].inc_list.append(rp) diff --git a/rdiff-backup/src/rpath.py b/rdiff-backup/src/rpath.py index 748cab4..5ed36f5 100644 --- a/rdiff-backup/src/rpath.py +++ b/rdiff-backup/src/rpath.py @@ -456,7 +456,6 @@ class RPath(RORPath): if stat.S_ISREG(mode): type = 'reg' - data['size'] = statblock[stat.ST_SIZE] elif stat.S_ISDIR(mode): type = 'dir' elif stat.S_ISCHR(mode): type = 'dev' @@ -471,6 +470,7 @@ class RPath(RORPath): elif stat.S_ISSOCK(mode): type = 'sock' else: raise RPathException("Unknown type for %s" % self.path) data['type'] = type + data['size'] = statblock[stat.ST_SIZE] data['perms'] = stat.S_IMODE(mode) data['uid'] = statblock[stat.ST_UID] data['gid'] = statblock[stat.ST_GID] @@ -754,7 +754,7 @@ class RPath(RORPath): if type == 'c': datatype = 'chr' elif type == 'b': datatype = 'blk' else: raise RPathException - self.data = {'type': datatype, 'devnums': (type, major, minor)} + self.setdata() def getRORPath(self, include_contents = None): """Return read only version of self""" diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index b1a1dc6..40acc2d 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -127,7 +127,12 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time): mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir], remote_schema) - _get_main().Restore(get_increment_rp(mirror_rp, time), dest_rp) + inc = get_increment_rp(mirror_rp, time) + if inc: + _get_main().Restore(get_increment_rp(mirror_rp, time), dest_rp) + else: # use alternate syntax + _get_main().restore_timestr = str(time) + _get_main().RestoreAsOf(mirror_rp, dest_rp) _get_main().cleanup() def get_increment_rp(mirror_rp, time): @@ -135,11 +140,9 @@ def get_increment_rp(mirror_rp, time): data_rp = mirror_rp.append("rdiff-backup-data") for filename in data_rp.listdir(): rp = data_rp.append(filename) - if (rp.isincfile() and rp.getincbase_str() == "increments" and - Time.stringtotime(rp.getinctime()) == time): - return rp - assert None, ("No increments.XXX.dir found in directory " - "%s with that time" % data_rp.path) + if rp.isincfile() and rp.getincbase_str() == "increments": + if Time.stringtotime(rp.getinctime()) == time: return rp + return None # Couldn't find appropriate increment def _reset_connections(src_rp, dest_rp): """Reset some global connection information""" diff --git a/rdiff-backup/testing/incrementtest.py b/rdiff-backup/testing/incrementtest.py index e53e532..96d7811 100644 --- a/rdiff-backup/testing/incrementtest.py +++ b/rdiff-backup/testing/incrementtest.py @@ -1,12 +1,12 @@ import unittest, os execfile("commontest.py") -rbexec("increment.py") +rbexec("main.py") lc = Globals.local_connection Globals.change_source_perms = 1 -Log.setverbosity(7) +Log.setverbosity(4) def getrp(ending): return RPath(lc, "testfiles/various_file_types/" + ending) @@ -162,4 +162,74 @@ class inctest(unittest.TestCase): out2.delete() out_gz.delete() +class inctest2(unittest.TestCase): + """Like inctest but contains more elaborate tests""" + def testStatistics(self): + """Test the writing of statistics + + The file sizes are approximate because the size of directories + could change with different file systems... + + """ + Globals.compression = 1 + Myrm("testfiles/output") + InternalBackup(1, 1, "testfiles/stattest1", "testfiles/output") + InternalBackup(1, 1, "testfiles/stattest2", "testfiles/output") + + inc_base = RPath(Globals.local_connection, + "testfiles/output/rdiff-backup-data/increments") + + incs = Restore.get_inclist(inc_base.append("subdir"). + append("directory_statistics")) + assert len(incs) == 1 + subdir_stats = self.parse_statistics(incs[0]) + assert subdir_stats.total_files == 2, subdir_stats.total_files + assert 350000 < subdir_stats.total_file_size < 450000, \ + subdir_stats.total_file_size + assert subdir_stats.changed_files == 2, subdir_stats.changed_files + assert 350000 < subdir_stats.changed_file_size < 450000, \ + subdir_stats.changed_file_size + assert 10 < subdir_stats.increment_file_size < 20000, \ + subdir_stats.increment_file_size + + incs = Restore.get_inclist(inc_base.append("directory_statistics")) + assert len(incs) == 1 + root_stats = self.parse_statistics(incs[0]) + assert root_stats.total_files == 6, root_stats.total_files + assert 650000 < root_stats.total_file_size < 750000, \ + root_stats.total_file_size + assert root_stats.changed_files == 4, root_stats.changed_files + assert 550000 < root_stats.changed_file_size < 650000, \ + root_stats.changed_file_size + assert 10 < root_stats.increment_file_size < 20000, \ + root_stats.increment_file_size + + def parse_statistics(self, statrp): + """Return StatObj from given statrp""" + assert statrp.isincfile() and statrp.getinctype() == "data" + s = StatObj() + fp = statrp.open("r") + for line in fp: + lsplit = line.split() + assert len(lsplit) == 2 + field, num = lsplit[0], long(lsplit[1]) + if field == "TotalFiles": s.total_files = num + elif field == "TotalFileSize": s.total_file_size = num + elif field == "ChangedFiles": s.changed_files = num + elif field == "ChangedFileSize": s.changed_file_size = num + elif field == "IncrementFileSize": s.increment_file_size = num + else: assert None, "Unrecognized field %s" % (field,) + assert not fp.close() + return s + + +class StatObj: + """Just hold various statistics""" + total_files = 0 + total_file_size = 0 + changed_files = 0 + changed_file_size = 0 + increment_file_size = 0 + + if __name__ == '__main__': unittest.main() diff --git a/rdiff-backup/testing/restoretest.py b/rdiff-backup/testing/restoretest.py index 1a314eb..aaf84b8 100644 --- a/rdiff-backup/testing/restoretest.py +++ b/rdiff-backup/testing/restoretest.py @@ -74,13 +74,8 @@ class RestoreTest(unittest.TestCase): assert inc.isincfile() inctime = Time.stringtotime(inc.getinctime()) rid1 = RestoreIncrementData(basename, incbase, incs) - rid2 = RestoreIncrementData(basename, incbase, incs) rid1.sortincseq(inctime, mirror_time) - rid2.sortincseq(inctime + 5, mirror_time) assert rid1.inc_list, rid1.inc_list - # Five seconds later shouldn't make a difference - assert rid1.inc_list == rid2.inc_list, (rid1.inc_list, - rid2.inc_list) # oldest increment should be exactly inctime ridtime = Time.stringtotime(rid1.inc_list[-1].getinctime()) assert ridtime == inctime, (ridtime, inctime) diff --git a/rdiff-backup/testing/robusttest.py b/rdiff-backup/testing/robusttest.py index 4f08e44..2a54952 100644 --- a/rdiff-backup/testing/robusttest.py +++ b/rdiff-backup/testing/robusttest.py @@ -76,7 +76,7 @@ class SaveStateTest(unittest.TestCase): SaveState.record_last_file_action(last_rorp).execute() sym_rp = RPath(Globals.local_connection, - "testfiles/robust/last-file-mirrored.%s.snapshot" % + "testfiles/robust/last-file-mirrored.%s.data" % Time.curtimestr) assert sym_rp.issym() assert sym_rp.readlink() == "increments/usr/local/bin/ls" diff --git a/rdiff-backup/testing/roottest.py b/rdiff-backup/testing/roottest.py index 29ea2be..3f60544 100644 --- a/rdiff-backup/testing/roottest.py +++ b/rdiff-backup/testing/roottest.py @@ -13,7 +13,7 @@ Globals.counter = 0 Log.setverbosity(4) class RootTest(unittest.TestCase): - dirlist1 = ["testfiles/root", "testfiles/noperms", "testfiles/increment4"] + dirlist1 = ["testfiles/root", "testfiles/various_file_types", "testfiles/increment4"] dirlist2 = ["testfiles/increment4", "testfiles/root", "testfiles/increment1"] def testLocal1(self): BackupRestoreSeries(1, 1, self.dirlist1) -- cgit v1.2.1