From 716fa96b9cbf2570e381de0abb475d341487c2d6 Mon Sep 17 00:00:00 2001 From: bescoto Date: Tue, 18 Feb 2003 08:18:45 +0000 Subject: Various modifications to backup, restore, and regress systems. This version passes many tests but not all of them. The backup patch system was copied to restore.py. git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@281 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/backup.py | 210 +++++++++++++++++++++++++--------- rdiff-backup/rdiff_backup/log.py | 22 ++-- rdiff-backup/rdiff_backup/metadata.py | 1 + rdiff-backup/rdiff_backup/regress.py | 24 +++- rdiff-backup/rdiff_backup/restore.py | 114 +++++++++++++++++- rdiff-backup/rdiff_backup/rorpiter.py | 18 --- rdiff-backup/rdiff_backup/rpath.py | 10 +- rdiff-backup/testing/commontest.py | 9 +- rdiff-backup/testing/regresstest.py | 70 +++++++++++- 9 files changed, 369 insertions(+), 109 deletions(-) diff --git a/rdiff-backup/rdiff_backup/backup.py b/rdiff-backup/rdiff_backup/backup.py index 2f3d362..78bfbe9 100644 --- a/rdiff-backup/rdiff_backup/backup.py +++ b/rdiff-backup/rdiff_backup/backup.py @@ -28,24 +28,22 @@ def Mirror(src_rpath, dest_rpath): SourceS = src_rpath.conn.backup.SourceStruct DestS = dest_rpath.conn.backup.DestinationStruct - DestS.init_statistics() source_rpiter = SourceS.get_source_select() - dest_sigiter = DestS.process_source_get_sigs(dest_rpath, source_rpiter, 0) + DestS.set_rorp_cache(dest_rpath, source_rpiter, 0) + dest_sigiter = DestS.get_sigs() source_diffiter = SourceS.get_diffs(dest_sigiter) DestS.patch(dest_rpath, source_diffiter) - DestS.write_statistics() def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath): """Mirror + put increments in tree based at inc_rpath""" SourceS = src_rpath.conn.backup.SourceStruct DestS = dest_rpath.conn.backup.DestinationStruct - DestS.init_statistics() source_rpiter = SourceS.get_source_select() - dest_sigiter = DestS.process_source_get_sigs(dest_rpath, source_rpiter, 1) + DestS.set_rorp_cache(dest_rpath, source_rpiter, 1) + dest_sigiter = DestS.get_sigs() source_diffiter = SourceS.get_diffs(dest_sigiter) DestS.patch_and_increment(dest_rpath, source_diffiter, inc_rpath) - DestS.write_statistics() class SourceStruct: @@ -99,14 +97,6 @@ static.MakeClass(SourceStruct) class DestinationStruct: """Hold info used by destination side when backing up""" - def init_statistics(cls): - """Set cls.stats to StatFileObj object""" - cls.statfileobj = statistics.init_statfileobj() - - def write_statistics(cls): - """Write statistics file""" - statistics.write_active_statfileobj() - def get_dest_select(cls, rpath, use_metadata = 1): """Return destination select rorpath iterator @@ -125,66 +115,175 @@ class DestinationStruct: sel.parse_rbdir_exclude() return sel.set_iter() - def dest_iter_filter(cls, dest_iter): - """Destination rorps pass through this - record stats""" - for dest_rorp in dest_iter: - cls.statfileobj.add_dest_file(dest_rorp) - Hardlink.add_rorp(dest_rorp, source = 0) - yield dest_rorp - - def src_iter_filter(cls, source_iter): - """Source rorps pass through this - record stats, write metadata""" - metadata.OpenMetadata() - for src_rorp in source_iter: - cls.statfileobj.add_source_file(src_rorp) - Hardlink.add_rorp(src_rorp, source = 1) - metadata.WriteMetadata(src_rorp) - yield src_rorp - metadata.CloseMetadata() + def set_rorp_cache(cls, baserp, source_iter, for_increment): + """Initialize cls.CCPP, the destination rorp cache - def process_source_get_sigs(cls, baserp, source_iter, for_increment): - """Process the source rorpiter and return signatures of dest dir - - Write all metadata to file, then return signatures of any - destination files that have changed. for_increment should be - true if we are mirror+incrementing, and false if we are just - mirroring. + for_increment should be true if we are mirror+incrementing, + false if we are just mirroring. """ - source_iter = cls.src_iter_filter(source_iter) - dest_iter = cls.dest_iter_filter(cls.get_dest_select(baserp, - for_increment)) - for index in rorpiter.get_dissimilar_indicies(source_iter, dest_iter, - cls.statfileobj): - dest_rp = baserp.new_index(index) - dest_sig = dest_rp.getRORPath() - if Globals.preserve_hardlinks and Hardlink.islinked(dest_rp): - dest_sig.flaglinked(Hardlink.get_link_index(dest_rp)) - elif dest_rp.isreg(): - dest_sig.setfile(Rdiff.get_signature(dest_rp)) - yield dest_sig + dest_iter = cls.get_dest_select(baserp, for_increment) + collated = rorpiter.Collate2Iters(source_iter, dest_iter) + cls.CCPP = CacheCollatedPostProcess(collated, + Globals.pipeline_max_length*2) + + def get_sigs(cls): + """Yield signatures of any changed destination files""" + for src_rorp, dest_rorp in cls.CCPP: + if (src_rorp and dest_rorp and src_rorp == dest_rorp and + (not Globals.preserve_hardlinks or + Hardlink.rorp_eq(src_rorp, dest_rorp))): continue + index = src_rorp and src_rorp.index or dest_rorp.index + cls.CCPP.flag_changed(index) + if (Globals.preserve_hardlinks and + Hardlink.islinked(src_rorp or dest_rorp)): + dest_sig = rpath.RORPath(index) + dest_sig.flaglinked(Hardlink.get_link_index(dest_sig)) + elif dest_rorp: + dest_sig = dest_rorp.getRORPath() + if dest_rorp.isreg(): + dest_sig.setfile(Rdiff.get_signature(dest_rorp)) + else: dest_sig = rpath.RORPath(index) + yield dest_sig def patch(cls, dest_rpath, source_diffiter, start_index = ()): """Patch dest_rpath with an rorpiter of diffs""" - ITR = rorpiter.IterTreeReducer(PatchITRB, [dest_rpath]) + ITR = rorpiter.IterTreeReducer(PatchITRB, [dest_rpath, cls.CCPP]) for diff in rorpiter.FillInIter(source_diffiter, dest_rpath): log.Log("Processing changed file " + diff.get_indexpath(), 5) ITR(diff.index, diff) ITR.Finish() + cls.CCPP.close() dest_rpath.setdata() def patch_and_increment(cls, dest_rpath, source_diffiter, inc_rpath): """Patch dest_rpath with rorpiter of diffs and write increments""" - ITR = rorpiter.IterTreeReducer(IncrementITRB, [dest_rpath, inc_rpath]) + ITR = rorpiter.IterTreeReducer(IncrementITRB, + [dest_rpath, inc_rpath, cls.CCPP]) for diff in rorpiter.FillInIter(source_diffiter, dest_rpath): log.Log("Processing changed file " + diff.get_indexpath(), 5) ITR(diff.index, diff) ITR.Finish() + cls.CCPP.close() dest_rpath.setdata() static.MakeClass(DestinationStruct) +class CacheCollatedPostProcess: + """Cache a collated iter of (source_rorp, dest_rp) pairs + + This is necessary for two reasons: + + 1. The patch function may need the original source_rorp or + dest_rp information, which is not present in the diff it + receives. + + 2. The metadata must match what is stored in the destination + directory. If there is an error we do not update the dest + directory for that file, and the old metadata is used. Thus + we cannot write any metadata until we know the file has been + procesed correctly. + + The class caches older source_rorps and dest_rps so the patch + function can retrieve them if necessary. The patch function can + also update the processed correctly flag. When an item falls out + of the cache, we assume it has been processed, and write the + metadata for it. + + """ + def __init__(self, collated_iter, cache_size): + """Initialize new CCWP.""" + self.iter = collated_iter # generates (source_rorp, dest_rorp) pairs + self.cache_size = cache_size + self.statfileobj = statistics.init_statfileobj() + metadata.OpenMetadata() + + # the following should map indicies to lists [source_rorp, + # dest_rorp, changed_flag, success_flag] where changed_flag + # should be true if the rorps are different, and success_flag + # should be true if dest_rorp has been successfully updated to + # source_rorp. They both default to false. + self.cache_dict = {} + self.cache_indicies = [] + + def __iter__(self): return self + + def next(self): + """Return next (source_rorp, dest_rorp) pair. StopIteration passed""" + source_rorp, dest_rorp = self.iter.next() + self.pre_process(source_rorp, dest_rorp) + index = source_rorp and source_rorp.index or dest_rorp.index + self.cache_dict[index] = [source_rorp, dest_rorp, 0, 0] + self.cache_indicies.append(index) + + if len(self.cache_indicies) > self.cache_size: self.shorten_cache() + return source_rorp, dest_rorp + + def pre_process(self, source_rorp, dest_rorp): + """Do initial processing on source_rorp and dest_rorp + + It will not be clear whether source_rorp and dest_rorp have + errors at this point, so don't do anything which assumes they + will be backed up correctly. + + """ + if source_rorp: Hardlink.add_rorp(source_rorp, source = 1) + if dest_rorp: Hardlink.add_rorp(dest_rorp, source = 0) + + def shorten_cache(self): + """Remove one element from cache, possibly adding it to metadata""" + first_index = self.cache_indicies[0] + del self.cache_indicies[0] + old_source_rorp, old_dest_rorp, changed_flag, success_flag = \ + self.cache_dict[first_index] + del self.cache_dict[first_index] + self.post_process(old_source_rorp, old_dest_rorp, + changed_flag, success_flag) + + def post_process(self, source_rorp, dest_rorp, changed, success): + """Post process source_rorp and dest_rorp. + + changed will be true if the files have changed. success will + be true if the files have been successfully updated (this is + always false for un-changed files). + + """ + if not changed or success: + self.statfileobj.add_source_file(source_rorp) + self.statfileobj.add_dest_file(dest_rorp) + if success: + self.statfileobj.add_changed(source_rorp, dest_rorp) + metadata_rorp = source_rorp + else: + metadata_rorp = dest_rorp + if changed: self.statfileobj.add_error() + if metadata_rorp and metadata_rorp.lstat(): + metadata.WriteMetadata(metadata_rorp) + + def flag_success(self, index): + """Signal that the file with given index was updated successfully""" + self.cache_dict[index][3] = 1 + + def flag_changed(self, index): + """Signal that the file with given index has changed""" + self.cache_dict[index][2] = 1 + + def get_rorps(self, index): + """Retrieve (source_rorp, dest_rorp) from cache""" + return self.cache_dict[index][:2] + + def get_source_rorp(self, index): + """Retrieve source_rorp with given index from cache""" + return self.cache_dict[index][0] + + def close(self): + """Process the remaining elements in the cache""" + while self.cache_indicies: self.shorten_cache() + metadata.CloseMetadata() + statistics.write_active_statfileobj() + + class PatchITRB(rorpiter.ITRBranch): """Patch an rpath with the given diff iters (use with IterTreeReducer) @@ -195,7 +294,7 @@ class PatchITRB(rorpiter.ITRBranch): contents. """ - def __init__(self, basis_root_rp): + def __init__(self, basis_root_rp, rorp_cache): """Set basis_root_rp, the base of the tree to be incremented""" self.basis_root_rp = basis_root_rp assert basis_root_rp.conn is Globals.local_connection @@ -267,7 +366,8 @@ class PatchITRB(rorpiter.ITRBranch): else: assert self.dir_replacement self.base_rp.rmdir() - rpath.rename(self.dir_replacement, self.base_rp) + if self.dir_replacement.lstat(): + rpath.rename(self.dir_replacement, self.base_rp) class IncrementITRB(PatchITRB): @@ -276,10 +376,10 @@ class IncrementITRB(PatchITRB): Like PatchITRB, but this time also write increments. """ - def __init__(self, basis_root_rp, inc_root_rp): + def __init__(self, basis_root_rp, inc_root_rp, rorp_cache): self.inc_root_rp = inc_root_rp self.cached_incrp = None - PatchITRB.__init__(self, basis_root_rp) + PatchITRB.__init__(self, basis_root_rp, rorp_cache) def get_incrp(self, index): """Return inc RPath by adding index to self.basis_root_rp""" diff --git a/rdiff-backup/rdiff_backup/log.py b/rdiff-backup/rdiff_backup/log.py index 956e578..421f78e 100644 --- a/rdiff-backup/rdiff_backup/log.py +++ b/rdiff-backup/rdiff_backup/log.py @@ -194,21 +194,21 @@ class ErrorLog: created. See the error policy file for more info. """ - log_fileobj = None - log_inc_rp = None + _log_fileobj = None + _log_inc_rp = None def open(cls, compress = 1): """Open the error log, prepare for writing""" - assert not cls.log_fileobj and not cls.log_inc_rp, "log already open" + assert not cls._log_fileobj and not cls._log_inc_rp, "log already open" if compress: typestr = 'data.gz' else: typestr = 'data' - cls.log_inc_rp = Global.rbdir.append("error_log.%s.%s" % - (Time.curtimestr, typestr)) - assert not cls.log_inc_rp.lstat(), "Error file already exists" - cls.log_fileobj = cls.log_inc_rp.open("wb", compress = compress) + cls._log_inc_rp = Global.rbdir.append("error_log.%s.%s" % + (Time.curtimestr, typestr)) + assert not cls._log_inc_rp.lstat(), "Error file already exists" + cls._log_fileobj = cls._log_inc_rp.open("wb", compress = compress) def isopen(cls): """True if the error log file is currently open""" - return cls.log_fileobj is not None + return cls._log_fileobj is not None def write(cls, error_type, rp, exc): """Add line to log file indicating error exc with file rp""" @@ -218,7 +218,7 @@ class ErrorLog: else: s = re.sub("\n", " ", s) s += "\n" - cls.log_fileobj.write(s) + cls._log_fileobj.write(s) def get_indexpath(cls, rp): """Return filename for logging. rp is a rpath, string, or tuple""" @@ -240,8 +240,8 @@ class ErrorLog: def close(cls): """Close the error log file""" - assert not cls.log_fileobj.close() - cls.log_fileobj = cls.log_inc_rp = None + assert not cls._log_fileobj.close() + cls._log_fileobj = cls._log_inc_rp = None static.MakeClass(ErrorLog) diff --git a/rdiff-backup/rdiff_backup/metadata.py b/rdiff-backup/rdiff_backup/metadata.py index 8b4e96d..a434024 100644 --- a/rdiff-backup/rdiff_backup/metadata.py +++ b/rdiff-backup/rdiff_backup/metadata.py @@ -280,6 +280,7 @@ def WriteMetadata(rorp): def CloseMetadata(): """Close the metadata file""" global metadata_rp, metadata_fileobj + assert metadata_fileobj, "Metadata file not open" try: fileno = metadata_fileobj.fileno() # will not work if GzipFile except AttributeError: fileno = metadata_fileobj.fileobj.fileno() os.fsync(fileno) diff --git a/rdiff-backup/rdiff_backup/regress.py b/rdiff-backup/rdiff_backup/regress.py index db64b0a..bfd5b62 100644 --- a/rdiff-backup/rdiff_backup/regress.py +++ b/rdiff-backup/rdiff_backup/regress.py @@ -34,7 +34,7 @@ recovered. """ from __future__ import generators -import Globals, restore, log, rorpiter, journal, TempFile +import Globals, restore, log, rorpiter, journal, TempFile, metadata, rpath # regress_time should be set to the time we want to regress back to # (usually the time of the last successful backup) @@ -43,6 +43,10 @@ regress_time = None # This should be set to the latest unsuccessful backup time unsuccessful_backup_time = None +# This is set by certain tests and allows overriding of global time +# variables. +time_override_mode = None + class RegressException(Exception): """Raised on any exception in regress process""" @@ -64,6 +68,9 @@ def Regress(mirror_rp): assert mirror_rp.conn is inc_rpath.conn is Globals.local_connection set_regress_time() set_restore_times() + ITR = rorpiter.IterTreeReducer(RegressITRB, []) + for rf in iterate_meta_rfs(mirror_rp, inc_rpath): ITR(rf.index, rf) + ITR.Finish() def set_regress_time(): """Set global regress_time to previous sucessful backup @@ -73,6 +80,10 @@ def set_regress_time(): """ global regress_time, unsuccessful_backup_time + if time_override_mode: + assert regress_time and unsuccessful_backup_time + return + curmir_incs = restore.get_inclist(Globals.rbdir.append("current_mirror")) assert len(curmir_incs) == 2, \ "Found %s current_mirror flags, expected 2" % len(curmir_incs) @@ -134,10 +145,10 @@ class RegressFile(restore.RestoreFile): """ def __init__(self, mirror_rp, inc_rp, inc_list): - restore.RestoreFile._init__(self, mirror_rp, inc_rp, inclist) + restore.RestoreFile.__init__(self, mirror_rp, inc_rp, inc_list) assert len(self.relevant_incs) <= 2, "Too many incs" if len(self.relevant_incs) == 2: - self.regress_inc = self.relevant.incs[-1] + self.regress_inc = self.relevant_incs[-1] else: self.regress_inc = None def set_metadata_rorp(self, metadata_rorp): @@ -178,7 +189,8 @@ class RegressITRB(rorpiter.ITRBranch): def fast_process(self, index, rf): """Process when nothing is a directory""" - if not rpath.cmp_attribs(rf.metadata_rorp, rf.mirror_rp): + if (not rf.metadata_rorp.lstat() or not rf.mirror_rp.lstat() or + not rpath.cmp_attribs(rf.metadata_rorp, rf.mirror_rp)): if rf.metadata_rorp.isreg(): self.restore_orig_regfile(rf) else: if rf.mirror_rp.lstat(): rf.mirror_rp.delete() @@ -242,8 +254,8 @@ class RegressITRB(rorpiter.ITRBranch): """ if args and args[0] and isinstance(args[0], tuple): - filename = os.path.join(*args[0]) - elif self.index: filename = os.path.join(*self.index) + filename = "/".join(args[0]) + elif self.index: filename = "/".join(*self.index) else: filename = "." log.Log("Error '%s' processing %s" % (exc, filename), 2) raise RegressException("Error during Regress") diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 53b64ea..54d29d2 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -22,7 +22,7 @@ from __future__ import generators import tempfile, os import Globals, Time, Rdiff, Hardlink, rorpiter, selection, rpath, \ - log, backup, static, robust, metadata + log, static, robust, metadata, statistics, TempFile # This should be set to selection.Select objects over the source and @@ -220,11 +220,18 @@ class TargetStruct: def patch(cls, target, diff_iter): """Patch target with the diffs from the mirror side - This function was already written for use when backing up, so - just use that. + This function and the associated ITRB is similar to the + patching code in backup.py, but they have different error + correction requirements, so it seemed easier to just repeat it + all in this module. """ - backup.DestinationStruct.patch(target, diff_iter) + ITR = rorpiter.IterTreeReducer(PatchITRB, [target]) + for diff in rorpiter.FillInIter(diff_iter, target): + log.Log("Processing changed file " + diff.get_indexpath(), 5) + ITR(diff.index, diff) + ITR.Finish() + target.setdata() static.MakeClass(TargetStruct) @@ -279,6 +286,7 @@ class CachedRF: assert new_rfs, "No RFs added for index %s" % index self.rf_list[0:0] = new_rfs + class RestoreFile: """Hold data about a single mirror file and its related increments @@ -390,8 +398,12 @@ class RestoreFile: def yield_sub_rfs(self): """Return RestoreFiles under current RestoreFile (which is dir)""" assert self.mirror_rp.isdir() or self.inc_rp.isdir() - mirror_iter = self.yield_mirrorrps(self.mirror_rp) - inc_pair_iter = self.yield_inc_complexes(self.inc_rp) + if self.mirror_rp.isdir(): + mirror_iter = self.yield_mirrorrps(self.mirror_rp) + else: mirror_iter = iter([]) + if self.inc_rp.isdir(): + inc_pair_iter = self.yield_inc_complexes(self.inc_rp) + else: inc_pair_iter = iter([]) collated = rorpiter.Collate2Iters(mirror_iter, inc_pair_iter) for mirror_rp, inc_pair in collated: @@ -405,6 +417,7 @@ class RestoreFile: def yield_mirrorrps(self, mirrorrp): """Yield mirrorrps underneath given mirrorrp""" + assert mirrorrp.isdir() for filename in robust.listrp(mirrorrp): rp = mirrorrp.append(filename) if rp.index != ('rdiff-backup-data',): yield rp @@ -440,3 +453,92 @@ class RestoreFile: keys = inc_dict.keys() keys.sort() for key in keys: yield inc_dict[key] + + +class PatchITRB(rorpiter.ITRBranch): + """Patch an rpath with the given diff iters (use with IterTreeReducer) + + The main complication here involves directories. We have to + finish processing the directory after what's in the directory, as + the directory may have inappropriate permissions to alter the + contents or the dir's mtime could change as we change the + contents. + + This code was originally taken from backup.py. However, because + of different error correction requirements, it is repeated here. + + """ + def __init__(self, basis_root_rp): + """Set basis_root_rp, the base of the tree to be incremented""" + self.basis_root_rp = basis_root_rp + assert basis_root_rp.conn is Globals.local_connection + self.statfileobj = (statistics.get_active_statfileobj() or + statistics.StatFileObj()) + self.dir_replacement, self.dir_update = None, None + self.cached_rp = None + + def get_rp_from_root(self, index): + """Return RPath by adding index to self.basis_root_rp""" + if not self.cached_rp or self.cached_rp.index != index: + self.cached_rp = self.basis_root_rp.new_index(index) + return self.cached_rp + + def can_fast_process(self, index, diff_rorp): + """True if diff_rorp and mirror are not directories""" + rp = self.get_rp_from_root(index) + return not diff_rorp.isdir() and not rp.isdir() + + def fast_process(self, index, diff_rorp): + """Patch base_rp with diff_rorp (case where neither is directory)""" + rp = self.get_rp_from_root(index) + tf = TempFile.new(rp) + self.patch_to_temp(rp, diff_rorp, tf) + rpath.rename(tf, rp) + + def patch_to_temp(self, basis_rp, diff_rorp, new): + """Patch basis_rp, writing output in new, which doesn't exist yet""" + if diff_rorp.isflaglinked(): + Hardlink.link_rp(diff_rorp, new, self.basis_root_rp) + elif diff_rorp.get_attached_filetype() == 'snapshot': + rpath.copy(diff_rorp, new) + else: + assert diff_rorp.get_attached_filetype() == 'diff' + Rdiff.patch_local(basis_rp, diff_rorp, new) + if new.lstat(): rpath.copy_attribs(diff_rorp, new) + + def start_process(self, index, diff_rorp): + """Start processing directory - record information for later""" + base_rp = self.base_rp = self.get_rp_from_root(index) + assert diff_rorp.isdir() or base_rp.isdir() or not base_rp.index + if diff_rorp.isdir(): self.prepare_dir(diff_rorp, base_rp) + else: self.set_dir_replacement(diff_rorp, base_rp) + + def set_dir_replacement(self, diff_rorp, base_rp): + """Set self.dir_replacement, which holds data until done with dir + + This is used when base_rp is a dir, and diff_rorp is not. + + """ + assert diff_rorp.get_attached_filetype() == 'snapshot' + self.dir_replacement = TempFile.new(base_rp) + rpath.copy_with_attribs(diff_rorp, self.dir_replacement) + if base_rp.isdir(): base_rp.chmod(0700) + + def prepare_dir(self, diff_rorp, base_rp): + """Prepare base_rp to turn into a directory""" + self.dir_update = diff_rorp.getRORPath() # make copy in case changes + if not base_rp.isdir(): + if base_rp.lstat(): base_rp.delete() + base_rp.mkdir() + base_rp.chmod(0700) + + def end_process(self): + """Finish processing directory""" + if self.dir_update: + assert self.base_rp.isdir() + rpath.copy_attribs(self.dir_update, self.base_rp) + else: + assert self.dir_replacement + self.base_rp.rmdir() + if self.dir_replacement.lstat(): + rpath.rename(self.dir_replacement, self.base_rp) diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py index 3849fdc..072478f 100644 --- a/rdiff-backup/rdiff_backup/rorpiter.py +++ b/rdiff-backup/rdiff_backup/rorpiter.py @@ -149,24 +149,6 @@ def Collate2Iters(riter1, riter2): yield (None, relem2) relem2 = None -def get_dissimilar_indicies(src_init_iter, dest_init_iter, statfileobj = None): - """Get dissimilar indicies given two rorpiters - - Returns an iterator which enumerates the indicies of the rorps - which are different on the source and destination ends. If - statfileobj is given, call add_changed on each pair of different - indicies. - - """ - collated = Collate2Iters(src_init_iter, dest_init_iter) - for src_rorp, dest_rorp in collated: - if (src_rorp and dest_rorp and src_rorp == dest_rorp and - (not Globals.preserve_hardlinks or - Hardlink.rorp_eq(src_rorp, dest_rorp))): continue - if statfileobj: statfileobj.add_changed(src_rorp, dest_rorp) - if not dest_rorp: yield src_rorp.index - else: yield dest_rorp.index - class IndexedTuple(UserList.UserList): """Like a tuple, but has .index diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 48e98ed..7d7c440 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -90,7 +90,7 @@ def copy(rpin, rpout, compress = 0): if rpout.lstat(): if rpin.isreg() or not cmp(rpin, rpout): - rpout.delete() # easier to write that compare + rpout.delete() # easier to write than compare else: return if rpin.isreg(): copy_reg_file(rpin, rpout, compress) @@ -177,7 +177,7 @@ def cmp_attribs(rp1, rp2): elif rp1.ischardev() and rp2.ischardev(): result = 1 else: result = (rp1.getmtime() == rp2.getmtime()) log.Log("Compare attribs of %s and %s: %s" % - (rp1.path, rp2.path, result), 7) + (rp1.get_indexpath(), rp2.get_indexpath(), result), 7) return result def copy_with_attribs(rpin, rpout, compress = 0): @@ -694,11 +694,7 @@ class RPath(RORPath): def delete(self): """Delete file at self.path. Recursively deletes directories.""" log.Log("Deleting %s" % self.path, 7) - self.setdata() - if not self.lstat(): - log.Log("Warning: %s does not exist---deleted in meantime?" - % (self.path,), 2) - elif self.isdir(): + if self.isdir(): try: self.rmdir() except os.error: shutil.rmtree(self.path) else: self.conn.os.unlink(self.path) diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index 57aad7b..c527e83 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -5,6 +5,7 @@ from rdiff_backup.rpath import RPath from rdiff_backup import Globals, Hardlink, SetConnections, Main, \ selection, lazy, Time, rpath +RBBin = "../rdiff-backup" SourceDir = "../rdiff_backup" AbsCurdir = os.getcwd() # Absolute path name of current directory AbsTFdir = AbsCurdir+"/testfiles" @@ -56,12 +57,14 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir, dest_dir = ("test2/tmp; ../../%s/rdiff-backup --server::../../%s" % (SourceDir, dest_dir)) - cmdargs = [SourceDir + "/rdiff-backup", extra_options] + cmdargs = [RBBin, extra_options] if not (source_local and dest_local): cmdargs.append("--remote-schema %s") if current_time: cmdargs.append("--current-time %s" % current_time) - - os.system(" ".join(cmdargs)) + cmdargs.extend([src_dir, dest_dir]) + cmdline = " ".join(cmdargs) + print "Executing: ", cmdline + assert not os.system(cmdline) def cmd_schemas2rps(schema_list, remote_schema): """Input list of file descriptions and the remote schema, return rps diff --git a/rdiff-backup/testing/regresstest.py b/rdiff-backup/testing/regresstest.py index 8a24958..cb13765 100644 --- a/rdiff-backup/testing/regresstest.py +++ b/rdiff-backup/testing/regresstest.py @@ -1,11 +1,75 @@ -"""regresstest - test the regress module. Not to be confused with the -regression tests.""" +"""regresstest - test the regress module. + +Not to be confused with the regression tests. + +""" import unittest from commontest import * +from rdiff_backup import regress + +Log.setverbosity(7) class RegressTest(unittest.TestCase): - XXX + regress_rp1 = rpath.RPath(Globals.local_connection, + "testfiles/regress_output1") + regress_rp2 = rpath.RPath(Globals.local_connection, + "testfiles/regress_output2") + + def make_output(self, level): + """Set up two rdiff-backup destination dir of level and level+1 + + testfiles/regress_output1 will be a copy of + testfiles/increment1 through testfiles/increment{level} + + testfiles/regress_output2 will have all everything backed up + in testfiles/regress_output1 + testfiles/increment{level+1}. + + The time of each increment will be 10000*level. + + """ + assert 1 <= level <= 3 + if self.regress_rp1.lstat(): Myrm(self.regress_rp1.path) + if self.regress_rp2.lstat(): Myrm(self.regress_rp2.path) + + # Make regress_output1 + Log("Making %s" % (self.regress_rp1.path,), 4) + for i in range(1, level+1): + rdiff_backup(1, 1, + "testfiles/increment%s" % (i,), + self.regress_rp1.path, + current_time = 10000*i) + + # Now make regress_output2 + Log("Making %s" % (self.regress_rp2.path,), 4) + assert not os.system("cp -a %s %s" % + (self.regress_rp1.path, self.regress_rp2.path)) + rdiff_backup(1, 1, + "testfiles/increment%s" % (level+1), + self.regress_rp2.path, + current_time = 10000*(level+1)) + self.regress_rp1.setdata() + self.regress_rp2.setdata() + + def test_full(self): + """Test regressing a full directory to older state + + Make two directories, one with one more backup in it. Then + regress the bigger one, and then make sure they compare the + same. + + """ + for level in range(1, 4): + self.make_output(level) + regress.regress_time = 10000*level + regress.unsuccessful_backup_time = 10000*(level+1) + regress.time_override_mode = 1 + Globals.rbdir = self.regress_rp2.append_path("rdiff-backup-data") + Log("######### Beginning regress ###########", 5) + regress.Regress(self.regress_rp2) + + assert CompareRecursive(self.regress_rp1, self.regress_rp2, + exclude_rbdir = 0) if __name__ == "__main__": unittest.main() -- cgit v1.2.1