diff options
Diffstat (limited to 'rdiff-backup/rdiff_backup/backup.py')
| -rw-r--r-- | rdiff-backup/rdiff_backup/backup.py | 234 |
1 files changed, 169 insertions, 65 deletions
diff --git a/rdiff-backup/rdiff_backup/backup.py b/rdiff-backup/rdiff_backup/backup.py index fc75099..f776988 100644 --- a/rdiff-backup/rdiff_backup/backup.py +++ b/rdiff-backup/rdiff_backup/backup.py @@ -17,38 +17,39 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA -"""High level functions for mirroring, mirror & inc, etc.""" +"""High level functions for mirroring and mirror+incrementing""" from __future__ import generators -import Globals, MiscStats, metadata, rorpiter, TempFile, Hardlink, \ - robust, increment, rpath, lazy, static, log, selection, Time, Rdiff - +import Globals, metadata, rorpiter, TempFile, Hardlink, robust, increment, \ + rpath, static, log, selection, Time, Rdiff, statistics def Mirror(src_rpath, dest_rpath): """Turn dest_rpath into a copy of src_rpath""" - SourceS = src_rpath.conn.highlevel.HLSourceStruct - DestS = dest_rpath.conn.highlevel.HLDestinationStruct + SourceS = src_rpath.conn.backup.SourceStruct + DestS = dest_rpath.conn.backup.DestinationStruct + DestS.init_statistics() source_rpiter = SourceS.get_source_select() - dest_sigiter = DestS.process_source_get_sigs(dest_rpath, - source_rpiter, 0) + dest_sigiter = DestS.process_source_get_sigs(dest_rpath, source_rpiter, 0) source_diffiter = SourceS.get_diffs(src_rpath, dest_sigiter) DestS.patch(dest_rpath, source_diffiter) + DestS.write_statistics() def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath): """Mirror + put increments in tree based at inc_rpath""" - SourceS = src_rpath.conn.highlevel.HLSourceStruct - DestS = dest_rpath.conn.highlevel.HLDestinationStruct + SourceS = src_rpath.conn.backup.SourceStruct + DestS = dest_rpath.conn.backup.DestinationStruct + DestS.init_statistics() source_rpiter = SourceS.get_source_select() - dest_sigiter = DestS.process_source_get_sigs(dest_rpath, - source_rpiter, 1) + dest_sigiter = DestS.process_source_get_sigs(dest_rpath, source_rpiter, 1) source_diffiter = SourceS.get_diffs(src_rpath, dest_sigiter) DestS.patch_and_increment(dest_rpath, source_diffiter, inc_rpath) + DestS.write_statistics() -class HLSourceStruct: - """Hold info used by HL on the source side""" +class SourceStruct: + """Hold info used on source side when backing up""" source_select = None # will be set to source Select iterator def set_source_select(cls, rpath, tuplelist, *filelists): """Initialize select object using tuplelist @@ -69,23 +70,36 @@ class HLSourceStruct: def get_diffs(cls, baserp, dest_sigiter): """Return diffs of any files with signature in dest_sigiter""" - for dest_sig in dest_sigiter: + def get_one_diff(dest_sig): src_rp = baserp.new_index(dest_sig.index) diff_rorp = src_rp.getRORPath() - if dest_sig.isflaglinked(): diff_rorp.flaglinked() + if dest_sig.isflaglinked(): + diff_rorp.flaglinked(dest_sig.get_link_flag()) elif dest_sig.isreg() and src_rp.isreg(): diff_rorp.setfile(Rdiff.get_delta_sigrp(dest_sig, src_rp)) diff_rorp.set_attached_filetype('diff') else: diff_rorp.set_attached_filetype('snapshot') if src_rp.isreg(): diff_rorp.setfile(src_rp.open("rb")) - yield diff_rorp + return diff_rorp + + for dest_sig in dest_sigiter: + diff = robust.check_common_error(None, get_one_diff, [dest_sig]) + if diff: yield diff -static.MakeClass(HLSourceStruct) +static.MakeClass(SourceStruct) -class HLDestinationStruct: - """Hold info used by HL on the destination side""" +class DestinationStruct: + """Hold info used by destination side when backing up""" + def init_statistics(cls): + """Set cls.stats to StatFileObj object""" + cls.statfileobj = statistics.init_statfileobj() + + def write_statistics(cls): + """Write statistics file""" + statistics.write_active_statfileobj() + def get_dest_select(cls, rpath, use_metadata = 1): """Return destination select rorpath iterator @@ -95,7 +109,7 @@ class HLDestinationStruct: """ if use_metadata: metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir, - Time.curtime) + Time.prevtime) if metadata_iter: return metadata_iter log.Log("Warning: Metadata file not found.\n" "Metadata will be read from filesystem.", 2) @@ -107,7 +121,7 @@ class HLDestinationStruct: def dest_iter_filter(cls, dest_iter): """Destination rorps pass through this - record stats""" for dest_rorp in dest_iter: - # XXX Statistics process + cls.statfileobj.add_dest_file(dest_rorp) Hardlink.add_rorp(dest_rorp, source = 0) yield dest_rorp @@ -115,9 +129,9 @@ class HLDestinationStruct: """Source rorps pass through this - record stats, write metadata""" metadata.OpenMetadata() for src_rorp in source_iter: + cls.statfileobj.add_source_file(src_rorp) Hardlink.add_rorp(src_rorp, source = 1) metadata.WriteMetadata(src_rorp) - #XXXX Statistics process yield src_rorp metadata.CloseMetadata() @@ -133,66 +147,156 @@ class HLDestinationStruct: source_iter = cls.src_iter_filter(source_iter) dest_iter = cls.dest_iter_filter(cls.get_dest_select(baserp, for_increment)) - for index in rorpiter.get_dissimilar_indicies(source_iter, dest_iter): + for index in rorpiter.get_dissimilar_indicies(source_iter, dest_iter, + cls.statfileobj): dest_rp = baserp.new_index(index) dest_sig = dest_rp.getRORPath() if Globals.preserve_hardlinks and Hardlink.islinked(dest_rp): - dest_sig.flaglinked() + dest_sig.flaglinked(Hardlink.get_link_index(dest_rp)) elif dest_rp.isreg(): dest_sig.setfile(Rdiff.get_signature(dest_rp)) yield dest_sig - def patch(cls, dest_rpath, source_diffiter): + def patch(cls, dest_rpath, source_diffiter, start_index = ()): """Patch dest_rpath with an rorpiter of diffs""" - ITR = rorpiter.IterTreeReducer(increment.PatchITRB, [dest_rpath]) + ITR = rorpiter.IterTreeReducer(PatchITRB, [dest_rpath]) for diff in rorpiter.FillInIter(source_diffiter, dest_rpath): + log.Log("Processing changed file " + diff.get_indexpath(), 5) ITR(diff.index, diff) ITR.Finish() dest_rpath.setdata() def patch_and_increment(cls, dest_rpath, source_diffiter, inc_rpath): """Patch dest_rpath with rorpiter of diffs and write increments""" - ITR = rorpiter.IterTreeReducer(increment.IncrementITRB, - [dest_rpath, inc_rpath]) + ITR = rorpiter.IterTreeReducer(IncrementITRB, [dest_rpath, inc_rpath]) for diff in rorpiter.FillInIter(source_diffiter, dest_rpath): + log.Log("Processing changed file " + diff.get_indexpath(), 5) ITR(diff.index, diff) ITR.Finish() dest_rpath.setdata() - def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): - """Apply diffs, write increment if necessary, and finalize""" - collated = rorpiter.CollateIterators(diffs, cls.initial_dsiter2) - #finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath) - finalizer, ITR = None, cls.get_ITR(inc_rpath) - MiscStats.open_dir_stats_file() - dsrp, finished_dsrp = None, None - - try: - for indexed_tuple in collated: - log.Log(lambda: "Processing %s" % str(indexed_tuple), 7) - diff_rorp, dsrp = indexed_tuple - index = indexed_tuple.index - if not dsrp: dsrp = cls.get_dsrp(dest_rpath, index) - if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None - ITR(index, diff_rorp, dsrp) - #finalizer(index, dsrp) - finished_dsrp = dsrp - ITR.Finish() - #finalizer.Finish() - except: cls.handle_last_error(finished_dsrp, finalizer, ITR) - - if Globals.preserve_hardlinks: Hardlink.final_writedata() - MiscStats.close_dir_stats_file() - MiscStats.write_session_statistics(ITR.root_branch) - - def handle_last_error(cls, dsrp, finalizer, ITR): - """If catch fatal error, try to checkpoint before exiting""" - log.Log.exception(1, 2) - robust.TracebackArchive.log() - #SaveState.checkpoint(ITR, finalizer, dsrp, 1) - #if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir) - #SaveState.touch_last_file_definitive() - raise - -static.MakeClass(HLDestinationStruct) +static.MakeClass(DestinationStruct) + + +class PatchITRB(rorpiter.ITRBranch): + """Patch an rpath with the given diff iters (use with IterTreeReducer) + + The main complication here involves directories. We have to + finish processing the directory after what's in the directory, as + the directory may have inappropriate permissions to alter the + contents or the dir's mtime could change as we change the + contents. + + """ + def __init__(self, basis_root_rp): + """Set basis_root_rp, the base of the tree to be incremented""" + self.basis_root_rp = basis_root_rp + assert basis_root_rp.conn is Globals.local_connection + self.statfileobj = (statistics.get_active_statfileobj() or + statistics.StatFileObj()) + self.dir_replacement, self.dir_update = None, None + self.cached_rp = None + + def get_rp_from_root(self, index): + """Return RPath by adding index to self.basis_root_rp""" + if not self.cached_rp or self.cached_rp.index != index: + self.cached_rp = self.basis_root_rp.new_index(index) + return self.cached_rp + + def can_fast_process(self, index, diff_rorp): + """True if diff_rorp and mirror are not directories""" + rp = self.get_rp_from_root(index) + return not diff_rorp.isdir() and not rp.isdir() + + def fast_process(self, index, diff_rorp): + """Patch base_rp with diff_rorp (case where neither is directory)""" + rp = self.get_rp_from_root(index) + tf = TempFile.new(rp) + self.patch_to_temp(rp, diff_rorp, tf) + tf.rename(rp) + + def patch_to_temp(self, basis_rp, diff_rorp, new): + """Patch basis_rp, writing output in new, which doesn't exist yet""" + if diff_rorp.isflaglinked(): + Hardlink.link_rp(diff_rorp, new, self.basis_root_rp) + elif diff_rorp.get_attached_filetype() == 'snapshot': + rpath.copy(diff_rorp, new) + else: + assert diff_rorp.get_attached_filetype() == 'diff' + Rdiff.patch_local(basis_rp, diff_rorp, new) + if new.lstat(): rpath.copy_attribs(diff_rorp, new) + + def start_process(self, index, diff_rorp): + """Start processing directory - record information for later""" + base_rp = self.base_rp = self.get_rp_from_root(index) + assert diff_rorp.isdir() or base_rp.isdir() or not base_rp.index + if diff_rorp.isdir(): self.prepare_dir(diff_rorp, base_rp) + else: self.set_dir_replacement(diff_rorp, base_rp) + + def set_dir_replacement(self, diff_rorp, base_rp): + """Set self.dir_replacement, which holds data until done with dir + + This is used when base_rp is a dir, and diff_rorp is not. + + """ + assert diff_rorp.get_attached_filetype() == 'snapshot' + self.dir_replacement = TempFile.new(base_rp) + rpath.copy_with_attribs(diff_rorp, self.dir_replacement) + if base_rp.isdir(): base_rp.chmod(0700) + + def prepare_dir(self, diff_rorp, base_rp): + """Prepare base_rp to turn into a directory""" + self.dir_update = diff_rorp.getRORPath() # make copy in case changes + if not base_rp.isdir(): + if base_rp.lstat(): base_rp.delete() + base_rp.mkdir() + base_rp.chmod(0700) + + def end_process(self): + """Finish processing directory""" + if self.dir_update: + assert self.base_rp.isdir() + rpath.copy_attribs(self.dir_update, self.base_rp) + else: + assert self.dir_replacement + self.base_rp.rmdir() + self.dir_replacement.rename(self.base_rp) + + +class IncrementITRB(PatchITRB): + """Patch an rpath with the given diff iters and write increments + + Like PatchITRB, but this time also write increments. + + """ + def __init__(self, basis_root_rp, inc_root_rp): + self.inc_root_rp = inc_root_rp + self.cached_incrp = None + PatchITRB.__init__(self, basis_root_rp) + + def get_incrp(self, index): + """Return inc RPath by adding index to self.basis_root_rp""" + if not self.cached_incrp or self.cached_incrp.index != index: + self.cached_incrp = self.inc_root_rp.new_index(index) + return self.cached_incrp + + def fast_process(self, index, diff_rorp): + """Patch base_rp with diff_rorp and write increment (neither is dir)""" + rp = self.get_rp_from_root(index) + tf = TempFile.new(rp) + self.patch_to_temp(rp, diff_rorp, tf) + increment.Increment(tf, rp, self.get_incrp(index)) + tf.rename(rp) + + def start_process(self, index, diff_rorp): + """Start processing directory""" + base_rp = self.base_rp = self.get_rp_from_root(index) + assert diff_rorp.isdir() or base_rp.isdir() + if diff_rorp.isdir(): + increment.Increment(diff_rorp, base_rp, self.get_incrp(index)) + self.prepare_dir(diff_rorp, base_rp) + else: + self.set_dir_replacement(diff_rorp, base_rp) + increment.Increment(self.dir_replacement, base_rp, + self.get_incrp(index)) |