diff options
author | ben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2002-05-25 18:54:30 +0000 |
---|---|---|
committer | ben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2002-05-25 18:54:30 +0000 |
commit | 7a82871f7b6d25464516dd659f3e98cc56211ff1 (patch) | |
tree | f0f6c1d0b040769c759e76eeee355776dd453eb8 /rdiff-backup/rdiff_backup | |
parent | 9b6416eaf5e1691d2e640a5da2a14b4efac38348 (diff) | |
download | rdiff-backup-7a82871f7b6d25464516dd659f3e98cc56211ff1.tar.gz |
Bugfixed the new statistics stuff
git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@108 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
Diffstat (limited to 'rdiff-backup/rdiff_backup')
-rw-r--r-- | rdiff-backup/rdiff_backup/highlevel.py | 65 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/increment.py | 70 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/robust.py | 36 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/statistics.py | 12 |
4 files changed, 112 insertions, 71 deletions
diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index 4a2b515..cc1335c 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -24,14 +24,12 @@ class HighLevel: accompanying diagram. """ - def Mirror(src_rpath, dest_rpath, checkpoint = 1, - session_info = None, write_finaldata = 1): + def Mirror(src_rpath, dest_rpath, inc_rpath = None, session_info = None): """Turn dest_rpath into a copy of src_rpath - Checkpoint true means to checkpoint periodically, otherwise - not. If session_info is given, try to resume Mirroring from - that point. If write_finaldata is true, save extra data files - like hardlink_data. If it is false, make a complete mirror. + If inc_rpath is true, then this is the initial mirroring of an + incremental backup, so checkpoint and write to data_dir. + Otherwise only mirror and don't create any extra files. """ SourceS = src_rpath.conn.HLSourceStruct @@ -42,8 +40,9 @@ class HighLevel: src_init_dsiter = SourceS.split_initial_dsiter() dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter) diffiter = SourceS.get_diffs_and_finalize(dest_sigiter) - DestS.patch_and_finalize(dest_rpath, diffiter, - checkpoint, write_finaldata) + if inc_rpath: + DestS.patch_w_datadir_writes(dest_rpath, diffiter, inc_rpath) + else: DestS.patch_and_finalize(dest_rpath, diffiter) dest_rpath.setdata() @@ -207,8 +206,13 @@ class HLDestinationStruct: iitr.override_changed() return iitr - def patch_and_finalize(cls, dest_rpath, diffs, - checkpoint = 1, write_finaldata = 1): + def get_MirrorITR(cls, inc_rpath): + """Return MirrorITR, starting from state if available""" + if cls._session_info and cls._session_info.ITR: + return cls._session_info.ITR + else: return MirrorITR(inc_rpath) + + def patch_and_finalize(cls, dest_rpath, diffs): """Apply diffs and finalize""" collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) finalizer = cls.get_finalizer() @@ -229,12 +233,36 @@ class HLDestinationStruct: while 1: try: dsrp = cls.check_skip_error(error_checked, dsrp) except StopIteration: break - if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) - except: cls.handle_last_error(dsrp, finalizer) + except: Log.exception(1) finalizer.Finish() - if Globals.preserve_hardlinks and write_finaldata: - Hardlink.final_writedata() - if checkpoint: SaveState.checkpoint_remove() + + def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath): + """Apply diffs and finalize, with checkpointing and statistics""" + collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) + finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath) + dsrp = None + + def error_checked(): + """Inner writing loop, check this for errors""" + indexed_tuple = collated.next() + Log("Processing %s" % str(indexed_tuple), 7) + diff_rorp, dsrp = indexed_tuple + if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index) + if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None + ITR(dsrp.index, diff_rorp, dsrp) + finalizer(dsrp.index, dsrp) + return dsrp + + try: + while 1: + try: dsrp = cls.check_skip_error(error_checked, dsrp) + except StopIteration: break + SaveState.checkpoint(ITR, finalizer, dsrp) + cls.check_skip_error(ITR.Finish, dsrp) + cls.check_skip_error(finalizer.Finish, dsrp) + except: cls.handle_last_error(dsrp, finalizer, ITR) + if Globals.preserve_hardlinks: Hardlink.final_writedata() + SaveState.checkpoint_remove() def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): """Apply diffs, write increment if necessary, and finalize""" @@ -258,7 +286,7 @@ class HLDestinationStruct: while 1: try: dsrp = cls.check_skip_error(error_checked, dsrp) except StopIteration: break - SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp) + SaveState.checkpoint(ITR, finalizer, dsrp) cls.check_skip_error(ITR.Finish, dsrp) cls.check_skip_error(finalizer.Finish, dsrp) except: cls.handle_last_error(dsrp, finalizer, ITR) @@ -285,11 +313,10 @@ class HLDestinationStruct: Log.exception(1,2) raise - def handle_last_error(cls, dsrp, finalizer, ITR = None): + def handle_last_error(cls, dsrp, finalizer, ITR): """If catch fatal error, try to checkpoint before exiting""" Log.exception(1) - if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) - else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) + SaveState.checkpoint(ITR, finalizer, dsrp, 1) if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir) SaveState.touch_last_file_definitive() raise diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index 499d709..18b137f 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -79,28 +79,28 @@ class Inc: RPath.copy_attribs(mirrordir, dirsign) return RobustAction(lambda: None, final, dirsign.delete) + def get_inc(rp, time, typestr): + """Return increment like rp but with time and typestr suffixes""" + addtostr = lambda s: "%s.%s.%s" % (s, Time.timetostring(time), typestr) + if rp.index: + incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] + + (addtostr(rp.index[-1]),)) + else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index) + if Globals.quoting_enabled: incrp.quote_path() + return incrp + def get_inc_ext(rp, typestr): - """Return RPath/DSRPath like rp but with inc/time extension + """Return increment with specified type and correct time If the file exists, then probably a previous backup has been aborted. We then keep asking FindTime to get a time later than the one that already has an inc file. """ - def get_newinc(timestr): - """Get new increment rp with given time suffix""" - addtostr = lambda s: "%s.%s.%s" % (s, timestr, typestr) - if rp.index: - incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] + - (addtostr(rp.index[-1]),)) - else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index) - if Globals.quoting_enabled: incrp.quote_path() - return incrp - inctime = 0 while 1: inctime = Resume.FindTime(rp.index, inctime) - incrp = get_newinc(Time.timetostring(inctime)) + incrp = Inc.get_inc(rp, inctime, typestr) if not incrp.lstat(): break Inc._inc_file = incrp return incrp @@ -109,7 +109,7 @@ MakeStatic(Inc) class IncrementITR(StatsITR): - """Patch and increment iterator of increment triples + """Patch and increment mirror directory This has to be an ITR because directories that have files in them changed are flagged with an increment marker. There are four @@ -208,7 +208,10 @@ class IncrementITR(StatsITR): def end_process(self): """Do final work when leaving a tree (directory)""" - diff_rorp, dsrp, incpref = self.diff_rorp, self.dsrp, self.incpref + try: diff_rorp, dsrp, incpref = self.diff_rorp, self.dsrp, self.incpref + except AttributeError: # This weren't set because of some error + return + if self.mirror_isdirectory: if not diff_rorp and not self.changed: return @@ -224,8 +227,9 @@ class IncrementITR(StatsITR): self.end_stats(diff_rorp, dsrp, Inc._inc_file) if self.incpref.isdir() and (self.mirror_isdirectory or dsrp.isdir()): - self.write_stats_to_rp(Inc.get_inc_ext( - self.incpref.append("directory_statistics"), "data")) + self.write_stats_to_rp(Inc.get_inc( + self.incpref.append("directory_statistics"), + Time.curtime, "data")) def branch_process(self, subinstance): """Update statistics, and the has_changed flag if change in branch""" @@ -233,3 +237,37 @@ class IncrementITR(StatsITR): self.add_file_stats(subinstance) +class MirrorITR(StatsITR): + """Like IncrementITR, but only patch mirror directory, don't increment""" + def __init__(self, inc_rpath): + """Set inc_rpath, an rpath of the base of the inc tree""" + self.inc_rpath = inc_rpath + StatsITR.__init__(self, inc_rpath) + + def start_process(self, index, diff_rorp, mirror_dsrp): + """Initialize statistics, do actual writing to mirror""" + self.start_stats(mirror_dsrp) + if diff_rorp and not diff_rorp.isplaceholder(): + RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute() + + self.incpref = self.inc_rpath.new_index(index) + if mirror_dsrp.isdir() and not self.incpref.lstat(): + self.incpref.mkdir() # holds the statistics files + + self.diff_rorp, self.mirror_dsrp = diff_rorp, mirror_dsrp + + def end_process(self): + """Update statistics when leaving""" + try: diff_rorp, mirror_dsrp = self.diff_rorp, self.mirror_dsrp + except AttributeError: # Some error above prevented these being set + return + + self.end_stats(self.diff_rorp, self.mirror_dsrp) + if self.incpref.isdir(): + self.write_stats_to_rp(Inc.get_inc( + self.incpref.append("directory_statistics"), + Time.curtime, "data")) + + def branch_process(self, subinstance): + """Update statistics with subdirectory results""" + self.add_file_stats(subinstance) diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py index b38e792..3795bd1 100644 --- a/rdiff-backup/rdiff_backup/robust.py +++ b/rdiff-backup/rdiff_backup/robust.py @@ -315,23 +315,16 @@ class SaveState: _last_checkpoint_time = 0 # time in seconds of last checkpoint _checkpoint_rp = None # RPath of checkpoint data pickle - def init_filenames(cls, incrementing): - """Set rpaths of markers. Assume rbdir already set. - - If incrementing, then indicate increment operation, otherwise - indicate mirror. - - """ + def init_filenames(cls): + """Set rpaths of markers. Assume rbdir already set.""" if not Globals.isbackup_writer: - return Globals.backup_writer.SaveState.init_filenames(incrementing) + return Globals.backup_writer.SaveState.init_filenames() assert Globals.local_connection is Globals.rbdir.conn, \ (Globals.rbdir.conn, Globals.backup_writer) - if incrementing: cls._last_file_sym = Globals.rbdir.append( + cls._last_file_sym = Globals.rbdir.append( "last-file-incremented.%s.data" % Time.curtimestr) - else: cls._last_file_sym = Globals.rbdir.append( - "last-file-mirrored.%s.data" % Time.curtimestr) cls._checkpoint_rp = Globals.rbdir.append( "checkpoint-data.%s.data" % Time.curtimestr) cls._last_file_definitive_rp = Globals.rbdir.append( @@ -367,8 +360,7 @@ class SaveState: else: return RobustAction(lambda: None, cls.touch_last_file, lambda exc: None) - def checkpoint_inc_backup(cls, ITR, finalizer, last_file_rorp, - override = None): + def checkpoint(cls, ITR, finalizer, last_file_rorp, override = None): """Save states of tree reducer and finalizer during inc backup If override is true, checkpoint even if one isn't due. @@ -384,20 +376,6 @@ class SaveState: state_string), cls.record_last_file_action(last_file_rorp)]).execute() - def checkpoint_mirror(cls, finalizer, last_file_rorp, override = None): - """For a mirror, only finalizer and last_file should be saved""" - if not override and not cls.checkpoint_needed(): return - if not cls._checkpoint_rp: - Log("Warning, _checkpoint_rp not set yet", 2) - return - - cls._last_checkpoint_time = time.time() - Log("Writing checkpoint time %s" % cls._last_checkpoint_time, 7) - state_string = cPickle.dumps(finalizer) - Robust.chain([Robust.destructive_write_action(cls._checkpoint_rp, - state_string), - cls.record_last_file_action(last_file_rorp)]).execute() - def checkpoint_needed(cls): """Returns true if another checkpoint is called for""" return (time.time() > cls._last_checkpoint_time + @@ -522,9 +500,7 @@ class Resume: def unpickle_checkpoint(cls, checkpoint_rp): """Read data from checkpoint_rp and return unpickled data - Return value is pair finalizer state for a mirror checkpoint, - and (patch increment ITR, finalizer state) for increment - checkpoint. + Return value is pair (patch increment ITR, finalizer state). """ fp = checkpoint_rp.open("rb") diff --git a/rdiff-backup/rdiff_backup/statistics.py b/rdiff-backup/rdiff_backup/statistics.py index e0b19bf..40eb184 100644 --- a/rdiff-backup/rdiff_backup/statistics.py +++ b/rdiff-backup/rdiff_backup/statistics.py @@ -38,21 +38,21 @@ class StatsObj: """Return string printing out statistics""" timelist = [] if self.StartTime is not None: - timelist.append("StartTime %s (%s)" % + timelist.append("StartTime %s (%s)\n" % (self.StartTime, Time.timetopretty(self.StartTime))) if self.EndTime is not None: - timelist.append("EndTime %s (%s)" % + timelist.append("EndTime %s (%s)\n" % (self.EndTime, Time.timetopretty(self.EndTime))) if self.StartTime is not None and self.EndTime is not None: if self.ElapsedTime is None: self.ElapsedTime = self.EndTime - self.StartTime - timelist.append("ElapsedTime %s (%s)" % + timelist.append("ElapsedTime %s (%s)\n" % (self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) - filelist = ["%s %s" % (attr, self.get_stat(attr)) + filelist = ["%s %s\n" % (attr, self.get_stat(attr)) for attr in self.stat_file_attrs if self.get_stat(attr) is not None] - return "\n".join(timelist + filelist) + return "".join(timelist + filelist) def init_stats_from_string(self, s): """Initialize attributes from string, return self for convenience""" @@ -157,7 +157,7 @@ class StatsITR(IterTreeReducer, StatsObj): self.DeletedFiles += 1 self.DeletedFileSize += self.mirror_base_size self.IncrementFileSize += inc_rp and inc_rp.getsize() or 0 - else: assert None # One of before and after should exist + def add_file_stats(self, subinstance): """Add all file statistics from subinstance to current totals""" |