diff options
Diffstat (limited to 'rdiff-backup/rdiff_backup/statistics.py')
| -rw-r--r-- | rdiff-backup/rdiff_backup/statistics.py | 182 |
1 files changed, 164 insertions, 18 deletions
diff --git a/rdiff-backup/rdiff_backup/statistics.py b/rdiff-backup/rdiff_backup/statistics.py index 8269456..c18f34a 100644 --- a/rdiff-backup/rdiff_backup/statistics.py +++ b/rdiff-backup/rdiff_backup/statistics.py @@ -16,13 +16,29 @@ class StatsObj: 'DeletedFiles', 'DeletedFileSize', 'ChangedFiles', 'ChangedSourceSize', 'ChangedMirrorSize', - 'IncrementFileSize') + 'IncrementFiles', 'IncrementFileSize') stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime') - stat_attrs = stat_time_attrs + stat_file_attrs + stat_attrs = ('Filename',) + stat_time_attrs + stat_file_attrs + + # Below, the second value in each pair is true iff the value + # indicates a number of bytes + stat_file_pairs = (('SourceFiles', None), ('SourceFileSize', 1), + ('MirrorFiles', None), ('MirrorFileSize', 1), + ('NewFiles', None), ('NewFileSize', 1), + ('DeletedFiles', None), ('DeletedFileSize', 1), + ('ChangedFiles', None), + ('ChangedSourceSize', 1), ('ChangedMirrorSize', 1), + ('IncrementFiles', None), ('IncrementFileSize', 1)) # Set all stats to None, indicating info not available for attr in stat_attrs: locals()[attr] = None + # This is used in get_byte_summary_string below + byte_abbrev_list = ((1024*1024*1024*1024, "TB"), + (1024*1024*1024, "GB"), + (1024*1024, "MB"), + (1024, "KB")) + def get_stat(self, attribute): """Get a statistic""" try: return self.__dict__[attribute] @@ -34,33 +50,89 @@ class StatsObj: """Set attribute to given value""" self.__dict__[attr] = value + def get_stats_line(self, index): + """Return one line abbreviated version of full stats string""" + file_attrs = map(lambda attr: str(self.get_stat(attr)), + self.stat_file_attrs) + if not index: filename = "." + else: + # use repr to quote newlines in relative filename, then + # take of leading and trailing quote. + filename = repr(apply(os.path.join, index))[1:-1] + return " ".join([filename,] + file_attrs) + + def set_stats_from_line(self, line): + """Set statistics from given line""" + def error(): raise StatsException("Bad line '%s'" % line) + if line[-1] == "\n": line = line[:-1] + lineparts = line.split(" ") + if len(lineparts) < len(stat_file_attrs): error() + for attr, val_string in zip(stat_file_attrs, + lineparts[-len(stat_file_attrs):]): + try: val = long(val_string) + except ValueError: + try: val = float(val_string) + except ValueError: error() + self.set_stat(attr, val) + return self + def get_stats_string(self): - """Return string printing out statistics""" + """Return extended string printing out statistics""" + return self.get_timestats_string() + self.get_filestats_string() + + def get_timestats_string(self): + """Return portion of statistics string dealing with time""" timelist = [] if self.StartTime is not None: - timelist.append("StartTime %s (%s)\n" % + timelist.append("StartTime %.2f (%s)\n" % (self.StartTime, Time.timetopretty(self.StartTime))) if self.EndTime is not None: - timelist.append("EndTime %s (%s)\n" % + timelist.append("EndTime %.2f (%s)\n" % (self.EndTime, Time.timetopretty(self.EndTime))) - if self.StartTime is not None and self.EndTime is not None: + if self.ElapsedTime or (self.StartTime is not None and + self.EndTime is not None): if self.ElapsedTime is None: self.ElapsedTime = self.EndTime - self.StartTime - timelist.append("ElapsedTime %s (%s)\n" % + timelist.append("ElapsedTime %.2f (%s)\n" % (self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) + return "".join(timelist) + + def get_filestats_string(self): + """Return portion of statistics string about files and bytes""" + def fileline(stat_file_pair): + """Return zero or one line of the string""" + attr, in_bytes = stat_file_pair + val = self.get_stat(attr) + if val is None: return "" + if in_bytes: + return "%s %s (%s)\n" % (attr, val, + self.get_byte_summary_string(val)) + else: return "%s %s\n" % (attr, val) + + return "".join(map(fileline, self.stat_file_pairs)) - filelist = ["%s %s\n" % (attr, self.get_stat(attr)) - for attr in self.stat_file_attrs - if self.get_stat(attr) is not None] - return "".join(timelist + filelist) + def get_byte_summary_string(self, byte_count): + """Turn byte count into human readable string like "7.23GB" """ + for abbrev_bytes, abbrev_string in self.byte_abbrev_list: + if byte_count >= abbrev_bytes: + # Now get 3 significant figures + abbrev_count = float(byte_count)/abbrev_bytes + if abbrev_count >= 100: precision = 0 + elif abbrev_count >= 10: precision = 1 + else: precision = 2 + return "%%.%df %s" % (precision, abbrev_string) \ + % (abbrev_count,) + byte_count = round(byte_count) + if byte_count == 1: return "1 byte" + else: return "%d bytes" % (byte_count,) def get_stats_logstring(self, title): """Like get_stats_string, but add header and footer""" - header = "-------------[ %s ]-------------" % title + header = "--------------[ %s ]--------------" % title footer = "-" * len(header) return "%s\n%s%s\n" % (header, self.get_stats_string(), footer) - def init_stats_from_string(self, s): + def set_stats_from_string(self, s): """Initialize attributes from string, return self for convenience""" def error(line): raise StatsException("Bad line '%s'" % line) @@ -91,7 +163,7 @@ class StatsObj: def read_stats_from_rp(self, rp): """Set statistics from rpath, return self for convenience""" fp = rp.open("r") - self.init_stats_from_string(fp.read()) + self.set_stats_from_string(fp.read()) fp.close() return self @@ -162,22 +234,96 @@ class StatsITR(IterTreeReducer, StatsObj): self.ChangedFiles += 1 self.ChangedSourceSize += mirror_dsrp.getsize() self.ChangedMirrorSize += self.mirror_base_size - self.IncrementFileSize += inc_rp and inc_rp.getsize() or 0 + if inc_rp: + self.IncrementFiles += 1 + self.IncrementFileSize += inc_rp.getsize() else: # new file was created self.NewFiles += 1 self.NewFileSize += mirror_dsrp.getsize() - self.IncrementFileSize += inc_rp and inc_rp.getsize() or 0 + if inc_rp: + self.IncrementFiles += 1 + self.IncrementFileSize += inc_rp.getsize() else: if self.mirror_base_exists: # file was deleted from mirror self.MirrorFiles += 1 self.MirrorFileSize += self.mirror_base_size self.DeletedFiles += 1 self.DeletedFileSize += self.mirror_base_size - self.IncrementFileSize += inc_rp and inc_rp.getsize() or 0 - + if inc_rp: + self.IncrementFiles += 1 + self.IncrementFileSize += inc_rp.getsize() def add_file_stats(self, subinstance): """Add all file statistics from subinstance to current totals""" for attr in self.stat_file_attrs: self.set_stat(attr, self.get_stat(attr) + subinstance.get_stat(attr)) + + +class Stats: + """Misc statistics methods, pertaining to dir and session stat files""" + # This is the RPath of the directory statistics file, and the + # associated open file. It will hold a line of statistics for + # each directory that is backed up. + _dir_stats_rp = None + _dir_stats_fp = None + + # This goes at the beginning of the directory statistics file and + # explains the format. + _dir_stats_header = """# rdiff-backup directory statistics file +# +# Each line is in the following format: +# RelativeDirName %s +""" % " ".join(StatsObj.stat_file_attrs) + + def open_dir_stats_file(cls): + """Open directory statistics file, write header""" + assert not cls._dir_stats_fp, "Directory file already open" + + if Globals.compression: suffix = "data.gz" + else: suffix = "data" + cls._dir_stats_rp = Inc.get_inc(Globals.rbdir.append( + "directory_statistics"), Time.curtime, suffix) + + if cls._dir_stats_rp.lstat(): + Log("Warning, statistics file %s already exists, appending", 2) + cls._dir_stats_fp = cls._dir_stats_rp.open("ab", + Globals.compression) + else: cls._dir_stats_fp = \ + cls._dir_stats_rp.open("wb", Globals.compression) + cls._dir_stats_fp.write(cls._dir_stats_header) + + def write_dir_stats_line(cls, statobj, index): + """Write info from statobj about rpath to statistics file""" + cls._dir_stats_fp.write(statobj.get_stats_line(index) +"\n") + + def close_dir_stats_file(cls): + """Close directory statistics file if its open""" + if cls._dir_stats_fp: + cls._dir_stats_fp.close() + cls._dir_stats_fp = None + + def write_session_statistics(cls, statobj): + """Write session statistics into file, log""" + stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"), + Time.curtime, "data") + statobj.StartTime = Time.curtime + statobj.EndTime = time.time() + + # include hardlink data and dir stats in size of increments + if Globals.preserve_hardlinks and Hardlink.final_inc: + # include hardlink data in size of increments + statobj.IncrementFiles += 1 + statobj.IncrementFileSize += Hardlink.final_inc.getsize() + if cls._dir_stats_rp and cls._dir_stats_rp.lstat(): + statobj.IncrementFiles += 1 + statobj.IncrementFileSize += cls._dir_stats_rp.getsize() + + statobj.write_stats_to_rp(stat_inc) + if Globals.print_statistics: + message = statobj.get_stats_logstring("Session statistics") + Log.log_to_file(message) + Globals.client_conn.sys.stdout.write(message) + +MakeClass(Stats) + |