diff options
| -rw-r--r-- | rdiff-backup/CHANGELOG | 7 | ||||
| -rw-r--r-- | rdiff-backup/rdiff-backup.1 | 13 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/header.py | 2 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/highlevel.py | 17 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/statistics.py | 19 | ||||
| -rw-r--r-- | rdiff-backup/src/globals.py | 11 | ||||
| -rw-r--r-- | rdiff-backup/src/hardlink.py | 16 | ||||
| -rw-r--r-- | rdiff-backup/src/header.py | 2 | ||||
| -rw-r--r-- | rdiff-backup/src/highlevel.py | 17 | ||||
| -rwxr-xr-x | rdiff-backup/src/main.py | 36 | ||||
| -rw-r--r-- | rdiff-backup/src/statistics.py | 19 |
11 files changed, 138 insertions, 21 deletions
diff --git a/rdiff-backup/CHANGELOG b/rdiff-backup/CHANGELOG index 8a6ff30..35ac39d 100644 --- a/rdiff-backup/CHANGELOG +++ b/rdiff-backup/CHANGELOG @@ -1,3 +1,10 @@ +New in v0.7.6 (2002/??/??) +-------------------------- + +Improved statistics support, and added --print-statistics and +--calculate-average switches. + + New in v0.7.5 (2002/05/21) -------------------------- diff --git a/rdiff-backup/rdiff-backup.1 b/rdiff-backup/rdiff-backup.1 index c22f662..05deb81 100644 --- a/rdiff-backup/rdiff-backup.1 +++ b/rdiff-backup/rdiff-backup.1 @@ -12,6 +12,9 @@ rdiff-backup \- local/remote mirror and incremental backup .BI "| --remove-older-than " time_interval } .BI [[[ user@ ] host2.foo ]:: destination_directory ] +.B rdiff-backup --calculate-average +.I statfile1 statfile2 ... + .SH DESCRIPTION .B rdiff-backup is a script, written in @@ -45,6 +48,11 @@ on other options, see the section on .B -b, --backup-mode Force backup mode even if first argument appears to be an increment file. .TP +.B --calculate-average +Enter calculate average mode. The arguments should be a number of +statistics files. rdiff-backup will print the average of the listed +statistics files and exit. +.TP .B --change-source-perms If this option is set, rdiff-backup will try to change the mode of any unreadable files or unreadable/unexecutable directories in the source @@ -199,6 +207,11 @@ or .B --list-increments switches. .TP +.B --print-statistics +If set, summary statistics will be printed after a successful backup +If not set, this information will still be available from the +session_statistics.<time>.data file. +.TP .BI "--quoting-char " char Use the specified character for quoting characters specified to be escaped by the diff --git a/rdiff-backup/rdiff_backup/header.py b/rdiff-backup/rdiff_backup/header.py index 16ad128..9ca40d2 100644 --- a/rdiff-backup/rdiff_backup/header.py +++ b/rdiff-backup/rdiff_backup/header.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.5 released May 21, 2002 +# Version 0.7.5.1 released May 25, 2002 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu> # # This program is licensed under the GNU General Public License (GPL). diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index cc1335c..a9e5ce2 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -262,6 +262,7 @@ class HLDestinationStruct: cls.check_skip_error(finalizer.Finish, dsrp) except: cls.handle_last_error(dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() + cls.write_statistics(ITR) SaveState.checkpoint_remove() def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): @@ -291,6 +292,7 @@ class HLDestinationStruct: cls.check_skip_error(finalizer.Finish, dsrp) except: cls.handle_last_error(dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() + cls.write_statistics(ITR) SaveState.checkpoint_remove() def check_skip_error(cls, thunk, dsrp): @@ -321,4 +323,19 @@ class HLDestinationStruct: SaveState.touch_last_file_definitive() raise + def write_statistics(cls, ITR): + """Write session statistics to file, log""" + stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"), + Time.curtime, "data") + ITR.StartTime = Time.curtime + ITR.EndTime = time.time() + if Globals.preserve_hardlinks and Hardlink.final_inc: + # include hardlink data in size of increments + ITR.IncrementFileSize += Hardlink.final_inc.getsize() + ITR.write_stats_to_rp(stat_inc) + if Globals.print_statistics: + message = ITR.get_stats_logstring("Session statistics") + Log.log_to_file(message) + Globals.client_conn.sys.stdout.write(message) + MakeClass(HLDestinationStruct) diff --git a/rdiff-backup/rdiff_backup/statistics.py b/rdiff-backup/rdiff_backup/statistics.py index 40eb184..8269456 100644 --- a/rdiff-backup/rdiff_backup/statistics.py +++ b/rdiff-backup/rdiff_backup/statistics.py @@ -54,6 +54,12 @@ class StatsObj: if self.get_stat(attr) is not None] return "".join(timelist + filelist) + def get_stats_logstring(self, title): + """Like get_stats_string, but add header and footer""" + header = "-------------[ %s ]-------------" % title + footer = "-" * len(header) + return "%s\n%s%s\n" % (header, self.get_stats_string(), footer) + def init_stats_from_string(self, s): """Initialize attributes from string, return self for convenience""" def error(line): raise StatsException("Bad line '%s'" % line) @@ -64,7 +70,12 @@ class StatsObj: if len(line_parts) < 2: error(line) attr, value_string = line_parts[:2] if not attr in self.stat_attrs: error(line) - try: self.set_stat(attr, long(value_string)) + try: + try: val1 = long(value_string) + except ValueError: val1 = None + val2 = float(value_string) + if val1 == val2: self.set_stat(attr, val1) # use integer val + else: self.set_stat(attr, val2) # use float except ValueError: error(line) return self @@ -111,6 +122,12 @@ class StatsObj: self.get_stat(attr)/float(len(statobj_list))) return self + def get_statsobj_copy(self): + """Return new StatsObj object with same stats as self""" + s = StatObj() + for attr in self.stat_attrs: s.set_stat(attr, self.get_stat(attr)) + return s + class StatsITR(IterTreeReducer, StatsObj): """Keep track of per directory statistics diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py index a06eb61..2a042a8 100644 --- a/rdiff-backup/src/globals.py +++ b/rdiff-backup/src/globals.py @@ -8,7 +8,7 @@ import re, os class Globals: # The current version of rdiff-backup - version = "0.7.5" + version = "0.7.5.1" # If this is set, use this value in seconds as the current time # instead of reading it from the clock. @@ -79,6 +79,12 @@ class Globals: # Connection of the backup writer backup_writer = None + # True if this process is the client invoked by the user + isclient = None + + # Connection of the client + client_conn = None + # This list is used by the set function below. When a new # connection is created with init_connection, its Globals class # will match this one for all the variables mentioned in this @@ -144,6 +150,9 @@ class Globals: # Determines whether or not ssh will be run with the -C switch ssh_compression = 1 + # If true, print statistics after successful backup + print_statistics = None + # On the reader and writer connections, the following will be # replaced by the source and mirror Select objects respectively. select_source, select_mirror = None, None diff --git a/rdiff-backup/src/hardlink.py b/rdiff-backup/src/hardlink.py index 7f97c4a..431207d 100644 --- a/rdiff-backup/src/hardlink.py +++ b/rdiff-backup/src/hardlink.py @@ -151,6 +151,7 @@ class Hardlink: fp = tf.open("wb", compress) cPickle.dump(dict, fp) assert not fp.close() + tf.setdata() Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute() def get_linkrp(cls, data_rpath, time, prefix): @@ -173,14 +174,17 @@ class Hardlink: def final_writedata(cls): """Write final checkpoint data to rbdir after successful backup""" - if not cls._src_index_indicies: return + if not cls._src_index_indicies: # no hardlinks, so writing unnecessary + cls.final_inc = None + return Log("Writing hard link data", 6) if Globals.compression: - rp = Globals.rbdir.append("hardlink_data.%s.data.gz" % - Time.curtimestr) - else: rp = Globals.rbdir.append("hardlink_data.%s.data" % - Time.curtimestr) - cls.write_linkdict(rp, cls._src_index_indicies, Globals.compression) + cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" % + Time.curtimestr) + else: cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data" % + Time.curtimestr) + cls.write_linkdict(cls.final_inc, + cls._src_index_indicies, Globals.compression) def retrieve_final(cls, time): """Set source index dictionary from hardlink_data file if avail""" diff --git a/rdiff-backup/src/header.py b/rdiff-backup/src/header.py index 16ad128..9ca40d2 100644 --- a/rdiff-backup/src/header.py +++ b/rdiff-backup/src/header.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.5 released May 21, 2002 +# Version 0.7.5.1 released May 25, 2002 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu> # # This program is licensed under the GNU General Public License (GPL). diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py index cc1335c..a9e5ce2 100644 --- a/rdiff-backup/src/highlevel.py +++ b/rdiff-backup/src/highlevel.py @@ -262,6 +262,7 @@ class HLDestinationStruct: cls.check_skip_error(finalizer.Finish, dsrp) except: cls.handle_last_error(dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() + cls.write_statistics(ITR) SaveState.checkpoint_remove() def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): @@ -291,6 +292,7 @@ class HLDestinationStruct: cls.check_skip_error(finalizer.Finish, dsrp) except: cls.handle_last_error(dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() + cls.write_statistics(ITR) SaveState.checkpoint_remove() def check_skip_error(cls, thunk, dsrp): @@ -321,4 +323,19 @@ class HLDestinationStruct: SaveState.touch_last_file_definitive() raise + def write_statistics(cls, ITR): + """Write session statistics to file, log""" + stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"), + Time.curtime, "data") + ITR.StartTime = Time.curtime + ITR.EndTime = time.time() + if Globals.preserve_hardlinks and Hardlink.final_inc: + # include hardlink data in size of increments + ITR.IncrementFileSize += Hardlink.final_inc.getsize() + ITR.write_stats_to_rp(stat_inc) + if Globals.print_statistics: + message = ITR.get_stats_logstring("Session statistics") + Log.log_to_file(message) + Globals.client_conn.sys.stdout.write(message) + MakeClass(HLDestinationStruct) diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py index d0d129c..c222fdc 100755 --- a/rdiff-backup/src/main.py +++ b/rdiff-backup/src/main.py @@ -24,16 +24,17 @@ class Main: except IOError: Log.FatalError("Error opening file %s" % filename) try: optlist, self.args = getopt.getopt(arglist, "blmr:sv:V", - ["backup-mode", "change-source-perms", - "chars-to-quote=", "checkpoint-interval=", - "current-time=", "exclude=", "exclude-device-files", - "exclude-filelist=", "exclude-filelist-stdin", - "exclude-mirror=", "exclude-regexp=", "force", - "include=", "include-filelist=", - "include-filelist-stdin", "include-regexp=", - "list-increments", "mirror-only", "no-compression", - "no-compression-regexp=", "no-hard-links", "no-resume", - "parsable-output", "quoting-char=", "remote-cmd=", + ["backup-mode", "calculate-average", + "change-source-perms", "chars-to-quote=", + "checkpoint-interval=", "current-time=", "exclude=", + "exclude-device-files", "exclude-filelist=", + "exclude-filelist-stdin", "exclude-mirror=", + "exclude-regexp=", "force", "include=", + "include-filelist=", "include-filelist-stdin", + "include-regexp=", "list-increments", "mirror-only", + "no-compression", "no-compression-regexp=", + "no-hard-links", "no-resume", "parsable-output", + "print-statistics", "quoting-char=", "remote-cmd=", "remote-schema=", "remove-older-than=", "restore-as-of=", "resume", "resume-window=", "server", "ssh-no-compression", "terminal-verbosity=", @@ -44,6 +45,8 @@ class Main: for opt, arg in optlist: if opt == "-b" or opt == "--backup-mode": self.action = "backup" + elif opt == "--calculate-average": + self.action = "calculate-average" elif opt == "--change-source-perms": Globals.set('change_source_perms', 1) elif opt == "--chars-to-quote": @@ -89,6 +92,8 @@ class Main: self.restore_timestr = arg self.action = "restore-as-of" elif opt == "--parsable-output": Globals.set('parsable_output', 1) + elif opt == "--print-statistics": + Globals.set('print_statistics', 1) elif opt == "--quoting-char": Globals.set('quoting_char', arg) Globals.set('quoting_enabled', 1) @@ -160,6 +165,8 @@ class Main: os.umask(077) Time.setcurtime(Globals.current_time) FilenameMapping.set_init_quote_vals() + Globals.set("isclient", 1) + SetConnections.UpdateGlobal("client_conn", Globals.local_connection) # This is because I originally didn't think compiled regexps # could be pickled, and so must be compiled on remote side. @@ -177,6 +184,7 @@ class Main: elif self.action == "test-server": SetConnections.TestConnections() elif self.action == "list-increments": self.ListIncrements(rps[0]) elif self.action == "remove-older-than": self.RemoveOlderThan(rps[0]) + elif self.action == "calculate-average": self.CalculateAverage(rps) else: raise AssertionError("Unknown action " + self.action) def cleanup(self): @@ -444,6 +452,14 @@ Try restoring from an increment file (the filenames look like else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp) + def CalculateAverage(self, rps): + """Print out the average of the given statistics files""" + statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps) + average_stats = StatsObj().set_to_average(statobjs) + print average_stats.get_stats_logstring( + "Average of %d stat files" % len(rps)) + + def RemoveOlderThan(self, rootrp): """Remove all increment files older than a certain time""" datadir = rootrp.append("rdiff-backup-data") diff --git a/rdiff-backup/src/statistics.py b/rdiff-backup/src/statistics.py index 40eb184..8269456 100644 --- a/rdiff-backup/src/statistics.py +++ b/rdiff-backup/src/statistics.py @@ -54,6 +54,12 @@ class StatsObj: if self.get_stat(attr) is not None] return "".join(timelist + filelist) + def get_stats_logstring(self, title): + """Like get_stats_string, but add header and footer""" + header = "-------------[ %s ]-------------" % title + footer = "-" * len(header) + return "%s\n%s%s\n" % (header, self.get_stats_string(), footer) + def init_stats_from_string(self, s): """Initialize attributes from string, return self for convenience""" def error(line): raise StatsException("Bad line '%s'" % line) @@ -64,7 +70,12 @@ class StatsObj: if len(line_parts) < 2: error(line) attr, value_string = line_parts[:2] if not attr in self.stat_attrs: error(line) - try: self.set_stat(attr, long(value_string)) + try: + try: val1 = long(value_string) + except ValueError: val1 = None + val2 = float(value_string) + if val1 == val2: self.set_stat(attr, val1) # use integer val + else: self.set_stat(attr, val2) # use float except ValueError: error(line) return self @@ -111,6 +122,12 @@ class StatsObj: self.get_stat(attr)/float(len(statobj_list))) return self + def get_statsobj_copy(self): + """Return new StatsObj object with same stats as self""" + s = StatObj() + for attr in self.stat_attrs: s.set_stat(attr, self.get_stat(attr)) + return s + class StatsITR(IterTreeReducer, StatsObj): """Keep track of per directory statistics |