From 38f302883e8e08457f9e4fd93d4ecdb6dff77de2 Mon Sep 17 00:00:00 2001 From: ben Date: Fri, 31 May 2002 08:24:24 +0000 Subject: Added resuming bug fixes git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@114 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/connection.py | 6 ++++-- rdiff-backup/rdiff_backup/header.py | 4 ++-- rdiff-backup/rdiff_backup/highlevel.py | 17 ++++++++++++----- rdiff-backup/rdiff_backup/robust.py | 26 ++++++++++++++++++++++---- rdiff-backup/rdiff_backup/selection.py | 2 +- rdiff-backup/rdiff_backup/statistics.py | 10 +++++++++- rdiff-backup/src/connection.py | 6 ++++-- rdiff-backup/src/globals.py | 11 ++++++++++- rdiff-backup/src/header.py | 4 ++-- rdiff-backup/src/highlevel.py | 17 ++++++++++++----- rdiff-backup/src/main.py | 2 ++ rdiff-backup/src/robust.py | 26 ++++++++++++++++++++++---- rdiff-backup/src/selection.py | 2 +- rdiff-backup/src/statistics.py | 10 +++++++++- 14 files changed, 112 insertions(+), 31 deletions(-) diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 54cf527..a92cab8 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -338,8 +338,10 @@ class PipeConnection(LowLevelPipeConnection): def extract_exception(self): """Return active exception""" - Log("Sending back exception: \n" + - "".join(traceback.format_tb(sys.exc_info()[2])), 2) + if Log.verbosity >= 5 or Log.term_verbosity >= 5: + Log("Sending back exception %s of type %s: \n%s" % + (sys.exc_info()[1], sys.exc_info()[0], + "".join(traceback.format_tb(sys.exc_info()[2]))), 5) return sys.exc_info()[1] def Server(self): diff --git a/rdiff-backup/rdiff_backup/header.py b/rdiff-backup/rdiff_backup/header.py index 4f27dc7..1ffdfdb 100644 --- a/rdiff-backup/rdiff_backup/header.py +++ b/rdiff-backup/rdiff_backup/header.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.5.4 released May 29, 2002 +# Version 0.7.6 released May 31, 2002 # Copyright (C) 2001, 2002 Ben Escoto # # This program is licensed under the GNU General Public License (GPL). @@ -14,6 +14,6 @@ # bugs or have any suggestions. from __future__ import nested_scopes, generators -import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno +import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno, signal diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index ae3fa53..c0b6b61 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -207,13 +207,18 @@ class HLDestinationStruct: else: iitr = IncrementITR(inc_rpath) iitr.override_changed() + Globals.ITR = iitr + iitr.Errors = 0 return iitr def get_MirrorITR(cls, inc_rpath): """Return MirrorITR, starting from state if available""" if cls._session_info and cls._session_info.ITR: return cls._session_info.ITR - else: return MirrorITR(inc_rpath) + ITR = MirrorITR(inc_rpath) + Globals.ITR = ITR + ITR.Errors = 0 + return ITR def patch_and_finalize(cls, dest_rpath, diffs): """Apply diffs and finalize""" @@ -244,7 +249,7 @@ class HLDestinationStruct: collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath) Stats.open_dir_stats_file() - dsrp = None + dsrp, finished_dsrp = None, None try: for indexed_tuple in collated: @@ -255,9 +260,10 @@ class HLDestinationStruct: ITR(dsrp.index, diff_rorp, dsrp) finalizer(dsrp.index, dsrp) SaveState.checkpoint(ITR, finalizer, dsrp) + finished_dsrp = dsrp ITR.Finish() finalizer.Finish() - except: cls.handle_last_error(dsrp, finalizer, ITR) + except: cls.handle_last_error(finished_dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() Stats.close_dir_stats_file() @@ -269,7 +275,7 @@ class HLDestinationStruct: collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath) Stats.open_dir_stats_file() - dsrp = None + dsrp, finished_dsrp = None, None try: for indexed_tuple in collated: @@ -281,9 +287,10 @@ class HLDestinationStruct: ITR(index, diff_rorp, dsrp) finalizer(index, dsrp) SaveState.checkpoint(ITR, finalizer, dsrp) + finished_dsrp = dsrp ITR.Finish() finalizer.Finish() - except: cls.handle_last_error(dsrp, finalizer, ITR) + except: cls.handle_last_error(finished_dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() Stats.close_dir_stats_file() diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py index d7040dc..fe1a316 100644 --- a/rdiff-backup/rdiff_backup/robust.py +++ b/rdiff-backup/rdiff_backup/robust.py @@ -1,4 +1,4 @@ -import tempfile, errno +import tempfile, errno, signal execfile("hardlink.py") ####################################################################### @@ -79,7 +79,7 @@ class RobustAction: class Robust: - """Contains various file operations made safer using tempfiles""" + """Contains various methods designed to make things safer""" null_action = RobustAction(None, None, None) def chain(*robust_action_list): """Return chain tying together a number of robust actions @@ -122,9 +122,9 @@ class Robust: final_vals.append(ra.final_func(init_val)) return final_vals def error(exc, ran_init, init_val): - for ra, init_val in zip(ras_with_started_inits, init_return_vals): + for ra, init_val in zip(ras_with_started_inits, init_vals): ra.error_handler(exc, 1, init_val) - for ra in ras_with_started_inits[len(init_return_vals):]: + for ra in ras_with_started_inits[len(init_vals):]: ra.error_handler(exc, None, None) return RobustAction(init, final, error) @@ -251,6 +251,10 @@ class Robust: 'ENOTDIR', 'ENAMETOOLONG', 'EINTR', 'ENOTEMPTY', 'EIO', 'ETXTBSY', 'ESRCH', 'EINVAL'])): Log.exception() + conn = Globals.backup_writer + if conn is not None: # increment error count + ITR_exists = conn.Globals.is_not_None('ITR') + if ITR_exists: conn.Globals.ITR.increment_stat('Errors') if error_handler: return error_handler(exc, *args) else: Log.exception(1, 2) @@ -265,9 +269,23 @@ class Robust: dir_listing.sort() return dir_listing + def signal_handler(signum, frame): + """This is called when signal signum is caught""" + raise SignalException(signum) + + def install_signal_handlers(): + """Install signal handlers on current connection""" + for signum in [signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM]: + signal.signal(signum, Robust.signal_handler) + MakeStatic(Robust) +class SignalException(Exception): + """SignalException(signum) means signal signum has been received""" + pass + + class TracebackArchive: """Save last 10 caught exceptions, so they can be printed if fatal""" _traceback_strings = [] diff --git a/rdiff-backup/rdiff_backup/selection.py b/rdiff-backup/rdiff_backup/selection.py index c26dfab..8b87af8 100644 --- a/rdiff-backup/rdiff_backup/selection.py +++ b/rdiff-backup/rdiff_backup/selection.py @@ -275,7 +275,7 @@ probably isn't what you meant.""" % if include is None: continue # later line may match return include - selection_function.exclude = something_excluded + selection_function.exclude = something_excluded or inc_default == 0 selection_function.name = "Filelist: " + filelist_name return selection_function diff --git a/rdiff-backup/rdiff_backup/statistics.py b/rdiff-backup/rdiff_backup/statistics.py index 7dff7a3..7f99c31 100644 --- a/rdiff-backup/rdiff_backup/statistics.py +++ b/rdiff-backup/rdiff_backup/statistics.py @@ -17,8 +17,10 @@ class StatsObj: 'ChangedFiles', 'ChangedSourceSize', 'ChangedMirrorSize', 'IncrementFiles', 'IncrementFileSize') + stat_misc_attrs = ('Errors',) stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime') - stat_attrs = ('Filename',) + stat_time_attrs + stat_file_attrs + stat_attrs = (('Filename',) + stat_time_attrs + + stat_misc_attrs + stat_file_attrs) # Below, the second value in each pair is true iff the value # indicates a number of bytes @@ -50,6 +52,10 @@ class StatsObj: """Set attribute to given value""" self.__dict__[attr] = value + def increment_stat(self, attr): + """Add 1 to value of attribute""" + self.__dict__[attr] = self.get_stat(attr) + 1 + def get_stats_line(self, index): """Return one line abbreviated version of full stats string""" file_attrs = map(lambda attr: str(self.get_stat(attr)), @@ -95,6 +101,8 @@ class StatsObj: self.ElapsedTime = self.EndTime - self.StartTime timelist.append("ElapsedTime %.2f (%s)\n" % (self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) + if self.Errors is not None: + timelist.append("Errors %d\n" % self.Errors) return "".join(timelist) def get_filestats_string(self): diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py index 54cf527..a92cab8 100644 --- a/rdiff-backup/src/connection.py +++ b/rdiff-backup/src/connection.py @@ -338,8 +338,10 @@ class PipeConnection(LowLevelPipeConnection): def extract_exception(self): """Return active exception""" - Log("Sending back exception: \n" + - "".join(traceback.format_tb(sys.exc_info()[2])), 2) + if Log.verbosity >= 5 or Log.term_verbosity >= 5: + Log("Sending back exception %s of type %s: \n%s" % + (sys.exc_info()[1], sys.exc_info()[0], + "".join(traceback.format_tb(sys.exc_info()[2]))), 5) return sys.exc_info()[1] def Server(self): diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py index a2797a4..9d81ad9 100644 --- a/rdiff-backup/src/globals.py +++ b/rdiff-backup/src/globals.py @@ -8,7 +8,7 @@ import re, os class Globals: # The current version of rdiff-backup - version = "0.7.5.4" + version = "0.7.6" # If this is set, use this value in seconds as the current time # instead of reading it from the clock. @@ -157,11 +157,20 @@ class Globals: # replaced by the source and mirror Select objects respectively. select_source, select_mirror = None, None + # On the backup writer connection, holds the main incrementing + # function. Access is provided to increment error counts. + ITR = None + def get(cls, name): """Return the value of something in this class""" return cls.__dict__[name] get = classmethod(get) + def is_not_None(cls, name): + """Returns true if value is not None""" + return cls.__dict__[name] is not None + is_not_None = classmethod(is_not_None) + def set(cls, name, val): """Set the value of something in this class diff --git a/rdiff-backup/src/header.py b/rdiff-backup/src/header.py index 4f27dc7..1ffdfdb 100644 --- a/rdiff-backup/src/header.py +++ b/rdiff-backup/src/header.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.5.4 released May 29, 2002 +# Version 0.7.6 released May 31, 2002 # Copyright (C) 2001, 2002 Ben Escoto # # This program is licensed under the GNU General Public License (GPL). @@ -14,6 +14,6 @@ # bugs or have any suggestions. from __future__ import nested_scopes, generators -import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno +import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno, signal diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py index ae3fa53..c0b6b61 100644 --- a/rdiff-backup/src/highlevel.py +++ b/rdiff-backup/src/highlevel.py @@ -207,13 +207,18 @@ class HLDestinationStruct: else: iitr = IncrementITR(inc_rpath) iitr.override_changed() + Globals.ITR = iitr + iitr.Errors = 0 return iitr def get_MirrorITR(cls, inc_rpath): """Return MirrorITR, starting from state if available""" if cls._session_info and cls._session_info.ITR: return cls._session_info.ITR - else: return MirrorITR(inc_rpath) + ITR = MirrorITR(inc_rpath) + Globals.ITR = ITR + ITR.Errors = 0 + return ITR def patch_and_finalize(cls, dest_rpath, diffs): """Apply diffs and finalize""" @@ -244,7 +249,7 @@ class HLDestinationStruct: collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath) Stats.open_dir_stats_file() - dsrp = None + dsrp, finished_dsrp = None, None try: for indexed_tuple in collated: @@ -255,9 +260,10 @@ class HLDestinationStruct: ITR(dsrp.index, diff_rorp, dsrp) finalizer(dsrp.index, dsrp) SaveState.checkpoint(ITR, finalizer, dsrp) + finished_dsrp = dsrp ITR.Finish() finalizer.Finish() - except: cls.handle_last_error(dsrp, finalizer, ITR) + except: cls.handle_last_error(finished_dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() Stats.close_dir_stats_file() @@ -269,7 +275,7 @@ class HLDestinationStruct: collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath) Stats.open_dir_stats_file() - dsrp = None + dsrp, finished_dsrp = None, None try: for indexed_tuple in collated: @@ -281,9 +287,10 @@ class HLDestinationStruct: ITR(index, diff_rorp, dsrp) finalizer(index, dsrp) SaveState.checkpoint(ITR, finalizer, dsrp) + finished_dsrp = dsrp ITR.Finish() finalizer.Finish() - except: cls.handle_last_error(dsrp, finalizer, ITR) + except: cls.handle_last_error(finished_dsrp, finalizer, ITR) if Globals.preserve_hardlinks: Hardlink.final_writedata() Stats.close_dir_stats_file() diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py index 6b1db51..6359450 100755 --- a/rdiff-backup/src/main.py +++ b/rdiff-backup/src/main.py @@ -174,6 +174,8 @@ class Main: Globals.postset_regexp('no_compression_regexp', Globals.no_compression_regexp_string) + for conn in Globals.connections: Robust.install_signal_handlers() + def take_action(self, rps): """Do whatever self.action says""" if self.action == "server": diff --git a/rdiff-backup/src/robust.py b/rdiff-backup/src/robust.py index d7040dc..fe1a316 100644 --- a/rdiff-backup/src/robust.py +++ b/rdiff-backup/src/robust.py @@ -1,4 +1,4 @@ -import tempfile, errno +import tempfile, errno, signal execfile("hardlink.py") ####################################################################### @@ -79,7 +79,7 @@ class RobustAction: class Robust: - """Contains various file operations made safer using tempfiles""" + """Contains various methods designed to make things safer""" null_action = RobustAction(None, None, None) def chain(*robust_action_list): """Return chain tying together a number of robust actions @@ -122,9 +122,9 @@ class Robust: final_vals.append(ra.final_func(init_val)) return final_vals def error(exc, ran_init, init_val): - for ra, init_val in zip(ras_with_started_inits, init_return_vals): + for ra, init_val in zip(ras_with_started_inits, init_vals): ra.error_handler(exc, 1, init_val) - for ra in ras_with_started_inits[len(init_return_vals):]: + for ra in ras_with_started_inits[len(init_vals):]: ra.error_handler(exc, None, None) return RobustAction(init, final, error) @@ -251,6 +251,10 @@ class Robust: 'ENOTDIR', 'ENAMETOOLONG', 'EINTR', 'ENOTEMPTY', 'EIO', 'ETXTBSY', 'ESRCH', 'EINVAL'])): Log.exception() + conn = Globals.backup_writer + if conn is not None: # increment error count + ITR_exists = conn.Globals.is_not_None('ITR') + if ITR_exists: conn.Globals.ITR.increment_stat('Errors') if error_handler: return error_handler(exc, *args) else: Log.exception(1, 2) @@ -265,9 +269,23 @@ class Robust: dir_listing.sort() return dir_listing + def signal_handler(signum, frame): + """This is called when signal signum is caught""" + raise SignalException(signum) + + def install_signal_handlers(): + """Install signal handlers on current connection""" + for signum in [signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM]: + signal.signal(signum, Robust.signal_handler) + MakeStatic(Robust) +class SignalException(Exception): + """SignalException(signum) means signal signum has been received""" + pass + + class TracebackArchive: """Save last 10 caught exceptions, so they can be printed if fatal""" _traceback_strings = [] diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py index c26dfab..8b87af8 100644 --- a/rdiff-backup/src/selection.py +++ b/rdiff-backup/src/selection.py @@ -275,7 +275,7 @@ probably isn't what you meant.""" % if include is None: continue # later line may match return include - selection_function.exclude = something_excluded + selection_function.exclude = something_excluded or inc_default == 0 selection_function.name = "Filelist: " + filelist_name return selection_function diff --git a/rdiff-backup/src/statistics.py b/rdiff-backup/src/statistics.py index 7dff7a3..7f99c31 100644 --- a/rdiff-backup/src/statistics.py +++ b/rdiff-backup/src/statistics.py @@ -17,8 +17,10 @@ class StatsObj: 'ChangedFiles', 'ChangedSourceSize', 'ChangedMirrorSize', 'IncrementFiles', 'IncrementFileSize') + stat_misc_attrs = ('Errors',) stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime') - stat_attrs = ('Filename',) + stat_time_attrs + stat_file_attrs + stat_attrs = (('Filename',) + stat_time_attrs + + stat_misc_attrs + stat_file_attrs) # Below, the second value in each pair is true iff the value # indicates a number of bytes @@ -50,6 +52,10 @@ class StatsObj: """Set attribute to given value""" self.__dict__[attr] = value + def increment_stat(self, attr): + """Add 1 to value of attribute""" + self.__dict__[attr] = self.get_stat(attr) + 1 + def get_stats_line(self, index): """Return one line abbreviated version of full stats string""" file_attrs = map(lambda attr: str(self.get_stat(attr)), @@ -95,6 +101,8 @@ class StatsObj: self.ElapsedTime = self.EndTime - self.StartTime timelist.append("ElapsedTime %.2f (%s)\n" % (self.ElapsedTime, Time.inttopretty(self.ElapsedTime))) + if self.Errors is not None: + timelist.append("Errors %d\n" % self.Errors) return "".join(timelist) def get_filestats_string(self): -- cgit v1.2.1