diff options
-rw-r--r-- | rdiff-backup/TODO | 4 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/Main.py | 23 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/Security.py | 5 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/compare.py | 52 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/hash.py | 14 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/restore.py | 28 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/robust.py | 2 | ||||
-rw-r--r-- | rdiff-backup/testing/commontest.py | 5 | ||||
-rw-r--r-- | rdiff-backup/testing/comparetest.py | 48 |
9 files changed, 150 insertions, 31 deletions
diff --git a/rdiff-backup/TODO b/rdiff-backup/TODO index 2be3440..2633ba6 100644 --- a/rdiff-backup/TODO +++ b/rdiff-backup/TODO @@ -1,4 +1,6 @@ ---verify switch for checking hashs, and hash check on restore +Do hash check on restore + +Don't copy metadata onto a hardlink twice For comparing, check source filesystem diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py index 0f8a060..62ab840 100644 --- a/rdiff-backup/rdiff_backup/Main.py +++ b/rdiff-backup/rdiff_backup/Main.py @@ -83,7 +83,8 @@ def parse_cmdlineoptions(arglist): "remove-older-than=", "restore-as-of=", "restrict=", "restrict-read-only=", "restrict-update-only=", "server", "ssh-no-compression", "terminal-verbosity=", "test-server", - "user-mapping-file=", "verbosity=", "version"]) + "user-mapping-file=", "verbosity=", "verify", + "verify-at-time=", "version"]) except getopt.error, e: commandline_error("Bad commandline options: %s" % str(e)) @@ -190,10 +191,12 @@ def parse_cmdlineoptions(arglist): elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg) elif opt == "--test-server": action = "test-server" elif opt == "--user-mapping-file": user_mapping_filename = arg + elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg) + elif opt == "--verify": action, restore_timestr = "verify", "now" + elif opt == "--verify-at-time": action, restore_timestr = "verify", arg elif opt == "-V" or opt == "--version": print "rdiff-backup " + Globals.version sys.exit(0) - elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg) else: Log.FatalError("Unknown option %s" % opt) def check_action(): @@ -202,7 +205,8 @@ def check_action(): arg_action_dict = {0: ['server'], 1: ['list-increments', 'list-increment-sizes', 'remove-older-than', 'list-at-time', - 'list-changed-since', 'check-destination-dir'], + 'list-changed-since', 'check-destination-dir', + 'verify'], 2: ['backup', 'restore', 'restore-as-of', 'compare', 'compare-hash', 'compare-full']} l = len(args) @@ -276,6 +280,7 @@ def take_action(rps): elif action == "restore": Restore(*rps) elif action == "restore-as-of": Restore(rps[0], rps[1], 1) elif action == "test-server": SetConnections.TestConnections() + elif action == "verify": Verify(rps[0]) else: raise AssertionError("Unknown action " + action) def cleanup(): @@ -722,6 +727,18 @@ def Compare(compare_type, src_rp, dest_rp, compare_time = None): compare_func = compare.Compare_full return_val = compare_func(src_rp, mirror_rp, inc_rp, compare_time) +def Verify(dest_rp, verify_time = None): + """Check the hashs of the regular files against mirror_metadata""" + global return_val + dest_rp = require_root_set(dest_rp, 1) + if not verify_time: + try: verify_time = Time.genstrtotime(restore_timestr) + except Time.TimeException, exc: Log.FatalError(str(exc)) + + mirror_rp = restore_root.new_index(restore_index) + inc_rp = Globals.rbdir.append_path("increments", restore_index) + return_val = dest_rp.conn.compare.Verify(mirror_rp, inc_rp, verify_time) + def CheckDest(dest_rp): """Check the destination directory, """ diff --git a/rdiff-backup/rdiff_backup/Security.py b/rdiff-backup/rdiff_backup/Security.py index 1e06d46..ba61c60 100644 --- a/rdiff-backup/rdiff_backup/Security.py +++ b/rdiff-backup/rdiff_backup/Security.py @@ -115,7 +115,7 @@ def set_security_level(action, cmdpairs): elif action in ["test-server", "list-increments", 'list-increment-sizes', "list-at-time", "list-changed-since", "calculate-average", "remove-older-than", "compare", - "compare-hash", "compare-full"]: + "compare-hash", "compare-full", "verify"]: sec_level = "minimal" rdir = tempfile.gettempdir() else: assert 0, "Unknown action %s" % action @@ -159,7 +159,8 @@ def set_allowed_requests(sec_level): "compare.DataSide.get_source_select", "compare.DataSide.compare_fast", "compare.DataSide.compare_hash", - "compare.DataSide.compare_full"]) + "compare.DataSide.compare_full", + "compare.Verify"]) if sec_level == "update-only" or sec_level == "all": l.extend(["log.Log.open_logfile_local", "log.Log.close_logfile_local", "log.ErrorLog.open", "log.ErrorLog.isopen", diff --git a/rdiff-backup/rdiff_backup/compare.py b/rdiff-backup/rdiff_backup/compare.py index 9ceff12..2395395 100644 --- a/rdiff-backup/rdiff_backup/compare.py +++ b/rdiff-backup/rdiff_backup/compare.py @@ -70,6 +70,35 @@ def Compare_full(src_rp, mirror_rp, inc_rp, compare_time): repo_side.close_rf_cache() return return_val +def Verify(mirror_rp, inc_rp, verify_time): + """Compute SHA1 sums of repository files and check against metadata""" + assert mirror_rp.conn is Globals.local_connection + repo_iter = RepoSide.init_and_get_iter(mirror_rp, inc_rp, verify_time) + base_index = RepoSide.mirror_base.index + + bad_files = 0 + for repo_rorp in repo_iter: + if not repo_rorp.isreg(): continue + if not repo_rorp.has_sha1(): + log.Log("Warning: Cannot find SHA1 digest for file %s,\n" + "perhaps because these feature was added in v1.1.1" + % (repo_rorp.get_indexpath(),), 2) + continue + fp = RepoSide.rf_cache.get_fp(base_index + repo_rorp.index) + computed_hash = hash.compute_sha1_fp(fp) + if computed_hash == repo_rorp.get_sha1(): + log.Log("Verified SHA1 digest of " + repo_rorp.get_indexpath(), 5) + else: + bad_files += 1 + log.Log("Warning: Computed SHA1 digest of %s\n %s\n" + "doesn't match recorded digest of\n %s\n" + "Your backup repository may be corrupted!" % + (repo_rorp.get_indexpath(), computed_hash, + repo_rorp.get_sha1()), 2) + RepoSide.close_rf_cache() + if not bad_files: log.Log("Every file verified successfully.", 3) + return bad_files + def print_reports(report_iter): """Given an iter of CompareReport objects, print them to screen""" assert not Globals.server @@ -80,7 +109,7 @@ def print_reports(report_iter): print "%s: %s" % (report.reason, indexpath) if not changed_files_found: - log.Log("No changes found. Directory matches archive data.", 2) + log.Log("No changes found. Directory matches archive data.", 3) return changed_files_found def get_basic_report(src_rp, repo_rorp, comp_data_func = None): @@ -112,6 +141,11 @@ def get_basic_report(src_rp, repo_rorp, comp_data_func = None): elif src_rp == repo_rorp: return None else: return CompareReport(index, "changed") +def log_success(src_rorp, mir_rorp = None): + """Log that src_rorp and mir_rorp compare successfully""" + path = src_rorp and src_rorp.get_indexpath() or mir_rorp.get_indexpath() + log.Log("Successful compare: %s" % (path,), 5) + class RepoSide(restore.MirrorStruct): """On the repository side, comparing is like restoring""" @@ -132,13 +166,14 @@ class RepoSide(restore.MirrorStruct): """ repo_iter = cls.init_and_get_iter(mirror_rp, inc_rp, compare_time) base_index = cls.mirror_base.index - for src_rp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter): - index = src_rp and src_rp.index or mir_rorp.index - if src_rp and mir_rorp: - if not src_rp.isreg() and src_rp == mir_rorp: + for src_rorp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter): + index = src_rorp and src_rorp.index or mir_rorp.index + if src_rorp and mir_rorp: + if not src_rorp.isreg() and src_rorp == mir_rorp: + log_success(src_rorp, mir_rorp) continue # They must be equal, nothing else to check - if (src_rp.isreg() and mir_rorp.isreg() and - src_rp.getsize() == mir_rorp.getsize()): + if (src_rorp.isreg() and mir_rorp.isreg() and + src_rorp.getsize() == mir_rorp.getsize()): mir_rorp.setfile(cls.rf_cache.get_fp(base_index + index)) mir_rorp.set_attached_filetype('snapshot') @@ -156,6 +191,7 @@ class DataSide(backup.SourceStruct): for src_rorp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter): report = get_basic_report(src_rorp, mir_rorp) if report: yield report + else: log_success(src_rorp, mir_rorp) def compare_hash(cls, repo_iter): """Like above, but also compare sha1 sums of any regular files""" @@ -174,6 +210,7 @@ class DataSide(backup.SourceStruct): for src_rp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter): report = get_basic_report(src_rp, mir_rorp, hashs_changed) if report: yield report + else: log_success(src_rp, mir_rorp) def compare_full(cls, src_root, repo_iter): """Given repo iter with full data attached, return report iter""" @@ -191,6 +228,7 @@ class DataSide(backup.SourceStruct): src_rp = src_root.new_index(repo_rorp.index) report = get_basic_report(src_rp, repo_rorp, data_changed) if report: yield report + else: log_success(repo_rorp) static.MakeClass(DataSide) diff --git a/rdiff-backup/rdiff_backup/hash.py b/rdiff-backup/rdiff_backup/hash.py index 3e7306f..0279f70 100644 --- a/rdiff-backup/rdiff_backup/hash.py +++ b/rdiff-backup/rdiff_backup/hash.py @@ -57,12 +57,16 @@ class Report: def compute_sha1(rp, compressed = 0): """Return the hex sha1 hash of given rpath""" assert rp.conn is Globals.local_connection # inefficient not to do locally - blocksize = Globals.blocksize - fp = FileWrapper(rp.open("r", compressed)) - while 1: - if not fp.read(blocksize): break - digest = fp.close().sha1_digest + digest = compute_sha1_fp(rp.open("r", compressed)) rp.set_sha1(digest) return digest +def compute_sha1_fp(fp, compressed = 0): + """Return hex sha1 hash of given file-like object""" + blocksize = Globals.blocksize + fw = FileWrapper(fp) + while 1: + if not fw.read(blocksize): break + return fw.close().sha1_digest + diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 26de579..8f0a5a5 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -459,6 +459,23 @@ class RestoreFile: def get_restore_fp(self): """Return file object of restored data""" + def get_fp(): + current_fp = self.get_first_fp() + for inc_diff in self.relevant_incs[1:]: + log.Log("Applying patch %s" % (inc_diff.get_indexpath(),), 7) + assert inc_diff.getinctype() == 'diff' + delta_fp = inc_diff.open("rb", inc_diff.isinccompressed()) + new_fp = tempfile.TemporaryFile() + Rdiff.write_patched_fp(current_fp, delta_fp, new_fp) + new_fp.seek(0) + current_fp = new_fp + return current_fp + + def error_handler(exc): + log.Log("Error reading %s, substituting empty file." % + (self.mirror_rp.path,), 2) + return cStringIO.StringIO('') + if not self.relevant_incs[-1].isreg(): log.Log("""Warning: Could not restore file %s! @@ -469,16 +486,7 @@ created. This error is probably caused by data loss in the rdiff-backup destination directory, or a bug in rdiff-backup""" % (self.mirror_rp.path, self.relevant_incs[-1].lstat()), 2) return cStringIO.StringIO('') - current_fp = self.get_first_fp() - for inc_diff in self.relevant_incs[1:]: - log.Log("Applying patch %s" % (inc_diff.get_indexpath(),), 7) - assert inc_diff.getinctype() == 'diff' - delta_fp = inc_diff.open("rb", inc_diff.isinccompressed()) - new_fp = tempfile.TemporaryFile() - Rdiff.write_patched_fp(current_fp, delta_fp, new_fp) - new_fp.seek(0) - current_fp = new_fp - return current_fp + return robust.check_common_error(error_handler, get_fp) def get_first_fp(self): """Return first file object from relevant inc list""" diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py index 8a8ca39..67221e3 100644 --- a/rdiff-backup/rdiff_backup/robust.py +++ b/rdiff-backup/rdiff_backup/robust.py @@ -48,7 +48,7 @@ def catch_error(exc): if isinstance(exc, exception_class): return 1 if (isinstance(exc, EnvironmentError) and # the invalid mode shows up in backups of /proc for some reason - (exc[0] == 'invalid mode: rb' or + (exc[0] in ('invalid mode: rb', 'Not a gzipped file') or errno.errorcode.has_key(exc[0]) and errno.errorcode[exc[0]] in ('EPERM', 'ENOENT', 'EACCES', 'EBUSY', 'EEXIST', 'ENOTDIR', diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index b205c0a..27c0406 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -60,7 +60,7 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir, """ if not source_local: src_dir = ("'cd test1; ../%s --server'::../%s" % (RBBin, src_dir)) - if not dest_local: + if dest_dir and not dest_local: dest_dir = ("'cd test2/tmp; ../../%s --server'::../../%s" % (RBBin, dest_dir)) @@ -68,7 +68,8 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir, if not (source_local and dest_local): cmdargs.append("--remote-schema %s") if current_time: cmdargs.append("--current-time %s" % current_time) - cmdargs.extend([src_dir, dest_dir]) + cmdargs.append(src_dir) + if dest_dir: cmdargs.append(dest_dir) cmdline = " ".join(cmdargs) print "Executing: ", cmdline ret_val = os.system(cmdline) diff --git a/rdiff-backup/testing/comparetest.py b/rdiff-backup/testing/comparetest.py index 1d0b67d..0459e49 100644 --- a/rdiff-backup/testing/comparetest.py +++ b/rdiff-backup/testing/comparetest.py @@ -98,5 +98,53 @@ class CompareTest(unittest.TestCase): """Test --compare-full of subdirectory, remote""" self.generic_selective_test(0, "--compare-full") + def verify(self, local): + """Used for the verify tests""" + def change_file(rp): + """Given rpath, open it, and change a byte in the middle""" + fp = rp.open("rb") + fp.seek(int(rp.getsize()/2)) + char = fp.read(1) + fp.close() + + fp = rp.open("wb") + fp.seek(int(rp.getsize()/2)) + if char == 'a': fp.write('b') + else: fp.write('a') + fp.close() + + def modify_diff(): + """Write to the stph_icons.h diff""" + l = [filename for filename in + os.listdir('testfiles/output/rdiff-backup-data/increments') + if filename.startswith('stph_icons.h')] + assert len(l) == 1, l + diff_rp = rpath.RPath(Globals.local_connection, + 'testfiles/output/rdiff-backup-data/increments/' + l[0]) + change_file(diff_rp) + + rdiff_backup(local, local, 'testfiles/output', None, + extra_options = "--verify") + rdiff_backup(local, local, 'testfiles/output', None, + extra_options = "--verify-at-time 10000") + modify_diff() + ret_val = rdiff_backup(local, local, 'testfiles/output', None, + extra_options = "--verify-at-time 10000", + check_return_val = 0) + assert ret_val, ret_val + change_file(rpath.RPath(Globals.local_connection, + 'testfiles/output/stph_icons.h')) + ret_val = rdiff_backup(local, local, 'testfiles/output', None, + extra_options = "--verify", check_return_val = 0) + assert ret_val, ret_val + + def testVerifyLocal(self): + """Test --verify of directory, local""" + self.verify(1) + + def testVerifyRemote(self): + """Test --verify remotely""" + self.verify(0) + if __name__ == "__main__": unittest.main() |