diff options
Diffstat (limited to 'rdiff-backup')
-rw-r--r-- | rdiff-backup/rdiff_backup/header.py | 11 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/highlevel.py | 9 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/increment.py | 23 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/restore.py | 9 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/robust.py | 5 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/rpath.py | 69 | ||||
-rw-r--r-- | rdiff-backup/src/globals.py | 26 | ||||
-rw-r--r-- | rdiff-backup/src/hardlink.py | 17 | ||||
-rw-r--r-- | rdiff-backup/src/header.py | 11 | ||||
-rw-r--r-- | rdiff-backup/src/highlevel.py | 9 | ||||
-rw-r--r-- | rdiff-backup/src/increment.py | 23 | ||||
-rwxr-xr-x | rdiff-backup/src/main.py | 14 | ||||
-rw-r--r-- | rdiff-backup/src/rdiff.py | 43 | ||||
-rw-r--r-- | rdiff-backup/src/restore.py | 9 | ||||
-rw-r--r-- | rdiff-backup/src/robust.py | 5 | ||||
-rw-r--r-- | rdiff-backup/src/rpath.py | 69 |
16 files changed, 252 insertions, 100 deletions
diff --git a/rdiff-backup/rdiff_backup/header.py b/rdiff-backup/rdiff_backup/header.py index 721e130..91ef2d2 100644 --- a/rdiff-backup/rdiff_backup/header.py +++ b/rdiff-backup/rdiff_backup/header.py @@ -1,18 +1,19 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.0 released March 21, 2002 -# Copyright (C) 2001 Ben Escoto <bescoto@stanford.edu> +# Version 0.7.1 released March 25, 2002 +# Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu> # # This program is licensed under the GNU General Public License (GPL). # Distributions of rdiff-backup usually include a copy of the GPL in a # file called COPYING. The GPL is also available online at # http://www.gnu.org/copyleft/gpl.html. # -# Please send mail to me or the mailing list if you find bugs or have -# any suggestions. +# See http://www.stanford.edu/~bescoto/rdiff-backup for more +# information. Please send mail to me or the mailing list if you find +# bugs or have any suggestions. from __future__ import nested_scopes, generators -import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile +import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index 7603c21..2b366f2 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -238,7 +238,7 @@ class HLDestinationStruct: try: while 1: - try: dsrp = cls.check_skip_error(error_checked) + try: dsrp = cls.check_skip_error(error_checked, dsrp) except StopIteration: break if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer) @@ -272,7 +272,7 @@ class HLDestinationStruct: try: while 1: - try: dsrp = cls.check_skip_error(error_checked) + try: dsrp = cls.check_skip_error(error_checked, dsrp) except StopIteration: break SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer, ITR) @@ -281,7 +281,7 @@ class HLDestinationStruct: if Globals.preserve_hardlinks: Hardlink.final_writedata() SaveState.checkpoint_remove() - def check_skip_error(cls, thunk): + def check_skip_error(cls, thunk, dsrp): """Run thunk, catch certain errors skip files""" try: return thunk() except (IOError, OSError, SkipFileException), exp: @@ -294,7 +294,8 @@ class HLDestinationStruct: 26] # Requested by Campbell (see list) - # happens on some NT systems ))): - Log("Skipping file", 2) + Log("Skipping file because of error after %s" % + (dsrp and dsrp.index,), 2) return None else: raise diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index a290d3c..1bbdd39 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -43,14 +43,27 @@ class Inc: def makesnapshot_action(mirror, incpref): """Copy mirror to incfile, since new is quite different""" - snapshotrp = Inc.get_inc_ext(incpref, "snapshot") - return Robust.copy_with_attribs_action(mirror, snapshotrp) + if (mirror.isreg() and Globals.compression and + not Globals.no_compression_regexp.match(mirror.path)): + snapshotrp = Inc.get_inc_ext(incpref, "snapshot.gz") + return Robust.copy_with_attribs_action(mirror, snapshotrp, 1) + else: + snapshotrp = Inc.get_inc_ext(incpref, "snapshot") + return Robust.copy_with_attribs_action(mirror, snapshotrp, None) def makediff_action(new, mirror, incpref): """Make incfile which is a diff new -> mirror""" - diff = Inc.get_inc_ext(incpref, "diff") - return Robust.chain([Rdiff.write_delta_action(new, mirror, diff), - Robust.copy_attribs_action(mirror, diff)]) + if (Globals.compression and + not Globals.no_compression_regexp.match(mirror.path)): + diff = Inc.get_inc_ext(incpref, "diff.gz") + return Robust.chain([Rdiff.write_delta_action(new, mirror, + diff, 1), + Robust.copy_attribs_action(mirror, diff)]) + else: + diff = Inc.get_inc_ext(incpref, "diff") + return Robust.chain([Rdiff.write_delta_action(new, mirror, + diff, None), + Robust.copy_attribs_action(mirror, diff)]) def makedir_action(mirrordir, incpref): """Make file indicating directory mirrordir has changed""" diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 9c7a42a..0e0d62e 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -69,14 +69,19 @@ class Restore: if inctype == "diff": if not target.lstat(): raise RestoreError("Bad increment sequence at " + inc.path) - Rdiff.patch_action(target, inc).execute() + Rdiff.patch_action(target, inc, + delta_compressed = inc.isinccompressed() + ).execute() elif inctype == "dir": if not target.isdir(): if target.lstat(): raise RestoreError("File %s already exists" % target.path) target.mkdir() elif inctype == "missing": return - elif inctype == "snapshot": RPath.copy(inc, target) + elif inctype == "snapshot": + if inc.isinccompressed(): + target.write_from_fileobj(inc.open("rb", compress = 1)) + else: RPath.copy(inc, target) else: raise RestoreError("Unknown inctype %s" % inctype) RPath.copy_attribs(inc, target) diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py index 206e9d5..17942d3 100644 --- a/rdiff-backup/rdiff_backup/robust.py +++ b/rdiff-backup/rdiff_backup/robust.py @@ -142,13 +142,14 @@ class Robust: tfl[0].rename(rpout) return RobustAction(init, final, lambda e: tfl[0] and tfl[0].delete()) - def copy_with_attribs_action(rorpin, rpout): + def copy_with_attribs_action(rorpin, rpout, compress = None): """Like copy_action but also copy attributes""" tfl = [None] # Need mutable object that init and final can access def init(): if not (rorpin.isdir() and rpout.isdir()): # already a dir tfl[0] = TempFileManager.new(rpout) - if rorpin.isreg(): tfl[0].write_from_fileobj(rorpin.open("rb")) + if rorpin.isreg(): + tfl[0].write_from_fileobj(rorpin.open("rb"), compress) else: RPath.copy(rorpin, tfl[0]) if tfl[0].lstat(): # Some files, like sockets, won't be created RPathStatic.copy_attribs(rorpin, tfl[0]) diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 3f26ade..56be1d1 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -1,5 +1,5 @@ execfile("connection.py") -import os, stat, re, sys, shutil +import os, stat, re, sys, shutil, gzip ####################################################################### # @@ -73,7 +73,7 @@ class RPathStatic: try: if rpout.conn is rpin.conn: rpout.conn.shutil.copyfile(rpin.path, rpout.path) - rpout.data = {'type': rpin.data['type']} + rpout.setdata() return except AttributeError: pass rpout.write_from_fileobj(rpin.open("rb")) @@ -648,44 +648,75 @@ class RPath(RORPath): """Return similar RPath but with new index""" return self.__class__(self.conn, self.base, index) - def open(self, mode): - """Return open file. Supports modes "w" and "r".""" - return self.conn.open(self.path, mode) + def open(self, mode, compress = None): + """Return open file. Supports modes "w" and "r". + + If compress is true, data written/read will be gzip + compressed/decompressed on the fly. + + """ + if compress: return self.conn.gzip.GzipFile(self.path, mode) + else: return self.conn.open(self.path, mode) + + def write_from_fileobj(self, fp, compress = None): + """Reads fp and writes to self.path. Closes both when done + + If compress is true, fp will be gzip compressed before being + written to self. - def write_from_fileobj(self, fp): - """Reads fp and writes to self.path. Closes both when done""" + """ Log("Writing file object to " + self.path, 7) assert not self.lstat(), "File %s already exists" % self.path - outfp = self.open("wb") + outfp = self.open("wb", compress = compress) RPath.copyfileobj(fp, outfp) if fp.close() or outfp.close(): raise RPathException("Error closing file") self.setdata() def isincfile(self): - """Return true if path looks like an increment file""" - dotsplit = self.path.split(".") - if len(dotsplit) < 3: return None - timestring, ext = dotsplit[-2:] + """Return true if path looks like an increment file + + Also sets various inc information used by the *inc* functions. + + """ + if self.index: dotsplit = self.index[-1].split(".") + else: dotsplit = self.base.split(".") + if dotsplit[-1] == "gz": + compressed = 1 + if len(dotsplit) < 4: return None + timestring, ext = dotsplit[-3:-1] + else: + compressed = None + if len(dotsplit) < 3: return None + timestring, ext = dotsplit[-2:] if Time.stringtotime(timestring) is None: return None - return (ext == "snapshot" or ext == "dir" or - ext == "missing" or ext == "diff") + if not (ext == "snapshot" or ext == "dir" or + ext == "missing" or ext == "diff"): return None + self.inc_timestr = timestring + self.inc_compressed = compressed + self.inc_type = ext + if compressed: self.inc_basestr = ".".join(dotsplit[:-3]) + else: self.inc_basestr = ".".join(dotsplit[:-2]) + return 1 + + def isinccompressed(self): + """Return true if inc file is compressed""" + return self.inc_compressed def getinctype(self): """Return type of an increment file""" - return self.path.split(".")[-1] + return self.inc_type def getinctime(self): """Return timestring of an increment file""" - return self.path.split(".")[-2] + return self.inc_timestr def getincbase(self): """Return the base filename of an increment file in rp form""" if self.index: return self.__class__(self.conn, self.base, self.index[:-1] + - ((".".join(self.index[-1].split(".")[:-2])),)) - else: return self.__class__(self.conn, - ".".join(self.base.split(".")[:-2]), ()) + (self.inc_basestr,)) + else: return self.__class__(self.conn, self.inc_basestr) def getincbase_str(self): """Return the base filename string of an increment file""" diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py index 5147299..195883b 100644 --- a/rdiff-backup/src/globals.py +++ b/rdiff-backup/src/globals.py @@ -8,7 +8,7 @@ import re, os class Globals: # The current version of rdiff-backup - version = "0.7.0" + version = "0.7.1" # If this is set, use this value in seconds as the current time # instead of reading it from the clock. @@ -132,6 +132,18 @@ class Globals: # hardlink information regardless. preserve_hardlinks = 1 + # If this is false, then rdiff-backup will not compress any + # increments. Default is to compress based on regexp below. + compression = 1 + + # Increments based on files whose names match this + # case-insensitive regular expression won't be compressed (applies + # to .snapshots and .diffs). The second below is the compiled + # version of the first. + no_compression_regexp_string = ".*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|" \ + "mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$" + no_compression_regexp = None + def get(cls, name): """Return the value of something in this class""" return cls.__dict__[name] @@ -181,3 +193,15 @@ class Globals: if mirror: Globals.exclude_mirror_regexps.append(compiled) else: Globals.exclude_regexps.append(compiled) add_regexp_local = classmethod(add_regexp_local) + + def postset_regexp(cls, name, re_string, flags = None): + """Compile re_string on all existing connections, set to name""" + for conn in Globals.connections: + conn.Globals.postset_regexp_local(name, re_string, flags) + postset_regexp = classmethod(postset_regexp) + + def postset_regexp_local(cls, name, re_string, flags): + """Set name to compiled re_string locally""" + if flags: cls.__dict__[name] = re.compile(re_string, flags) + else: cls.__dict__[name] = re.compile(re_string) + postset_regexp_local = classmethod(postset_regexp_local) diff --git a/rdiff-backup/src/hardlink.py b/rdiff-backup/src/hardlink.py index ee5248e..deecd68 100644 --- a/rdiff-backup/src/hardlink.py +++ b/rdiff-backup/src/hardlink.py @@ -129,7 +129,7 @@ class Hardlink: cls.get_indicies(src_rorp, 1)[0]) dest_rpath.hardlink(dest_link_rpath.path) - def write_linkdict(cls, rpath, dict): + def write_linkdict(cls, rpath, dict, compress = None): """Write link data to the rbdata dir It is stored as the a big pickled dictionary dated to match @@ -140,7 +140,7 @@ class Hardlink: rpath.conn is Globals.local_connection) tf = TempFileManager.new(rpath) def init(): - fp = tf.open("wb") + fp = tf.open("wb", compress) cPickle.dump(dict, fp) assert not fp.close() Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute() @@ -158,18 +158,21 @@ class Hardlink: """Return index dictionary written by write_linkdata at time""" rp = cls.get_linkrp(data_rpath, time, prefix) if not rp: return None - fp = rp.open("rb") + fp = rp.open("rb", rp.isinccompressed()) index_dict = cPickle.load(fp) assert not fp.close() return index_dict def final_writedata(cls): """Write final checkpoint data to rbdir after successful backup""" - if cls._src_index_indicies: - Log("Writing hard link data", 6) - rp = Globals.rbdir.append("hardlink_data.%s.snapshot" % + if not cls._src_index_indicies: return + Log("Writing hard link data", 6) + if Globals.compression: + rp = Globals.rbdir.append("hardlink_data.%s.snapshot.gz" % Time.curtimestr) - cls.write_linkdict(rp, cls._src_index_indicies) + else: rp = Globals.rbdir.append("hardlink_data.%s.snapshot" % + Time.curtimestr) + cls.write_linkdict(rp, cls._src_index_indicies, Globals.compression) def retrieve_final(cls, time): """Set source index dictionary from hardlink_data file if avail""" diff --git a/rdiff-backup/src/header.py b/rdiff-backup/src/header.py index 721e130..91ef2d2 100644 --- a/rdiff-backup/src/header.py +++ b/rdiff-backup/src/header.py @@ -1,18 +1,19 @@ #!/usr/bin/env python # # rdiff-backup -- Mirror files while keeping incremental changes -# Version 0.7.0 released March 21, 2002 -# Copyright (C) 2001 Ben Escoto <bescoto@stanford.edu> +# Version 0.7.1 released March 25, 2002 +# Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu> # # This program is licensed under the GNU General Public License (GPL). # Distributions of rdiff-backup usually include a copy of the GPL in a # file called COPYING. The GPL is also available online at # http://www.gnu.org/copyleft/gpl.html. # -# Please send mail to me or the mailing list if you find bugs or have -# any suggestions. +# See http://www.stanford.edu/~bescoto/rdiff-backup for more +# information. Please send mail to me or the mailing list if you find +# bugs or have any suggestions. from __future__ import nested_scopes, generators -import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile +import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py index 7603c21..2b366f2 100644 --- a/rdiff-backup/src/highlevel.py +++ b/rdiff-backup/src/highlevel.py @@ -238,7 +238,7 @@ class HLDestinationStruct: try: while 1: - try: dsrp = cls.check_skip_error(error_checked) + try: dsrp = cls.check_skip_error(error_checked, dsrp) except StopIteration: break if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer) @@ -272,7 +272,7 @@ class HLDestinationStruct: try: while 1: - try: dsrp = cls.check_skip_error(error_checked) + try: dsrp = cls.check_skip_error(error_checked, dsrp) except StopIteration: break SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer, ITR) @@ -281,7 +281,7 @@ class HLDestinationStruct: if Globals.preserve_hardlinks: Hardlink.final_writedata() SaveState.checkpoint_remove() - def check_skip_error(cls, thunk): + def check_skip_error(cls, thunk, dsrp): """Run thunk, catch certain errors skip files""" try: return thunk() except (IOError, OSError, SkipFileException), exp: @@ -294,7 +294,8 @@ class HLDestinationStruct: 26] # Requested by Campbell (see list) - # happens on some NT systems ))): - Log("Skipping file", 2) + Log("Skipping file because of error after %s" % + (dsrp and dsrp.index,), 2) return None else: raise diff --git a/rdiff-backup/src/increment.py b/rdiff-backup/src/increment.py index a290d3c..1bbdd39 100644 --- a/rdiff-backup/src/increment.py +++ b/rdiff-backup/src/increment.py @@ -43,14 +43,27 @@ class Inc: def makesnapshot_action(mirror, incpref): """Copy mirror to incfile, since new is quite different""" - snapshotrp = Inc.get_inc_ext(incpref, "snapshot") - return Robust.copy_with_attribs_action(mirror, snapshotrp) + if (mirror.isreg() and Globals.compression and + not Globals.no_compression_regexp.match(mirror.path)): + snapshotrp = Inc.get_inc_ext(incpref, "snapshot.gz") + return Robust.copy_with_attribs_action(mirror, snapshotrp, 1) + else: + snapshotrp = Inc.get_inc_ext(incpref, "snapshot") + return Robust.copy_with_attribs_action(mirror, snapshotrp, None) def makediff_action(new, mirror, incpref): """Make incfile which is a diff new -> mirror""" - diff = Inc.get_inc_ext(incpref, "diff") - return Robust.chain([Rdiff.write_delta_action(new, mirror, diff), - Robust.copy_attribs_action(mirror, diff)]) + if (Globals.compression and + not Globals.no_compression_regexp.match(mirror.path)): + diff = Inc.get_inc_ext(incpref, "diff.gz") + return Robust.chain([Rdiff.write_delta_action(new, mirror, + diff, 1), + Robust.copy_attribs_action(mirror, diff)]) + else: + diff = Inc.get_inc_ext(incpref, "diff") + return Robust.chain([Rdiff.write_delta_action(new, mirror, + diff, None), + Robust.copy_attribs_action(mirror, diff)]) def makedir_action(mirrordir, incpref): """Make file indicating directory mirrordir has changed""" diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py index ba8302f..3e00a7e 100755 --- a/rdiff-backup/src/main.py +++ b/rdiff-backup/src/main.py @@ -27,7 +27,8 @@ class Main: "include-from-stdin", "terminal-verbosity=", "exclude-device-files", "resume", "no-resume", "resume-window=", "windows-time-format", - "checkpoint-interval=", "no-hard-links", "current-time="]) + "checkpoint-interval=", "no-hard-links", "current-time=", + "no-compression", "no-compression-regexp="]) except getopt.error: self.commandline_error("Error parsing commandline options") @@ -45,11 +46,14 @@ class Main: elif opt == "--exclude-mirror": self.exclude_mirror_regstrs.append(arg) elif opt == "--force": self.force = 1 - elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == "--include-from-stdin": Globals.include_from_stdin = 1 elif opt == "-l" or opt == "--list-increments": self.action = "list-increments" elif opt == "-m" or opt == "--mirror-only": self.action = "mirror" + elif opt == "--no-compression": Globals.set("compression", None) + elif opt == "--no-compression-regexp": + Globals.set("no_compression_regexp_string", arg) + elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == '--no-resume': Globals.resume = 0 elif opt == "--remote-cmd": self.remote_cmd = arg elif opt == "--remote-schema": self.remote_schema = arg @@ -103,7 +107,7 @@ class Main: sys.exit(1) def misc_setup(self, rps): - """Set default change ownership flag, umask, excludes""" + """Set default change ownership flag, umask, regular expressions""" if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or (len(rps) < 2 and os.getuid() == 0)): # Allow change_ownership if destination connection is root @@ -116,6 +120,8 @@ class Main: Globals.add_regexp(regex_string, None) for regex_string in self.exclude_mirror_regstrs: Globals.add_regexp(regex_string, 1) + Globals.postset_regexp('no_compression_regexp', + Globals.no_compression_regexp_string, re.I) def take_action(self, rps): """Do whatever self.action says""" @@ -250,7 +256,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2) """Warning: duplicate current_mirror files found. Perhaps something went wrong during your last backup? Using """ + mirrorrps[-1].path, 2) - timestr = self.datadir.append(mirrorrps[-1].path).getinctime() + timestr = mirrorrps[-1].getinctime() return Time.stringtotime(timestr) def backup_touch_curmirror(self, rpin, rpout): diff --git a/rdiff-backup/src/rdiff.py b/rdiff-backup/src/rdiff.py index 56ebeb5..3ad80b3 100644 --- a/rdiff-backup/src/rdiff.py +++ b/rdiff-backup/src/rdiff.py @@ -36,37 +36,52 @@ class Rdiff: return rp_new.conn.RdiffPopen(['rdiff', 'delta', rp_signature.path, rp_new.path]) - def write_delta_action(basis, new, delta): - """Return action writing delta which brings basis to new""" + def write_delta_action(basis, new, delta, compress = None): + """Return action writing delta which brings basis to new + + If compress is true, the output of rdiff will be gzipped + before written to delta. + + """ sig_tf = TempFileManager.new(new, None) delta_tf = TempFileManager.new(delta) def init(): Log("Writing delta %s from %s -> %s" % (basis.path, new.path, delta.path), 7) sig_tf.write_from_fileobj(Rdiff.get_signature(basis)) - delta_tf.write_from_fileobj(Rdiff.get_delta(sig_tf, new)) + delta_tf.write_from_fileobj(Rdiff.get_delta(sig_tf, new), compress) sig_tf.delete() return Robust.make_tf_robustaction(init, (sig_tf, delta_tf), (None, delta)) - def write_delta(basis, new, delta): + def write_delta(basis, new, delta, compress = None): """Write rdiff delta which brings basis to new""" - Rdiff.write_delta_action(basis, new, delta).execute() + Rdiff.write_delta_action(basis, new, delta, compress).execute() - def patch_action(rp_basis, rp_delta, rp_out = None, out_tf = None): + def patch_action(rp_basis, rp_delta, rp_out = None, + out_tf = None, delta_compressed = None): """Return RobustAction which patches rp_basis with rp_delta If rp_out is None, put output in rp_basis. Will use TempFile - out_tf it is specified. + out_tf it is specified. If delta_compressed is true, the + delta file will be decompressed before processing with rdiff. """ if not rp_out: rp_out = rp_basis else: assert rp_out.conn is rp_basis.conn - if not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath) - and rp_basis.conn is rp_delta.conn): - return Rdiff.patch_fileobj_action(rp_basis, rp_delta.open('rb'), - rp_out, out_tf) - + if (delta_compressed or + not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath) + and rp_basis.conn is rp_delta.conn)): + if delta_compressed: + assert isinstance(rp_delta, RPath) + return Rdiff.patch_fileobj_action(rp_basis, + rp_delta.open('rb', 1), + rp_out, out_tf) + else: return Rdiff.patch_fileobj_action(rp_basis, + rp_delta.open('rb'), + rp_out, out_tf) + + # Files are uncompressed on same connection, run rdiff if out_tf is None: out_tf = TempFileManager.new(rp_out) def init(): Log("Patching %s using %s to %s via %s" % @@ -79,8 +94,8 @@ class Rdiff: RdiffException("Error running %s" % cmdlist) return Robust.make_tf_robustaction(init, (out_tf,), (rp_out,)) - def patch_fileobj_action(rp_basis, delta_fileobj, - rp_out = None, out_tf = None): + def patch_fileobj_action(rp_basis, delta_fileobj, rp_out = None, + out_tf = None, delta_compressed = None): """Like patch_action but diff is given in fileobj form Nest a writing of a tempfile with the actual patching to diff --git a/rdiff-backup/src/restore.py b/rdiff-backup/src/restore.py index 9c7a42a..0e0d62e 100644 --- a/rdiff-backup/src/restore.py +++ b/rdiff-backup/src/restore.py @@ -69,14 +69,19 @@ class Restore: if inctype == "diff": if not target.lstat(): raise RestoreError("Bad increment sequence at " + inc.path) - Rdiff.patch_action(target, inc).execute() + Rdiff.patch_action(target, inc, + delta_compressed = inc.isinccompressed() + ).execute() elif inctype == "dir": if not target.isdir(): if target.lstat(): raise RestoreError("File %s already exists" % target.path) target.mkdir() elif inctype == "missing": return - elif inctype == "snapshot": RPath.copy(inc, target) + elif inctype == "snapshot": + if inc.isinccompressed(): + target.write_from_fileobj(inc.open("rb", compress = 1)) + else: RPath.copy(inc, target) else: raise RestoreError("Unknown inctype %s" % inctype) RPath.copy_attribs(inc, target) diff --git a/rdiff-backup/src/robust.py b/rdiff-backup/src/robust.py index 206e9d5..17942d3 100644 --- a/rdiff-backup/src/robust.py +++ b/rdiff-backup/src/robust.py @@ -142,13 +142,14 @@ class Robust: tfl[0].rename(rpout) return RobustAction(init, final, lambda e: tfl[0] and tfl[0].delete()) - def copy_with_attribs_action(rorpin, rpout): + def copy_with_attribs_action(rorpin, rpout, compress = None): """Like copy_action but also copy attributes""" tfl = [None] # Need mutable object that init and final can access def init(): if not (rorpin.isdir() and rpout.isdir()): # already a dir tfl[0] = TempFileManager.new(rpout) - if rorpin.isreg(): tfl[0].write_from_fileobj(rorpin.open("rb")) + if rorpin.isreg(): + tfl[0].write_from_fileobj(rorpin.open("rb"), compress) else: RPath.copy(rorpin, tfl[0]) if tfl[0].lstat(): # Some files, like sockets, won't be created RPathStatic.copy_attribs(rorpin, tfl[0]) diff --git a/rdiff-backup/src/rpath.py b/rdiff-backup/src/rpath.py index 3f26ade..56be1d1 100644 --- a/rdiff-backup/src/rpath.py +++ b/rdiff-backup/src/rpath.py @@ -1,5 +1,5 @@ execfile("connection.py") -import os, stat, re, sys, shutil +import os, stat, re, sys, shutil, gzip ####################################################################### # @@ -73,7 +73,7 @@ class RPathStatic: try: if rpout.conn is rpin.conn: rpout.conn.shutil.copyfile(rpin.path, rpout.path) - rpout.data = {'type': rpin.data['type']} + rpout.setdata() return except AttributeError: pass rpout.write_from_fileobj(rpin.open("rb")) @@ -648,44 +648,75 @@ class RPath(RORPath): """Return similar RPath but with new index""" return self.__class__(self.conn, self.base, index) - def open(self, mode): - """Return open file. Supports modes "w" and "r".""" - return self.conn.open(self.path, mode) + def open(self, mode, compress = None): + """Return open file. Supports modes "w" and "r". + + If compress is true, data written/read will be gzip + compressed/decompressed on the fly. + + """ + if compress: return self.conn.gzip.GzipFile(self.path, mode) + else: return self.conn.open(self.path, mode) + + def write_from_fileobj(self, fp, compress = None): + """Reads fp and writes to self.path. Closes both when done + + If compress is true, fp will be gzip compressed before being + written to self. - def write_from_fileobj(self, fp): - """Reads fp and writes to self.path. Closes both when done""" + """ Log("Writing file object to " + self.path, 7) assert not self.lstat(), "File %s already exists" % self.path - outfp = self.open("wb") + outfp = self.open("wb", compress = compress) RPath.copyfileobj(fp, outfp) if fp.close() or outfp.close(): raise RPathException("Error closing file") self.setdata() def isincfile(self): - """Return true if path looks like an increment file""" - dotsplit = self.path.split(".") - if len(dotsplit) < 3: return None - timestring, ext = dotsplit[-2:] + """Return true if path looks like an increment file + + Also sets various inc information used by the *inc* functions. + + """ + if self.index: dotsplit = self.index[-1].split(".") + else: dotsplit = self.base.split(".") + if dotsplit[-1] == "gz": + compressed = 1 + if len(dotsplit) < 4: return None + timestring, ext = dotsplit[-3:-1] + else: + compressed = None + if len(dotsplit) < 3: return None + timestring, ext = dotsplit[-2:] if Time.stringtotime(timestring) is None: return None - return (ext == "snapshot" or ext == "dir" or - ext == "missing" or ext == "diff") + if not (ext == "snapshot" or ext == "dir" or + ext == "missing" or ext == "diff"): return None + self.inc_timestr = timestring + self.inc_compressed = compressed + self.inc_type = ext + if compressed: self.inc_basestr = ".".join(dotsplit[:-3]) + else: self.inc_basestr = ".".join(dotsplit[:-2]) + return 1 + + def isinccompressed(self): + """Return true if inc file is compressed""" + return self.inc_compressed def getinctype(self): """Return type of an increment file""" - return self.path.split(".")[-1] + return self.inc_type def getinctime(self): """Return timestring of an increment file""" - return self.path.split(".")[-2] + return self.inc_timestr def getincbase(self): """Return the base filename of an increment file in rp form""" if self.index: return self.__class__(self.conn, self.base, self.index[:-1] + - ((".".join(self.index[-1].split(".")[:-2])),)) - else: return self.__class__(self.conn, - ".".join(self.base.split(".")[:-2]), ()) + (self.inc_basestr,)) + else: return self.__class__(self.conn, self.inc_basestr) def getincbase_str(self): """Return the base filename string of an increment file""" |