Added support for gzipped increments

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@23 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

6 files changed, 89 insertions, 37 deletions

diff --git a/rdiff-backup/rdiff_backup/header.py b/rdiff-backup/rdiff_backup/header.py
index 721e130..91ef2d2 100644
--- a/rdiff-backup/rdiff_backup/header.py
+++ b/rdiff-backup/rdiff_backup/header.py
@@ -1,18 +1,19 @@
 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.0 released March 21, 2002
-# Copyright (C) 2001 Ben Escoto <bescoto@stanford.edu>
+# Version 0.7.1 released March 25, 2002
+# Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).
 # Distributions of rdiff-backup usually include a copy of the GPL in a
 # file called COPYING.  The GPL is also available online at
 # http://www.gnu.org/copyleft/gpl.html.
 #
-# Please send mail to me or the mailing list if you find bugs or have
-# any suggestions.
+# See http://www.stanford.edu/~bescoto/rdiff-backup for more
+# information.  Please send mail to me or the mailing list if you find
+# bugs or have any suggestions.
 
 from __future__ import nested_scopes, generators
-import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile
+import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip
 
 
diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py
index 7603c21..2b366f2 100644
--- a/rdiff-backup/rdiff_backup/highlevel.py
+++ b/rdiff-backup/rdiff_backup/highlevel.py
@@ -238,7 +238,7 @@ class HLDestinationStruct:
 
 		try:
 			while 1:
-				try: dsrp = cls.check_skip_error(error_checked)
+				try: dsrp = cls.check_skip_error(error_checked, dsrp)
 				except StopIteration: break
 				if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer)
@@ -272,7 +272,7 @@ class HLDestinationStruct:
 
 		try:
 			while 1:
-				try: dsrp = cls.check_skip_error(error_checked)
+				try: dsrp = cls.check_skip_error(error_checked, dsrp)
 				except StopIteration: break
 				SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
@@ -281,7 +281,7 @@ class HLDestinationStruct:
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		SaveState.checkpoint_remove()
 
-	def check_skip_error(cls, thunk):
+	def check_skip_error(cls, thunk, dsrp):
 		"""Run thunk, catch certain errors skip files"""
 		try: return thunk()
 		except (IOError, OSError, SkipFileException), exp:
@@ -294,7 +294,8 @@ class HLDestinationStruct:
 							 26] # Requested by Campbell (see list) -
                                  # happens on some NT systems
 				 ))):
-				Log("Skipping file", 2)
+				Log("Skipping file because of error after %s" %
+					(dsrp and dsrp.index,), 2)
 				return None
 			else: raise
 
diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py
index a290d3c..1bbdd39 100644
--- a/rdiff-backup/rdiff_backup/increment.py
+++ b/rdiff-backup/rdiff_backup/increment.py
@@ -43,14 +43,27 @@ class Inc:
 		
 	def makesnapshot_action(mirror, incpref):
 		"""Copy mirror to incfile, since new is quite different"""
-		snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
-		return Robust.copy_with_attribs_action(mirror, snapshotrp)
+		if (mirror.isreg() and Globals.compression and
+			not Globals.no_compression_regexp.match(mirror.path)):
+			snapshotrp = Inc.get_inc_ext(incpref, "snapshot.gz")
+			return Robust.copy_with_attribs_action(mirror, snapshotrp, 1)
+		else:
+			snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
+			return Robust.copy_with_attribs_action(mirror, snapshotrp, None)
 
 	def makediff_action(new, mirror, incpref):
 		"""Make incfile which is a diff new -> mirror"""
-		diff = Inc.get_inc_ext(incpref, "diff")
-		return Robust.chain([Rdiff.write_delta_action(new, mirror, diff),
-							 Robust.copy_attribs_action(mirror, diff)])
+		if (Globals.compression and
+			not Globals.no_compression_regexp.match(mirror.path)):
+			diff = Inc.get_inc_ext(incpref, "diff.gz")
+			return Robust.chain([Rdiff.write_delta_action(new, mirror,
+														  diff, 1),
+								 Robust.copy_attribs_action(mirror, diff)])
+		else: 
+			diff = Inc.get_inc_ext(incpref, "diff")
+			return Robust.chain([Rdiff.write_delta_action(new, mirror,
+														  diff, None),
+								 Robust.copy_attribs_action(mirror, diff)])
 
 	def makedir_action(mirrordir, incpref):
 		"""Make file indicating directory mirrordir has changed"""
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 9c7a42a..0e0d62e 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -69,14 +69,19 @@ class Restore:
 		if inctype == "diff":
 			if not target.lstat():
 				raise RestoreError("Bad increment sequence at " + inc.path)
-			Rdiff.patch_action(target, inc).execute()
+			Rdiff.patch_action(target, inc,
+							   delta_compressed = inc.isinccompressed()
+							   ).execute()
 		elif inctype == "dir":
 			if not target.isdir():
 				if target.lstat():
 					raise RestoreError("File %s already exists" % target.path)
 				target.mkdir()
 		elif inctype == "missing": return
-		elif inctype == "snapshot": RPath.copy(inc, target)
+		elif inctype == "snapshot":
+			if inc.isinccompressed():
+				target.write_from_fileobj(inc.open("rb", compress = 1))
+			else: RPath.copy(inc, target)
 		else: raise RestoreError("Unknown inctype %s" % inctype)
 		RPath.copy_attribs(inc, target)
 
diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py
index 206e9d5..17942d3 100644
--- a/rdiff-backup/rdiff_backup/robust.py
+++ b/rdiff-backup/rdiff_backup/robust.py
@@ -142,13 +142,14 @@ class Robust:
 				tfl[0].rename(rpout)
 		return RobustAction(init, final, lambda e: tfl[0] and tfl[0].delete())
 
-	def copy_with_attribs_action(rorpin, rpout):
+	def copy_with_attribs_action(rorpin, rpout, compress = None):
 		"""Like copy_action but also copy attributes"""
 		tfl = [None] # Need mutable object that init and final can access
 		def init(): 
 			if not (rorpin.isdir() and rpout.isdir()): # already a dir
 				tfl[0] = TempFileManager.new(rpout)
-				if rorpin.isreg(): tfl[0].write_from_fileobj(rorpin.open("rb"))
+				if rorpin.isreg():
+					tfl[0].write_from_fileobj(rorpin.open("rb"), compress)
 				else: RPath.copy(rorpin, tfl[0])
 				if tfl[0].lstat(): # Some files, like sockets, won't be created
 					RPathStatic.copy_attribs(rorpin, tfl[0])
diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py
index 3f26ade..56be1d1 100644
--- a/rdiff-backup/rdiff_backup/rpath.py
+++ b/rdiff-backup/rdiff_backup/rpath.py
@@ -1,5 +1,5 @@
 execfile("connection.py")
-import os, stat, re, sys, shutil
+import os, stat, re, sys, shutil, gzip
 
 #######################################################################
 #
@@ -73,7 +73,7 @@ class RPathStatic:
 		try:
 			if rpout.conn is rpin.conn:
 				rpout.conn.shutil.copyfile(rpin.path, rpout.path)
-				rpout.data = {'type': rpin.data['type']}
+				rpout.setdata()
 				return
 		except AttributeError: pass
 		rpout.write_from_fileobj(rpin.open("rb"))
@@ -648,44 +648,75 @@ class RPath(RORPath):
 		"""Return similar RPath but with new index"""
 		return self.__class__(self.conn, self.base, index)
 
-	def open(self, mode):
-		"""Return open file.  Supports modes "w" and "r"."""
-		return self.conn.open(self.path, mode)
+	def open(self, mode, compress = None):
+		"""Return open file.  Supports modes "w" and "r".
+
+		If compress is true, data written/read will be gzip
+		compressed/decompressed on the fly.
+
+		"""
+		if compress: return self.conn.gzip.GzipFile(self.path, mode)
+		else: return self.conn.open(self.path, mode)
+
+	def write_from_fileobj(self, fp, compress = None):
+		"""Reads fp and writes to self.path.  Closes both when done
+
+		If compress is true, fp will be gzip compressed before being
+		written to self.
 
-	def write_from_fileobj(self, fp):
-		"""Reads fp and writes to self.path.  Closes both when done"""
+		"""
 		Log("Writing file object to " + self.path, 7)
 		assert not self.lstat(), "File %s already exists" % self.path
-		outfp = self.open("wb")
+		outfp = self.open("wb", compress = compress)
 		RPath.copyfileobj(fp, outfp)
 		if fp.close() or outfp.close():
 			raise RPathException("Error closing file")
 		self.setdata()
 
 	def isincfile(self):
-		"""Return true if path looks like an increment file"""
-		dotsplit = self.path.split(".")
-		if len(dotsplit) < 3: return None
-		timestring, ext = dotsplit[-2:]
+		"""Return true if path looks like an increment file
+
+		Also sets various inc information used by the *inc* functions.
+
+		"""
+		if self.index: dotsplit = self.index[-1].split(".")
+		else: dotsplit = self.base.split(".")
+		if dotsplit[-1] == "gz":
+			compressed = 1
+			if len(dotsplit) < 4: return None
+			timestring, ext = dotsplit[-3:-1]
+		else:
+			compressed = None
+			if len(dotsplit) < 3: return None
+			timestring, ext = dotsplit[-2:]
 		if Time.stringtotime(timestring) is None: return None
-		return (ext == "snapshot" or ext == "dir" or
-				ext == "missing" or ext == "diff")
+		if not (ext == "snapshot" or ext == "dir" or
+				ext == "missing" or ext == "diff"): return None
+		self.inc_timestr = timestring
+		self.inc_compressed = compressed
+		self.inc_type = ext
+		if compressed: self.inc_basestr = ".".join(dotsplit[:-3])
+		else: self.inc_basestr = ".".join(dotsplit[:-2])
+		return 1
+
+	def isinccompressed(self):
+		"""Return true if inc file is compressed"""
+		return self.inc_compressed
 
 	def getinctype(self):
 		"""Return type of an increment file"""
-		return self.path.split(".")[-1]
+		return self.inc_type
 
 	def getinctime(self):
 		"""Return timestring of an increment file"""
-		return self.path.split(".")[-2]
+		return self.inc_timestr
 	
 	def getincbase(self):
 		"""Return the base filename of an increment file in rp form"""
 		if self.index:
 			return self.__class__(self.conn, self.base, self.index[:-1] +
-							 ((".".join(self.index[-1].split(".")[:-2])),))
-		else: return self.__class__(self.conn,
-									".".join(self.base.split(".")[:-2]), ())
+								  (self.inc_basestr,))
+		else: return self.__class__(self.conn, self.inc_basestr)
 
 	def getincbase_str(self):
 		"""Return the base filename string of an increment file"""