Now choose blocksize based on size of file

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@353 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

3 files changed, 27 insertions, 13 deletions

diff --git a/rdiff-backup/rdiff_backup/Rdiff.py b/rdiff-backup/rdiff_backup/Rdiff.py
index ee688c1..0070ceb 100644
--- a/rdiff-backup/rdiff_backup/Rdiff.py
+++ b/rdiff-backup/rdiff_backup/Rdiff.py
@@ -23,10 +23,24 @@ import os, librsync
 import Globals, log, static, TempFile, rpath
 
 
-def get_signature(rp):
+def get_signature(rp, blocksize = None):
 	"""Take signature of rpin file and return in file object"""
-	log.Log("Getting signature of %s" % rp.get_indexpath(), 7)
-	return librsync.SigFile(rp.open("rb"))
+	if not blocksize: blocksize = find_blocksize(rp.getsize())
+	log.Log("Getting signature of %s with blocksize %s" %
+			(rp.get_indexpath(), blocksize), 7)
+	return librsync.SigFile(rp.open("rb"), blocksize)
+
+def find_blocksize(file_len):
+	"""Return a reasonable block size to use on files of length file_len
+
+	If the block size is too big, deltas will be bigger than is
+	necessary.  If the block size is too small, making deltas and
+	patching can take a really long time.
+
+	"""
+	if file_len < 1024000: return 512 # set minimum of 512 bytes
+	else: # Split file into about 2000 pieces, rounding to 512
+		return long((file_len/(2000*512))*512)
 
 def get_delta_sigfileobj(sig_fileobj, rp_new):
 	"""Like get_delta but signature is in a file object"""
@@ -43,8 +57,7 @@ def write_delta(basis, new, delta, compress = None):
 	"""Write rdiff delta which brings basis to new"""
 	log.Log("Writing delta %s from %s -> %s" %
 			(basis.path, new.path, delta.path), 7)
-	sigfile = librsync.SigFile(basis.open("rb"))
-	deltafile = librsync.DeltaFile(sigfile, new.open("rb"))
+	deltafile = librsync.DeltaFile(get_signature(basis), new.open("rb"))
 	delta.write_from_fileobj(deltafile, compress)
 
 def write_patched_fp(basis_fp, delta_fp, out_fp):
@@ -79,8 +92,8 @@ def patch_local(rp_basis, rp_delta, outrp = None, delta_compressed = None):
 def copy_local(rpin, rpout, rpnew = None):
 	"""Write rpnew == rpin using rpout as basis.  rpout and rpnew local"""
 	assert rpout.conn is Globals.local_connection
-	sigfile = librsync.SigFile(rpout.open("rb"))
-	deltafile = rpin.conn.librsync.DeltaFile(sigfile, rpin.open("rb"))
+	deltafile = rpin.conn.librsync.DeltaFile(get_signature(rpout),
+											 rpin.open("rb"))
 	patched_file = librsync.PatchedFile(rpout.open("rb"), deltafile)
 
 	if rpnew: rpnew.write_from_fileobj(patched_file)
diff --git a/rdiff-backup/rdiff_backup/_librsyncmodule.c b/rdiff-backup/rdiff_backup/_librsyncmodule.c
index 9b24d6a..7a78580 100644
--- a/rdiff-backup/rdiff_backup/_librsyncmodule.c
+++ b/rdiff-backup/rdiff_backup/_librsyncmodule.c
@@ -41,15 +41,16 @@ static PyObject*
 _librsync_new_sigmaker(PyObject* self, PyObject* args)
 {
   _librsync_SigMakerObject* sm;
+  long blocklen;
   
-  if (!PyArg_ParseTuple(args,":new_sigmaker"))
+  if (!PyArg_ParseTuple(args, "l:new_sigmaker", &blocklen))
 	return NULL;
 
   sm = PyObject_New(_librsync_SigMakerObject, &_librsync_SigMakerType);
   if (sm == NULL) return NULL;
   sm->x_attr = NULL;
 
-  sm->sig_job = rs_sig_begin((size_t)RS_DEFAULT_BLOCK_LEN,
+  sm->sig_job = rs_sig_begin((size_t)blocklen,
 							 (size_t)RS_DEFAULT_STRONG_LEN);
   return (PyObject*)sm;
 }
diff --git a/rdiff-backup/rdiff_backup/librsync.py b/rdiff-backup/rdiff_backup/librsync.py
index eb5e235..a8e37ca 100644
--- a/rdiff-backup/rdiff_backup/librsync.py
+++ b/rdiff-backup/rdiff_backup/librsync.py
@@ -108,7 +108,7 @@ class LikeFile:
 
 class SigFile(LikeFile):
 	"""File-like object which incrementally generates a librsync signature"""
-	def __init__(self, infile):
+	def __init__(self, infile, blocksize = _librsync.RS_DEFAULT_BLOCK_LEN):
 		"""SigFile initializer - takes basis file
 
 		basis file only needs to have read() and close() methods.  It
@@ -116,7 +116,7 @@ class SigFile(LikeFile):
 
 		"""
 		LikeFile.__init__(self, infile)
-		try: self.maker = _librsync.new_sigmaker()
+		try: self.maker = _librsync.new_sigmaker(blocksize)
 		except _librsync.librsyncError, e: raise librsyncError(str(e))
 
 class DeltaFile(LikeFile):
@@ -163,9 +163,9 @@ class SigGenerator:
 	module, not filelike object
 
 	"""
-	def __init__(self):
+	def __init__(self, blocksize = _librsync.RS_DEFAULT_BLOCK_LEN):
 		"""Return new signature instance"""
-		try: self.sig_maker = _librsync.new_sigmaker()
+		try: self.sig_maker = _librsync.new_sigmaker(blocksize)
 		except _librsync.librsyncError, e: raise librsyncError(str(e))
 		self.gotsig = None
 		self.buffer = ""