summaryrefslogtreecommitdiff
path: root/rdiff-backup
diff options
context:
space:
mode:
Diffstat (limited to 'rdiff-backup')
-rw-r--r--rdiff-backup/rdiff_backup/Rdiff.py27
-rw-r--r--rdiff-backup/rdiff_backup/_librsyncmodule.c5
-rw-r--r--rdiff-backup/rdiff_backup/librsync.py8
-rw-r--r--rdiff-backup/testing/librsynctest.py52
-rw-r--r--rdiff-backup/testing/rdifftest.py4
5 files changed, 63 insertions, 33 deletions
diff --git a/rdiff-backup/rdiff_backup/Rdiff.py b/rdiff-backup/rdiff_backup/Rdiff.py
index ee688c1..0070ceb 100644
--- a/rdiff-backup/rdiff_backup/Rdiff.py
+++ b/rdiff-backup/rdiff_backup/Rdiff.py
@@ -23,10 +23,24 @@ import os, librsync
import Globals, log, static, TempFile, rpath
-def get_signature(rp):
+def get_signature(rp, blocksize = None):
"""Take signature of rpin file and return in file object"""
- log.Log("Getting signature of %s" % rp.get_indexpath(), 7)
- return librsync.SigFile(rp.open("rb"))
+ if not blocksize: blocksize = find_blocksize(rp.getsize())
+ log.Log("Getting signature of %s with blocksize %s" %
+ (rp.get_indexpath(), blocksize), 7)
+ return librsync.SigFile(rp.open("rb"), blocksize)
+
+def find_blocksize(file_len):
+ """Return a reasonable block size to use on files of length file_len
+
+ If the block size is too big, deltas will be bigger than is
+ necessary. If the block size is too small, making deltas and
+ patching can take a really long time.
+
+ """
+ if file_len < 1024000: return 512 # set minimum of 512 bytes
+ else: # Split file into about 2000 pieces, rounding to 512
+ return long((file_len/(2000*512))*512)
def get_delta_sigfileobj(sig_fileobj, rp_new):
"""Like get_delta but signature is in a file object"""
@@ -43,8 +57,7 @@ def write_delta(basis, new, delta, compress = None):
"""Write rdiff delta which brings basis to new"""
log.Log("Writing delta %s from %s -> %s" %
(basis.path, new.path, delta.path), 7)
- sigfile = librsync.SigFile(basis.open("rb"))
- deltafile = librsync.DeltaFile(sigfile, new.open("rb"))
+ deltafile = librsync.DeltaFile(get_signature(basis), new.open("rb"))
delta.write_from_fileobj(deltafile, compress)
def write_patched_fp(basis_fp, delta_fp, out_fp):
@@ -79,8 +92,8 @@ def patch_local(rp_basis, rp_delta, outrp = None, delta_compressed = None):
def copy_local(rpin, rpout, rpnew = None):
"""Write rpnew == rpin using rpout as basis. rpout and rpnew local"""
assert rpout.conn is Globals.local_connection
- sigfile = librsync.SigFile(rpout.open("rb"))
- deltafile = rpin.conn.librsync.DeltaFile(sigfile, rpin.open("rb"))
+ deltafile = rpin.conn.librsync.DeltaFile(get_signature(rpout),
+ rpin.open("rb"))
patched_file = librsync.PatchedFile(rpout.open("rb"), deltafile)
if rpnew: rpnew.write_from_fileobj(patched_file)
diff --git a/rdiff-backup/rdiff_backup/_librsyncmodule.c b/rdiff-backup/rdiff_backup/_librsyncmodule.c
index 9b24d6a..7a78580 100644
--- a/rdiff-backup/rdiff_backup/_librsyncmodule.c
+++ b/rdiff-backup/rdiff_backup/_librsyncmodule.c
@@ -41,15 +41,16 @@ static PyObject*
_librsync_new_sigmaker(PyObject* self, PyObject* args)
{
_librsync_SigMakerObject* sm;
+ long blocklen;
- if (!PyArg_ParseTuple(args,":new_sigmaker"))
+ if (!PyArg_ParseTuple(args, "l:new_sigmaker", &blocklen))
return NULL;
sm = PyObject_New(_librsync_SigMakerObject, &_librsync_SigMakerType);
if (sm == NULL) return NULL;
sm->x_attr = NULL;
- sm->sig_job = rs_sig_begin((size_t)RS_DEFAULT_BLOCK_LEN,
+ sm->sig_job = rs_sig_begin((size_t)blocklen,
(size_t)RS_DEFAULT_STRONG_LEN);
return (PyObject*)sm;
}
diff --git a/rdiff-backup/rdiff_backup/librsync.py b/rdiff-backup/rdiff_backup/librsync.py
index eb5e235..a8e37ca 100644
--- a/rdiff-backup/rdiff_backup/librsync.py
+++ b/rdiff-backup/rdiff_backup/librsync.py
@@ -108,7 +108,7 @@ class LikeFile:
class SigFile(LikeFile):
"""File-like object which incrementally generates a librsync signature"""
- def __init__(self, infile):
+ def __init__(self, infile, blocksize = _librsync.RS_DEFAULT_BLOCK_LEN):
"""SigFile initializer - takes basis file
basis file only needs to have read() and close() methods. It
@@ -116,7 +116,7 @@ class SigFile(LikeFile):
"""
LikeFile.__init__(self, infile)
- try: self.maker = _librsync.new_sigmaker()
+ try: self.maker = _librsync.new_sigmaker(blocksize)
except _librsync.librsyncError, e: raise librsyncError(str(e))
class DeltaFile(LikeFile):
@@ -163,9 +163,9 @@ class SigGenerator:
module, not filelike object
"""
- def __init__(self):
+ def __init__(self, blocksize = _librsync.RS_DEFAULT_BLOCK_LEN):
"""Return new signature instance"""
- try: self.sig_maker = _librsync.new_sigmaker()
+ try: self.sig_maker = _librsync.new_sigmaker(blocksize)
except _librsync.librsyncError, e: raise librsyncError(str(e))
self.gotsig = None
self.buffer = ""
diff --git a/rdiff-backup/testing/librsynctest.py b/rdiff-backup/testing/librsynctest.py
index 54dc4dd..2c56a8a 100644
--- a/rdiff-backup/testing/librsynctest.py
+++ b/rdiff-backup/testing/librsynctest.py
@@ -1,16 +1,22 @@
import unittest, random
from commontest import *
-from rdiff_backup import librsync
-
-def MakeRandomFile(path):
- """Writes a random file of length between 10000 and 100000"""
- fp = open(path, "w")
- randseq = []
- for i in xrange(random.randrange(5000, 30000)):
- randseq.append(chr(random.randrange(256)))
- fp.write("".join(randseq))
- fp.close()
+from rdiff_backup import librsync, log
+
+def MakeRandomFile(path, length = None):
+ """Writes a random file of given length, or random len if unspecified"""
+ if not length: length = random.randrange(5000, 100000)
+ fp = open(path, "wb")
+ fp_random = open('/dev/urandom', 'rb')
+ # Old slow way, may still be of use on systems without /dev/urandom
+ #randseq = []
+ #for i in xrange(random.randrange(5000, 30000)):
+ # randseq.append(chr(random.randrange(256)))
+ #fp.write("".join(randseq))
+ fp.write(fp_random.read(length))
+
+ fp.close()
+ fp_random.close()
class LibrsyncTest(unittest.TestCase):
"""Test various librsync wrapper functions"""
@@ -20,23 +26,33 @@ class LibrsyncTest(unittest.TestCase):
sig = RPath(Globals.local_connection, "testfiles/signature")
sig2 = RPath(Globals.local_connection, "testfiles/signature2")
delta = RPath(Globals.local_connection, "testfiles/delta")
- def testSigFile(self):
- """Make sure SigFile generates same data as rdiff"""
- for i in range(5):
- MakeRandomFile(self.basis.path)
- self.sig.delete()
- assert not os.system("rdiff signature %s %s" %
- (self.basis.path, self.sig.path))
+ def sig_file_test_helper(self, blocksize, iterations, file_len = None):
+ """Compare SigFile output to rdiff output at given blocksize"""
+ for i in range(iterations):
+ MakeRandomFile(self.basis.path, file_len)
+ if self.sig.lstat(): self.sig.delete()
+ assert not os.system("rdiff -b %s signature %s %s" %
+ (blocksize, self.basis.path, self.sig.path))
fp = self.sig.open("rb")
rdiff_sig = fp.read()
fp.close()
- sf = librsync.SigFile(self.basis.open("rb"))
+ sf = librsync.SigFile(self.basis.open("rb"), blocksize)
librsync_sig = sf.read()
sf.close()
assert rdiff_sig == librsync_sig, \
(len(rdiff_sig), len(librsync_sig))
+
+ def testSigFile(self):
+ """Make sure SigFile generates same data as rdiff, blocksize 512"""
+ self.sig_file_test_helper(512, 5)
+
+ def testSigFile2(self):
+ """Test SigFile like above, but try various blocksize"""
+ self.sig_file_test_helper(2048, 1, 60000)
+ self.sig_file_test_helper(7168, 1, 6000)
+ self.sig_file_test_helper(204800, 1, 40*1024*1024)
def testSigGenerator(self):
"""Test SigGenerator, make sure it's same as SigFile"""
diff --git a/rdiff-backup/testing/rdifftest.py b/rdiff-backup/testing/rdifftest.py
index 6079f1a..d5bde4c 100644
--- a/rdiff-backup/testing/rdifftest.py
+++ b/rdiff-backup/testing/rdifftest.py
@@ -2,7 +2,7 @@ import unittest, random
from commontest import *
from rdiff_backup import Globals, Rdiff, selection, log, rpath
-Log.setverbosity(6)
+Log.setverbosity(7)
def MakeRandomFile(path):
"""Writes a random file of length between 10000 and 100000"""
@@ -28,7 +28,7 @@ class RdiffTest(unittest.TestCase):
sig = rpath.RPath(self.lc,
"testfiles/various_file_types/regular_file.sig")
sigfp = sig.open("r")
- rfsig = Rdiff.get_signature(RPath(self.lc, "testfiles/various_file_types/regular_file"))
+ rfsig = Rdiff.get_signature(RPath(self.lc, "testfiles/various_file_types/regular_file"), 2048)
assert rpath.cmpfileobj(sigfp, rfsig)
sigfp.close()
rfsig.close()