From 3617a1f5be2c8c90d2e3c98a642279c58981c3d7 Mon Sep 17 00:00:00 2001 From: bescoto Date: Tue, 22 Jul 2003 09:07:43 +0000 Subject: Now choose blocksize based on size of file git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@353 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/rdiff_backup/Rdiff.py | 27 ++++++++++++++++++++------- rdiff-backup/rdiff_backup/_librsyncmodule.c | 5 +++-- rdiff-backup/rdiff_backup/librsync.py | 8 ++++---- 3 files changed, 27 insertions(+), 13 deletions(-) (limited to 'rdiff-backup/rdiff_backup') diff --git a/rdiff-backup/rdiff_backup/Rdiff.py b/rdiff-backup/rdiff_backup/Rdiff.py index ee688c1..0070ceb 100644 --- a/rdiff-backup/rdiff_backup/Rdiff.py +++ b/rdiff-backup/rdiff_backup/Rdiff.py @@ -23,10 +23,24 @@ import os, librsync import Globals, log, static, TempFile, rpath -def get_signature(rp): +def get_signature(rp, blocksize = None): """Take signature of rpin file and return in file object""" - log.Log("Getting signature of %s" % rp.get_indexpath(), 7) - return librsync.SigFile(rp.open("rb")) + if not blocksize: blocksize = find_blocksize(rp.getsize()) + log.Log("Getting signature of %s with blocksize %s" % + (rp.get_indexpath(), blocksize), 7) + return librsync.SigFile(rp.open("rb"), blocksize) + +def find_blocksize(file_len): + """Return a reasonable block size to use on files of length file_len + + If the block size is too big, deltas will be bigger than is + necessary. If the block size is too small, making deltas and + patching can take a really long time. + + """ + if file_len < 1024000: return 512 # set minimum of 512 bytes + else: # Split file into about 2000 pieces, rounding to 512 + return long((file_len/(2000*512))*512) def get_delta_sigfileobj(sig_fileobj, rp_new): """Like get_delta but signature is in a file object""" @@ -43,8 +57,7 @@ def write_delta(basis, new, delta, compress = None): """Write rdiff delta which brings basis to new""" log.Log("Writing delta %s from %s -> %s" % (basis.path, new.path, delta.path), 7) - sigfile = librsync.SigFile(basis.open("rb")) - deltafile = librsync.DeltaFile(sigfile, new.open("rb")) + deltafile = librsync.DeltaFile(get_signature(basis), new.open("rb")) delta.write_from_fileobj(deltafile, compress) def write_patched_fp(basis_fp, delta_fp, out_fp): @@ -79,8 +92,8 @@ def patch_local(rp_basis, rp_delta, outrp = None, delta_compressed = None): def copy_local(rpin, rpout, rpnew = None): """Write rpnew == rpin using rpout as basis. rpout and rpnew local""" assert rpout.conn is Globals.local_connection - sigfile = librsync.SigFile(rpout.open("rb")) - deltafile = rpin.conn.librsync.DeltaFile(sigfile, rpin.open("rb")) + deltafile = rpin.conn.librsync.DeltaFile(get_signature(rpout), + rpin.open("rb")) patched_file = librsync.PatchedFile(rpout.open("rb"), deltafile) if rpnew: rpnew.write_from_fileobj(patched_file) diff --git a/rdiff-backup/rdiff_backup/_librsyncmodule.c b/rdiff-backup/rdiff_backup/_librsyncmodule.c index 9b24d6a..7a78580 100644 --- a/rdiff-backup/rdiff_backup/_librsyncmodule.c +++ b/rdiff-backup/rdiff_backup/_librsyncmodule.c @@ -41,15 +41,16 @@ static PyObject* _librsync_new_sigmaker(PyObject* self, PyObject* args) { _librsync_SigMakerObject* sm; + long blocklen; - if (!PyArg_ParseTuple(args,":new_sigmaker")) + if (!PyArg_ParseTuple(args, "l:new_sigmaker", &blocklen)) return NULL; sm = PyObject_New(_librsync_SigMakerObject, &_librsync_SigMakerType); if (sm == NULL) return NULL; sm->x_attr = NULL; - sm->sig_job = rs_sig_begin((size_t)RS_DEFAULT_BLOCK_LEN, + sm->sig_job = rs_sig_begin((size_t)blocklen, (size_t)RS_DEFAULT_STRONG_LEN); return (PyObject*)sm; } diff --git a/rdiff-backup/rdiff_backup/librsync.py b/rdiff-backup/rdiff_backup/librsync.py index eb5e235..a8e37ca 100644 --- a/rdiff-backup/rdiff_backup/librsync.py +++ b/rdiff-backup/rdiff_backup/librsync.py @@ -108,7 +108,7 @@ class LikeFile: class SigFile(LikeFile): """File-like object which incrementally generates a librsync signature""" - def __init__(self, infile): + def __init__(self, infile, blocksize = _librsync.RS_DEFAULT_BLOCK_LEN): """SigFile initializer - takes basis file basis file only needs to have read() and close() methods. It @@ -116,7 +116,7 @@ class SigFile(LikeFile): """ LikeFile.__init__(self, infile) - try: self.maker = _librsync.new_sigmaker() + try: self.maker = _librsync.new_sigmaker(blocksize) except _librsync.librsyncError, e: raise librsyncError(str(e)) class DeltaFile(LikeFile): @@ -163,9 +163,9 @@ class SigGenerator: module, not filelike object """ - def __init__(self): + def __init__(self, blocksize = _librsync.RS_DEFAULT_BLOCK_LEN): """Return new signature instance""" - try: self.sig_maker = _librsync.new_sigmaker() + try: self.sig_maker = _librsync.new_sigmaker(blocksize) except _librsync.librsyncError, e: raise librsyncError(str(e)) self.gotsig = None self.buffer = "" -- cgit v1.2.1