From 78f5e4609358ebb57a4cdf8b21e1fb38237b43c4 Mon Sep 17 00:00:00 2001 From: owsla Date: Sun, 20 Jan 2008 16:38:12 +0000 Subject: Improve the handling of directories with many small files by flushing the pipeline more often. Fixes a bug where Globals.pipeline_max_length*4-1 files are in a directory causes KeyError. git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@874 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/CHANGELOG | 4 ++++ rdiff-backup/rdiff_backup/backup.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/rdiff-backup/CHANGELOG b/rdiff-backup/CHANGELOG index 3792c1a..4503218 100644 --- a/rdiff-backup/CHANGELOG +++ b/rdiff-backup/CHANGELOG @@ -1,6 +1,10 @@ New in v1.1.16 (????/??/??) --------------------------- +Improve the handling of directories with many small files when backing-up +over a network connection. Thanks to Austin Clements for the test case. +(Andrew Ferguson) + Change high-bit permissions test to check both files and directories. Improves rdiff-backup's support for AFS and closes Debian bug #450409. (Patch from Marc Horowitz) diff --git a/rdiff-backup/rdiff_backup/backup.py b/rdiff-backup/rdiff_backup/backup.py index 2b139f4..0cdad16 100644 --- a/rdiff-backup/rdiff_backup/backup.py +++ b/rdiff-backup/rdiff_backup/backup.py @@ -161,18 +161,18 @@ class DestinationStruct: every so often so it doesn't get congested on destination end. """ - flush_threshold = int(Globals.pipeline_max_length/2) - num_rorps_skipped = 0 + flush_threshold = Globals.pipeline_max_length - 2 + num_rorps_seen = 0 for src_rorp, dest_rorp in cls.CCPP: - if (src_rorp and dest_rorp and src_rorp == dest_rorp and + if (Globals.backup_reader is not Globals.backup_writer): + num_rorps_seen += 1 + if (num_rorps_seen > flush_threshold): + num_rorps_seen = 0 + yield iterfile.MiscIterFlushRepeat + if not (src_rorp and dest_rorp and src_rorp == dest_rorp and (not Globals.preserve_hardlinks or Hardlink.rorp_eq(src_rorp, dest_rorp))): - num_rorps_skipped += 1 - if (Globals.backup_reader is not Globals.backup_writer and - num_rorps_skipped > flush_threshold): - num_rorps_skipped = 0 - yield iterfile.MiscIterFlushRepeat - else: + index = src_rorp and src_rorp.index or dest_rorp.index sig = cls.get_one_sig(dest_base_rpath, index, src_rorp, dest_rorp) -- cgit v1.2.1