Restoring/regressing now uses less memory

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/branches/r0-12@480 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

3 files changed, 37 insertions, 25 deletions

diff --git a/rdiff-backup/CHANGELOG b/rdiff-backup/CHANGELOG
index 07a390b..6889ca2 100644
--- a/rdiff-backup/CHANGELOG
+++ b/rdiff-backup/CHANGELOG
@@ -8,6 +8,9 @@ Added --list-increment-sizes option from the development branch.
 Previously this option was in the man page but was omitted in the code
 (thanks to Michael Schwendt for report).
 
+Restoring/regressing should now take less memory, especially on large
+directories.  CPU usage may slightly increase, however.
+
 
 New in v0.12.5 (2003/09/27)
 ---------------------------
diff --git a/rdiff-backup/rdiff-backup.1 b/rdiff-backup/rdiff-backup.1
index 9552f59..0210424 100644
--- a/rdiff-backup/rdiff-backup.1
+++ b/rdiff-backup/rdiff-backup.1
@@ -136,8 +136,7 @@ See the
 section for more information.
 .TP
 .B --exclude-special-files
-Exclude all device files, fifos, sockets, and symlinks.  This option
-is implied by --windows-mode.
+Exclude all device files, fifos, sockets, and symlinks.
 .TP
 .B --force
 Authorize the updating or overwriting of a destination path.
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 34813df..0e2e5c0 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -513,29 +513,39 @@ as data loss may result.\n""" % (self.mirror_rp.get_indexpath(),), 2)
 
 		"""
 		if not inc_rpath.isdir(): return
-		inc_dict = {} # dictionary of basenames:IndexedTuples(index, inc_list)
-		dirlist = robust.listrp(inc_rpath)
-
-		def affirm_dict_indexed(basename):
-			"""Make sure the rid dictionary has given basename as key"""
-			if not inc_dict.has_key(basename):
-				sub_inc_rp = inc_rpath.append(basename)
-				inc_dict[basename] = rorpiter.IndexedTuple(sub_inc_rp.index,
-														   (sub_inc_rp, []))
-
-		def add_to_dict(filename):
-			"""Add filename to the inc tuple dictionary"""
-			rp = inc_rpath.append(filename)
-			if rp.isincfile() and rp.getinctype() != 'data':
-				basename = rp.getincbase_str()
-				affirm_dict_indexed(basename)
-				inc_dict[basename][1].append(rp)
-			elif rp.isdir(): affirm_dict_indexed(filename)
-
-		for filename in dirlist: add_to_dict(filename)
-		keys = inc_dict.keys()
-		keys.sort()
-		for key in keys: yield inc_dict[key]
+
+		def get_inc_pairs():
+			"""Return unsorted list of (basename, inc_filenames) pairs"""
+			inc_dict = {} # dictionary of basenames:inc_filenames
+			dirlist = robust.listrp(inc_rpath)
+
+			def add_to_dict(filename):
+				"""Add filename to the inc tuple dictionary"""
+				rp = inc_rpath.append(filename)
+				if rp.isincfile() and rp.getinctype() != 'data':
+					basename = rp.getincbase_str()
+					inc_filename_list = inc_dict.setdefault(basename, [])
+					inc_filename_list.append(filename)
+				elif rp.isdir(): inc_dict.setdefault(filename, [])
+
+			for filename in dirlist: add_to_dict(filename)
+			return inc_dict.items()
+
+		def inc_filenames2incrps(filenames):
+			"""Map list of filenames into increment rps"""
+			l = []
+			for filename in filenames:
+				rp = inc_rpath.append(filename)
+				assert rp.isincfile(), rp.path
+				l.append(rp)
+			return l
+
+		items = get_inc_pairs()
+		items.sort() # Sorting on basis of basename now
+		for (basename, inc_filenames) in items:
+			sub_inc_rpath = inc_rpath.append(basename)
+			yield rorpiter.IndexedTuple(sub_inc_rpath.index,
+					  (sub_inc_rpath, inc_filenames2incrps(inc_filenames)))
 
 
 class PatchITRB(rorpiter.ITRBranch):