Added metadata diffing, and an iterfile hash bugfix

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@669 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

5 files changed, 118 insertions, 24 deletions

diff --git a/rdiff-backup/testing/backuptest.py b/rdiff-backup/testing/backuptest.py
index cdcfefc..f5cec9b 100644
--- a/rdiff-backup/testing/backuptest.py
+++ b/rdiff-backup/testing/backuptest.py
@@ -6,7 +6,7 @@ class RemoteMirrorTest(unittest.TestCase):
 	"""Test mirroring"""
 	def setUp(self):
 		"""Start server"""
-		Log.setverbosity(3)
+		Log.setverbosity(5)
 		Globals.change_source_perms = 1
 		SetConnections.UpdateGlobal('checkpoint_interval', 3)
 		user_group.init_user_mapping()
diff --git a/rdiff-backup/testing/eas_aclstest.py b/rdiff-backup/testing/eas_aclstest.py
index 4d6535f..0fc3a37 100644
--- a/rdiff-backup/testing/eas_aclstest.py
+++ b/rdiff-backup/testing/eas_aclstest.py
@@ -141,16 +141,16 @@ user.empty
 
 		# Now write records corresponding to above rps into file
 		Globals.rbdir = tempdir
-		Time.setcurtime(10000)
-		ExtendedAttributesFile.open_file()
+		man = metadata.PatchDiffMan()
+		writer = man.get_ea_writer('snapshot', 10000)
 		for rp in [self.ea_testdir1, rp1, rp2, rp3]:
 			ea = ExtendedAttributes(rp.index)
 			ea.read_from_rp(rp)
-			ExtendedAttributesFile.write_object(ea)
-		ExtendedAttributesFile.close_file()
+			writer.write_object(ea)
+		writer.close()
 
 		# Read back records and compare
-		ea_iter = ExtendedAttributesFile.get_objects_at_time(tempdir, 10000)
+		ea_iter = man.get_eas_at_time(10000, None)
 		assert ea_iter, "No extended_attributes.<time> file found"
 		sample_ea_reread = ea_iter.next()
 		assert sample_ea_reread == self.sample_ea
@@ -357,22 +357,23 @@ other::---
 	def testIterate(self):
 		"""Test writing several records and then reading them back"""
 		self.make_backup_dirs()
+		self.make_temp()
 		rp1 = self.acl_testdir1.append('1')
 		rp2 = self.acl_testdir1.append('2')
 		rp3 = self.acl_testdir1.append('3')
 
 		# Now write records corresponding to above rps into file
 		Globals.rbdir = tempdir
-		Time.setcurtime(10000)
-		AccessControlListFile.open_file()
+		man = metadata.PatchDiffMan()
+		writer = man.get_acl_writer('snapshot', 10000)
 		for rp in [self.acl_testdir1, rp1, rp2, rp3]:
 			acl = AccessControlLists(rp.index)
 			acl.read_from_rp(rp)
-			AccessControlListFile.write_object(acl)
-		AccessControlListFile.close_file()
+			writer.write_object(acl)
+		writer.close()
 
 		# Read back records and compare
-		acl_iter = AccessControlListFile.get_objects_at_time(tempdir, 10000)
+		acl_iter = man.get_acls_at_time(10000, None)
 		assert acl_iter, "No acl file found"
 		dir_acl_reread = acl_iter.next()
 		assert dir_acl_reread == self.dir_acl
diff --git a/rdiff-backup/testing/hashtest.py b/rdiff-backup/testing/hashtest.py
index ad993f1..d21a24a 100644
--- a/rdiff-backup/testing/hashtest.py
+++ b/rdiff-backup/testing/hashtest.py
@@ -1,5 +1,5 @@
 import unittest, StringIO
-from rdiff_backup import hash, rpath, regress, restore, metadata
+from rdiff_backup import hash
 from commontest import *
 
 class HashTest(unittest.TestCase):
@@ -81,7 +81,7 @@ class HashTest(unittest.TestCase):
 		in_rp1, hashlist1, in_rp2, hashlist2 = self.make_dirs()
 		Myrm("testfiles/output")
 
-		rdiff_backup(1, 1, in_rp1.path, "testfiles/output", 10000, "-v5")
+		rdiff_backup(1, 1, in_rp1.path, "testfiles/output", 10000, "-v3")
 		meta_prefix = rpath.RPath(Globals.local_connection,
 					  "testfiles/output/rdiff-backup-data/mirror_metadata")
 		incs = restore.get_inclist(meta_prefix)
@@ -90,14 +90,53 @@ class HashTest(unittest.TestCase):
 		hashlist = self.extract_hashs(metadata_rp)
 		assert hashlist == hashlist1, (hashlist1, hashlist)
 
-		rdiff_backup(1, 1, in_rp2.path, "testfiles/output", 20000, "-v7")
+		rdiff_backup(1, 1, in_rp2.path, "testfiles/output", 20000, "-v3")
 		incs = restore.get_inclist(meta_prefix)
 		assert len(incs) == 2
-		metadata_rp.delete() # easy way to find the other one
-		incs = restore.get_inclist(meta_prefix)
-		assert len(incs) == 1
-		hashlist = self.extract_hashs(incs[0])
+		if incs[0].getinctype() == 'snapshot': inc = incs[0]
+		else: inc = incs[1]
+		hashlist = self.extract_hashs(inc)
 		assert hashlist == hashlist2, (hashlist2, hashlist)
 
+	def test_rorpiter_xfer(self):
+		"""Test if hashes are transferred in files, rorpiter"""
+		#log.Log.setverbosity(5)
+		Globals.security_level = 'override'
+		conn = SetConnections.init_connection('python ./server.py .')
+		assert conn.reval("lambda x: x+1", 4) == 5 # connection sanity check
+
+		fp = hash.FileWrapper(StringIO.StringIO(self.s1))
+		conn.Globals.set('tmp_file', fp)
+		fp_remote = conn.Globals.get('tmp_file')
+		assert fp_remote.read() == self.s1
+		assert fp_remote.close().sha1_digest == self.s1_hash
+
+		# Tested xfer of file, now test xfer of files in rorpiter
+		root = MakeOutputDir()
+		rp1 = root.append('s1')
+		rp1.write_string(self.s1)
+		rp2 = root.append('s2')
+		rp2.write_string(self.s2)
+		rp1.setfile(hash.FileWrapper(rp1.open('rb')))
+		rp2.setfile(hash.FileWrapper(rp2.open('rb')))
+		rpiter = iter([rp1, rp2])
+
+		conn.Globals.set('tmp_conn_iter', rpiter)
+		remote_iter = conn.Globals.get('tmp_conn_iter')
+
+		rorp1 = remote_iter.next()
+		fp = rorp1.open('rb')
+		assert fp.read() == self.s1, fp.read()
+		ret_val = fp.close()
+		assert isinstance(ret_val, hash.Report), ret_val
+		assert ret_val.sha1_digest == self.s1_hash
+		rorp2 = remote_iter.next()
+		fp2 = rorp1.open('rb')
+		assert fp2.close().sha1_digest == self.s2_hash
+
+		conn.quit()
+
+
+from rdiff_backup import rpath, regress, restore, metadata, log, Globals
 
 if __name__ == "__main__": unittest.main()
diff --git a/rdiff-backup/testing/metadatatest.py b/rdiff-backup/testing/metadatatest.py
index becfecc..8da750d 100644
--- a/rdiff-backup/testing/metadatatest.py
+++ b/rdiff-backup/testing/metadatatest.py
@@ -1,6 +1,6 @@
 import unittest, os, cStringIO, time
 from rdiff_backup.metadata import *
-from rdiff_backup import rpath, connection, Globals, selection
+from rdiff_backup import rpath, connection, Globals, selection, lazy
 
 tempdir = rpath.RPath(Globals.local_connection, "testfiles/output")
 
@@ -151,7 +151,9 @@ class MetadataTest(unittest.TestCase):
 		diff1 = [rp1, rp4]
 		diff2 = [rp1new, rp2, zero]
 
-		output = patch(iter(current), iter(diff1), iter(diff2))
+		Globals.rbdir = tempdir
+		output = PatchDiffMan().iterate_patched_meta(
+			            [iter(current), iter(diff1), iter(diff2)])
 		out1 = output.next()
 		assert out1 is rp1new, out1
 		out2 = output.next()
@@ -160,5 +162,57 @@ class MetadataTest(unittest.TestCase):
 		assert out3 is rp3, out3
 		self.assertRaises(StopIteration, output.next)
 
+	def test_meta_patch_cycle(self):
+		"""Create various metadata rorps, diff them, then compare"""
+		def write_dir_to_meta(manager, rp, time):
+			"""Record the metadata under rp to a mirror_metadata file"""
+			metawriter = man.get_meta_writer('snapshot', time)
+			for rorp in selection.Select(rp).set_iter():
+				metawriter.write_object(rorp)
+			metawriter.close()
+
+		def compare(man, rootrp, time):
+			assert lazy.Iter.equal(selection.Select(rootrp).set_iter(),
+								   man.get_meta_at_time(time, None))
+
+
+		self.make_temp()
+		Globals.rbdir = tempdir
+		man = PatchDiffMan()
+		inc1 = rpath.RPath(Globals.local_connection, "testfiles/increment1")
+		inc2 = rpath.RPath(Globals.local_connection, "testfiles/increment2")
+		inc3 = rpath.RPath(Globals.local_connection, "testfiles/increment3")
+		inc4 = rpath.RPath(Globals.local_connection, "testfiles/increment4")
+		write_dir_to_meta(man, inc1, 10000)
+		compare(man, inc1, 10000)
+		write_dir_to_meta(man, inc2, 20000)
+		compare(man, inc2, 20000)
+		man.ConvertMetaToDiff()
+		man = PatchDiffMan()
+		write_dir_to_meta(man, inc3, 30000)
+		compare(man, inc3, 30000)
+		man.ConvertMetaToDiff()
+		man = PatchDiffMan()
+		man.max_diff_chain = 3
+		write_dir_to_meta(man, inc4, 40000)
+		compare(man, inc4, 40000)
+		man.ConvertMetaToDiff()
+
+		man = PatchDiffMan()
+		l = man.sorted_meta_inclist()
+		assert l[0].getinctype() == 'snapshot'
+		assert l[0].getinctime() == 40000
+		assert l[1].getinctype() == 'snapshot'
+		assert l[1].getinctime() == 30000
+		assert l[2].getinctype() == 'diff'
+		assert l[2].getinctime() == 20000
+		assert l[3].getinctype() == 'diff'
+		assert l[3].getinctime() == 10000
+
+		compare(man, inc1, 10000)
+		compare(man, inc2, 20000)
+		compare(man, inc3, 30000)
+		compare(man, inc4, 40000)
+		
 
 if __name__ == "__main__": unittest.main()
diff --git a/rdiff-backup/testing/restoretest.py b/rdiff-backup/testing/restoretest.py
index 395becb..bdcd2a6 100644
--- a/rdiff-backup/testing/restoretest.py
+++ b/rdiff-backup/testing/restoretest.py
@@ -2,7 +2,7 @@ import unittest
 from commontest import *
 from rdiff_backup import log, restore, Globals, rpath, TempFile
 
-Log.setverbosity(5)
+Log.setverbosity(3)
 lc = Globals.local_connection
 tempdir = rpath.RPath(Globals.local_connection, "testfiles/output")
 restore_base_rp = rpath.RPath(Globals.local_connection,
@@ -31,8 +31,8 @@ class RestoreFileComparer:
 		"""Restore file, make sure it is the same at time t"""
 		log.Log("Checking result at time %s" % (t,), 7)
 		tf = TempFile.new(tempdir.append("foo"))
-		restore._mirror_time = mirror_time
-		restore._rest_time = t
+		restore.MirrorStruct._mirror_time = mirror_time
+		restore.MirrorStruct._rest_time = t
 		self.rf.set_relevant_incs()
 		out_rorpath = self.rf.get_attribs().getRORPath()
 		correct_result = self.time_rp_dict[t]
@@ -59,7 +59,7 @@ class RestoreTimeTest(unittest.TestCase):
 		rdiff-backup-data directory already being laid out.
 
 		"""
-		restore._mirror_time = None # Reset
+		restore.MirrorStruct._mirror_time = None # Reset
 		Globals.rbdir = rpath.RPath(lc,
 									"testfiles/restoretest3/rdiff-backup-data")
 		assert Time.genstrtotime("0B") == Time.time_from_session(0)