summaryrefslogtreecommitdiff
path: root/rdiff-backup/testing/commontest.py
blob: 1e6fa82d9472ceffd8652a529c8e60b3f633cf4b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
"""commontest - Some functions and constants common to several test cases"""
import os, sys
from rdiff_backup.log import Log
from rdiff_backup.rpath import RPath
from rdiff_backup import Globals, Hardlink, SetConnections, Main, \
	 selection, lazy, Time, rpath

SourceDir = "../rdiff_backup"
AbsCurdir = os.getcwd() # Absolute path name of current directory
AbsTFdir = AbsCurdir+"/testfiles"
MiscDir = "../misc"
__no_execute__ = 1 # Keeps the actual rdiff-backup program from running

def Myrm(dirstring):
	"""Run myrm on given directory string"""
	root_rp = rpath.RPath(Globals.local_connection, dirstring)
	for rp in selection.Select(root_rp).set_iter():
		if rp.isdir(): rp.chmod(0700) # otherwise may not be able to remove
	assert not os.system("rm -rf %s" % (dirstring,))

def Make():
	"""Make sure the rdiff-backup script in the source dir is up-to-date"""
	os.chdir(SourceDir)
	os.system("python ./Make")
	os.chdir(AbsCurdir)

def MakeOutputDir():
	"""Initialize the testfiles/output directory"""
	Myrm("testfiles/output")
	rp = rpath.RPath(Globals.local_connection, "testfiles/output")
	rp.mkdir()
	return rp

def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
				 current_time = None, extra_options = ""):
	"""Run rdiff-backup with the given options

	source_local and dest_local are boolean values.  If either is
	false, then rdiff-backup will be run pretending that src_dir and
	dest_dir, respectively, are remote.  The server process will be
	run in directories test1 and test2/tmp respectively.

	src_dir and dest_dir are the source and destination
	(mirror) directories, relative to the testing directory.

	If current time is true, add the --current-time option with the
	given number of seconds.

	extra_options are just added to the command line.

	"""
	if not source_local:
		src_dir = ("cd test1; ../%s/rdiff-backup --server::../%s" %
				   (SourceDir, src_dir))
	if not dest_local:
		dest_dir = ("test2/tmp; ../../%s/rdiff-backup --server::../../%s" %
					(SourceDir, dest_dir))

	cmdargs = [SourceDir + "/rdiff-backup", extra_options]
	if not (source_local and dest_local): cmdargs.append("--remote-schema %s")

	if current_time: cmdargs.append("--current-time %s" % current_time)

	os.system(" ".join(cmdargs))	

def cmd_schemas2rps(schema_list, remote_schema):
	"""Input list of file descriptions and the remote schema, return rps

	File descriptions should be strings of the form 'hostname.net::foo'

	"""
	return map(SetConnections.cmdpair2rp,
			   SetConnections.get_cmd_pairs(schema_list, remote_schema))

def InternalBackup(source_local, dest_local, src_dir, dest_dir,
				   current_time = None):
	"""Backup src to dest internally

	This is like rdiff_backup but instead of running a separate
	rdiff-backup script, use the separate *.py files.  This way the
	script doesn't have to be rebuild constantly, and stacktraces have
	correct line/file references.

	"""
	Globals.current_time = current_time
	#_reset_connections()
	remote_schema = '%s'

	if not source_local:
		src_dir = "cd test1; python ../server.py ../%s::../%s" % \
				  (SourceDir, src_dir)
	if not dest_local:
		dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
				   % (SourceDir, dest_dir)

	rpin, rpout = cmd_schemas2rps([src_dir, dest_dir], remote_schema)
	Main.misc_setup([rpin, rpout])
	Main.Backup(rpin, rpout)
	Main.cleanup()

def InternalMirror(source_local, dest_local, src_dir, dest_dir):
	"""Mirror src to dest internally

	like InternalBackup, but only mirror.  Do this through
	InternalBackup, but then delete rdiff-backup-data directory.

	"""
	# Save attributes of root to restore later
	src_root = rpath.RPath(Globals.local_connection, src_dir)
	dest_root = rpath.RPath(Globals.local_connection, dest_dir)
	dest_rbdir = dest_root.append("rdiff-backup-data")
	dest_incdir = dest_rbdir.append("increments")

	# We need to create these directories or else failure because
	# --force option not given.
	if not dest_root.lstat(): dest_root.mkdir()
	if not dest_rbdir.lstat(): dest_rbdir.mkdir()
	if not dest_incdir.lstat(): dest_incdir.mkdir()
	
	InternalBackup(source_local, dest_local, src_dir, dest_dir)
	dest_root.setdata()
	Myrm(dest_rbdir.path)
	# Restore old attributes
	rpath.copy_attribs(src_root, dest_root)

def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
	"""Restore mirror_dir to dest_dir at given time

	This will automatically find the increments.XXX.dir representing
	the time specified.  The mirror_dir and dest_dir are relative to
	the testing directory and will be modified for remote trials.

	"""
	Main.force = 1
	remote_schema = '%s'
	#_reset_connections()
	if not mirror_local:
		mirror_dir = "cd test1; python ../server.py ../%s::../%s" % \
					 (SourceDir, mirror_dir)
	if not dest_local:
		dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
				   % (SourceDir, dest_dir)

	mirror_rp, dest_rp = cmd_schemas2rps([mirror_dir, dest_dir], remote_schema)
	Main.misc_setup([mirror_rp, dest_rp])
	inc = get_increment_rp(mirror_rp, time)
	if inc: Main.Restore(get_increment_rp(mirror_rp, time), dest_rp)
	else: # use alternate syntax
		Main.restore_timestr = str(time)
		Main.RestoreAsOf(mirror_rp, dest_rp)
	Main.cleanup()

def get_increment_rp(mirror_rp, time):
	"""Return increment rp matching time in seconds"""
	data_rp = mirror_rp.append("rdiff-backup-data")
	if not data_rp.isdir(): return None
	for filename in data_rp.listdir():
		rp = data_rp.append(filename)
		if rp.isincfile() and rp.getincbase_str() == "increments":
			if Time.stringtotime(rp.getinctime()) == time: return rp
	return None # Couldn't find appropriate increment

def _reset_connections(src_rp, dest_rp):
	"""Reset some global connection information"""
	Globals.isbackup_reader = Globals.isbackup_writer = None
	#Globals.connections = [Globals.local_connection]
	#Globals.connection_dict = {0: Globals.local_connection}
	SetConnections.UpdateGlobal('rbdir', None)
	Main.misc_setup([src_rp, dest_rp])

def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
					 equality_func = None, exclude_rbdir = 1,
					 ignore_tmp_files = None):
	"""Compare src_rp and dest_rp, which can be directories

	This only compares file attributes, not the actual data.  This
	will overwrite the hardlink dictionaries if compare_hardlinks is
	specified.

	"""
	if compare_hardlinks: reset_hardlink_dicts()
	src_rp.setdata()
	dest_rp.setdata()

	Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
											   compare_hardlinks), 3)
	src_select = selection.Select(src_rp)
	dest_select = selection.Select(dest_rp)

	if ignore_tmp_files:
		# Ignoring temp files can be useful when we want to check the
		# correctness of a backup which aborted in the middle.  In
		# these cases it is OK to have tmp files lying around.
		src_select.add_selection_func(src_select.regexp_get_sf(
			".*rdiff-backup.tmp.[^/]+$", 0))
		dest_select.add_selection_func(dest_select.regexp_get_sf(
			".*rdiff-backup.tmp.[^/]+$", 0))

	if exclude_rbdir:
		src_select.parse_rbdir_exclude()
		dest_select.parse_rbdir_exclude()
	else:
		# include rdiff-backup-data/increments
		src_select.add_selection_func(src_select.glob_get_tuple_sf(
			('rdiff-backup-data', 'increments'), 1))
		dest_select.add_selection_func(dest_select.glob_get_tuple_sf(
			('rdiff-backup-data', 'increments'), 1))
		
		# but exclude rdiff-backup-data
		src_select.add_selection_func(src_select.glob_get_tuple_sf(
			('rdiff-backup-data',), 0))
		dest_select.add_selection_func(dest_select.glob_get_tuple_sf(
			('rdiff-backup-data',), 0))

	dsiter1, dsiter2 = src_select.set_iter(), dest_select.set_iter()

	def hardlink_equal(src_rorp, dest_rorp):
		if not src_rorp.equal_verbose(dest_rorp): return None
		if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
		Log("%s: %s" % (dest_rorp.index,
						Hardlink.get_indicies(dest_rorp, None)), 3)
		return None

	def rbdir_equal(src_rorp, dest_rorp):
		"""Like hardlink_equal, but make allowances for data directories"""
		if not src_rorp.index and not dest_rorp.index: return 1
		if (src_rorp.index and src_rorp.index[0] == 'rdiff-backup-data' and
			src_rorp.index == dest_rorp.index):
			# Don't compare dirs - they don't carry significant info
			if dest_rorp.isdir() and src_rorp.isdir(): return 1
			if dest_rorp.isreg() and src_rorp.isreg():
				# Don't compare gzipped files because it is apparently
				# non-deterministic.
				if dest_rorp.index[-1].endswith('gz'): return 1
				# Don't compare .missing increments because they don't matter
				if dest_rorp.index[-1].endswith('.missing'): return 1
		if compare_hardlinks:
			if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
		elif src_rorp.equal_verbose(dest_rorp): return 1
		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
		Log("%s: %s" % (dest_rorp.index,
						Hardlink.get_indicies(dest_rorp, None)), 3)
		return None

	if equality_func: result = lazy.Iter.equal(dsiter1, dsiter2,
											   1, equality_func)
	elif compare_hardlinks:
		dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1)
		dsiter2 = Hardlink.add_rorp_iter(dsiter2, None)		
		if exclude_rbdir:
			result = lazy.Iter.equal(dsiter1, dsiter2, 1, hardlink_equal)
		else: result = lazy.Iter.equal(dsiter1, dsiter2, 1, rbdir_equal)
	elif not exclude_rbdir:
		result = lazy.Iter.equal(dsiter1, dsiter2, 1, rbdir_equal)
	else: result = lazy.Iter.equal(dsiter1, dsiter2, 1)

	for i in dsiter1: pass # make sure all files processed anyway
	for i in dsiter2: pass
	return result

def reset_hardlink_dicts():
	"""Clear the hardlink dictionaries"""
	Hardlink._src_inode_indicies = {}
	Hardlink._src_index_indicies = {}
	Hardlink._dest_inode_indicies = {}
	Hardlink._dest_index_indicies = {}
	Hardlink._restore_index_path = {}

def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
						compare_hardlinks = 1,
						dest_dirname = "testfiles/output",
						restore_dirname = "testfiles/rest_out",
						compare_backups = 1):
	"""Test backing up/restoring of a series of directories

	The dirnames correspond to a single directory at different times.
	After each backup, the dest dir will be compared.  After the whole
	set, each of the earlier directories will be recovered to the
	restore_dirname and compared.

	"""
	Globals.set('preserve_hardlinks', compare_hardlinks)
	time = 10000
	dest_rp = rpath.RPath(Globals.local_connection, dest_dirname)
	restore_rp = rpath.RPath(Globals.local_connection, restore_dirname)
	
	Myrm(dest_dirname)
	for dirname in list_of_dirnames:
		src_rp = rpath.RPath(Globals.local_connection, dirname)
		reset_hardlink_dicts()
		_reset_connections(src_rp, dest_rp)

		InternalBackup(source_local, dest_local, dirname, dest_dirname, time)
		time += 10000
		_reset_connections(src_rp, dest_rp)
		if compare_backups:
			assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)

	time = 10000
	for dirname in list_of_dirnames[:-1]:
		reset_hardlink_dicts()
		Myrm(restore_dirname)
		InternalRestore(dest_local, source_local, dest_dirname,
						restore_dirname, time)
		src_rp = rpath.RPath(Globals.local_connection, dirname)
		assert CompareRecursive(src_rp, restore_rp)

		# Restore should default back to newest time older than it
		# with a backup then.
		if time == 20000: time = 21000

		time += 10000

def MirrorTest(source_local, dest_local, list_of_dirnames,
			   compare_hardlinks = 1,
			   dest_dirname = "testfiles/output"):
	"""Mirror each of list_of_dirnames, and compare after each"""
	Globals.set('preserve_hardlinks', compare_hardlinks)
	dest_rp = rpath.RPath(Globals.local_connection, dest_dirname)

	Myrm(dest_dirname)
	for dirname in list_of_dirnames:
		src_rp = rpath.RPath(Globals.local_connection, dirname)
		reset_hardlink_dicts()
		_reset_connections(src_rp, dest_rp)

		InternalMirror(source_local, dest_local, dirname, dest_dirname)
		_reset_connections(src_rp, dest_rp)
		assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)