1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
|
"""commontest - Some functions and constants common to several test cases"""
import os
SourceDir = "../src"
AbsCurdir = os.getcwd() # Absolute path name of current directory
AbsTFdir = AbsCurdir+"/testfiles"
MiscDir = "../misc"
__no_execute__ = 1 # Keeps the actual rdiff-backup program from running
def rbexec(src_file):
"""Changes to the source directory, execfile src_file, return"""
os.chdir(SourceDir)
execfile(src_file, globals())
os.chdir(AbsCurdir)
def Myrm(dirstring):
"""Run myrm on given directory string"""
assert not os.system("%s/myrm %s" % (MiscDir, dirstring))
def Make():
"""Make sure the rdiff-backup script in the source dir is up-to-date"""
os.chdir(SourceDir)
os.system("python ./Make")
os.chdir(AbsCurdir)
def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
current_time = None, extra_options = ""):
"""Run rdiff-backup with the given options
source_local and dest_local are boolean values. If either is
false, then rdiff-backup will be run pretending that src_dir and
dest_dir, respectively, are remote. The server process will be
run in directories test1 and test2/tmp respectively.
src_dir and dest_dir are the source and destination
(mirror) directories, relative to the testing directory.
If current time is true, add the --current-time option with the
given number of seconds.
extra_options are just added to the command line.
"""
if not source_local:
src_dir = ("cd test1; ../%s/rdiff-backup --server::../%s" %
(SourceDir, src_dir))
if not dest_local:
dest_dir = ("test2/tmp; ../../%s/rdiff-backup --server::../../%s" %
(SourceDir, dest_dir))
cmdargs = [SourceDir + "/rdiff-backup", extra_options]
if not (source_local and dest_local): cmdargs.append("--remote-schema %s")
if current_time: cmdargs.append("--current-time %s" % current_time)
os.system(" ".join(cmdargs))
def InternalBackup(source_local, dest_local, src_dir, dest_dir,
current_time = None):
"""Backup src to dest internally
This is like rdiff_backup but instead of running a separate
rdiff-backup script, use the separate *.py files. This way the
script doesn't have to be rebuild constantly, and stacktraces have
correct line/file references.
"""
Globals.current_time = current_time
#_reset_connections()
remote_schema = '%s'
if not source_local:
src_dir = "cd test1; python ../server.py ../%s::../%s" % \
(SourceDir, src_dir)
if not dest_local:
dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
% (SourceDir, dest_dir)
rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
_get_main().misc_setup([rpin, rpout])
_get_main().Backup(rpin, rpout)
_get_main().cleanup()
def InternalMirror(source_local, dest_local, src_dir, dest_dir,
checkpointing = None):
"""Mirror src to dest internally, like InternalBackup"""
remote_schema = '%s'
if not source_local:
src_dir = "cd test1; python ../server.py ../%s::../%s" % \
(SourceDir, src_dir)
if not dest_local:
dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
% (SourceDir, dest_dir)
rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
_get_main().misc_setup([rpin, rpout])
_get_main().backup_init_select(rpin, rpout)
if not rpout.lstat(): rpout.mkdir()
if checkpointing: # rdiff-backup-data must exist to checkpoint
data_dir = rpout.append("rdiff-backup-data")
if not data_dir.lstat(): data_dir.mkdir()
SetConnections.UpdateGlobal('rbdir', data_dir)
HighLevel.Mirror(rpin, rpout, checkpointing)
_get_main().cleanup()
def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
"""Restore mirror_dir to dest_dir at given time
This will automatically find the increments.XXX.dir representing
the time specified. The mirror_dir and dest_dir are relative to
the testing directory and will be modified for remote trials.
"""
remote_schema = '%s'
#_reset_connections()
if not mirror_local:
mirror_dir = "cd test1; python ../server.py ../%s::../%s" % \
(SourceDir, mirror_dir)
if not dest_local:
dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
% (SourceDir, dest_dir)
mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir],
remote_schema)
_get_main().Restore(get_increment_rp(mirror_rp, time), dest_rp)
_get_main().cleanup()
def get_increment_rp(mirror_rp, time):
"""Return increment rp matching time in seconds"""
data_rp = mirror_rp.append("rdiff-backup-data")
for filename in data_rp.listdir():
rp = data_rp.append(filename)
if (rp.isincfile() and rp.getincbase_str() == "increments" and
Time.stringtotime(rp.getinctime()) == time):
return rp
assert None, ("No increments.XXX.dir found in directory "
"%s with that time" % data_rp.path)
def _reset_connections(src_rp, dest_rp):
"""Reset some global connection information"""
Globals.isbackup_reader = Globals.isbackup_writer = None
#Globals.connections = [Globals.local_connection]
#Globals.connection_dict = {0: Globals.local_connection}
SetConnections.UpdateGlobal('rbdir', None)
_get_main().misc_setup([src_rp, dest_rp])
def _get_main():
"""Set Globals.Main if it doesn't exist, and return"""
try: return Globals.Main
except AttributeError:
Globals.Main = Main()
return Globals.Main
def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
equality_func = None):
"""Compare src_rp and dest_rp, which can be directories
This only compares file attributes, not the actual data. This
will overwrite the hardlink dictionaries if compare_hardlinks is
specified.
"""
if compare_hardlinks: reset_hardlink_dicts()
src_rp.setdata()
dest_rp.setdata()
Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
compare_hardlinks), 3)
src_select, dest_select = Select(src_rp, 1), Select(dest_rp, None)
src_select.parse_rbdir_exclude()
dest_select.parse_rbdir_exclude()
src_select.set_iter()
dest_select.set_iter()
dsiter1, dsiter2 = src_select.iterate_with_finalizer(), \
dest_select.iterate_with_finalizer()
def hardlink_equal(src_rorp, dest_rorp):
if src_rorp != dest_rorp: return None
if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
Log("%s: %s" % (dest_rorp.index,
Hardlink.get_indicies(dest_rorp, None)), 3)
return None
if compare_hardlinks:
dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1)
dsiter2 = Hardlink.add_rorp_iter(dsiter2, None)
result = Iter.equal(dsiter1, dsiter2, 1, hardlink_equal)
elif equality_func: result = Iter.equal(dsiter1, dsiter2, 1, equality_func)
else: result = Iter.equal(dsiter1, dsiter2, 1)
for i in dsiter1: pass # make sure all files processed anyway
for i in dsiter2: pass
return result
def reset_hardlink_dicts():
"""Clear the hardlink dictionaries"""
Hardlink._src_inode_indicies = {}
Hardlink._src_index_indicies = {}
Hardlink._dest_inode_indicies = {}
Hardlink._dest_index_indicies = {}
def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
compare_hardlinks = 1,
dest_dirname = "testfiles/output",
restore_dirname = "testfiles/rest_out"):
"""Test backing up/restoring of a series of directories
The dirnames correspond to a single directory at different times.
After each backup, the dest dir will be compared. After the whole
set, each of the earlier directories will be recovered to the
restore_dirname and compared.
"""
Globals.set('preserve_hardlinks', compare_hardlinks)
time = 10000
dest_rp = RPath(Globals.local_connection, dest_dirname)
restore_rp = RPath(Globals.local_connection, restore_dirname)
os.system(MiscDir + "/myrm " + dest_dirname)
for dirname in list_of_dirnames:
src_rp = RPath(Globals.local_connection, dirname)
reset_hardlink_dicts()
_reset_connections(src_rp, dest_rp)
InternalBackup(source_local, dest_local, dirname, dest_dirname, time)
time += 10000
_reset_connections(src_rp, dest_rp)
assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)
time = 10000
for dirname in list_of_dirnames[:-1]:
reset_hardlink_dicts()
os.system(MiscDir + "/myrm " + restore_dirname)
InternalRestore(dest_local, source_local, dest_dirname,
restore_dirname, time)
src_rp = RPath(Globals.local_connection, dirname)
assert CompareRecursive(src_rp, restore_rp)
time += 10000
def MirrorTest(source_local, dest_local, list_of_dirnames,
compare_hardlinks = 1,
dest_dirname = "testfiles/output"):
"""Mirror each of list_of_dirnames, and compare after each"""
Globals.set('preserve_hardlinks', compare_hardlinks)
dest_rp = RPath(Globals.local_connection, dest_dirname)
os.system(MiscDir + "/myrm " + dest_dirname)
for dirname in list_of_dirnames:
src_rp = RPath(Globals.local_connection, dirname)
reset_hardlink_dicts()
_reset_connections(src_rp, dest_rp)
InternalMirror(source_local, dest_local, dirname, dest_dirname)
_reset_connections(src_rp, dest_rp)
assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)
|