From a2e3c38d72877dd9142d802e76047e10cf490e19 Mon Sep 17 00:00:00 2001 From: ben Date: Thu, 21 Mar 2002 07:34:29 +0000 Subject: Added hardlink support, refactored some test cases git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@7 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109 --- rdiff-backup/TODO | 8 ++ rdiff-backup/rdiff-backup.1 | 12 ++ rdiff-backup/rdiff_backup/connection.py | 13 +- rdiff-backup/rdiff_backup/highlevel.py | 59 +++++--- rdiff-backup/rdiff_backup/increment.py | 9 +- rdiff-backup/rdiff_backup/log.py | 2 + rdiff-backup/rdiff_backup/restore.py | 23 +++- rdiff-backup/rdiff_backup/robust.py | 21 ++- rdiff-backup/rdiff_backup/rorpiter.py | 18 ++- rdiff-backup/rdiff_backup/rpath.py | 73 ++++++---- rdiff-backup/src/Make | 7 +- rdiff-backup/src/connection.py | 13 +- rdiff-backup/src/globals.py | 11 ++ rdiff-backup/src/highlevel.py | 59 +++++--- rdiff-backup/src/increment.py | 9 +- rdiff-backup/src/log.py | 2 + rdiff-backup/src/main.py | 37 ++--- rdiff-backup/src/rdiff.py | 2 +- rdiff-backup/src/restore.py | 23 +++- rdiff-backup/src/robust.py | 21 ++- rdiff-backup/src/rorpiter.py | 18 ++- rdiff-backup/src/rpath.py | 73 ++++++---- rdiff-backup/testing/commontest.py | 231 +++++++++++++++++++++++++++++++- rdiff-backup/testing/finaltest.py | 16 +-- rdiff-backup/testing/highleveltest.py | 55 ++------ rdiff-backup/testing/regressiontest.py | 59 ++++---- rdiff-backup/testing/roottest.py | 4 +- rdiff-backup/testing/rorpitertest.py | 36 ++--- rdiff-backup/testing/server.py | 32 ++++- 29 files changed, 695 insertions(+), 251 deletions(-) diff --git a/rdiff-backup/TODO b/rdiff-backup/TODO index 510fca0..c380515 100644 --- a/rdiff-backup/TODO +++ b/rdiff-backup/TODO @@ -6,3 +6,11 @@ hardlinks Don't produce stack trace which looks like crash/include file name in logging stats + +Michael S. Muegel suggestion: "--char-translate source-char +replacement-string" for use between windows/unix conversions, e.g. ':' +to _colon_. Also distinguish new vs changed update in lvl 5 logging. + +CB: Log the filenames skipped (not just excluded) for various reasons + + diff --git a/rdiff-backup/rdiff-backup.1 b/rdiff-backup/rdiff-backup.1 index 1c4a589..1f26ba4 100644 --- a/rdiff-backup/rdiff-backup.1 +++ b/rdiff-backup/rdiff-backup.1 @@ -51,6 +51,11 @@ permissions and mtimes afterwards. This option controls every how many seconds rdiff-backup checkpoints its current status. The default is 20. .TP +.BI "--current-time " seconds +This option is useful mainly for testing. If set, rdiff-backup will +it for the current time instead of consulting the clock. The argument +is the number of seconds since the epoch. +.TP .BI "--exclude " regexp Exclude files matching regexp. This argument can be used multiple times. .TP @@ -67,6 +72,13 @@ automatically excluded. Authorize overwriting of a destination directory. rdiff-backup will generally tell you if it needs this. .TP +.BI --hard-links +Preserve hard links from source to mirror directories. No increment +files will themselves be hard linked, but a hard link database will be +written so that hard links from any dataset will be recreated if +originally present. If many hard linked files are present, this +option can drastically increase memory usage. +.TP .B "-l, --list-increments" List the number and date of partial incremental backups contained in the specified destination directory. diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 83fc874..4c87979 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -6,11 +6,9 @@ import types, os, tempfile, cPickle, shutil, traceback # connection - Code that deals with remote execution # -class ConnectionError(Exception): - pass +class ConnectionError(Exception): pass -class ConnectionQuit(Exception): - pass +class ConnectionQuit(Exception): pass class Connection: @@ -433,6 +431,10 @@ class VirtualFile: return cls.vfiles[id].read(length) readfromid = classmethod(readfromid) + def readlinefromid(cls, id): + return cls.vfiles[id].readline() + readlinefromid = classmethod(readlinefromid) + def writetoid(cls, id, buffer): return cls.vfiles[id].write(buffer) writetoid = classmethod(writetoid) @@ -460,6 +462,9 @@ class VirtualFile: def read(self, length = -1): return self.connection.VirtualFile.readfromid(self.id, length) + def readline(self): + return self.connection.VirtualFile.readlinefromid(self.id) + def write(self, buf): return self.connection.VirtualFile.writetoid(self.id, buf) diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index 55fe007..7603c21 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -1,5 +1,5 @@ from __future__ import generators -execfile("filelist.py") +execfile("manage.py") ####################################################################### # @@ -61,12 +61,19 @@ class HighLevel: dest_rpath.setdata() inc_rpath.setdata() - def Restore(rest_time, mirror_base, baseinc_tup, target_base): + def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base): """Like Restore.RestoreRecursive but check arguments""" + if (Globals.preserve_hardlinks != 0 and + Hardlink.retrieve_final(rest_time)): + Log("Hard link information found, attempting to preserve " + "hard links.", 4) + SetConnections.UpdateGlobal('preserve_hardlinks', 1) + else: SetConnections.UpdateGlobal('preserve_hardlinks', None) + if not isinstance(target_base, DSRPath): target_base = DSRPath(target_base.conn, target_base.base, target_base.index, target_base.data) - Restore.RestoreRecursive(rest_time, mirror_base, + Restore.RestoreRecursive(rest_time, mirror_base, rel_index, baseinc_tup, target_base) MakeStatic(HighLevel) @@ -154,27 +161,38 @@ class HLDestinationStruct: """ collated = RORPIter.CollateIterators(src_init_iter, dest_init_iter) + def compare(src_rorp, dest_dsrp): + """Return dest_dsrp if they are different, None if the same""" + if not dest_dsrp: + dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) + if dest_dsrp.lstat(): + Log("Warning: Found unexpected destination file %s, " + "not processing it." % dest_dsrp.path, 2) + return None + elif (src_rorp and src_rorp == dest_dsrp and + (not Globals.preserve_hardlinks or + Hardlink.rorp_eq(src_rorp, dest_dsrp))): + return None + if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp): + dest_dsrp.flaglinked() + return dest_dsrp + def generate_dissimilar(): counter = 0 for src_rorp, dest_dsrp in collated: - if not dest_dsrp: - dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) - if dsrp.lstat(): - Log("Warning: Found unexpected destination file %s." - % dsrp.path, 2) - if DestructiveStepping.isexcluded(dsrp, None): continue + if Globals.preserve_hardlinks: + if src_rorp: Hardlink.add_rorp(src_rorp, 1) + if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None) + dsrp = compare(src_rorp, dest_dsrp) + if dsrp: counter = 0 yield dsrp - elif not src_rorp or not src_rorp == dest_dsrp: + elif counter == 20: + placeholder = RORPath(src_rorp.index) + placeholder.make_placeholder() counter = 0 - yield dest_dsrp - else: # source and destinition both exist and are same - if counter == 20: - placeholder = RORPath(src_rorp.index) - placeholder.make_placeholder() - counter = 0 - yield placeholder - else: counter += 1 + yield placeholder + else: counter += 1 return generate_dissimilar() def get_sigs(cls, baserp, src_init_iter): @@ -225,6 +243,8 @@ class HLDestinationStruct: if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer) finalizer.getresult() + if Globals.preserve_hardlinks and Globals.rbdir: + Hardlink.final_writedata() if checkpoint: SaveState.checkpoint_remove() def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): @@ -258,6 +278,7 @@ class HLDestinationStruct: except: cls.handle_last_error(dsrp, finalizer, ITR) ITR.getresult() finalizer.getresult() + if Globals.preserve_hardlinks: Hardlink.final_writedata() SaveState.checkpoint_remove() def check_skip_error(cls, thunk): @@ -282,6 +303,8 @@ class HLDestinationStruct: Log.exception(1) if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) + if Globals.preserve_hardlinks: + Hardlink.final_checkpoint(Globals.rbdir) SaveState.touch_last_file_definitive() raise diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index 4ed6a39..a290d3c 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -141,11 +141,14 @@ class Inc: """ if diff_rorp: - if dsrp.isreg() and diff_rorp.isreg(): + if diff_rorp.isreg() and (dsrp.isreg() or + diff_rorp.isflaglinked()): tf = TempFileManager.new(dsrp) def init_thunk(): - Rdiff.patch_with_attribs_action(dsrp, diff_rorp, - tf).execute() + if diff_rorp.isflaglinked(): + Hardlink.link_rp(diff_rorp, tf, dsrp) + else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp, + tf).execute() Inc.Increment_action(tf, dsrp, incpref).execute() Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute() diff --git a/rdiff-backup/rdiff_backup/log.py b/rdiff-backup/rdiff_backup/log.py index 5416fd2..4605875 100644 --- a/rdiff-backup/rdiff_backup/log.py +++ b/rdiff-backup/rdiff_backup/log.py @@ -40,6 +40,7 @@ class Logger: write commands off to it. """ + assert not self.log_file_open for conn in Globals.connections: conn.Log.open_logfile_allconn(rpath.conn) rpath.conn.Log.open_logfile_local(rpath) @@ -71,6 +72,7 @@ class Logger: """Run by logging connection - close logfile""" assert self.log_file_conn is Globals.local_connection assert not self.logfp.close() + self.log_file_local = None def format(self, message, verbosity): """Format the message, possibly adding date information""" diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 1f7d24e..9c7a42a 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -10,11 +10,12 @@ import tempfile class RestoreError(Exception): pass class Restore: - def RestoreFile(rest_time, rpbase, inclist, rptarget): + def RestoreFile(rest_time, rpbase, mirror_rel_index, inclist, rptarget): """Non-recursive restore function rest_time is the time in seconds to restore to, rpbase is the base name of the file being restored, + mirror_rel_index is the same as in RestoreRecursive, inclist is a list of rpaths containing all the relevant increments, and rptarget is the rpath that will be written with the restored file. @@ -25,6 +26,12 @@ class Restore: Log("Restoring %s with increments %s to %s" % (rpbase and rpbase.path, Restore.inclist2str(inclist), rptarget.path), 5) + + if (Globals.preserve_hardlinks and + Hardlink.restore_link(mirror_rel_index, rptarget)): + RPath.copy_attribs(inclist and inclist[-1] or rpbase, rptarget) + return + if not inclist or inclist[0].getinctype() == "diff": assert rpbase and rpbase.lstat(), \ "No base to go with incs %s" % Restore.inclist2str(inclist) @@ -73,14 +80,23 @@ class Restore: else: raise RestoreError("Unknown inctype %s" % inctype) RPath.copy_attribs(inc, target) - def RestoreRecursive(rest_time, mirror_base, baseinc_tup, target_base): + def RestoreRecursive(rest_time, mirror_base, mirror_rel_index, + baseinc_tup, target_base): """Recursive restore function. rest_time is the time in seconds to restore to; + mirror_base is an rpath of the mirror directory corresponding to the one to be restored; + + mirror_rel_index is the index of the mirror_base relative to + the root of the mirror directory. (The mirror_base itself + always has index (), as its index must match that of + target_base.) + baseinc_tup is the inc tuple (incdir, list of incs) to be restored; + and target_base in the dsrp of the target directory. """ @@ -99,7 +115,8 @@ class Restore: inclist = inc_tup[1] target = target_base.new_index(inc_tup.index) DestructiveStepping.initialize(target, None) - Restore.RestoreFile(rest_time, mirror, inclist, target) + Restore.RestoreFile(rest_time, mirror, mirror_rel_index, + inclist, target) target_finalizer(target) if mirror: mirror_finalizer(mirror) target_finalizer.getresult() diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py index c23ff6a..206e9d5 100644 --- a/rdiff-backup/rdiff_backup/robust.py +++ b/rdiff-backup/rdiff_backup/robust.py @@ -1,5 +1,5 @@ import tempfile -execfile("rpath.py") +execfile("hardlink.py") ####################################################################### # @@ -258,6 +258,16 @@ class TempFile(RPath): rp_dest.chmod(self.getperms()) self.chmod(0700) RPathStatic.rename(self, rp_dest) + + # Sometimes this just seems to fail silently, as in one + # hardlinked twin is moved over the other. So check to make + # sure below. + self.setdata() + if self.lstat(): + rp_dest.delete() + RPathStatic.rename(self, rp_dest) + self.setdata() + if self.lstat(): raise OSError("Cannot rename tmp file correctly") TempFileManager.remove_listing(self) def delete(self): @@ -283,7 +293,8 @@ class SaveState: return Globals.backup_writer.SaveState.init_filenames(incrementing) assert Globals.local_connection is Globals.rbdir.conn, \ - Globals.rbdir.conn + (Globals.rbdir.conn, Globals.backup_writer) + if incrementing: cls._last_file_sym = Globals.rbdir.append( "last-file-incremented.%s.snapshot" % Time.curtimestr) else: cls._last_file_sym = Globals.rbdir.append( @@ -362,6 +373,7 @@ class SaveState: def checkpoint_remove(cls): """Remove all checkpointing data after successful operation""" for rp in Resume.get_relevant_rps(): rp.delete() + if Globals.preserve_hardlinks: Hardlink.remove_all_checkpoints() MakeClass(SaveState) @@ -506,6 +518,11 @@ class Resume: Log("Resuming aborted backup dated %s" % Time.timetopretty(si.time), 2) Time.setcurtime(si.time) + if Globals.preserve_hardlinks: + if (not si.last_definitive or not + Hardlink.retrieve_checkpoint(Globals.rbdir, si.time)): + Log("Hardlink information not successfully " + "recovered.", 2) return si else: Log("Last backup dated %s was aborted, but we aren't " diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py index 5740ef8..e98fa13 100644 --- a/rdiff-backup/rdiff_backup/rorpiter.py +++ b/rdiff-backup/rdiff_backup/rorpiter.py @@ -61,7 +61,9 @@ class RORPIter: if rp.isplaceholder(): yield rp else: rorp = rp.getRORPath() - if rp.isreg(): rorp.setfile(Rdiff.get_signature(rp)) + if rp.isreg(): + if rp.isflaglinked(): rorp.flaglinked() + else: rorp.setfile(Rdiff.get_signature(rp)) yield rorp def GetSignatureIter(base_rp): @@ -172,7 +174,12 @@ class RORPIter: def diffonce(sig_rorp, new_rp): """Return one diff rorp, based from signature rorp and orig rp""" - if sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg(): + if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked(): + if new_rp: diff_rorp = new_rp.getRORPath() + else: diff_rorp = RORPath(sig_rorp.index) + diff_rorp.flaglinked() + return diff_rorp + elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg(): diff_rorp = new_rp.getRORPath() diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"), new_rp)) @@ -201,7 +208,12 @@ class RORPIter: if not diff_rorp.lstat(): return RobustAction(lambda: None, basisrp.delete, lambda e: None) - if basisrp and basisrp.isreg() and diff_rorp.isreg(): + if Globals.preserve_hardlinks and diff_rorp.isflaglinked(): + if not basisrp: basisrp = base_rp.new_index(diff_rorp.index) + return RobustAction(lambda: None, + lambda: Hardlink.link_rp(diff_rorp, basisrp), + lambda e: None) + elif basisrp and basisrp.isreg() and diff_rorp.isreg(): assert diff_rorp.get_attached_filetype() == 'diff' return Rdiff.patch_with_attribs_action(basisrp, diff_rorp) else: # Diff contains whole file, just copy it over diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 4e6cc8f..6f81347 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -179,23 +179,6 @@ class RPathStatic: try: return tuple(os.lstat(filename)) except os.error: return None - def cmp_recursive(rp1, rp2): - """True if rp1 and rp2 are at the base of same directories - - Includes only attributes, no file data. This function may not - be used in rdiff-backup but it comes in handy in the unit - tests. - - """ - rp1.setdata() - rp2.setdata() - dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer, - [rp1, rp2], [1, None]) - result = Iter.equal(dsiter1, dsiter2, 1) - for i in dsiter1: pass # make sure all files processed anyway - for i in dsiter2: pass - return result - MakeStatic(RPathStatic) @@ -215,15 +198,20 @@ class RORPath(RPathStatic): self.file = None def __eq__(self, other): - """Signal two files equivalent""" - if not Globals.change_ownership or self.issym() and other.issym(): - # Don't take file ownership into account when comparing - data1, data2 = self.data.copy(), other.data.copy() - for d in (data1, data2): - for key in ('uid', 'gid'): - if d.has_key(key): del d[key] - return self.index == other.index and data1 == data2 - else: return self.index == other.index and self.data == other.data + """True iff the two rorpaths are equivalent""" + if self.index != other.index: return None + + for key in self.data.keys(): # compare dicts key by key + if ((key == 'uid' or key == 'gid') and + (not Globals.change_ownership or self.issym())): + # Don't compare gid/uid for symlinks or if not change_ownership + pass + elif key == 'devloc' or key == 'inode' or key == 'nlink': pass + elif (not other.data.has_key(key) or + self.data[key] != other.data[key]): return None + return 1 + + def __ne__(self, other): return not self.__eq__(other) def __str__(self): """Pretty print file statistics""" @@ -324,6 +312,18 @@ class RORPath(RPathStatic): """Return modification time in seconds""" return self.data['mtime'] + def getinode(self): + """Return inode number of file""" + return self.data['inode'] + + def getdevloc(self): + """Device number file resides on""" + return self.data['devloc'] + + def getnumlinks(self): + """Number of places inode is linked to""" + return self.data['nlink'] + def readlink(self): """Wrapper around os.readlink()""" return self.data['linkname'] @@ -352,6 +352,19 @@ class RORPath(RPathStatic): """Set the type of the attached file""" self.data['filetype'] = type + def isflaglinked(self): + """True if rorp is a signature/diff for a hardlink file + + This indicates that a file's data need not be transferred + because it is hardlinked on the remote side. + + """ + return self.data.has_key('linked') + + def flaglinked(self): + """Signal that rorp is a signature/diff for a hardlink file""" + self.data['linked'] = 1 + def open(self, mode): """Return file type object if any was given using self.setfile""" if mode != "rb": raise RPathException("Bad mode %s" % mode) @@ -447,6 +460,9 @@ class RPath(RORPath): data['perms'] = stat.S_IMODE(mode) data['uid'] = statblock[stat.ST_UID] data['gid'] = statblock[stat.ST_GID] + data['inode'] = statblock[stat.ST_INO] + data['devloc'] = statblock[stat.ST_DEV] + data['nlink'] = statblock[stat.ST_NLINK] if not (type == 'sym' or type == 'dev'): # mtimes on symlinks and dev files don't work consistently @@ -522,6 +538,11 @@ class RPath(RORPath): self.setdata() assert self.issym() + def hardlink(self, linkpath): + """Make self into a hardlink joined to linkpath""" + self.conn.os.link(linkpath, self.path) + self.setdata() + def mkfifo(self): """Make a fifo at self.path""" self.conn.os.mkfifo(self.path) diff --git a/rdiff-backup/src/Make b/rdiff-backup/src/Make index cadf9ea..cc7c69f 100755 --- a/rdiff-backup/src/Make +++ b/rdiff-backup/src/Make @@ -21,11 +21,10 @@ def mystrip(filename): files = ["globals.py", "static.py", "lazy.py", "log.py", "ttime.py", - "iterfile.py", "rlist.py", "rdiff.py", "connection.py", - "rpath.py", "robust.py", "rorpiter.py", + "iterfile.py", "rdiff.py", "connection.py", "rpath.py", + "hardlink.py", "robust.py", "rorpiter.py", "destructive_stepping.py", "increment.py", "restore.py", - "manage.py", "filelist.py", "highlevel.py", - "setconnections.py", "main.py"] + "manage.py", "highlevel.py", "setconnections.py", "main.py"] os.system("cp header.py rdiff-backup") diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py index 83fc874..4c87979 100644 --- a/rdiff-backup/src/connection.py +++ b/rdiff-backup/src/connection.py @@ -6,11 +6,9 @@ import types, os, tempfile, cPickle, shutil, traceback # connection - Code that deals with remote execution # -class ConnectionError(Exception): - pass +class ConnectionError(Exception): pass -class ConnectionQuit(Exception): - pass +class ConnectionQuit(Exception): pass class Connection: @@ -433,6 +431,10 @@ class VirtualFile: return cls.vfiles[id].read(length) readfromid = classmethod(readfromid) + def readlinefromid(cls, id): + return cls.vfiles[id].readline() + readlinefromid = classmethod(readlinefromid) + def writetoid(cls, id, buffer): return cls.vfiles[id].write(buffer) writetoid = classmethod(writetoid) @@ -460,6 +462,9 @@ class VirtualFile: def read(self, length = -1): return self.connection.VirtualFile.readfromid(self.id, length) + def readline(self): + return self.connection.VirtualFile.readlinefromid(self.id) + def write(self, buf): return self.connection.VirtualFile.writetoid(self.id, buf) diff --git a/rdiff-backup/src/globals.py b/rdiff-backup/src/globals.py index d9cd64a..5511bf9 100644 --- a/rdiff-backup/src/globals.py +++ b/rdiff-backup/src/globals.py @@ -10,6 +10,10 @@ class Globals: # The current version of rdiff-backup version = "0.6.0" + # If this is set, use this value in seconds as the current time + # instead of reading it from the clock. + current_time = None + # This determines how many bytes to read at a time when copying blocksize = 32768 @@ -121,6 +125,13 @@ class Globals: # under MS windows NT. time_separator = ":" + # If true, then hardlinks will be preserved to mirror and recorded + # in the increments directory. There is also a difference here + # between None and 0. When restoring, None or 1 means to preserve + # hardlinks iff can find a hardlink dictionary. 0 means ignore + # hardlink information regardless. + preserve_hardlinks = 1 + def get(cls, name): """Return the value of something in this class""" return cls.__dict__[name] diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py index 55fe007..7603c21 100644 --- a/rdiff-backup/src/highlevel.py +++ b/rdiff-backup/src/highlevel.py @@ -1,5 +1,5 @@ from __future__ import generators -execfile("filelist.py") +execfile("manage.py") ####################################################################### # @@ -61,12 +61,19 @@ class HighLevel: dest_rpath.setdata() inc_rpath.setdata() - def Restore(rest_time, mirror_base, baseinc_tup, target_base): + def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base): """Like Restore.RestoreRecursive but check arguments""" + if (Globals.preserve_hardlinks != 0 and + Hardlink.retrieve_final(rest_time)): + Log("Hard link information found, attempting to preserve " + "hard links.", 4) + SetConnections.UpdateGlobal('preserve_hardlinks', 1) + else: SetConnections.UpdateGlobal('preserve_hardlinks', None) + if not isinstance(target_base, DSRPath): target_base = DSRPath(target_base.conn, target_base.base, target_base.index, target_base.data) - Restore.RestoreRecursive(rest_time, mirror_base, + Restore.RestoreRecursive(rest_time, mirror_base, rel_index, baseinc_tup, target_base) MakeStatic(HighLevel) @@ -154,27 +161,38 @@ class HLDestinationStruct: """ collated = RORPIter.CollateIterators(src_init_iter, dest_init_iter) + def compare(src_rorp, dest_dsrp): + """Return dest_dsrp if they are different, None if the same""" + if not dest_dsrp: + dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) + if dest_dsrp.lstat(): + Log("Warning: Found unexpected destination file %s, " + "not processing it." % dest_dsrp.path, 2) + return None + elif (src_rorp and src_rorp == dest_dsrp and + (not Globals.preserve_hardlinks or + Hardlink.rorp_eq(src_rorp, dest_dsrp))): + return None + if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp): + dest_dsrp.flaglinked() + return dest_dsrp + def generate_dissimilar(): counter = 0 for src_rorp, dest_dsrp in collated: - if not dest_dsrp: - dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) - if dsrp.lstat(): - Log("Warning: Found unexpected destination file %s." - % dsrp.path, 2) - if DestructiveStepping.isexcluded(dsrp, None): continue + if Globals.preserve_hardlinks: + if src_rorp: Hardlink.add_rorp(src_rorp, 1) + if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None) + dsrp = compare(src_rorp, dest_dsrp) + if dsrp: counter = 0 yield dsrp - elif not src_rorp or not src_rorp == dest_dsrp: + elif counter == 20: + placeholder = RORPath(src_rorp.index) + placeholder.make_placeholder() counter = 0 - yield dest_dsrp - else: # source and destinition both exist and are same - if counter == 20: - placeholder = RORPath(src_rorp.index) - placeholder.make_placeholder() - counter = 0 - yield placeholder - else: counter += 1 + yield placeholder + else: counter += 1 return generate_dissimilar() def get_sigs(cls, baserp, src_init_iter): @@ -225,6 +243,8 @@ class HLDestinationStruct: if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer) finalizer.getresult() + if Globals.preserve_hardlinks and Globals.rbdir: + Hardlink.final_writedata() if checkpoint: SaveState.checkpoint_remove() def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): @@ -258,6 +278,7 @@ class HLDestinationStruct: except: cls.handle_last_error(dsrp, finalizer, ITR) ITR.getresult() finalizer.getresult() + if Globals.preserve_hardlinks: Hardlink.final_writedata() SaveState.checkpoint_remove() def check_skip_error(cls, thunk): @@ -282,6 +303,8 @@ class HLDestinationStruct: Log.exception(1) if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) + if Globals.preserve_hardlinks: + Hardlink.final_checkpoint(Globals.rbdir) SaveState.touch_last_file_definitive() raise diff --git a/rdiff-backup/src/increment.py b/rdiff-backup/src/increment.py index 4ed6a39..a290d3c 100644 --- a/rdiff-backup/src/increment.py +++ b/rdiff-backup/src/increment.py @@ -141,11 +141,14 @@ class Inc: """ if diff_rorp: - if dsrp.isreg() and diff_rorp.isreg(): + if diff_rorp.isreg() and (dsrp.isreg() or + diff_rorp.isflaglinked()): tf = TempFileManager.new(dsrp) def init_thunk(): - Rdiff.patch_with_attribs_action(dsrp, diff_rorp, - tf).execute() + if diff_rorp.isflaglinked(): + Hardlink.link_rp(diff_rorp, tf, dsrp) + else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp, + tf).execute() Inc.Increment_action(tf, dsrp, incpref).execute() Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute() diff --git a/rdiff-backup/src/log.py b/rdiff-backup/src/log.py index 5416fd2..4605875 100644 --- a/rdiff-backup/src/log.py +++ b/rdiff-backup/src/log.py @@ -40,6 +40,7 @@ class Logger: write commands off to it. """ + assert not self.log_file_open for conn in Globals.connections: conn.Log.open_logfile_allconn(rpath.conn) rpath.conn.Log.open_logfile_local(rpath) @@ -71,6 +72,7 @@ class Logger: """Run by logging connection - close logfile""" assert self.log_file_conn is Globals.local_connection assert not self.logfp.close() + self.log_file_local = None def format(self, message, verbosity): """Format the message, possibly adding date information""" diff --git a/rdiff-backup/src/main.py b/rdiff-backup/src/main.py index 24455f6..2721f34 100755 --- a/rdiff-backup/src/main.py +++ b/rdiff-backup/src/main.py @@ -1,6 +1,6 @@ #!/usr/bin/python -execfile("highlevel.py") +execfile("setconnections.py") import getopt, sys, re ####################################################################### @@ -27,7 +27,7 @@ class Main: "include-from-stdin", "terminal-verbosity=", "exclude-device-files", "resume", "no-resume", "resume-window=", "windows-time-format", - "checkpoint-interval="]) + "checkpoint-interval=", "hard-links", "current-time="]) except getopt.error: self.commandline_error("Error parsing commandline options") @@ -37,12 +37,15 @@ class Main: Globals.set('change_source_perms', 1) elif opt == "--checkpoint-interval": Globals.set_integer('checkpoint_interval', arg) + elif opt == "--current-time": + Globals.set_integer('current_time', arg) elif opt == "--exclude": self.exclude_regstrs.append(arg) elif opt == "--exclude-device-files": Globals.set('exclude_device_files', 1) elif opt == "--exclude-mirror": self.exclude_mirror_regstrs.append(arg) elif opt == "--force": self.force = 1 + elif opt == "--hard-links": Globals.set('preserve_hardlinks', 1) elif opt == "--include-from-stdin": Globals.include_from_stdin = 1 elif opt == "-l" or opt == "--list-increments": self.action = "list-increments" @@ -167,7 +170,7 @@ rdiff-backup with the --force option if you want to mirror anyway.""" % """Backup, possibly incrementally, src_path to dest_path.""" SetConnections.BackupInitConnections(rpin.conn, rpout.conn) self.backup_init_dirs(rpin, rpout) - Time.setcurtime() + Time.setcurtime(Globals.current_time) RSI = Resume.ResumeCheck() if self.prevtime: Time.setprevtime(self.prevtime) @@ -213,6 +216,7 @@ rdiff-backup with the --force option.""" % rpout.path) if not self.datadir.lstat(): self.datadir.mkdir() Globals.add_regexp(self.datadir.path, 1) Globals.add_regexp(rpin.append("rdiff-backup-data").path, None) + if Log.verbosity > 0: Log.open_logfile(self.datadir.append("backup.log")) self.backup_warn_if_infinite_regress(rpin, rpout) @@ -271,10 +275,10 @@ went wrong during your last backup? Using """ + mirrorrps[-1].path, 2) Log("Starting Restore", 5) rpin, rpout = self.restore_check_paths(src_rp, dest_rp) inc_tup = self.restore_get_inctup(rpin) - mirror_base = self.restore_get_mirror(rpin) + mirror_base, mirror_rel_index = self.restore_get_mirror(rpin) rtime = Time.stringtotime(rpin.getinctime()) Log.open_logfile(self.datadir.append("restore.log")) - HighLevel.Restore(rtime, mirror_base, inc_tup, rpout) + HighLevel.Restore(rtime, mirror_base, mirror_rel_index, inc_tup, rpout) def restore_check_paths(self, rpin, rpout): """Check paths and return pair of corresponding rps""" @@ -306,7 +310,7 @@ Try restoring from an increment file (the filenames look like return IndexedTuple((), (incbase, inclist)) def restore_get_mirror(self, rpin): - """Return mirror file and set the data dir + """Return (mirror file, relative index) and set the data dir The idea here is to keep backing up on the path until we find something named "rdiff-backup-data". Then use that as a @@ -314,6 +318,9 @@ Try restoring from an increment file (the filenames look like increment file is pointed to in a funny way, using symlinks or somesuch. + The mirror file will have index (), so also return the index + relative to the rootrp. + """ pathcomps = os.path.join(rpin.conn.os.getcwd(), rpin.getincbase().path).split("/") @@ -323,7 +330,7 @@ Try restoring from an increment file (the filenames look like break else: Log.FatalError("Unable to find rdiff-backup-data dir") - self.datadir = datadirrp + Globals.rbdir = self.datadir = datadirrp Globals.add_regexp(self.datadir.path, 1) rootrp = RPath(rpin.conn, "/".join(pathcomps[:i])) if not rootrp.lstat(): @@ -332,14 +339,12 @@ Try restoring from an increment file (the filenames look like else: Log("Using root mirror %s" % rootrp.path, 6) from_datadir = pathcomps[i+1:] - if len(from_datadir) == 1: result = rootrp - elif len(from_datadir) > 1: - result = RPath(rootrp.conn, apply(os.path.join, - [rootrp.path] + from_datadir[1:])) - else: raise RestoreError("Problem finding mirror file") - - Log("Using mirror file %s" % result.path, 6) - return result + if not from_datadir: raise RestoreError("Problem finding mirror file") + rel_index = tuple(from_datadir[1:]) + mirrorrp = RPath(rootrp.conn, + apply(os.path.join, (rootrp.path,) + rel_index)) + Log("Using mirror file %s" % mirrorrp.path, 6) + return (mirrorrp, rel_index) def ListIncrements(self, rootrp): @@ -396,6 +401,6 @@ Try finding the increments first using --list-increments.""") -if __name__ == "__main__": +if __name__ == "__main__" and not globals().has_key('__no_execute__'): Globals.Main = Main() Globals.Main.Main() diff --git a/rdiff-backup/src/rdiff.py b/rdiff-backup/src/rdiff.py index c27d4f2..56ebeb5 100644 --- a/rdiff-backup/src/rdiff.py +++ b/rdiff-backup/src/rdiff.py @@ -1,4 +1,4 @@ -execfile("rlist.py") +execfile("iterfile.py") import os, popen2 ####################################################################### diff --git a/rdiff-backup/src/restore.py b/rdiff-backup/src/restore.py index 1f7d24e..9c7a42a 100644 --- a/rdiff-backup/src/restore.py +++ b/rdiff-backup/src/restore.py @@ -10,11 +10,12 @@ import tempfile class RestoreError(Exception): pass class Restore: - def RestoreFile(rest_time, rpbase, inclist, rptarget): + def RestoreFile(rest_time, rpbase, mirror_rel_index, inclist, rptarget): """Non-recursive restore function rest_time is the time in seconds to restore to, rpbase is the base name of the file being restored, + mirror_rel_index is the same as in RestoreRecursive, inclist is a list of rpaths containing all the relevant increments, and rptarget is the rpath that will be written with the restored file. @@ -25,6 +26,12 @@ class Restore: Log("Restoring %s with increments %s to %s" % (rpbase and rpbase.path, Restore.inclist2str(inclist), rptarget.path), 5) + + if (Globals.preserve_hardlinks and + Hardlink.restore_link(mirror_rel_index, rptarget)): + RPath.copy_attribs(inclist and inclist[-1] or rpbase, rptarget) + return + if not inclist or inclist[0].getinctype() == "diff": assert rpbase and rpbase.lstat(), \ "No base to go with incs %s" % Restore.inclist2str(inclist) @@ -73,14 +80,23 @@ class Restore: else: raise RestoreError("Unknown inctype %s" % inctype) RPath.copy_attribs(inc, target) - def RestoreRecursive(rest_time, mirror_base, baseinc_tup, target_base): + def RestoreRecursive(rest_time, mirror_base, mirror_rel_index, + baseinc_tup, target_base): """Recursive restore function. rest_time is the time in seconds to restore to; + mirror_base is an rpath of the mirror directory corresponding to the one to be restored; + + mirror_rel_index is the index of the mirror_base relative to + the root of the mirror directory. (The mirror_base itself + always has index (), as its index must match that of + target_base.) + baseinc_tup is the inc tuple (incdir, list of incs) to be restored; + and target_base in the dsrp of the target directory. """ @@ -99,7 +115,8 @@ class Restore: inclist = inc_tup[1] target = target_base.new_index(inc_tup.index) DestructiveStepping.initialize(target, None) - Restore.RestoreFile(rest_time, mirror, inclist, target) + Restore.RestoreFile(rest_time, mirror, mirror_rel_index, + inclist, target) target_finalizer(target) if mirror: mirror_finalizer(mirror) target_finalizer.getresult() diff --git a/rdiff-backup/src/robust.py b/rdiff-backup/src/robust.py index c23ff6a..206e9d5 100644 --- a/rdiff-backup/src/robust.py +++ b/rdiff-backup/src/robust.py @@ -1,5 +1,5 @@ import tempfile -execfile("rpath.py") +execfile("hardlink.py") ####################################################################### # @@ -258,6 +258,16 @@ class TempFile(RPath): rp_dest.chmod(self.getperms()) self.chmod(0700) RPathStatic.rename(self, rp_dest) + + # Sometimes this just seems to fail silently, as in one + # hardlinked twin is moved over the other. So check to make + # sure below. + self.setdata() + if self.lstat(): + rp_dest.delete() + RPathStatic.rename(self, rp_dest) + self.setdata() + if self.lstat(): raise OSError("Cannot rename tmp file correctly") TempFileManager.remove_listing(self) def delete(self): @@ -283,7 +293,8 @@ class SaveState: return Globals.backup_writer.SaveState.init_filenames(incrementing) assert Globals.local_connection is Globals.rbdir.conn, \ - Globals.rbdir.conn + (Globals.rbdir.conn, Globals.backup_writer) + if incrementing: cls._last_file_sym = Globals.rbdir.append( "last-file-incremented.%s.snapshot" % Time.curtimestr) else: cls._last_file_sym = Globals.rbdir.append( @@ -362,6 +373,7 @@ class SaveState: def checkpoint_remove(cls): """Remove all checkpointing data after successful operation""" for rp in Resume.get_relevant_rps(): rp.delete() + if Globals.preserve_hardlinks: Hardlink.remove_all_checkpoints() MakeClass(SaveState) @@ -506,6 +518,11 @@ class Resume: Log("Resuming aborted backup dated %s" % Time.timetopretty(si.time), 2) Time.setcurtime(si.time) + if Globals.preserve_hardlinks: + if (not si.last_definitive or not + Hardlink.retrieve_checkpoint(Globals.rbdir, si.time)): + Log("Hardlink information not successfully " + "recovered.", 2) return si else: Log("Last backup dated %s was aborted, but we aren't " diff --git a/rdiff-backup/src/rorpiter.py b/rdiff-backup/src/rorpiter.py index 5740ef8..e98fa13 100644 --- a/rdiff-backup/src/rorpiter.py +++ b/rdiff-backup/src/rorpiter.py @@ -61,7 +61,9 @@ class RORPIter: if rp.isplaceholder(): yield rp else: rorp = rp.getRORPath() - if rp.isreg(): rorp.setfile(Rdiff.get_signature(rp)) + if rp.isreg(): + if rp.isflaglinked(): rorp.flaglinked() + else: rorp.setfile(Rdiff.get_signature(rp)) yield rorp def GetSignatureIter(base_rp): @@ -172,7 +174,12 @@ class RORPIter: def diffonce(sig_rorp, new_rp): """Return one diff rorp, based from signature rorp and orig rp""" - if sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg(): + if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked(): + if new_rp: diff_rorp = new_rp.getRORPath() + else: diff_rorp = RORPath(sig_rorp.index) + diff_rorp.flaglinked() + return diff_rorp + elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg(): diff_rorp = new_rp.getRORPath() diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"), new_rp)) @@ -201,7 +208,12 @@ class RORPIter: if not diff_rorp.lstat(): return RobustAction(lambda: None, basisrp.delete, lambda e: None) - if basisrp and basisrp.isreg() and diff_rorp.isreg(): + if Globals.preserve_hardlinks and diff_rorp.isflaglinked(): + if not basisrp: basisrp = base_rp.new_index(diff_rorp.index) + return RobustAction(lambda: None, + lambda: Hardlink.link_rp(diff_rorp, basisrp), + lambda e: None) + elif basisrp and basisrp.isreg() and diff_rorp.isreg(): assert diff_rorp.get_attached_filetype() == 'diff' return Rdiff.patch_with_attribs_action(basisrp, diff_rorp) else: # Diff contains whole file, just copy it over diff --git a/rdiff-backup/src/rpath.py b/rdiff-backup/src/rpath.py index 4e6cc8f..6f81347 100644 --- a/rdiff-backup/src/rpath.py +++ b/rdiff-backup/src/rpath.py @@ -179,23 +179,6 @@ class RPathStatic: try: return tuple(os.lstat(filename)) except os.error: return None - def cmp_recursive(rp1, rp2): - """True if rp1 and rp2 are at the base of same directories - - Includes only attributes, no file data. This function may not - be used in rdiff-backup but it comes in handy in the unit - tests. - - """ - rp1.setdata() - rp2.setdata() - dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer, - [rp1, rp2], [1, None]) - result = Iter.equal(dsiter1, dsiter2, 1) - for i in dsiter1: pass # make sure all files processed anyway - for i in dsiter2: pass - return result - MakeStatic(RPathStatic) @@ -215,15 +198,20 @@ class RORPath(RPathStatic): self.file = None def __eq__(self, other): - """Signal two files equivalent""" - if not Globals.change_ownership or self.issym() and other.issym(): - # Don't take file ownership into account when comparing - data1, data2 = self.data.copy(), other.data.copy() - for d in (data1, data2): - for key in ('uid', 'gid'): - if d.has_key(key): del d[key] - return self.index == other.index and data1 == data2 - else: return self.index == other.index and self.data == other.data + """True iff the two rorpaths are equivalent""" + if self.index != other.index: return None + + for key in self.data.keys(): # compare dicts key by key + if ((key == 'uid' or key == 'gid') and + (not Globals.change_ownership or self.issym())): + # Don't compare gid/uid for symlinks or if not change_ownership + pass + elif key == 'devloc' or key == 'inode' or key == 'nlink': pass + elif (not other.data.has_key(key) or + self.data[key] != other.data[key]): return None + return 1 + + def __ne__(self, other): return not self.__eq__(other) def __str__(self): """Pretty print file statistics""" @@ -324,6 +312,18 @@ class RORPath(RPathStatic): """Return modification time in seconds""" return self.data['mtime'] + def getinode(self): + """Return inode number of file""" + return self.data['inode'] + + def getdevloc(self): + """Device number file resides on""" + return self.data['devloc'] + + def getnumlinks(self): + """Number of places inode is linked to""" + return self.data['nlink'] + def readlink(self): """Wrapper around os.readlink()""" return self.data['linkname'] @@ -352,6 +352,19 @@ class RORPath(RPathStatic): """Set the type of the attached file""" self.data['filetype'] = type + def isflaglinked(self): + """True if rorp is a signature/diff for a hardlink file + + This indicates that a file's data need not be transferred + because it is hardlinked on the remote side. + + """ + return self.data.has_key('linked') + + def flaglinked(self): + """Signal that rorp is a signature/diff for a hardlink file""" + self.data['linked'] = 1 + def open(self, mode): """Return file type object if any was given using self.setfile""" if mode != "rb": raise RPathException("Bad mode %s" % mode) @@ -447,6 +460,9 @@ class RPath(RORPath): data['perms'] = stat.S_IMODE(mode) data['uid'] = statblock[stat.ST_UID] data['gid'] = statblock[stat.ST_GID] + data['inode'] = statblock[stat.ST_INO] + data['devloc'] = statblock[stat.ST_DEV] + data['nlink'] = statblock[stat.ST_NLINK] if not (type == 'sym' or type == 'dev'): # mtimes on symlinks and dev files don't work consistently @@ -522,6 +538,11 @@ class RPath(RORPath): self.setdata() assert self.issym() + def hardlink(self, linkpath): + """Make self into a hardlink joined to linkpath""" + self.conn.os.link(linkpath, self.path) + self.setdata() + def mkfifo(self): """Make a fifo at self.path""" self.conn.os.mkfifo(self.path) diff --git a/rdiff-backup/testing/commontest.py b/rdiff-backup/testing/commontest.py index 5cd66d7..ac6f109 100644 --- a/rdiff-backup/testing/commontest.py +++ b/rdiff-backup/testing/commontest.py @@ -1,10 +1,11 @@ -"""commontest - Some functions and constants common to all test cases""" +"""commontest - Some functions and constants common to several test cases""" import os SourceDir = "../src" AbsCurdir = os.getcwd() # Absolute path name of current directory AbsTFdir = AbsCurdir+"/testfiles" MiscDir = "../misc" +__no_execute__ = 1 # Keeps the actual rdiff-backup program from running def rbexec(src_file): """Changes to the source directory, execfile src_file, return""" @@ -17,3 +18,231 @@ def Make(): os.chdir(SourceDir) os.system("python ./Make") os.chdir(AbsCurdir) + +def rdiff_backup(source_local, dest_local, src_dir, dest_dir, + current_time = None, extra_options = ""): + """Run rdiff-backup with the given options + + source_local and dest_local are boolean values. If either is + false, then rdiff-backup will be run pretending that src_dir and + dest_dir, respectively, are remote. The server process will be + run in directories test1 and test2/tmp respectively. + + src_dir and dest_dir are the source and destination + (mirror) directories, relative to the testing directory. + + If current time is true, add the --current-time option with the + given number of seconds. + + extra_options are just added to the command line. + + """ + if not source_local: + src_dir = ("cd test1; ../%s/rdiff-backup --server::../%s" % + (SourceDir, src_dir)) + if not dest_local: + dest_dir = ("test2/tmp; ../../%s/rdiff-backup --server::../../%s" % + (SourceDir, dest_dir)) + + cmdargs = [SourceDir + "/rdiff-backup", extra_options] + if not (source_local and dest_local): cmdargs.append("--remote-schema %s") + + if current_time: cmdargs.append("--current-time %s" % current_time) + + os.system(" ".join(cmdargs)) + +def InternalBackup(source_local, dest_local, src_dir, dest_dir, + current_time = None): + """Backup src to dest internally + + This is like rdiff_backup but instead of running a separate + rdiff-backup script, use the separate *.py files. This way the + script doesn't have to be rebuild constantly, and stacktraces have + correct line/file references. + + """ + Globals.current_time = current_time + #_reset_connections() + remote_schema = '%s' + + if not source_local: + src_dir = "cd test1; python ../server.py ../%s::../%s" % \ + (SourceDir, src_dir) + if not dest_local: + dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ + % (SourceDir, dest_dir) + + rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema) + _get_main().Backup(rpin, rpout) + _get_main().cleanup() + +def InternalMirror(source_local, dest_local, src_dir, dest_dir, + checkpointing = None): + """Mirror src to dest internally, like InternalBackup""" + remote_schema = '%s' + + if not source_local: + src_dir = "cd test1; python ../server.py ../%s::../%s" % \ + (SourceDir, src_dir) + if not dest_local: + dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ + % (SourceDir, dest_dir) + + rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema) + if not rpout.lstat(): rpout.mkdir() + if checkpointing: # rdiff-backup-data must exist to checkpoint + data_dir = rpout.append("rdiff-backup-data") + if not data_dir.lstat(): data_dir.mkdir() + Globals.add_regexp(data_dir.path, 1) + SetConnections.UpdateGlobal('rbdir', data_dir) + HighLevel.Mirror(rpin, rpout, checkpointing) + _get_main().cleanup() + +def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time): + """Restore mirror_dir to dest_dir at given time + + This will automatically find the increments.XXX.dir representing + the time specified. The mirror_dir and dest_dir are relative to + the testing directory and will be modified for remote trials. + + """ + remote_schema = '%s' + #_reset_connections() + if not mirror_local: + mirror_dir = "cd test1; python ../server.py ../%s::../%s" % \ + (SourceDir, mirror_dir) + if not dest_local: + dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \ + % (SourceDir, dest_dir) + + mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir], + remote_schema) + + def get_increment_rp(time): + """Return increment rp matching time""" + data_rp = mirror_rp.append("rdiff-backup-data") + for filename in data_rp.listdir(): + rp = data_rp.append(filename) + if (rp.isincfile() and rp.getincbase_str() == "increments" and + Time.stringtotime(rp.getinctime()) == time): + return rp + assert None, ("No increments.XXX.dir found in directory " + "%s with that time" % data_rp.path) + + _get_main().Restore(get_increment_rp(time), dest_rp) + _get_main().cleanup() + +def _reset_connections(src_rp, dest_rp): + """Reset some global connection information""" + Globals.isbackup_reader = Globals.isbackup_writer = None + #Globals.connections = [Globals.local_connection] + #Globals.connection_dict = {0: Globals.local_connection} + SetConnections.UpdateGlobal('rbdir', None) + SetConnections.UpdateGlobal('exclude_regexps', []) + SetConnections.UpdateGlobal('exclude_mirror_regexps', []) + Globals.add_regexp(dest_rp.append("rdiff-backup-data").path, 1) + Globals.add_regexp(src_rp.append("rdiff-backup-data").path, None) + +def _get_main(): + """Set Globals.Main if it doesn't exist, and return""" + try: return Globals.Main + except AttributeError: + Globals.Main = Main() + return Globals.Main + +def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1): + """Compare src_rp and dest_rp, which can be directories + + This only compares file attributes, not the actual data. This + will overwrite the hardlink dictionaries if compare_hardlinks is + specified. + + """ + if compare_hardlinks: reset_hardlink_dicts() + src_rp.setdata() + dest_rp.setdata() + + Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path, + compare_hardlinks), 3) + dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer, + [src_rp, dest_rp], [1, None]) + + def hardlink_equal(src_rorp, dest_rorp): + if src_rorp != dest_rorp: return None + if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1 + Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3) + Log("%s: %s" % (dest_rorp.index, + Hardlink.get_indicies(dest_rorp, None)), 3) + return None + + if compare_hardlinks: + dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1) + dsiter2 = Hardlink.add_rorp_iter(dsiter2, None) + result = Iter.equal(dsiter1, dsiter2, 1, hardlink_equal) + else: result = Iter.equal(dsiter1, dsiter2, 1) + + for i in dsiter1: pass # make sure all files processed anyway + for i in dsiter2: pass + return result + +def reset_hardlink_dicts(): + """Clear the hardlink dictionaries""" + Hardlink._src_inode_indicies = {} + Hardlink._src_index_indicies = {} + Hardlink._dest_inode_indicies = {} + Hardlink._dest_index_indicies = {} + +def BackupRestoreSeries(source_local, dest_local, list_of_dirnames, + compare_hardlinks = 1, + dest_dirname = "testfiles/output", + restore_dirname = "testfiles/rest_out"): + """Test backing up/restoring of a series of directories + + The dirnames correspond to a single directory at different times. + After each backup, the dest dir will be compared. After the whole + set, each of the earlier directories will be recovered to the + restore_dirname and compared. + + """ + Globals.set('preserve_hardlinks', compare_hardlinks) + time = 10000 + dest_rp = RPath(Globals.local_connection, dest_dirname) + restore_rp = RPath(Globals.local_connection, restore_dirname) + + os.system(MiscDir + "/myrm " + dest_dirname) + for dirname in list_of_dirnames: + src_rp = RPath(Globals.local_connection, dirname) + reset_hardlink_dicts() + _reset_connections(src_rp, dest_rp) + + InternalBackup(source_local, dest_local, dirname, dest_dirname, time) + time += 10000 + _reset_connections(src_rp, dest_rp) + assert CompareRecursive(src_rp, dest_rp, compare_hardlinks) + + time = 10000 + for dirname in list_of_dirnames[:-1]: + reset_hardlink_dicts() + os.system(MiscDir + "/myrm " + restore_dirname) + InternalRestore(dest_local, source_local, dest_dirname, + restore_dirname, time) + src_rp = RPath(Globals.local_connection, dirname) + assert CompareRecursive(src_rp, restore_rp) + time += 10000 + +def MirrorTest(source_local, dest_local, list_of_dirnames, + compare_hardlinks = 1, + dest_dirname = "testfiles/output"): + """Mirror each of list_of_dirnames, and compare after each""" + Globals.set('preserve_hardlinks', compare_hardlinks) + dest_rp = RPath(Globals.local_connection, dest_dirname) + + os.system(MiscDir + "/myrm " + dest_dirname) + for dirname in list_of_dirnames: + src_rp = RPath(Globals.local_connection, dirname) + reset_hardlink_dicts() + _reset_connections(src_rp, dest_rp) + + InternalMirror(source_local, dest_local, dirname, dest_dirname) + _reset_connections(src_rp, dest_rp) + assert CompareRecursive(src_rp, dest_rp, compare_hardlinks) diff --git a/rdiff-backup/testing/finaltest.py b/rdiff-backup/testing/finaltest.py index c92a7d1..a5ca5e1 100644 --- a/rdiff-backup/testing/finaltest.py +++ b/rdiff-backup/testing/finaltest.py @@ -73,22 +73,22 @@ class PathSetter(unittest.TestCase): # Backing up increment1 self.exec_rb('testfiles/increment1', 'testfiles/output') - assert RPathStatic.cmp_recursive(Local.inc1rp, Local.rpout) + assert CompareRecursive(Local.inc1rp, Local.rpout) time.sleep(1) # Backing up increment2 self.exec_rb('testfiles/increment2', 'testfiles/output') - assert RPathStatic.cmp_recursive(Local.inc2rp, Local.rpout) + assert CompareRecursive(Local.inc2rp, Local.rpout) time.sleep(1) # Backing up increment3 self.exec_rb('testfiles/increment3', 'testfiles/output') - assert RPathStatic.cmp_recursive(Local.inc3rp, Local.rpout) + assert CompareRecursive(Local.inc3rp, Local.rpout) time.sleep(1) # Backing up increment4 self.exec_rb('testfiles/increment4', 'testfiles/output') - assert RPathStatic.cmp_recursive(Local.inc4rp, Local.rpout) + assert CompareRecursive(Local.inc4rp, Local.rpout) # Getting restore rps inc_paths = self.getinc_paths("increments.", @@ -97,22 +97,22 @@ class PathSetter(unittest.TestCase): # Restoring increment1 self.exec_rb(inc_paths[0], 'testfiles/restoretarget1') - assert RPathStatic.cmp_recursive(Local.inc1rp, Local.rpout1) + assert CompareRecursive(Local.inc1rp, Local.rpout1) # Restoring increment2 self.exec_rb(inc_paths[1], 'testfiles/restoretarget2') - assert RPathStatic.cmp_recursive(Local.inc2rp, Local.rpout2) + assert CompareRecursive(Local.inc2rp, Local.rpout2) # Restoring increment3 self.exec_rb(inc_paths[2], 'testfiles/restoretarget3') - assert RPathStatic.cmp_recursive(Local.inc3rp, Local.rpout3) + assert CompareRecursive(Local.inc3rp, Local.rpout3) # Test restoration of a few random files vft_paths = self.getinc_paths("various_file_types.", "testfiles/output/rdiff-backup-data/increments") self.exec_rb(vft_paths[1], 'testfiles/vft_out') self.refresh(Local.vft_in, Local.vft_out) - assert RPathStatic.cmp_recursive(Local.vft_in, Local.vft_out) + assert CompareRecursive(Local.vft_in, Local.vft_out) timbar_paths = self.getinc_paths("timbar.pyc.", "testfiles/output/rdiff-backup-data/increments") diff --git a/rdiff-backup/testing/highleveltest.py b/rdiff-backup/testing/highleveltest.py index b1e6f8d..3bd10c3 100644 --- a/rdiff-backup/testing/highleveltest.py +++ b/rdiff-backup/testing/highleveltest.py @@ -1,7 +1,7 @@ import unittest execfile("commontest.py") -rbexec("setconnections.py") +rbexec("main.py") class RemoteMirrorTest(unittest.TestCase): @@ -10,66 +10,27 @@ class RemoteMirrorTest(unittest.TestCase): """Start server""" Log.setverbosity(7) Globals.change_source_perms = 1 - self.conn = SetConnections.init_connection("./server.py") - - self.inrp = RPath(Globals.local_connection, "testfiles/various_file_types") - self.outrp = RPath(self.conn, "testfiles/output") - self.rbdir = RPath(self.conn, "testfiles/output/rdiff-backup-data") - SetConnections.UpdateGlobal('rbdir', self.rbdir) - self.inc1 = RPath(Globals.local_connection, "testfiles/increment1") - self.inc2 = RPath(Globals.local_connection, "testfiles/increment2") - self.inc3 = RPath(Globals.local_connection, "testfiles/increment3") - self.inc4 = RPath(Globals.local_connection, "testfiles/increment4") - - SetConnections.BackupInitConnections(Globals.local_connection, - self.conn) SetConnections.UpdateGlobal('checkpoint_interval', 3) def testMirror(self): """Testing simple mirror""" - if self.outrp.lstat(): self.outrp.delete() - HighLevel.Mirror(self.inrp, self.outrp, None) - self.outrp.setdata() - assert RPath.cmp_recursive(self.inrp, self.outrp) + MirrorTest(None, None, ["testfiles/increment1"]) def testMirror2(self): """Test mirror with larger data set""" - if self.outrp.lstat(): self.outrp.delete() - for rp in [self.inc1, self.inc2, self.inc3, self.inc4]: - rp.setdata() - print "----------------- Starting ", rp.path - HighLevel.Mirror(rp, self.outrp, None) - #if rp is self.inc2: assert 0 - assert RPath.cmp_recursive(rp, self.outrp) - self.outrp.setdata() + MirrorTest(1, None, ['testfiles/increment1', 'testfiles/increment2', + 'testfiles/increment3', 'testfiles/increment4']) def testMirrorWithCheckpointing(self): """Like testMirror but this time checkpoint""" - if self.outrp.lstat(): self.outrp.delete() - self.outrp.mkdir() - self.rbdir.mkdir() - Globals.add_regexp("testfiles/output/rdiff-backup-data", 1) - Time.setcurtime() - SaveState.init_filenames(None) - HighLevel.Mirror(self.inrp, self.outrp, 1) - self.outrp.setdata() - assert RPath.cmp_recursive(self.inrp, self.outrp) + MirrorTest(None, None, ["testfiles/increment1"], 1) def testMirrorWithCheckpointing2(self): """Larger data set""" - if self.outrp.lstat(): os.system(MiscDir+"/myrm %s" % self.outrp.path) - self.outrp.setdata() - self.outrp.mkdir() - self.rbdir.mkdir() - Globals.add_regexp("testfiles/output/rdiff-backup-data", 1) - Time.setcurtime() - SaveState.init_filenames(None) - for rp in [self.inc1, self.inc2, self.inc3, self.inc4]: - print "----------------- Starting ", rp.path - HighLevel.Mirror(rp, self.outrp, 1) - assert RPath.cmp_recursive(rp, self.outrp) + MirrorTest(1, None, ['testfiles/increment1', 'testfiles/increment2', + 'testfiles/increment3', 'testfiles/increment4'], + 1) - def tearDown(self): SetConnections.CloseConnections() if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/regressiontest.py b/rdiff-backup/testing/regressiontest.py index 5d4d27e..354cbbb 100644 --- a/rdiff-backup/testing/regressiontest.py +++ b/rdiff-backup/testing/regressiontest.py @@ -1,7 +1,7 @@ import unittest, os execfile("commontest.py") -rbexec("setconnections.py") +rbexec("main.py") """Regression tests @@ -128,27 +128,28 @@ class PathSetter(unittest.TestCase): SetConnections.CloseConnections() -class IncrementTest(PathSetter): +class IncrementTest1(unittest.TestCase): + dirlist = ["testfiles/increment1", "testfiles/increment2", + "testfiles/increment3", "testfiles/increment4"] + def testLocalinc(self): """Test self.incrementing, and then restoring, local""" - self.setPathnames(None, None, None, None) - self.runtest() + BackupRestoreSeries(1, 1, self.dirlist) def test_remote_src(self): """Increment/Restore when source directory is remote""" - self.setPathnames('test1', '../', None, None) - self.runtest() + BackupRestoreSeries(None, 1, self.dirlist) def test_remote_dest(self): """Increment/Restore when target directory is remote""" - self.setPathnames(None, None, 'test2', '../') - self.runtest() + BackupRestoreSeries(1, None, self.dirlist) def test_remote_both(self): """Increment/Restore when both directories are remote""" - self.setPathnames('test1', '../', 'test2/tmp', '../../') - self.runtest() - + BackupRestoreSeries(None, None, self.dirlist) + + +class IncrementTest2(PathSetter): def OldtestRecoveryLocal(self): """Test to see if rdiff-backup can continue with bad increment""" os.system(MiscDir+'/myrm testfiles/recovery_out_backup') @@ -188,43 +189,43 @@ class IncrementTest(PathSetter): Time.setcurtime() SaveState.init_filenames(1) HighLevel.Mirror(self.inc1rp, self.rpout) - assert RPath.cmp_recursive(Local.inc1rp, Local.rpout) + assert CompareRecursive(Local.inc1rp, Local.rpout) Time.setcurtime() Time.setprevtime(999500000) HighLevel.Mirror_and_increment(self.inc2rp, self.rpout, self.rpout_inc) - assert RPath.cmp_recursive(Local.inc2rp, Local.rpout) + assert CompareRecursive(Local.inc2rp, Local.rpout) Time.setcurtime() Time.setprevtime(999510000) HighLevel.Mirror_and_increment(self.inc3rp, self.rpout, self.rpout_inc) - assert RPath.cmp_recursive(Local.inc3rp, Local.rpout) + assert CompareRecursive(Local.inc3rp, Local.rpout) Time.setcurtime() Time.setprevtime(999520000) HighLevel.Mirror_and_increment(self.inc4rp, self.rpout, self.rpout_inc) - assert RPath.cmp_recursive(Local.inc4rp, Local.rpout) + assert CompareRecursive(Local.inc4rp, Local.rpout) print "Restoring to self.inc4" HighLevel.Restore(999530000, self.rpout, self.get_inctup(), self.rpout4) - assert RPath.cmp_recursive(Local.inc4rp, Local.rpout4) + assert CompareRecursive(Local.inc4rp, Local.rpout4) print "Restoring to self.inc3" HighLevel.Restore(999520000, self.rpout, self.get_inctup(), self.rpout3) - assert RPath.cmp_recursive(Local.inc3rp, Local.rpout3) + assert CompareRecursive(Local.inc3rp, Local.rpout3) print "Restoring to self.inc2" HighLevel.Restore(999510000, self.rpout, self.get_inctup(), self.rpout2) - assert RPath.cmp_recursive(Local.inc2rp, Local.rpout2) + assert CompareRecursive(Local.inc2rp, Local.rpout2) print "Restoring to self.inc1" HighLevel.Restore(999500000, self.rpout, self.get_inctup(), self.rpout1) - assert RPath.cmp_recursive(Local.inc1rp, Local.rpout1) + assert CompareRecursive(Local.inc1rp, Local.rpout1) def get_inctup(self): """Return inc tuples as expected by Restore.RestoreRecursive @@ -288,7 +289,7 @@ class MirrorTest(PathSetter): SaveState.init_filenames(None) HighLevel.Mirror(self.noperms, self.noperms_out, None) # Can't compare because we don't have the permissions to do it right - #assert RPath.cmp_recursive(Local.noperms, Local.noperms_out) + #assert CompareRecursive(Local.noperms, Local.noperms_out) def testNopermsRemote(self): "No permissions mirroring (remote)" @@ -296,7 +297,7 @@ class MirrorTest(PathSetter): Time.setcurtime() SaveState.init_filenames(None) HighLevel.Mirror(self.noperms, self.noperms_out, checkpoint=None) - #assert RPath.cmp_recursive(Local.noperms, Local.noperms_out) + #assert CompareRecursive(Local.noperms, Local.noperms_out) def testPermSkipLocal(self): """Test to see if rdiff-backup will skip unreadable files""" @@ -330,7 +331,7 @@ class MirrorTest(PathSetter): self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # add uid/gid info HighLevel.Mirror(self.rootfiles, self.rootfiles_out) - assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out) + assert CompareRecursive(Local.rootfiles, Local.rootfiles_out) Globals.change_ownership = None self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # remove that info @@ -343,7 +344,7 @@ class MirrorTest(PathSetter): self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # add uid/gid info HighLevel.Mirror(self.rootfiles, self.rootfiles_out) - assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out) + assert CompareRecursive(Local.rootfiles, Local.rootfiles_out) for coon in Globals.connections: conn.Globals.set('change_ownership', None) self.refresh(self.rootfiles, self.rootfiles_out, @@ -356,11 +357,11 @@ class MirrorTest(PathSetter): self.refresh(self.rootfiles2, self.rootfiles_out2, Local.rootfiles2, Local.rootfiles_out2) # add uid/gid info HighLevel.Mirror(self.rootfiles2, self.rootfiles_out2) - assert RPath.cmp_recursive(Local.rootfiles2, Local.rootfiles_out2) + assert CompareRecursive(Local.rootfiles2, Local.rootfiles_out2) self.refresh(self.rootfiles2, self.rootfiles_out2, Local.rootfiles2, Local.rootfiles_out2) # remove that info HighLevel.Mirror(self.rootfiles21, self.rootfiles_out2) - assert RPath.cmp_recursive(Local.rootfiles21, Local.rootfiles_out2) + assert CompareRecursive(Local.rootfiles21, Local.rootfiles_out2) self.refresh(self.rootfiles21, self.rootfiles_out2, Local.rootfiles21, Local.rootfiles_out2) # remove that info Globals.change_source_perms = 1 @@ -387,12 +388,12 @@ class MirrorTest(PathSetter): SaveState.init_filenames(None) assert self.rbdir.lstat() HighLevel.Mirror(self.inc1rp, self.rpout) - assert RPath.cmp_recursive(Local.inc1rp, Local.rpout) + assert CompareRecursive(Local.inc1rp, Local.rpout) self.deleteoutput() HighLevel.Mirror(self.inc2rp, self.rpout) - assert RPath.cmp_recursive(Local.inc2rp, Local.rpout) + assert CompareRecursive(Local.inc2rp, Local.rpout) def run_partial_test(self): os.system("cp -a testfiles/increment3 testfiles/output") @@ -402,9 +403,9 @@ class MirrorTest(PathSetter): SaveState.init_filenames(None) HighLevel.Mirror(self.inc1rp, self.rpout) #RPath.copy_attribs(self.inc1rp, self.rpout) - assert RPath.cmp_recursive(Local.inc1rp, Local.rpout) + assert CompareRecursive(Local.inc1rp, Local.rpout) HighLevel.Mirror(self.inc2rp, self.rpout) - assert RPath.cmp_recursive(Local.inc2rp, Local.rpout) + assert CompareRecursive(Local.inc2rp, Local.rpout) if __name__ == "__main__": unittest.main() diff --git a/rdiff-backup/testing/roottest.py b/rdiff-backup/testing/roottest.py index 18a0afc..2de29f0 100644 --- a/rdiff-backup/testing/roottest.py +++ b/rdiff-backup/testing/roottest.py @@ -136,7 +136,7 @@ class MirrorTest(PathSetter): self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # add uid/gid info HighLevel.Mirror(self.rootfiles, self.rootfiles_out) - assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out) + assert CompareRecursive(Local.rootfiles, Local.rootfiles_out) Globals.change_ownership = None self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # remove that info @@ -151,7 +151,7 @@ class MirrorTest(PathSetter): self.refresh(self.rootfiles, self.rootfiles_out, Local.rootfiles, Local.rootfiles_out) # add uid/gid info HighLevel.Mirror(self.rootfiles, self.rootfiles_out) - assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out) + assert CompareRecursive(Local.rootfiles, Local.rootfiles_out) for coon in Globals.connections: conn.Globals.set('change_ownership', None) self.refresh(self.rootfiles, self.rootfiles_out, diff --git a/rdiff-backup/testing/rorpitertest.py b/rdiff-backup/testing/rorpitertest.py index b9d558f..28ac54b 100644 --- a/rdiff-backup/testing/rorpitertest.py +++ b/rdiff-backup/testing/rorpitertest.py @@ -62,25 +62,25 @@ class RORPIterTest(unittest.TestCase): RORPIter.PatchIter(self.output, RORPIter.FromFile(diff_file)) turninto(self.inc1rp) - assert self.compare_rps(self.output, self.inc1rp) + RPath.copy_attribs(self.inc1rp, self.output) # Update time + assert self.compare_no_times(self.inc1rp, self.output) turninto(self.inc2rp) - assert self.compare_rps(self.output, self.inc2rp) - - def compare_rps(self, rp1, rp2): - """True if rp1 and rp2 are equal in some sense""" - def RawIter(rp): - """Get raw iterator of file stats based an rp1""" - return RORPIter.ToRaw(Iter.map(lambda rp2: rp2.getRORPath(), - RORPIter.IterateRPaths(rp))) - ri1 = RawIter(rp1) - ri2 = RawIter(rp2) - try: - rorp1 = ri1.next() - rorp2 = ri2.next() - assert rorp1 == rorp2, "%s %s" % (rorp1, rorp2) - except StopIteration: pass - return 1 - # return Iter.equal(RawIter(rp1), RawIter(rp2)) + RPath.copy_attribs(self.inc2rp, self.output) + assert self.compare_no_times(self.inc2rp, self.output) + + def compare_no_times(self, src_rp, dest_rp): + """Compare but disregard directories attributes""" + dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer, + [src_rp, dest_rp], [1, None]) + + def equal(src_rorp, dest_rorp): + return ((src_rorp.isdir() and dest_rorp.isdir()) or + src_rorp == dest_rorp) + + result = Iter.equal(dsiter1, dsiter2, 1, equal) + for i in dsiter1: pass # make sure all files processed anyway + for i in dsiter2: pass + return result class IndexedTupleTest(unittest.TestCase): diff --git a/rdiff-backup/testing/server.py b/rdiff-backup/testing/server.py index b30c745..17e11ca 100755 --- a/rdiff-backup/testing/server.py +++ b/rdiff-backup/testing/server.py @@ -1,12 +1,30 @@ #!/usr/bin/env python -import sys -execfile("commontest.py") -rbexec("setconnections.py") +import sys, os -def Test_SetConnGlobals(conn, name, val): - """Used in unittesting - set one of specified connection's Global vars""" - conn.Globals.set(name, val) +__doc__ = """ -Log.setverbosity(9) +This starts an rdiff-backup server using the existing source files. +If not run from the source directory, the only argument should be +the directory the source files are in. +""" + +def print_usage(): + print "Usage: server.py [path to source files]", __doc__ + +if len(sys.argv) > 2: + print_usage() + sys.exit(1) + +try: + if len(sys.argv) == 2: + olddir = os.getcwd() + os.chdir(sys.argv[1]) + execfile("setconnections.py") + if len(sys.argv) == 2: os.chdir(olddir) +except (OSError, IOError): + print_usage() + raise + +#Log.setverbosity(9) PipeConnection(sys.stdin, sys.stdout).Server() -- cgit v1.2.1