diff options
author | ben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2002-03-21 07:34:29 +0000 |
---|---|---|
committer | ben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109> | 2002-03-21 07:34:29 +0000 |
commit | a2e3c38d72877dd9142d802e76047e10cf490e19 (patch) | |
tree | fd912dd37d0afe96adf760606d65f7b302c0678e /rdiff-backup/rdiff_backup | |
parent | 8c37a5bdfdd46d5cfad6e9d67925ddef9ca382bf (diff) | |
download | rdiff-backup-a2e3c38d72877dd9142d802e76047e10cf490e19.tar.gz |
Added hardlink support, refactored some test cases
git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@7 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
Diffstat (limited to 'rdiff-backup/rdiff_backup')
-rw-r--r-- | rdiff-backup/rdiff_backup/connection.py | 13 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/highlevel.py | 59 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/increment.py | 9 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/log.py | 2 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/restore.py | 23 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/robust.py | 21 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/rorpiter.py | 18 | ||||
-rw-r--r-- | rdiff-backup/rdiff_backup/rpath.py | 73 |
8 files changed, 159 insertions, 59 deletions
diff --git a/rdiff-backup/rdiff_backup/connection.py b/rdiff-backup/rdiff_backup/connection.py index 83fc874..4c87979 100644 --- a/rdiff-backup/rdiff_backup/connection.py +++ b/rdiff-backup/rdiff_backup/connection.py @@ -6,11 +6,9 @@ import types, os, tempfile, cPickle, shutil, traceback # connection - Code that deals with remote execution # -class ConnectionError(Exception): - pass +class ConnectionError(Exception): pass -class ConnectionQuit(Exception): - pass +class ConnectionQuit(Exception): pass class Connection: @@ -433,6 +431,10 @@ class VirtualFile: return cls.vfiles[id].read(length) readfromid = classmethod(readfromid) + def readlinefromid(cls, id): + return cls.vfiles[id].readline() + readlinefromid = classmethod(readlinefromid) + def writetoid(cls, id, buffer): return cls.vfiles[id].write(buffer) writetoid = classmethod(writetoid) @@ -460,6 +462,9 @@ class VirtualFile: def read(self, length = -1): return self.connection.VirtualFile.readfromid(self.id, length) + def readline(self): + return self.connection.VirtualFile.readlinefromid(self.id) + def write(self, buf): return self.connection.VirtualFile.writetoid(self.id, buf) diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index 55fe007..7603c21 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -1,5 +1,5 @@ from __future__ import generators -execfile("filelist.py") +execfile("manage.py") ####################################################################### # @@ -61,12 +61,19 @@ class HighLevel: dest_rpath.setdata() inc_rpath.setdata() - def Restore(rest_time, mirror_base, baseinc_tup, target_base): + def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base): """Like Restore.RestoreRecursive but check arguments""" + if (Globals.preserve_hardlinks != 0 and + Hardlink.retrieve_final(rest_time)): + Log("Hard link information found, attempting to preserve " + "hard links.", 4) + SetConnections.UpdateGlobal('preserve_hardlinks', 1) + else: SetConnections.UpdateGlobal('preserve_hardlinks', None) + if not isinstance(target_base, DSRPath): target_base = DSRPath(target_base.conn, target_base.base, target_base.index, target_base.data) - Restore.RestoreRecursive(rest_time, mirror_base, + Restore.RestoreRecursive(rest_time, mirror_base, rel_index, baseinc_tup, target_base) MakeStatic(HighLevel) @@ -154,27 +161,38 @@ class HLDestinationStruct: """ collated = RORPIter.CollateIterators(src_init_iter, dest_init_iter) + def compare(src_rorp, dest_dsrp): + """Return dest_dsrp if they are different, None if the same""" + if not dest_dsrp: + dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) + if dest_dsrp.lstat(): + Log("Warning: Found unexpected destination file %s, " + "not processing it." % dest_dsrp.path, 2) + return None + elif (src_rorp and src_rorp == dest_dsrp and + (not Globals.preserve_hardlinks or + Hardlink.rorp_eq(src_rorp, dest_dsrp))): + return None + if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp): + dest_dsrp.flaglinked() + return dest_dsrp + def generate_dissimilar(): counter = 0 for src_rorp, dest_dsrp in collated: - if not dest_dsrp: - dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) - if dsrp.lstat(): - Log("Warning: Found unexpected destination file %s." - % dsrp.path, 2) - if DestructiveStepping.isexcluded(dsrp, None): continue + if Globals.preserve_hardlinks: + if src_rorp: Hardlink.add_rorp(src_rorp, 1) + if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None) + dsrp = compare(src_rorp, dest_dsrp) + if dsrp: counter = 0 yield dsrp - elif not src_rorp or not src_rorp == dest_dsrp: + elif counter == 20: + placeholder = RORPath(src_rorp.index) + placeholder.make_placeholder() counter = 0 - yield dest_dsrp - else: # source and destinition both exist and are same - if counter == 20: - placeholder = RORPath(src_rorp.index) - placeholder.make_placeholder() - counter = 0 - yield placeholder - else: counter += 1 + yield placeholder + else: counter += 1 return generate_dissimilar() def get_sigs(cls, baserp, src_init_iter): @@ -225,6 +243,8 @@ class HLDestinationStruct: if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) except: cls.handle_last_error(dsrp, finalizer) finalizer.getresult() + if Globals.preserve_hardlinks and Globals.rbdir: + Hardlink.final_writedata() if checkpoint: SaveState.checkpoint_remove() def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): @@ -258,6 +278,7 @@ class HLDestinationStruct: except: cls.handle_last_error(dsrp, finalizer, ITR) ITR.getresult() finalizer.getresult() + if Globals.preserve_hardlinks: Hardlink.final_writedata() SaveState.checkpoint_remove() def check_skip_error(cls, thunk): @@ -282,6 +303,8 @@ class HLDestinationStruct: Log.exception(1) if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) + if Globals.preserve_hardlinks: + Hardlink.final_checkpoint(Globals.rbdir) SaveState.touch_last_file_definitive() raise diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index 4ed6a39..a290d3c 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -141,11 +141,14 @@ class Inc: """ if diff_rorp: - if dsrp.isreg() and diff_rorp.isreg(): + if diff_rorp.isreg() and (dsrp.isreg() or + diff_rorp.isflaglinked()): tf = TempFileManager.new(dsrp) def init_thunk(): - Rdiff.patch_with_attribs_action(dsrp, diff_rorp, - tf).execute() + if diff_rorp.isflaglinked(): + Hardlink.link_rp(diff_rorp, tf, dsrp) + else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp, + tf).execute() Inc.Increment_action(tf, dsrp, incpref).execute() Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute() diff --git a/rdiff-backup/rdiff_backup/log.py b/rdiff-backup/rdiff_backup/log.py index 5416fd2..4605875 100644 --- a/rdiff-backup/rdiff_backup/log.py +++ b/rdiff-backup/rdiff_backup/log.py @@ -40,6 +40,7 @@ class Logger: write commands off to it. """ + assert not self.log_file_open for conn in Globals.connections: conn.Log.open_logfile_allconn(rpath.conn) rpath.conn.Log.open_logfile_local(rpath) @@ -71,6 +72,7 @@ class Logger: """Run by logging connection - close logfile""" assert self.log_file_conn is Globals.local_connection assert not self.logfp.close() + self.log_file_local = None def format(self, message, verbosity): """Format the message, possibly adding date information""" diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 1f7d24e..9c7a42a 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -10,11 +10,12 @@ import tempfile class RestoreError(Exception): pass class Restore: - def RestoreFile(rest_time, rpbase, inclist, rptarget): + def RestoreFile(rest_time, rpbase, mirror_rel_index, inclist, rptarget): """Non-recursive restore function rest_time is the time in seconds to restore to, rpbase is the base name of the file being restored, + mirror_rel_index is the same as in RestoreRecursive, inclist is a list of rpaths containing all the relevant increments, and rptarget is the rpath that will be written with the restored file. @@ -25,6 +26,12 @@ class Restore: Log("Restoring %s with increments %s to %s" % (rpbase and rpbase.path, Restore.inclist2str(inclist), rptarget.path), 5) + + if (Globals.preserve_hardlinks and + Hardlink.restore_link(mirror_rel_index, rptarget)): + RPath.copy_attribs(inclist and inclist[-1] or rpbase, rptarget) + return + if not inclist or inclist[0].getinctype() == "diff": assert rpbase and rpbase.lstat(), \ "No base to go with incs %s" % Restore.inclist2str(inclist) @@ -73,14 +80,23 @@ class Restore: else: raise RestoreError("Unknown inctype %s" % inctype) RPath.copy_attribs(inc, target) - def RestoreRecursive(rest_time, mirror_base, baseinc_tup, target_base): + def RestoreRecursive(rest_time, mirror_base, mirror_rel_index, + baseinc_tup, target_base): """Recursive restore function. rest_time is the time in seconds to restore to; + mirror_base is an rpath of the mirror directory corresponding to the one to be restored; + + mirror_rel_index is the index of the mirror_base relative to + the root of the mirror directory. (The mirror_base itself + always has index (), as its index must match that of + target_base.) + baseinc_tup is the inc tuple (incdir, list of incs) to be restored; + and target_base in the dsrp of the target directory. """ @@ -99,7 +115,8 @@ class Restore: inclist = inc_tup[1] target = target_base.new_index(inc_tup.index) DestructiveStepping.initialize(target, None) - Restore.RestoreFile(rest_time, mirror, inclist, target) + Restore.RestoreFile(rest_time, mirror, mirror_rel_index, + inclist, target) target_finalizer(target) if mirror: mirror_finalizer(mirror) target_finalizer.getresult() diff --git a/rdiff-backup/rdiff_backup/robust.py b/rdiff-backup/rdiff_backup/robust.py index c23ff6a..206e9d5 100644 --- a/rdiff-backup/rdiff_backup/robust.py +++ b/rdiff-backup/rdiff_backup/robust.py @@ -1,5 +1,5 @@ import tempfile -execfile("rpath.py") +execfile("hardlink.py") ####################################################################### # @@ -258,6 +258,16 @@ class TempFile(RPath): rp_dest.chmod(self.getperms()) self.chmod(0700) RPathStatic.rename(self, rp_dest) + + # Sometimes this just seems to fail silently, as in one + # hardlinked twin is moved over the other. So check to make + # sure below. + self.setdata() + if self.lstat(): + rp_dest.delete() + RPathStatic.rename(self, rp_dest) + self.setdata() + if self.lstat(): raise OSError("Cannot rename tmp file correctly") TempFileManager.remove_listing(self) def delete(self): @@ -283,7 +293,8 @@ class SaveState: return Globals.backup_writer.SaveState.init_filenames(incrementing) assert Globals.local_connection is Globals.rbdir.conn, \ - Globals.rbdir.conn + (Globals.rbdir.conn, Globals.backup_writer) + if incrementing: cls._last_file_sym = Globals.rbdir.append( "last-file-incremented.%s.snapshot" % Time.curtimestr) else: cls._last_file_sym = Globals.rbdir.append( @@ -362,6 +373,7 @@ class SaveState: def checkpoint_remove(cls): """Remove all checkpointing data after successful operation""" for rp in Resume.get_relevant_rps(): rp.delete() + if Globals.preserve_hardlinks: Hardlink.remove_all_checkpoints() MakeClass(SaveState) @@ -506,6 +518,11 @@ class Resume: Log("Resuming aborted backup dated %s" % Time.timetopretty(si.time), 2) Time.setcurtime(si.time) + if Globals.preserve_hardlinks: + if (not si.last_definitive or not + Hardlink.retrieve_checkpoint(Globals.rbdir, si.time)): + Log("Hardlink information not successfully " + "recovered.", 2) return si else: Log("Last backup dated %s was aborted, but we aren't " diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py index 5740ef8..e98fa13 100644 --- a/rdiff-backup/rdiff_backup/rorpiter.py +++ b/rdiff-backup/rdiff_backup/rorpiter.py @@ -61,7 +61,9 @@ class RORPIter: if rp.isplaceholder(): yield rp else: rorp = rp.getRORPath() - if rp.isreg(): rorp.setfile(Rdiff.get_signature(rp)) + if rp.isreg(): + if rp.isflaglinked(): rorp.flaglinked() + else: rorp.setfile(Rdiff.get_signature(rp)) yield rorp def GetSignatureIter(base_rp): @@ -172,7 +174,12 @@ class RORPIter: def diffonce(sig_rorp, new_rp): """Return one diff rorp, based from signature rorp and orig rp""" - if sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg(): + if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked(): + if new_rp: diff_rorp = new_rp.getRORPath() + else: diff_rorp = RORPath(sig_rorp.index) + diff_rorp.flaglinked() + return diff_rorp + elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg(): diff_rorp = new_rp.getRORPath() diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"), new_rp)) @@ -201,7 +208,12 @@ class RORPIter: if not diff_rorp.lstat(): return RobustAction(lambda: None, basisrp.delete, lambda e: None) - if basisrp and basisrp.isreg() and diff_rorp.isreg(): + if Globals.preserve_hardlinks and diff_rorp.isflaglinked(): + if not basisrp: basisrp = base_rp.new_index(diff_rorp.index) + return RobustAction(lambda: None, + lambda: Hardlink.link_rp(diff_rorp, basisrp), + lambda e: None) + elif basisrp and basisrp.isreg() and diff_rorp.isreg(): assert diff_rorp.get_attached_filetype() == 'diff' return Rdiff.patch_with_attribs_action(basisrp, diff_rorp) else: # Diff contains whole file, just copy it over diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 4e6cc8f..6f81347 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -179,23 +179,6 @@ class RPathStatic: try: return tuple(os.lstat(filename)) except os.error: return None - def cmp_recursive(rp1, rp2): - """True if rp1 and rp2 are at the base of same directories - - Includes only attributes, no file data. This function may not - be used in rdiff-backup but it comes in handy in the unit - tests. - - """ - rp1.setdata() - rp2.setdata() - dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer, - [rp1, rp2], [1, None]) - result = Iter.equal(dsiter1, dsiter2, 1) - for i in dsiter1: pass # make sure all files processed anyway - for i in dsiter2: pass - return result - MakeStatic(RPathStatic) @@ -215,15 +198,20 @@ class RORPath(RPathStatic): self.file = None def __eq__(self, other): - """Signal two files equivalent""" - if not Globals.change_ownership or self.issym() and other.issym(): - # Don't take file ownership into account when comparing - data1, data2 = self.data.copy(), other.data.copy() - for d in (data1, data2): - for key in ('uid', 'gid'): - if d.has_key(key): del d[key] - return self.index == other.index and data1 == data2 - else: return self.index == other.index and self.data == other.data + """True iff the two rorpaths are equivalent""" + if self.index != other.index: return None + + for key in self.data.keys(): # compare dicts key by key + if ((key == 'uid' or key == 'gid') and + (not Globals.change_ownership or self.issym())): + # Don't compare gid/uid for symlinks or if not change_ownership + pass + elif key == 'devloc' or key == 'inode' or key == 'nlink': pass + elif (not other.data.has_key(key) or + self.data[key] != other.data[key]): return None + return 1 + + def __ne__(self, other): return not self.__eq__(other) def __str__(self): """Pretty print file statistics""" @@ -324,6 +312,18 @@ class RORPath(RPathStatic): """Return modification time in seconds""" return self.data['mtime'] + def getinode(self): + """Return inode number of file""" + return self.data['inode'] + + def getdevloc(self): + """Device number file resides on""" + return self.data['devloc'] + + def getnumlinks(self): + """Number of places inode is linked to""" + return self.data['nlink'] + def readlink(self): """Wrapper around os.readlink()""" return self.data['linkname'] @@ -352,6 +352,19 @@ class RORPath(RPathStatic): """Set the type of the attached file""" self.data['filetype'] = type + def isflaglinked(self): + """True if rorp is a signature/diff for a hardlink file + + This indicates that a file's data need not be transferred + because it is hardlinked on the remote side. + + """ + return self.data.has_key('linked') + + def flaglinked(self): + """Signal that rorp is a signature/diff for a hardlink file""" + self.data['linked'] = 1 + def open(self, mode): """Return file type object if any was given using self.setfile""" if mode != "rb": raise RPathException("Bad mode %s" % mode) @@ -447,6 +460,9 @@ class RPath(RORPath): data['perms'] = stat.S_IMODE(mode) data['uid'] = statblock[stat.ST_UID] data['gid'] = statblock[stat.ST_GID] + data['inode'] = statblock[stat.ST_INO] + data['devloc'] = statblock[stat.ST_DEV] + data['nlink'] = statblock[stat.ST_NLINK] if not (type == 'sym' or type == 'dev'): # mtimes on symlinks and dev files don't work consistently @@ -522,6 +538,11 @@ class RPath(RORPath): self.setdata() assert self.issym() + def hardlink(self, linkpath): + """Make self into a hardlink joined to linkpath""" + self.conn.os.link(linkpath, self.path) + self.setdata() + def mkfifo(self): """Make a fifo at self.path""" self.conn.os.mkfifo(self.path) |