diff options
Diffstat (limited to 'rdiff-backup/rdiff_backup')
| -rw-r--r-- | rdiff-backup/rdiff_backup/Globals.py | 20 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/Hardlink.py | 81 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/Main.py | 54 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/Rdiff.py | 2 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/TempFile.py | 100 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/highlevel.py | 332 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/increment.py | 124 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/metadata.py | 19 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/restore.py | 9 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/rorpiter.py | 190 | ||||
| -rw-r--r-- | rdiff-backup/rdiff_backup/rpath.py | 45 |
11 files changed, 498 insertions, 478 deletions
diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py index 50271d3..bb4195c 100644 --- a/rdiff-backup/rdiff_backup/Globals.py +++ b/rdiff-backup/rdiff_backup/Globals.py @@ -150,9 +150,9 @@ ssh_compression = 1 # If true, print statistics after successful backup print_statistics = None -# On the reader and writer connections, the following will be -# replaced by the source and mirror Select objects respectively. -select_source, select_mirror = None, None +# On the writer connection, the following will be set to the mirror +# Select iterator. +select_mirror = None # On the backup writer connection, holds the root incrementing branch # object. Access is provided to increment error counts. @@ -246,18 +246,4 @@ def postset_regexp_local(name, re_string, flags): if flags: globals()[name] = re.compile(re_string, flags) else: globals()[name] = re.compile(re_string) -def set_select(source, Sel_Obj, rpath, tuplelist, quote_mode, *filelists): - """Initialize select object using tuplelist - - Note that each list in filelists must each be passed as - separate arguments, so each is recognized as a file by the - connection. Otherwise we will get an error because a list - containing files can't be pickled. - - """ - global select_source, select_mirror - sel = Sel_Obj(rpath, quote_mode) - sel.ParseArgs(tuplelist, filelists) - if source: select_source = sel - else: select_mirror = sel diff --git a/rdiff-backup/rdiff_backup/Hardlink.py b/rdiff-backup/rdiff_backup/Hardlink.py index ec375fd..6ff9fbd 100644 --- a/rdiff-backup/rdiff_backup/Hardlink.py +++ b/rdiff-backup/rdiff_backup/Hardlink.py @@ -36,18 +36,88 @@ import Globals, Time, TempFile, rpath, log, robust # In all of these lists of indicies are the values. The keys in # _inode_ ones are (inode, devloc) pairs. -_src_inode_indicies = {} -_dest_inode_indicies = {} +_src_inode_indicies = None +_dest_inode_indicies = None # The keys for these two are just indicies. They share values # with the earlier dictionaries. -_src_index_indicies = {} -_dest_index_indicies = {} +_src_index_indicies = None +_dest_index_indicies = None # When a linked file is restored, its path is added to this dict, # so it can be found when later paths being restored are linked to # it. -_restore_index_path = {} +_restore_index_path = None + +def initialize_dictionaries(): + """Set all the hard link dictionaries to empty""" + global _src_inode_indicies, _dest_inode_indicies + global _src_index_indicies, _dest_index_indicies, _restore_index_path + _src_inode_indicies = {} + _dest_inode_indicies = {} + _src_index_indicies = {} + _dest_index_indicies = {} + _restore_index_path = {} + +def clear_dictionaries(): + """Delete all dictionaries""" + global _src_inode_indicies, _dest_inode_indicies + global _src_index_indicies, _dest_index_indicies, _restore_index_path + _src_inode_indicies = _dest_inode_indicies = None + _src_index_indicies = _dest_index_indicies = _restore_index_path = None + + +# The keys of this dictionary are (inode, devloc) pairs on the source +# side. The values are (numlinks, index) pairs, where numlinks are +# the number of files currently linked to this spot, and index is the +# index of the first file so linked. +_src_inode_index_dict = {} +_dest_inode_index_dict = {} + + +#def rorp_eq(src_rorp, dest_rorp): +# """Return true if source and dest rorp are equal as far as hardlinking +# +# This also processes the src_rorp, adding it if necessary to the +# inode dictionary. +# +# """ +# if not src_rorp.isreg(): return 1 # only reg files can be hard linked +# if src_rorp.getnumlinks() == 1: return dest_rorp.getnumlinks() == 1 +# +# src_linked_index = process_rorp(src_rorp, _src_inode_index_dict) +# if dest_rorp.getnumlinks() == 1: return 0 +# dest_linked_index = process_rorp(dest_rorp, _dest_inode_index_dict) +# return src_linked_index == dest_linked_index + +def process_rorp(rorp, inode_dict): + """Add inode info and returns index src_rorp is linked to, or None""" + key_pair = (rorp.getinode(), rorp.getdevloc()) + try: num, linked_index = inode_dict[key_pair] + except KeyError: + inode_dict[key_pair] = (1, src_rorp.index) + return None + inode_dict[key_pair] = (num+1, linked_index) + + if num+1 == src_rorp.getnumlinks(): del _inode_index_dict[key_pair] + else: _inode_index_dict[key_pair] = (num+1, linked_index) + return linked_index + +def get_linked_index(src_rorp): + """Return the index a src_rorp is linked to, or None + + Also deletes the src_rorp's entry in the dictionary if we have + accumulated all the hard link references. + + """ + key_pair = (rorp.getinode(), rorp.getdevloc()) + try: num, linked_index = _src_inode_index_dict[key_pair] + except KeyError: return None + if num == src_rorp.getnumlinks(): + del _src_inode_index_dict[key_pair] + + + def get_inode_key(rorp): """Return rorp's key for _inode_ dictionaries""" @@ -100,7 +170,6 @@ def rorp_eq(src_rorp, dest_rorp): indicies. """ - if not src_rorp.index == dest_rorp.index: return None if (not src_rorp.isreg() or not dest_rorp.isreg() or src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1): return 1 # Hard links don't apply diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py index d6b977c..6ffa355 100644 --- a/rdiff-backup/rdiff_backup/Main.py +++ b/rdiff-backup/rdiff_backup/Main.py @@ -23,7 +23,8 @@ from __future__ import generators import getopt, sys, re, os from log import Log import Globals, Time, SetConnections, selection, robust, rpath, \ - manage, highlevel, connection, restore, FilenameMapping, Security + manage, highlevel, connection, restore, FilenameMapping, \ + Security, Hardlink action = None @@ -108,8 +109,7 @@ def parse_cmdlineoptions(arglist): elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0) elif opt == "--null-separator": Globals.set("null_separator", 1) elif opt == "--parsable-output": Globals.set('parsable_output', 1) - elif opt == "--print-statistics": - Globals.set('print_statistics', 1) + elif opt == "--print-statistics": Globals.set('print_statistics', 1) elif opt == "--quoting-char": Globals.set('quoting_char', arg) Globals.set('quoting_enabled', 1) @@ -195,13 +195,11 @@ def misc_setup(rps): Time.setcurtime(Globals.current_time) FilenameMapping.set_init_quote_vals() SetConnections.UpdateGlobal("client_conn", Globals.local_connection) - - # This is because I originally didn't think compiled regexps - # could be pickled, and so must be compiled on remote side. Globals.postset_regexp('no_compression_regexp', Globals.no_compression_regexp_string) - - for conn in Globals.connections: robust.install_signal_handlers() + for conn in Globals.connections: + conn.robust.install_signal_handlers() + conn.Hardlink.initialize_dictionaries() def take_action(rps): """Do whatever action says""" @@ -238,20 +236,18 @@ def Main(arglist): def Backup(rpin, rpout): """Backup, possibly incrementally, src_path to dest_path.""" SetConnections.BackupInitConnections(rpin.conn, rpout.conn) - backup_init_select(rpin, rpout) + backup_set_select(rpin) backup_init_dirs(rpin, rpout) if prevtime: Time.setprevtime(prevtime) - highlevel.HighLevel.Mirror_and_increment(rpin, rpout, incdir) - else: highlevel.HighLevel.Mirror(rpin, rpout, incdir) + highlevel.Mirror_and_increment(rpin, rpout, incdir) + else: highlevel.Mirror(rpin, rpout) rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout) -def backup_init_select(rpin, rpout): - """Create Select objects on source and dest connections""" - rpin.conn.Globals.set_select(1, selection.Select, - rpin, select_opts, None, *select_files) - rpout.conn.Globals.set_select(0, selection.Select, - rpout, select_mirror_opts, 1) +def backup_set_select(rpin): + """Create Select objects on source connection""" + rpin.conn.highlevel.HLSourceStruct.set_source_select(rpin, select_opts, + *select_files) def backup_init_dirs(rpin, rpout): """Make sure rpin and rpout are valid, init data dir and logging""" @@ -277,10 +273,14 @@ def backup_init_dirs(rpin, rpout): if rpout.isdir() and not rpout.listdir(): # rpout is empty dir rpout.chmod(0700) # just make sure permissions aren't too lax elif not datadir.lstat() and not force: Log.FatalError( -"""Destination directory %s exists, but does not look like a -rdiff-backup directory. Running rdiff-backup like this could mess up -what is currently in it. If you want to update or overwrite it, run -rdiff-backup with the --force option.""" % rpout.path) +"""Destination directory + +%s + +exists, but does not look like a rdiff-backup directory. Running +rdiff-backup like this could mess up what is currently in it. If you +want to update or overwrite it, run rdiff-backup with the --force +option.""" % rpout.path) if not rpout.lstat(): try: rpout.mkdir() @@ -410,9 +410,10 @@ def restore_init_select(rpin, rpout): the restore operation isn't. """ - Globals.set_select(1, selection.Select, rpin, select_mirror_opts, None) - Globals.set_select(0, selection.Select, - rpout, select_opts, None, *select_files) + restore._select_mirror = selection.Select(rpin) + restore._select_mirror.ParseArgs(select_mirror_opts, []) + restore._select_mirror.parse_rbdir_exclude() + restore._select_source = selection.Select(rpout) def restore_get_root(rpin): """Return (mirror root, index) and set the data dir @@ -540,7 +541,6 @@ def ListChangedSince(rp): root_rid = restore.RestoreIncrementData(index, inc_rpath, inc_list) for rid in get_rids_recursive(root_rid): if rid.inc_list: - if not rid.index: path = "." - else: path = "/".join(rid.index) - print "%-11s: %s" % (determineChangeType(rid.inc_list), path) + print "%-11s: %s" % (determineChangeType(rid.inc_list), + rid.get_indexpath()) diff --git a/rdiff-backup/rdiff_backup/Rdiff.py b/rdiff-backup/rdiff_backup/Rdiff.py index 23bfda3..39dcb8a 100644 --- a/rdiff-backup/rdiff_backup/Rdiff.py +++ b/rdiff-backup/rdiff_backup/Rdiff.py @@ -42,7 +42,7 @@ def get_delta_sigfileobj(sig_fileobj, rp_new): def get_delta_sigrp(rp_signature, rp_new): """Take signature rp and new rp, return delta file object""" Log("Getting delta of %s with signature %s" % - (rp_new.path, rp_signature.path), 7) + (rp_new.path, rp_signature.get_indexpath()), 7) return librsync.DeltaFile(rp_signature.open("rb"), rp_new.open("rb")) def write_delta_action(basis, new, delta, compress = None): diff --git a/rdiff-backup/rdiff_backup/TempFile.py b/rdiff-backup/rdiff_backup/TempFile.py new file mode 100644 index 0000000..2b18920 --- /dev/null +++ b/rdiff-backup/rdiff_backup/TempFile.py @@ -0,0 +1,100 @@ +# Copyright 2002 Ben Escoto +# +# This file is part of rdiff-backup. +# +# rdiff-backup is free software; you can redistribute it and/or modify +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# rdiff-backup is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with rdiff-backup; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +# USA + +"""Manage temp files""" + +import os +import Globals, rpath + +# This is a connection-specific list of temp files, to be cleaned +# up before rdiff-backup exits. +_tempfiles = [] + +# To make collisions less likely, this gets put in the file name +# and incremented whenever a new file is requested. +_tfindex = 0 + +def new(rp_base, same_dir = 1): + """Return new tempfile that isn't in use. + + If same_dir, tempfile will be in same directory as rp_base. + Otherwise, use tempfile module to get filename. + + """ + conn = rp_base.conn + if conn is not Globals.local_connection: + return conn.TempFile.new(rp_base, same_dir) + + def find_unused(conn, dir): + """Find an unused tempfile with connection conn in directory dir""" + global _tfindex, tempfiles + while 1: + if _tfindex > 100000000: + Log("Resetting index", 2) + _tfindex = 0 + tf = TempFile(conn, os.path.join(dir, + "rdiff-backup.tmp.%d" % _tfindex)) + _tfindex = _tfindex+1 + if not tf.lstat(): return tf + + if same_dir: tf = find_unused(conn, rp_base.dirsplit()[0]) + else: tf = TempFile(conn, tempfile.mktemp()) + _tempfiles.append(tf) + return tf + +def remove_listing(tempfile): + """Remove listing of tempfile""" + if Globals.local_connection is not tempfile.conn: + tempfile.conn.TempFile.remove_listing(tempfile) + elif tempfile in _tempfiles: _tempfiles.remove(tempfile) + +def delete_all(): + """Delete all remaining tempfiles""" + for tf in _tempfiles[:]: tf.delete() + + +class TempFile(rpath.RPath): + """Like an RPath, but keep track of which ones are still here""" + def rename(self, rp_dest): + """Rename temp file to permanent location, possibly overwriting""" + if not self.lstat(): # "Moving" empty file, so just delete + if rp_dest.lstat(): rp_dest.delete() + remove_listing(self) + return + + if self.isdir() and not rp_dest.isdir(): + # Cannot move a directory directly over another file + rp_dest.delete() + rpath.rename(self, rp_dest) + + # Sometimes this just seems to fail silently, as in one + # hardlinked twin is moved over the other. So check to make + # sure below. + self.setdata() + if self.lstat(): + rp_dest.delete() + rpath.rename(self, rp_dest) + self.setdata() + if self.lstat(): raise OSError("Cannot rename tmp file correctly") + remove_listing(self) + + def delete(self): + rpath.RPath.delete(self) + remove_listing(self) + diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py index f93388f..fc75099 100644 --- a/rdiff-backup/rdiff_backup/highlevel.py +++ b/rdiff-backup/rdiff_backup/highlevel.py @@ -20,246 +20,144 @@ """High level functions for mirroring, mirror & inc, etc.""" from __future__ import generators -import Globals, MiscStats, metadata, rorpiter, TempFile, \ - Hardlink, robust, increment, rpath, lazy, static, log +import Globals, MiscStats, metadata, rorpiter, TempFile, Hardlink, \ + robust, increment, rpath, lazy, static, log, selection, Time, Rdiff -class HighLevel: - """High level static functions +def Mirror(src_rpath, dest_rpath): + """Turn dest_rpath into a copy of src_rpath""" + SourceS = src_rpath.conn.highlevel.HLSourceStruct + DestS = dest_rpath.conn.highlevel.HLDestinationStruct - The design of some of these functions is represented on the - accompanying diagram. + source_rpiter = SourceS.get_source_select() + dest_sigiter = DestS.process_source_get_sigs(dest_rpath, + source_rpiter, 0) + source_diffiter = SourceS.get_diffs(src_rpath, dest_sigiter) + DestS.patch(dest_rpath, source_diffiter) - """ - def Mirror(src_rpath, dest_rpath, inc_rpath): - """Turn dest_rpath into a copy of src_rpath +def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath): + """Mirror + put increments in tree based at inc_rpath""" + SourceS = src_rpath.conn.highlevel.HLSourceStruct + DestS = dest_rpath.conn.highlevel.HLDestinationStruct - If inc_rpath is true, then this is the initial mirroring of an - incremental backup, so checkpoint and write to data_dir. - Otherwise only mirror and don't create any extra files. - - """ - SourceS = src_rpath.conn.highlevel.HLSourceStruct - DestS = dest_rpath.conn.highlevel.HLDestinationStruct - - src_init_dsiter = SourceS.split_initial_dsiter() - dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter) - diffiter = SourceS.get_diffs_and_finalize(dest_sigiter) - DestS.patch_w_datadir_writes(dest_rpath, diffiter, inc_rpath) - - dest_rpath.setdata() - - def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath, - session_info = None): - """Mirror + put increments in tree based at inc_rpath""" - SourceS = src_rpath.conn.highlevel.HLSourceStruct - DestS = dest_rpath.conn.highlevel.HLDestinationStruct - - src_init_dsiter = SourceS.split_initial_dsiter() - dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter) - diffiter = SourceS.get_diffs_and_finalize(dest_sigiter) - DestS.patch_increment_and_finalize(dest_rpath, diffiter, inc_rpath) - - dest_rpath.setdata() - inc_rpath.setdata() - -static.MakeStatic(HighLevel) + source_rpiter = SourceS.get_source_select() + dest_sigiter = DestS.process_source_get_sigs(dest_rpath, + source_rpiter, 1) + source_diffiter = SourceS.get_diffs(src_rpath, dest_sigiter) + DestS.patch_and_increment(dest_rpath, source_diffiter, inc_rpath) class HLSourceStruct: """Hold info used by HL on the source side""" - def split_initial_dsiter(cls): - """Set iterators of all dsrps from rpath, returning one""" - dsiter = Globals.select_source.set_iter() - initial_dsiter1, cls.initial_dsiter2 = lazy.Iter.multiplex(dsiter, 2) - return initial_dsiter1 + source_select = None # will be set to source Select iterator + def set_source_select(cls, rpath, tuplelist, *filelists): + """Initialize select object using tuplelist - def get_diffs_and_finalize(cls, sigiter): - """Return diffs and finalize any dsrp changes remaining - - Return a rorpiterator with files included of signatures of - dissimilar files. + Note that each list in filelists must each be passed as + separate arguments, so each is recognized as a file by the + connection. Otherwise we will get an error because a list + containing files can't be pickled. """ - collated = rorpiter.CollateIterators(cls.initial_dsiter2, sigiter) - def error_handler(exc, dest_sig, rp): - log.Log("Error %s producing a diff of %s" % - (exc, rp and rp.path), 2) - return None - - def diffs(): - for rp, dest_sig in collated: - if dest_sig: - if dest_sig.isplaceholder(): yield dest_sig - else: - diff = robust.check_common_error( - error_handler, rorpiter.diffonce, [dest_sig, rp]) - if diff: yield diff - return diffs() + sel = selection.Select(rpath) + sel.ParseArgs(tuplelist, filelists) + cls.source_select = sel.set_iter() + + def get_source_select(cls): + """Return source select iterator, set by set_source_select""" + return cls.source_select + + def get_diffs(cls, baserp, dest_sigiter): + """Return diffs of any files with signature in dest_sigiter""" + for dest_sig in dest_sigiter: + src_rp = baserp.new_index(dest_sig.index) + diff_rorp = src_rp.getRORPath() + if dest_sig.isflaglinked(): diff_rorp.flaglinked() + elif dest_sig.isreg() and src_rp.isreg(): + diff_rorp.setfile(Rdiff.get_delta_sigrp(dest_sig, src_rp)) + diff_rorp.set_attached_filetype('diff') + else: + diff_rorp.set_attached_filetype('snapshot') + if src_rp.isreg(): diff_rorp.setfile(src_rp.open("rb")) + yield diff_rorp static.MakeClass(HLSourceStruct) class HLDestinationStruct: """Hold info used by HL on the destination side""" - _session_info = None # set to si if resuming - def split_initial_dsiter(cls): - """Set initial_dsiters (iteration of all rps from rpath)""" - result, cls.initial_dsiter2 = \ - lazy.Iter.multiplex(Globals.select_mirror.set_iter(), 2) - return result - - def get_dissimilar(cls, baserp, src_init_iter, dest_init_iter): - """Get dissimilars - - Returns an iterator which enumerates the dsrps which are - different on the source and destination ends. The dsrps do - not necessarily exist on the destination end. - - Also, to prevent the system from getting backed up on the - remote end, if we don't get enough dissimilars, stick in a - placeholder every so often, like fiber. The more - placeholders, the more bandwidth used, but if there aren't - enough, lots of memory will be used because files will be - accumulating on the source side. How much will accumulate - will depend on the Globals.conn_bufsize value. + def get_dest_select(cls, rpath, use_metadata = 1): + """Return destination select rorpath iterator - """ - collated = rorpiter.CollateIterators(src_init_iter, dest_init_iter) - def compare(src_rorp, dest_dsrp): - """Return dest_dsrp if they are different, None if the same""" - if not dest_dsrp: - dest_dsrp = cls.get_dsrp(baserp, src_rorp.index) - if dest_dsrp.lstat(): - log.Log("Warning: Found unexpected destination file %s, " - "not processing it." % dest_dsrp.path, 2) - return None - elif (src_rorp and src_rorp == dest_dsrp and - (not Globals.preserve_hardlinks or - Hardlink.rorp_eq(src_rorp, dest_dsrp))): - return None - if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp): - dest_dsrp.flaglinked() - return dest_dsrp - - def generate_dissimilar(): - counter = 0 - for src_rorp, dest_dsrp in collated: - if Globals.preserve_hardlinks: - if src_rorp: Hardlink.add_rorp(src_rorp, 1) - if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None) - dsrp = compare(src_rorp, dest_dsrp) - if dsrp: - counter = 0 - yield dsrp - elif counter == 20: - placeholder = rpath.RORPath(src_rorp.index) - placeholder.make_placeholder() - counter = 0 - yield placeholder - else: counter += 1 - return generate_dissimilar() - - def get_sigs(cls, baserp, src_init_iter): - """Return signatures of all dissimilar files - - Also writes all metadata to the metadata file. + If metadata file doesn't exist, select all files on + destination except rdiff-backup-data directory. """ - dest_iters1 = cls.split_initial_dsiter() - def duplicate_with_write(src_init_iter): - """Return iterator but write metadata of what passes through""" - metadata.OpenMetadata() - for rorp in src_init_iter: - metadata.WriteMetadata(rorp) - yield rorp - metadata.CloseMetadata() - dup = duplicate_with_write(src_init_iter) - dissimilars = cls.get_dissimilar(baserp, dup, dest_iters1) - return rorpiter.Signatures(dissimilars) - - def get_dsrp(cls, dest_rpath, index): - """Return initialized rpath based on dest_rpath with given index""" - rp = rpath.RPath(dest_rpath.conn, dest_rpath.base, index) - if Globals.quoting_enabled: rp.quote_path() - return rp - - def get_finalizer(cls): - """Return finalizer, starting from session info if necessary""" - old_finalizer = cls._session_info and cls._session_info.finalizer - if old_finalizer: return old_finalizer - else: return rorpiter.IterTreeReducer( - rorpiter.DestructiveSteppingFinalizer, []) - - def get_ITR(cls, inc_rpath): - """Return ITR, starting from state if necessary""" - if cls._session_info and cls._session_info.ITR: - return cls._session_info.ITR - else: - iitr = rorpiter.IterTreeReducer(increment.IncrementITRB, - [inc_rpath]) - iitr.root_branch.override_changed() - Globals.ITRB = iitr.root_branch - iitr.root_branch.Errors = 0 - return iitr - - def get_MirrorITR(cls, inc_rpath): - """Return MirrorITR, starting from state if available""" - if cls._session_info and cls._session_info.ITR: - return cls._session_info.ITR - ITR = rorpiter.IterTreeReducer(increment.MirrorITRB, [inc_rpath]) - Globals.ITRB = ITR.root_branch - ITR.root_branch.Errors = 0 - return ITR - - def patch_and_finalize(cls, dest_rpath, diffs): - """Apply diffs and finalize""" - collated = rorpiter.CollateIterators(diffs, cls.initial_dsiter2) - #finalizer = cls.get_finalizer() - diff_rorp, rp = None, None - - def patch(diff_rorp, dsrp): - if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index) - if diff_rorp and not diff_rorp.isplaceholder(): - rorpiter.patchonce_action(None, dsrp, diff_rorp).execute() - return dsrp - - def error_handler(exc, diff_rorp, dsrp): - filename = dsrp and dsrp.path or os.path.join(*diff_rorp.index) - log.Log("Error: %s processing file %s" % (exc, filename), 2) - - for indexed_tuple in collated: - log.Log(lambda: "Processing %s" % str(indexed_tuple), 7) - diff_rorp, dsrp = indexed_tuple - dsrp = robust.check_common_error(error_handler, patch, - [diff_rorp, dsrp]) - #finalizer(dsrp.index, dsrp) - #finalizer.Finish() - - def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath): - """Apply diffs and finalize, with checkpointing and statistics""" - collated = rorpiter.CollateIterators(diffs, cls.initial_dsiter2) - #finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath) - finalizer, ITR = None, cls.get_MirrorITR(inc_rpath) - MiscStats.open_dir_stats_file() - dsrp, finished_dsrp = None, None + if use_metadata: + metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir, + Time.curtime) + if metadata_iter: return metadata_iter + log.Log("Warning: Metadata file not found.\n" + "Metadata will be read from filesystem.", 2) + + sel = selection.Select(rpath) + sel.parse_rbdir_exclude() + return sel.set_iter() + + def dest_iter_filter(cls, dest_iter): + """Destination rorps pass through this - record stats""" + for dest_rorp in dest_iter: + # XXX Statistics process + Hardlink.add_rorp(dest_rorp, source = 0) + yield dest_rorp + + def src_iter_filter(cls, source_iter): + """Source rorps pass through this - record stats, write metadata""" + metadata.OpenMetadata() + for src_rorp in source_iter: + Hardlink.add_rorp(src_rorp, source = 1) + metadata.WriteMetadata(src_rorp) + #XXXX Statistics process + yield src_rorp + metadata.CloseMetadata() + + def process_source_get_sigs(cls, baserp, source_iter, for_increment): + """Process the source rorpiter and return signatures of dest dir + + Write all metadata to file, then return signatures of any + destination files that have changed. for_increment should be + true if we are mirror+incrementing, and false if we are just + mirroring. - try: - for indexed_tuple in collated: - log.Log(lambda: "Processing %s" % str(indexed_tuple), 7) - diff_rorp, dsrp = indexed_tuple - if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index) - if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None - ITR(dsrp.index, diff_rorp, dsrp) - #finalizer(dsrp.index, dsrp) - finished_dsrp = dsrp - ITR.Finish() - #finalizer.Finish() - except: cls.handle_last_error(finished_dsrp, finalizer, ITR) + """ + source_iter = cls.src_iter_filter(source_iter) + dest_iter = cls.dest_iter_filter(cls.get_dest_select(baserp, + for_increment)) + for index in rorpiter.get_dissimilar_indicies(source_iter, dest_iter): + dest_rp = baserp.new_index(index) + dest_sig = dest_rp.getRORPath() + if Globals.preserve_hardlinks and Hardlink.islinked(dest_rp): + dest_sig.flaglinked() + elif dest_rp.isreg(): + dest_sig.setfile(Rdiff.get_signature(dest_rp)) + yield dest_sig + + def patch(cls, dest_rpath, source_diffiter): + """Patch dest_rpath with an rorpiter of diffs""" + ITR = rorpiter.IterTreeReducer(increment.PatchITRB, [dest_rpath]) + for diff in rorpiter.FillInIter(source_diffiter, dest_rpath): + ITR(diff.index, diff) + ITR.Finish() + dest_rpath.setdata() - if Globals.preserve_hardlinks: Hardlink.final_writedata() - MiscStats.close_dir_stats_file() - MiscStats.write_session_statistics(ITR.root_branch) + def patch_and_increment(cls, dest_rpath, source_diffiter, inc_rpath): + """Patch dest_rpath with rorpiter of diffs and write increments""" + ITR = rorpiter.IterTreeReducer(increment.IncrementITRB, + [dest_rpath, inc_rpath]) + for diff in rorpiter.FillInIter(source_diffiter, dest_rpath): + ITR(diff.index, diff) + ITR.Finish() + dest_rpath.setdata() def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath): """Apply diffs, write increment if necessary, and finalize""" diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py index 46afd42..a11dd6a 100644 --- a/rdiff-backup/rdiff_backup/increment.py +++ b/rdiff-backup/rdiff_backup/increment.py @@ -285,10 +285,128 @@ class IncrementITRB(statistics.ITRB): self.add_file_stats(branch) +class PatchITRB(statistics.ITRB): + """Patch an rpath with the given diff iters (use with IterTreeReducer) + + The main complication here involves directories. We have to + finish processing the directory after what's in the directory, as + the directory may have inappropriate permissions to alter the + contents or the dir's mtime could change as we change the + contents. + + """ + def __init__(self, basis_root_rp): + """Set basis_root_rp, the base of the tree to be incremented""" + self.basis_root_rp = basis_root_rp + assert basis_root_rp.conn is Globals.local_connection + #statistics.ITRB.__init__(self) + self.dir_replacement, self.dir_update = None, None + self.cached_rp = None + + def get_rp_from_root(self, index): + """Return RPath by adding index to self.basis_root_rp""" + if not self.cached_rp or self.cached_rp.index != index: + self.cached_rp = self.basis_root_rp.new_index(index) + return self.cached_rp + + def can_fast_process(self, index, diff_rorp): + """True if diff_rorp and mirror are not directories""" + rp = self.get_rp_from_root(index) + return not diff_rorp.isdir() and not rp.isdir() + + def fast_process(self, index, diff_rorp): + """Patch base_rp with diff_rorp (case where neither is directory)""" + rp = self.get_rp_from_root(index) + tf = TempFile.new(rp) + self.patch_to_temp(rp, diff_rorp, tf) + tf.rename(rp) + + def patch_to_temp(self, basis_rp, diff_rorp, new): + """Patch basis_rp, writing output in new, which doesn't exist yet""" + if diff_rorp.isflaglinked(): + Hardlink.link_rp(diff_rorp, new, self.basis_root_rp) + elif diff_rorp.get_attached_filetype() == 'snapshot': + rpath.copy(diff_rorp, new) + else: + assert diff_rorp.get_attached_filetype() == 'diff' + Rdiff.patch_local(basis_rp, diff_rorp, new) + if new.lstat(): rpath.copy_attribs(diff_rorp, new) + + def start_process(self, index, diff_rorp): + """Start processing directory - record information for later""" + base_rp = self.base_rp = self.get_rp_from_root(index) + assert diff_rorp.isdir() or base_rp.isdir() + if diff_rorp.isdir(): self.prepare_dir(diff_rorp, base_rp) + else: self.set_dir_replacement(diff_rorp, base_rp) + + def set_dir_replacement(self, diff_rorp, base_rp): + """Set self.dir_replacement, which holds data until done with dir + + This is used when base_rp is a dir, and diff_rorp is not. + + """ + assert diff_rorp.get_attached_filetype() == 'snapshot' + self.dir_replacement = TempFile.new(base_rp) + rpath.copy_with_attribs(diff_rorp, self.dir_replacement) + + def prepare_dir(self, diff_rorp, base_rp): + """Prepare base_rp to turn into a directory""" + self.dir_update = diff_rorp.getRORPath() # make copy in case changes + if not base_rp.isdir(): + if base_rp.lstat(): base_rp.delete() + base_rp.mkdir() + base_rp.chmod(0700) + + def end_process(self): + """Finish processing directory""" + if self.dir_update: + assert self.base_rp.isdir() + rpath.copy_attribs(self.dir_update, self.base_rp) + else: + assert self.dir_replacement and self.base_rp.isdir() + self.base_rp.rmdir() + self.dir_replacement.rename(self.base_rp) + + +class IncrementITRB(PatchITRB): + """Patch an rpath with the given diff iters and write increments + + Like PatchITRB, but this time also write increments. + + """ + def __init__(self, basis_root_rp, inc_root_rp): + self.inc_root_rp = inc_root_rp + self.cached_incrp = None + PatchITRB.__init__(self, basis_root_rp) + + def get_incrp(self, index): + """Return inc RPath by adding index to self.basis_root_rp""" + if not self.cached_incrp or self.cached_incrp.index != index: + self.cached_incrp = self.inc_root_rp.new_index(index) + return self.cached_incrp + + def fast_process(self, index, diff_rorp): + """Patch base_rp with diff_rorp and write increment (neither is dir)""" + rp = self.get_rp_from_root(index) + tf = TempFile.new(rp) + self.patch_to_temp(rp, diff_rorp, tf) + Increment(tf, rp, self.get_incrp(index)) + tf.rename(rp) + + def start_process(self, index, diff_rorp): + """Start processing directory""" + base_rp = self.base_rp = self.get_rp_from_root(index) + assert diff_rorp.isdir() or base_rp.isdir() + if diff_rorp.isdir(): + Increment(diff_rorp, base_rp, self.get_incrp(index)) + self.prepare_dir(diff_rorp, base_rp) + else: + self.set_dir_replacement(diff_rorp, base_rp) + Increment(self.dir_replacement, base_rp, self.get_incrp(index)) + + class MirrorITRB(statistics.ITRB): """Like IncrementITR, but only patch mirror directory, don't increment""" - # This is always None since no increments will be created - incrp = None def __init__(self, inc_rpath): """Set inc_rpath, an rpath of the base of the inc tree""" self.inc_rpath = inc_rpath @@ -330,5 +448,3 @@ class MirrorITRB(statistics.ITRB): if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio) self.add_file_stats(branch) - - diff --git a/rdiff-backup/rdiff_backup/metadata.py b/rdiff-backup/rdiff_backup/metadata.py index d8e7216..de66588 100644 --- a/rdiff-backup/rdiff_backup/metadata.py +++ b/rdiff-backup/rdiff_backup/metadata.py @@ -56,7 +56,7 @@ field names and values. from __future__ import generators import re, gzip -import log, Globals, rpath, Time +import log, Globals, rpath, Time, robust class ParsingError(Exception): """This is raised when bad or unparsable data is received""" @@ -259,7 +259,7 @@ class rorp_extractor: metadata_rp = None metadata_fileobj = None def OpenMetadata(rp = None, compress = 1): - """Open the Metadata file for writing""" + """Open the Metadata file for writing, return metadata fileobj""" global metadata_rp, metadata_fileobj assert not metadata_fileobj, "Metadata file already open" if rp: metadata_rp = rp @@ -276,13 +276,13 @@ def WriteMetadata(rorp): def CloseMetadata(): """Close the metadata file""" - global metadata_fileobj + global metadata_rp, metadata_fileobj result = metadata_fileobj.close() metadata_fileobj = None metadata_rp.setdata() return result -def GetMetadata(rp = None, restrict_index = None, compressed = None): +def GetMetadata(rp, restrict_index = None, compressed = None): """Return iterator of metadata from given metadata file rp""" if compressed is None: if rp.isincfile(): @@ -294,16 +294,17 @@ def GetMetadata(rp = None, restrict_index = None, compressed = None): if restrict_index is None: return rorp_extractor(fileobj).iterate() else: return rorp_extractor(fileobj).iterate_starting_with(restrict_index) -def GetMetadata_at_time(rpdir, time, restrict_index = None, rplist = None): - """Scan through rpdir, finding metadata file at given time, iterate +def GetMetadata_at_time(rbdir, time, restrict_index = None, rblist = None): + """Scan through rbdir, finding metadata file at given time, iterate - If rplist is given, use that instead of listing rpdir. Time here + If rdlist is given, use that instead of listing rddir. Time here is exact, we don't take the next one older or anything. Returns None if no matching metadata found. """ - if rplist is None: rplist = map(lambda x: rpdir.append(x), rpdir.listdir()) - for rp in rplist: + if rblist is None: rblist = map(lambda x: rbdir.append(x), + robust.listrp(rbdir)) + for rp in rblist: if (rp.isincfile() and rp.getinctype() == "data" and rp.getincbase_str() == "mirror_metadata"): if Time.stringtotime(rp.getinctime()) == time: diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py index 40720a4..c3a2f6c 100644 --- a/rdiff-backup/rdiff_backup/restore.py +++ b/rdiff-backup/rdiff_backup/restore.py @@ -26,6 +26,11 @@ import Globals, Time, Rdiff, Hardlink, FilenameMapping, SetConnections, \ rorpiter, selection, destructive_stepping, rpath, lazy +# This should be set to selection.Select objects over the source and +# mirror directories respectively. +_select_source = None +_select_mirror = None + class RestoreError(Exception): pass def Restore(inc_rpath, mirror, target, rest_time): @@ -151,10 +156,10 @@ def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time): the source directory. """ - select_result = Globals.select_mirror.Select(target) + select_result = _select_mirror.Select(target) if select_result == 0: return - if mirrorrp and not Globals.select_source.Select(mirrorrp): + if mirrorrp and not _select_source.Select(mirrorrp): mirrorrp = None rcd = RestoreCombinedData(rid, mirrorrp, target) diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py index 875ab1e..f8ba83c 100644 --- a/rdiff-backup/rdiff_backup/rorpiter.py +++ b/rdiff-backup/rdiff_backup/rorpiter.py @@ -29,8 +29,9 @@ files), where files is the number of files attached (usually 1 or """ from __future__ import generators -import tempfile, UserList, types, librsync, Globals, Rdiff, \ - Hardlink, robust, log, static, rpath, iterfile, TempFile +import os, tempfile, UserList, types +import librsync, Globals, Rdiff, Hardlink, robust, log, static, \ + rpath, iterfile, TempFile class RORPIterException(Exception): pass @@ -60,39 +61,6 @@ def FromFile(fileobj): """Recover rorp iterator from file interface""" return FromRaw(iterfile.IterWrappingFile(fileobj)) -def IterateRPaths(base_rp): - """Return an iterator yielding RPaths with given base rp""" - yield base_rp - if base_rp.isdir(): - dirlisting = base_rp.listdir() - dirlisting.sort() - for filename in dirlisting: - for rp in IterateRPaths(base_rp.append(filename)): - yield rp - -def Signatures(rp_iter): - """Yield signatures of rpaths in given rp_iter""" - def error_handler(exc, rp): - log.Log("Error generating signature for %s" % rp.path) - return None - - for rp in rp_iter: - if rp.isplaceholder(): yield rp - else: - rorp = rp.getRORPath() - if rp.isreg(): - if rp.isflaglinked(): rorp.flaglinked() - else: - fp = robust.check_common_error( - error_handler, Rdiff.get_signature, (rp,)) - if fp: rorp.setfile(fp) - else: continue - yield rorp - -def GetSignatureIter(base_rp): - """Return a signature iterator recurring over the base_rp""" - return Signatures(IterateRPaths(base_rp)) - def CollateIterators(*rorp_iters): """Collate RORPath iterators by index @@ -151,28 +119,28 @@ def Collate2Iters(riter1, riter2): if not relem1: try: relem1 = riter1.next() except StopIteration: - if relem2: yield IndexedTuple(index2, (None, relem2)) + if relem2: yield (None, relem2) for relem2 in riter2: - yield IndexedTuple(relem2.index, (None, relem2)) + yield (None, relem2) break index1 = relem1.index if not relem2: try: relem2 = riter2.next() except StopIteration: - if relem1: yield IndexedTuple(index1, (relem1, None)) + if relem1: yield (relem1, None) for relem1 in riter1: - yield IndexedTuple(relem1.index, (relem1, None)) + yield (relem1, None) break index2 = relem2.index if index1 < index2: - yield IndexedTuple(index1, (relem1, None)) + yield (relem1, None) relem1 = None elif index1 == index2: - yield IndexedTuple(index1, (relem1, relem2)) + yield (relem1, relem2) relem1, relem2 = None, None else: # index2 is less - yield IndexedTuple(index2, (None, relem2)) + yield (None, relem2) relem2 = None def getnext(iter): @@ -181,6 +149,21 @@ def getnext(iter): except StopIteration: raise RORPIterException("Unexpected end to iter") return next +def get_dissimilar_indicies(src_init_iter, dest_init_iter): + """Get dissimilar indicies given two rorpiters + + Returns an iterator which enumerates the indicies of the rorps + which are different on the source and destination ends. + + """ + collated = Collate2Iters(src_init_iter, dest_init_iter) + for src_rorp, dest_rorp in collated: + if not src_rorp: yield dest_rorp.index + elif not dest_rorp: yield src_rorp.index + elif not src_rorp == dest_rorp: yield dest_rorp.index + elif (Globals.preserve_hardlinks and not + Hardlink.rorp_eq(src_rorp, dest_rorp)): yield dest_rorp.index + def GetDiffIter(sig_iter, new_iter): """Return delta iterator from sig_iter to new_iter @@ -225,13 +208,6 @@ def diffonce(sig_rorp, new_rp): return diff_rorp else: return new_rp.getRORPath() -def PatchIter(base_rp, diff_iter): - """Patch the appropriate rps in basis_iter using diff_iter""" - basis_iter = IterateRPaths(base_rp) - collated_iter = CollateIterators(basis_iter, diff_iter) - for basisrp, diff_rorp in collated_iter: - patchonce_action(base_rp, basisrp, diff_rorp).execute() - def patchonce_action(base_rp, basisrp, diff_rorp): """Return action patching basisrp using diff_rorp""" assert diff_rorp, "Missing diff index %s" % basisrp.index @@ -293,91 +269,6 @@ class IndexedTuple(UserList.UserList): return "(%s).%s" % (", ".join(map(str, self.data)), self.index) -class DirHandler: - """Handle directories when entering and exiting in mirror - - The problem is that we may need to write to a directory that may - have only read and exec permissions. Also, when leaving a - directory tree, we may have modified the directory and thus - changed the mod and access times. These need to be updated when - leaving. - - """ - def __init__(self, rootrp): - """DirHandler initializer - call with root rpath of mirror dir""" - self.rootrp = rootrp - assert rootrp.index == () - self.cur_dir_index = None # Current directory we have descended into - self.last_index = None # last index processed - - # This dictionary maps indicies to (rpath, (atime, mtime), - # perms) triples. Either or both of the time pair and perms - # can be None, which means not to update the times or the - # perms when leaving. We don't have to update the perms if we - # didn't have to change them in the first place. If a - # directory is explicitly given, then we don't have to update - # anything because it will be done by the normal process. - self.index_dict = {} - - def process_old_directories(self, new_dir_index): - """Update times/permissions for directories we are leaving - - Returns greatest index of the current index that has been seen - before (i.e. no need to process up to then as new dir). - - """ - if self.cur_dir_index is None: return -1 # no previous directory - - i = len(self.cur_dir_index) - while 1: - if new_dir_index[:i] == self.cur_dir_index[:i]: - return i - self.process_old_dir(self.cur_dir_index[:i]) - i-=1 - - def process_old_dir(self, dir_index): - """Process outstanding changes for given dir index""" - rpath, times, perms = self.index_dict[dir_index] - if times: apply(rpath.settime, times) - if perms: rpath.chmod(perms) - - def init_new_dirs(self, rpath, new_dir_index, common_dir_index): - """Initialize any new directories - - Record the time, and change permissions if no write access. - Use rpath if it is given to access permissions and times. - - """ - for i in range(common_dir_index, len(new_dir_index)): - process_index = new_dir_index[:i] - if rpath.index == process_index: - self.index_dict[process_index] = (None, None, None) - else: - new_rpath = self.rootrp.new_index(process_index) - if new_rpath.hasfullperms(): perms = None - else: perms = new_rpath.getperms() - times = (new_rpath.getatime(), new_rpath.getmtime()) - self.index_dict[process_index] = new_rpath, times, perms - - def __call__(self, rpath): - """Given rpath, process containing directories""" - if rpath.isdir(): new_dir_index = rpath.index - elif not rpath.index: return # no directory contains root - else: new_dir_index = rpath.index[:-1] - - common_dir_index = self.process_old_directories(new_dir_index) - self.init_new_dirs(rpath, new_dir_index, common_dir_index) - self.cur_dir_index = new_dir_index - - def Finish(self): - """Process any remaining directories""" - indicies = self.index_dict.keys() - indicies.sort() - assert len(indicies) >= 1, indicies - indicies.reverse() - map(self.process_old_dir, indicies) - - def FillInIter(rpiter, rootrp): """Given ordered rpiter and rootrp, fill in missing indicies with rpaths @@ -396,13 +287,15 @@ def FillInIter(rpiter, rootrp): del first_rp old_index = cur_index - # Now do the others (1,2,3) (1,4,5) + # Now do all the other elements for rp in rpiter: cur_index = rp.index if not cur_index[:-1] == old_index[:-1]: # Handle special case quickly for i in range(1, len(cur_index)): # i==0 case already handled if cur_index[:i] != old_index[:i]: - yield rootrp.new_index(cur_index[:i]) + filler_rp = rootrp.new_index(cur_index[:i]) + assert filler_rp.isdir(), "This shouldn't be possible" + yield filler_rp yield rp old_index = cur_index @@ -514,9 +407,6 @@ class ITRBranch: subclasses this one will probably fill in these functions to do more. - It is important that this class be pickable, so keep that in mind - when subclassing (this is used to resume failed sessions). - """ base_index = index = None finished = None @@ -566,25 +456,3 @@ class ITRBranch: (os.path.join(*index),), 2) -class DestructiveSteppingFinalizer(ITRBranch): - """Finalizer that can work on an iterator of dsrpaths - - The reason we have to use an IterTreeReducer is that some files - should be updated immediately, but for directories we sometimes - need to update all the files in the directory before finally - coming back to it. - - """ - dsrpath = None - def start_process(self, index, dsrpath): - self.dsrpath = dsrpath - - def end_process(self): - if self.dsrpath: self.dsrpath.write_changes() - - def can_fast_process(self, index, dsrpath): - return not self.dsrpath.isdir() - - def fast_process(self, index, dsrpath): - if self.dsrpath: self.dsrpath.write_changes() - diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py index 98914ec..d0b1156 100644 --- a/rdiff-backup/rdiff_backup/rpath.py +++ b/rdiff-backup/rdiff_backup/rpath.py @@ -71,8 +71,8 @@ def cmpfileobj(fp1, fp2): def check_for_files(*rps): """Make sure that all the rps exist, raise error if not""" for rp in rps: - if not rp.lstat(): - raise RPathException("File %s does not exist" % rp.path) + if not rp.lstat(): raise RPathException("File %s does not exist" + % rp.get_indexpath()) def move(rpin, rpout): """Move rpin to rpout, renaming if possible""" @@ -85,7 +85,8 @@ def copy(rpin, rpout): """Copy RPath rpin to rpout. Works for symlinks, dirs, etc.""" log.Log("Regular copying %s to %s" % (rpin.index, rpout.path), 6) if not rpin.lstat(): - raise RPathException, ("File %s does not exist" % rpin.index) + if rpout.lstat(): rpout.delete() + return if rpout.lstat(): if rpin.isreg() or not cmp(rpin, rpout): @@ -181,7 +182,7 @@ def cmp_attribs(rp1, rp2): def copy_with_attribs(rpin, rpout): """Copy file and then copy over attributes""" copy(rpin, rpout) - copy_attribs(rpin, rpout) + if rpin.lstat(): copy_attribs(rpin, rpout) def quick_cmp_with_attribs(rp1, rp2): """Quicker version of cmp_with_attribs @@ -204,9 +205,11 @@ def rename(rp_source, rp_dest): assert rp_source.conn is rp_dest.conn log.Log(lambda: "Renaming %s to %s" % (rp_source.path, rp_dest.path), 7) - rp_source.conn.os.rename(rp_source.path, rp_dest.path) - rp_dest.data = rp_source.data - rp_source.data = {'type': None} + if not rp_source.lstat(): rp_dest.delete() + else: + rp_source.conn.os.rename(rp_source.path, rp_dest.path) + rp_dest.data = rp_source.data + rp_source.data = {'type': None} def tupled_lstat(filename): """Like os.lstat, but return only a tuple, or None if os.error @@ -286,8 +289,6 @@ class RORPath: (not Globals.change_ownership or self.issym())): # Don't compare gid/uid for symlinks or if not change_ownership pass - elif key == 'mtime': - log.Log("%s differs only in mtime, skipping" % (self.path,), 2) elif key == 'atime' and not Globals.preserve_atime: pass elif key == 'devloc' or key == 'inode' or key == 'nlink': pass elif key == 'size' and not self.isreg(): pass @@ -319,28 +320,10 @@ class RORPath: """Reproduce RORPath from __getstate__ output""" self.index, self.data = rorp_state - def get_rorpath(self): + def getRORPath(self): """Return new rorpath based on self""" return RORPath(self.index, self.data.copy()) - def make_placeholder(self): - """Make rorp into a placeholder - - This object doesn't contain any information about the file, - but, when passed along, may show where the previous stages are - in their processing. It is the RORPath equivalent of fiber. - This placeholder size, in conjunction with the placeholder - threshold in Highlevel .. generate_dissimilar seem to yield an - OK tradeoff between unnecessary placeholders and lots of - memory usage, but I'm not sure exactly why. - - """ - self.data = {'placeholder': " "*500} - - def isplaceholder(self): - """True if the object is a placeholder""" - return self.data.has_key('placeholder') - def lstat(self): """Returns type of file @@ -863,12 +846,6 @@ class RPath(RORPath): else: raise RPathException self.setdata() - def getRORPath(self, include_contents = None): - """Return read only version of self""" - rorp = RORPath(self.index, self.data) - if include_contents: rorp.setfile(self.open("rb")) - return rorp - class RPathFileHook: """Look like a file, but add closing hook""" |