summaryrefslogtreecommitdiff
path: root/rdiff-backup/rdiff_backup
diff options
context:
space:
mode:
Diffstat (limited to 'rdiff-backup/rdiff_backup')
-rw-r--r--rdiff-backup/rdiff_backup/Globals.py20
-rw-r--r--rdiff-backup/rdiff_backup/Hardlink.py81
-rw-r--r--rdiff-backup/rdiff_backup/Main.py54
-rw-r--r--rdiff-backup/rdiff_backup/Rdiff.py2
-rw-r--r--rdiff-backup/rdiff_backup/TempFile.py100
-rw-r--r--rdiff-backup/rdiff_backup/highlevel.py332
-rw-r--r--rdiff-backup/rdiff_backup/increment.py124
-rw-r--r--rdiff-backup/rdiff_backup/metadata.py19
-rw-r--r--rdiff-backup/rdiff_backup/restore.py9
-rw-r--r--rdiff-backup/rdiff_backup/rorpiter.py190
-rw-r--r--rdiff-backup/rdiff_backup/rpath.py45
11 files changed, 498 insertions, 478 deletions
diff --git a/rdiff-backup/rdiff_backup/Globals.py b/rdiff-backup/rdiff_backup/Globals.py
index 50271d3..bb4195c 100644
--- a/rdiff-backup/rdiff_backup/Globals.py
+++ b/rdiff-backup/rdiff_backup/Globals.py
@@ -150,9 +150,9 @@ ssh_compression = 1
# If true, print statistics after successful backup
print_statistics = None
-# On the reader and writer connections, the following will be
-# replaced by the source and mirror Select objects respectively.
-select_source, select_mirror = None, None
+# On the writer connection, the following will be set to the mirror
+# Select iterator.
+select_mirror = None
# On the backup writer connection, holds the root incrementing branch
# object. Access is provided to increment error counts.
@@ -246,18 +246,4 @@ def postset_regexp_local(name, re_string, flags):
if flags: globals()[name] = re.compile(re_string, flags)
else: globals()[name] = re.compile(re_string)
-def set_select(source, Sel_Obj, rpath, tuplelist, quote_mode, *filelists):
- """Initialize select object using tuplelist
-
- Note that each list in filelists must each be passed as
- separate arguments, so each is recognized as a file by the
- connection. Otherwise we will get an error because a list
- containing files can't be pickled.
-
- """
- global select_source, select_mirror
- sel = Sel_Obj(rpath, quote_mode)
- sel.ParseArgs(tuplelist, filelists)
- if source: select_source = sel
- else: select_mirror = sel
diff --git a/rdiff-backup/rdiff_backup/Hardlink.py b/rdiff-backup/rdiff_backup/Hardlink.py
index ec375fd..6ff9fbd 100644
--- a/rdiff-backup/rdiff_backup/Hardlink.py
+++ b/rdiff-backup/rdiff_backup/Hardlink.py
@@ -36,18 +36,88 @@ import Globals, Time, TempFile, rpath, log, robust
# In all of these lists of indicies are the values. The keys in
# _inode_ ones are (inode, devloc) pairs.
-_src_inode_indicies = {}
-_dest_inode_indicies = {}
+_src_inode_indicies = None
+_dest_inode_indicies = None
# The keys for these two are just indicies. They share values
# with the earlier dictionaries.
-_src_index_indicies = {}
-_dest_index_indicies = {}
+_src_index_indicies = None
+_dest_index_indicies = None
# When a linked file is restored, its path is added to this dict,
# so it can be found when later paths being restored are linked to
# it.
-_restore_index_path = {}
+_restore_index_path = None
+
+def initialize_dictionaries():
+ """Set all the hard link dictionaries to empty"""
+ global _src_inode_indicies, _dest_inode_indicies
+ global _src_index_indicies, _dest_index_indicies, _restore_index_path
+ _src_inode_indicies = {}
+ _dest_inode_indicies = {}
+ _src_index_indicies = {}
+ _dest_index_indicies = {}
+ _restore_index_path = {}
+
+def clear_dictionaries():
+ """Delete all dictionaries"""
+ global _src_inode_indicies, _dest_inode_indicies
+ global _src_index_indicies, _dest_index_indicies, _restore_index_path
+ _src_inode_indicies = _dest_inode_indicies = None
+ _src_index_indicies = _dest_index_indicies = _restore_index_path = None
+
+
+# The keys of this dictionary are (inode, devloc) pairs on the source
+# side. The values are (numlinks, index) pairs, where numlinks are
+# the number of files currently linked to this spot, and index is the
+# index of the first file so linked.
+_src_inode_index_dict = {}
+_dest_inode_index_dict = {}
+
+
+#def rorp_eq(src_rorp, dest_rorp):
+# """Return true if source and dest rorp are equal as far as hardlinking
+#
+# This also processes the src_rorp, adding it if necessary to the
+# inode dictionary.
+#
+# """
+# if not src_rorp.isreg(): return 1 # only reg files can be hard linked
+# if src_rorp.getnumlinks() == 1: return dest_rorp.getnumlinks() == 1
+#
+# src_linked_index = process_rorp(src_rorp, _src_inode_index_dict)
+# if dest_rorp.getnumlinks() == 1: return 0
+# dest_linked_index = process_rorp(dest_rorp, _dest_inode_index_dict)
+# return src_linked_index == dest_linked_index
+
+def process_rorp(rorp, inode_dict):
+ """Add inode info and returns index src_rorp is linked to, or None"""
+ key_pair = (rorp.getinode(), rorp.getdevloc())
+ try: num, linked_index = inode_dict[key_pair]
+ except KeyError:
+ inode_dict[key_pair] = (1, src_rorp.index)
+ return None
+ inode_dict[key_pair] = (num+1, linked_index)
+
+ if num+1 == src_rorp.getnumlinks(): del _inode_index_dict[key_pair]
+ else: _inode_index_dict[key_pair] = (num+1, linked_index)
+ return linked_index
+
+def get_linked_index(src_rorp):
+ """Return the index a src_rorp is linked to, or None
+
+ Also deletes the src_rorp's entry in the dictionary if we have
+ accumulated all the hard link references.
+
+ """
+ key_pair = (rorp.getinode(), rorp.getdevloc())
+ try: num, linked_index = _src_inode_index_dict[key_pair]
+ except KeyError: return None
+ if num == src_rorp.getnumlinks():
+ del _src_inode_index_dict[key_pair]
+
+
+
def get_inode_key(rorp):
"""Return rorp's key for _inode_ dictionaries"""
@@ -100,7 +170,6 @@ def rorp_eq(src_rorp, dest_rorp):
indicies.
"""
- if not src_rorp.index == dest_rorp.index: return None
if (not src_rorp.isreg() or not dest_rorp.isreg() or
src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1):
return 1 # Hard links don't apply
diff --git a/rdiff-backup/rdiff_backup/Main.py b/rdiff-backup/rdiff_backup/Main.py
index d6b977c..6ffa355 100644
--- a/rdiff-backup/rdiff_backup/Main.py
+++ b/rdiff-backup/rdiff_backup/Main.py
@@ -23,7 +23,8 @@ from __future__ import generators
import getopt, sys, re, os
from log import Log
import Globals, Time, SetConnections, selection, robust, rpath, \
- manage, highlevel, connection, restore, FilenameMapping, Security
+ manage, highlevel, connection, restore, FilenameMapping, \
+ Security, Hardlink
action = None
@@ -108,8 +109,7 @@ def parse_cmdlineoptions(arglist):
elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
elif opt == "--null-separator": Globals.set("null_separator", 1)
elif opt == "--parsable-output": Globals.set('parsable_output', 1)
- elif opt == "--print-statistics":
- Globals.set('print_statistics', 1)
+ elif opt == "--print-statistics": Globals.set('print_statistics', 1)
elif opt == "--quoting-char":
Globals.set('quoting_char', arg)
Globals.set('quoting_enabled', 1)
@@ -195,13 +195,11 @@ def misc_setup(rps):
Time.setcurtime(Globals.current_time)
FilenameMapping.set_init_quote_vals()
SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
-
- # This is because I originally didn't think compiled regexps
- # could be pickled, and so must be compiled on remote side.
Globals.postset_regexp('no_compression_regexp',
Globals.no_compression_regexp_string)
-
- for conn in Globals.connections: robust.install_signal_handlers()
+ for conn in Globals.connections:
+ conn.robust.install_signal_handlers()
+ conn.Hardlink.initialize_dictionaries()
def take_action(rps):
"""Do whatever action says"""
@@ -238,20 +236,18 @@ def Main(arglist):
def Backup(rpin, rpout):
"""Backup, possibly incrementally, src_path to dest_path."""
SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
- backup_init_select(rpin, rpout)
+ backup_set_select(rpin)
backup_init_dirs(rpin, rpout)
if prevtime:
Time.setprevtime(prevtime)
- highlevel.HighLevel.Mirror_and_increment(rpin, rpout, incdir)
- else: highlevel.HighLevel.Mirror(rpin, rpout, incdir)
+ highlevel.Mirror_and_increment(rpin, rpout, incdir)
+ else: highlevel.Mirror(rpin, rpout)
rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout)
-def backup_init_select(rpin, rpout):
- """Create Select objects on source and dest connections"""
- rpin.conn.Globals.set_select(1, selection.Select,
- rpin, select_opts, None, *select_files)
- rpout.conn.Globals.set_select(0, selection.Select,
- rpout, select_mirror_opts, 1)
+def backup_set_select(rpin):
+ """Create Select objects on source connection"""
+ rpin.conn.highlevel.HLSourceStruct.set_source_select(rpin, select_opts,
+ *select_files)
def backup_init_dirs(rpin, rpout):
"""Make sure rpin and rpout are valid, init data dir and logging"""
@@ -277,10 +273,14 @@ def backup_init_dirs(rpin, rpout):
if rpout.isdir() and not rpout.listdir(): # rpout is empty dir
rpout.chmod(0700) # just make sure permissions aren't too lax
elif not datadir.lstat() and not force: Log.FatalError(
-"""Destination directory %s exists, but does not look like a
-rdiff-backup directory. Running rdiff-backup like this could mess up
-what is currently in it. If you want to update or overwrite it, run
-rdiff-backup with the --force option.""" % rpout.path)
+"""Destination directory
+
+%s
+
+exists, but does not look like a rdiff-backup directory. Running
+rdiff-backup like this could mess up what is currently in it. If you
+want to update or overwrite it, run rdiff-backup with the --force
+option.""" % rpout.path)
if not rpout.lstat():
try: rpout.mkdir()
@@ -410,9 +410,10 @@ def restore_init_select(rpin, rpout):
the restore operation isn't.
"""
- Globals.set_select(1, selection.Select, rpin, select_mirror_opts, None)
- Globals.set_select(0, selection.Select,
- rpout, select_opts, None, *select_files)
+ restore._select_mirror = selection.Select(rpin)
+ restore._select_mirror.ParseArgs(select_mirror_opts, [])
+ restore._select_mirror.parse_rbdir_exclude()
+ restore._select_source = selection.Select(rpout)
def restore_get_root(rpin):
"""Return (mirror root, index) and set the data dir
@@ -540,7 +541,6 @@ def ListChangedSince(rp):
root_rid = restore.RestoreIncrementData(index, inc_rpath, inc_list)
for rid in get_rids_recursive(root_rid):
if rid.inc_list:
- if not rid.index: path = "."
- else: path = "/".join(rid.index)
- print "%-11s: %s" % (determineChangeType(rid.inc_list), path)
+ print "%-11s: %s" % (determineChangeType(rid.inc_list),
+ rid.get_indexpath())
diff --git a/rdiff-backup/rdiff_backup/Rdiff.py b/rdiff-backup/rdiff_backup/Rdiff.py
index 23bfda3..39dcb8a 100644
--- a/rdiff-backup/rdiff_backup/Rdiff.py
+++ b/rdiff-backup/rdiff_backup/Rdiff.py
@@ -42,7 +42,7 @@ def get_delta_sigfileobj(sig_fileobj, rp_new):
def get_delta_sigrp(rp_signature, rp_new):
"""Take signature rp and new rp, return delta file object"""
Log("Getting delta of %s with signature %s" %
- (rp_new.path, rp_signature.path), 7)
+ (rp_new.path, rp_signature.get_indexpath()), 7)
return librsync.DeltaFile(rp_signature.open("rb"), rp_new.open("rb"))
def write_delta_action(basis, new, delta, compress = None):
diff --git a/rdiff-backup/rdiff_backup/TempFile.py b/rdiff-backup/rdiff_backup/TempFile.py
new file mode 100644
index 0000000..2b18920
--- /dev/null
+++ b/rdiff-backup/rdiff_backup/TempFile.py
@@ -0,0 +1,100 @@
+# Copyright 2002 Ben Escoto
+#
+# This file is part of rdiff-backup.
+#
+# rdiff-backup is free software; you can redistribute it and/or modify
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# rdiff-backup is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with rdiff-backup; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+
+"""Manage temp files"""
+
+import os
+import Globals, rpath
+
+# This is a connection-specific list of temp files, to be cleaned
+# up before rdiff-backup exits.
+_tempfiles = []
+
+# To make collisions less likely, this gets put in the file name
+# and incremented whenever a new file is requested.
+_tfindex = 0
+
+def new(rp_base, same_dir = 1):
+ """Return new tempfile that isn't in use.
+
+ If same_dir, tempfile will be in same directory as rp_base.
+ Otherwise, use tempfile module to get filename.
+
+ """
+ conn = rp_base.conn
+ if conn is not Globals.local_connection:
+ return conn.TempFile.new(rp_base, same_dir)
+
+ def find_unused(conn, dir):
+ """Find an unused tempfile with connection conn in directory dir"""
+ global _tfindex, tempfiles
+ while 1:
+ if _tfindex > 100000000:
+ Log("Resetting index", 2)
+ _tfindex = 0
+ tf = TempFile(conn, os.path.join(dir,
+ "rdiff-backup.tmp.%d" % _tfindex))
+ _tfindex = _tfindex+1
+ if not tf.lstat(): return tf
+
+ if same_dir: tf = find_unused(conn, rp_base.dirsplit()[0])
+ else: tf = TempFile(conn, tempfile.mktemp())
+ _tempfiles.append(tf)
+ return tf
+
+def remove_listing(tempfile):
+ """Remove listing of tempfile"""
+ if Globals.local_connection is not tempfile.conn:
+ tempfile.conn.TempFile.remove_listing(tempfile)
+ elif tempfile in _tempfiles: _tempfiles.remove(tempfile)
+
+def delete_all():
+ """Delete all remaining tempfiles"""
+ for tf in _tempfiles[:]: tf.delete()
+
+
+class TempFile(rpath.RPath):
+ """Like an RPath, but keep track of which ones are still here"""
+ def rename(self, rp_dest):
+ """Rename temp file to permanent location, possibly overwriting"""
+ if not self.lstat(): # "Moving" empty file, so just delete
+ if rp_dest.lstat(): rp_dest.delete()
+ remove_listing(self)
+ return
+
+ if self.isdir() and not rp_dest.isdir():
+ # Cannot move a directory directly over another file
+ rp_dest.delete()
+ rpath.rename(self, rp_dest)
+
+ # Sometimes this just seems to fail silently, as in one
+ # hardlinked twin is moved over the other. So check to make
+ # sure below.
+ self.setdata()
+ if self.lstat():
+ rp_dest.delete()
+ rpath.rename(self, rp_dest)
+ self.setdata()
+ if self.lstat(): raise OSError("Cannot rename tmp file correctly")
+ remove_listing(self)
+
+ def delete(self):
+ rpath.RPath.delete(self)
+ remove_listing(self)
+
diff --git a/rdiff-backup/rdiff_backup/highlevel.py b/rdiff-backup/rdiff_backup/highlevel.py
index f93388f..fc75099 100644
--- a/rdiff-backup/rdiff_backup/highlevel.py
+++ b/rdiff-backup/rdiff_backup/highlevel.py
@@ -20,246 +20,144 @@
"""High level functions for mirroring, mirror & inc, etc."""
from __future__ import generators
-import Globals, MiscStats, metadata, rorpiter, TempFile, \
- Hardlink, robust, increment, rpath, lazy, static, log
+import Globals, MiscStats, metadata, rorpiter, TempFile, Hardlink, \
+ robust, increment, rpath, lazy, static, log, selection, Time, Rdiff
-class HighLevel:
- """High level static functions
+def Mirror(src_rpath, dest_rpath):
+ """Turn dest_rpath into a copy of src_rpath"""
+ SourceS = src_rpath.conn.highlevel.HLSourceStruct
+ DestS = dest_rpath.conn.highlevel.HLDestinationStruct
- The design of some of these functions is represented on the
- accompanying diagram.
+ source_rpiter = SourceS.get_source_select()
+ dest_sigiter = DestS.process_source_get_sigs(dest_rpath,
+ source_rpiter, 0)
+ source_diffiter = SourceS.get_diffs(src_rpath, dest_sigiter)
+ DestS.patch(dest_rpath, source_diffiter)
- """
- def Mirror(src_rpath, dest_rpath, inc_rpath):
- """Turn dest_rpath into a copy of src_rpath
+def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath):
+ """Mirror + put increments in tree based at inc_rpath"""
+ SourceS = src_rpath.conn.highlevel.HLSourceStruct
+ DestS = dest_rpath.conn.highlevel.HLDestinationStruct
- If inc_rpath is true, then this is the initial mirroring of an
- incremental backup, so checkpoint and write to data_dir.
- Otherwise only mirror and don't create any extra files.
-
- """
- SourceS = src_rpath.conn.highlevel.HLSourceStruct
- DestS = dest_rpath.conn.highlevel.HLDestinationStruct
-
- src_init_dsiter = SourceS.split_initial_dsiter()
- dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
- diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
- DestS.patch_w_datadir_writes(dest_rpath, diffiter, inc_rpath)
-
- dest_rpath.setdata()
-
- def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath,
- session_info = None):
- """Mirror + put increments in tree based at inc_rpath"""
- SourceS = src_rpath.conn.highlevel.HLSourceStruct
- DestS = dest_rpath.conn.highlevel.HLDestinationStruct
-
- src_init_dsiter = SourceS.split_initial_dsiter()
- dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
- diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
- DestS.patch_increment_and_finalize(dest_rpath, diffiter, inc_rpath)
-
- dest_rpath.setdata()
- inc_rpath.setdata()
-
-static.MakeStatic(HighLevel)
+ source_rpiter = SourceS.get_source_select()
+ dest_sigiter = DestS.process_source_get_sigs(dest_rpath,
+ source_rpiter, 1)
+ source_diffiter = SourceS.get_diffs(src_rpath, dest_sigiter)
+ DestS.patch_and_increment(dest_rpath, source_diffiter, inc_rpath)
class HLSourceStruct:
"""Hold info used by HL on the source side"""
- def split_initial_dsiter(cls):
- """Set iterators of all dsrps from rpath, returning one"""
- dsiter = Globals.select_source.set_iter()
- initial_dsiter1, cls.initial_dsiter2 = lazy.Iter.multiplex(dsiter, 2)
- return initial_dsiter1
+ source_select = None # will be set to source Select iterator
+ def set_source_select(cls, rpath, tuplelist, *filelists):
+ """Initialize select object using tuplelist
- def get_diffs_and_finalize(cls, sigiter):
- """Return diffs and finalize any dsrp changes remaining
-
- Return a rorpiterator with files included of signatures of
- dissimilar files.
+ Note that each list in filelists must each be passed as
+ separate arguments, so each is recognized as a file by the
+ connection. Otherwise we will get an error because a list
+ containing files can't be pickled.
"""
- collated = rorpiter.CollateIterators(cls.initial_dsiter2, sigiter)
- def error_handler(exc, dest_sig, rp):
- log.Log("Error %s producing a diff of %s" %
- (exc, rp and rp.path), 2)
- return None
-
- def diffs():
- for rp, dest_sig in collated:
- if dest_sig:
- if dest_sig.isplaceholder(): yield dest_sig
- else:
- diff = robust.check_common_error(
- error_handler, rorpiter.diffonce, [dest_sig, rp])
- if diff: yield diff
- return diffs()
+ sel = selection.Select(rpath)
+ sel.ParseArgs(tuplelist, filelists)
+ cls.source_select = sel.set_iter()
+
+ def get_source_select(cls):
+ """Return source select iterator, set by set_source_select"""
+ return cls.source_select
+
+ def get_diffs(cls, baserp, dest_sigiter):
+ """Return diffs of any files with signature in dest_sigiter"""
+ for dest_sig in dest_sigiter:
+ src_rp = baserp.new_index(dest_sig.index)
+ diff_rorp = src_rp.getRORPath()
+ if dest_sig.isflaglinked(): diff_rorp.flaglinked()
+ elif dest_sig.isreg() and src_rp.isreg():
+ diff_rorp.setfile(Rdiff.get_delta_sigrp(dest_sig, src_rp))
+ diff_rorp.set_attached_filetype('diff')
+ else:
+ diff_rorp.set_attached_filetype('snapshot')
+ if src_rp.isreg(): diff_rorp.setfile(src_rp.open("rb"))
+ yield diff_rorp
static.MakeClass(HLSourceStruct)
class HLDestinationStruct:
"""Hold info used by HL on the destination side"""
- _session_info = None # set to si if resuming
- def split_initial_dsiter(cls):
- """Set initial_dsiters (iteration of all rps from rpath)"""
- result, cls.initial_dsiter2 = \
- lazy.Iter.multiplex(Globals.select_mirror.set_iter(), 2)
- return result
-
- def get_dissimilar(cls, baserp, src_init_iter, dest_init_iter):
- """Get dissimilars
-
- Returns an iterator which enumerates the dsrps which are
- different on the source and destination ends. The dsrps do
- not necessarily exist on the destination end.
-
- Also, to prevent the system from getting backed up on the
- remote end, if we don't get enough dissimilars, stick in a
- placeholder every so often, like fiber. The more
- placeholders, the more bandwidth used, but if there aren't
- enough, lots of memory will be used because files will be
- accumulating on the source side. How much will accumulate
- will depend on the Globals.conn_bufsize value.
+ def get_dest_select(cls, rpath, use_metadata = 1):
+ """Return destination select rorpath iterator
- """
- collated = rorpiter.CollateIterators(src_init_iter, dest_init_iter)
- def compare(src_rorp, dest_dsrp):
- """Return dest_dsrp if they are different, None if the same"""
- if not dest_dsrp:
- dest_dsrp = cls.get_dsrp(baserp, src_rorp.index)
- if dest_dsrp.lstat():
- log.Log("Warning: Found unexpected destination file %s, "
- "not processing it." % dest_dsrp.path, 2)
- return None
- elif (src_rorp and src_rorp == dest_dsrp and
- (not Globals.preserve_hardlinks or
- Hardlink.rorp_eq(src_rorp, dest_dsrp))):
- return None
- if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp):
- dest_dsrp.flaglinked()
- return dest_dsrp
-
- def generate_dissimilar():
- counter = 0
- for src_rorp, dest_dsrp in collated:
- if Globals.preserve_hardlinks:
- if src_rorp: Hardlink.add_rorp(src_rorp, 1)
- if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None)
- dsrp = compare(src_rorp, dest_dsrp)
- if dsrp:
- counter = 0
- yield dsrp
- elif counter == 20:
- placeholder = rpath.RORPath(src_rorp.index)
- placeholder.make_placeholder()
- counter = 0
- yield placeholder
- else: counter += 1
- return generate_dissimilar()
-
- def get_sigs(cls, baserp, src_init_iter):
- """Return signatures of all dissimilar files
-
- Also writes all metadata to the metadata file.
+ If metadata file doesn't exist, select all files on
+ destination except rdiff-backup-data directory.
"""
- dest_iters1 = cls.split_initial_dsiter()
- def duplicate_with_write(src_init_iter):
- """Return iterator but write metadata of what passes through"""
- metadata.OpenMetadata()
- for rorp in src_init_iter:
- metadata.WriteMetadata(rorp)
- yield rorp
- metadata.CloseMetadata()
- dup = duplicate_with_write(src_init_iter)
- dissimilars = cls.get_dissimilar(baserp, dup, dest_iters1)
- return rorpiter.Signatures(dissimilars)
-
- def get_dsrp(cls, dest_rpath, index):
- """Return initialized rpath based on dest_rpath with given index"""
- rp = rpath.RPath(dest_rpath.conn, dest_rpath.base, index)
- if Globals.quoting_enabled: rp.quote_path()
- return rp
-
- def get_finalizer(cls):
- """Return finalizer, starting from session info if necessary"""
- old_finalizer = cls._session_info and cls._session_info.finalizer
- if old_finalizer: return old_finalizer
- else: return rorpiter.IterTreeReducer(
- rorpiter.DestructiveSteppingFinalizer, [])
-
- def get_ITR(cls, inc_rpath):
- """Return ITR, starting from state if necessary"""
- if cls._session_info and cls._session_info.ITR:
- return cls._session_info.ITR
- else:
- iitr = rorpiter.IterTreeReducer(increment.IncrementITRB,
- [inc_rpath])
- iitr.root_branch.override_changed()
- Globals.ITRB = iitr.root_branch
- iitr.root_branch.Errors = 0
- return iitr
-
- def get_MirrorITR(cls, inc_rpath):
- """Return MirrorITR, starting from state if available"""
- if cls._session_info and cls._session_info.ITR:
- return cls._session_info.ITR
- ITR = rorpiter.IterTreeReducer(increment.MirrorITRB, [inc_rpath])
- Globals.ITRB = ITR.root_branch
- ITR.root_branch.Errors = 0
- return ITR
-
- def patch_and_finalize(cls, dest_rpath, diffs):
- """Apply diffs and finalize"""
- collated = rorpiter.CollateIterators(diffs, cls.initial_dsiter2)
- #finalizer = cls.get_finalizer()
- diff_rorp, rp = None, None
-
- def patch(diff_rorp, dsrp):
- if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
- if diff_rorp and not diff_rorp.isplaceholder():
- rorpiter.patchonce_action(None, dsrp, diff_rorp).execute()
- return dsrp
-
- def error_handler(exc, diff_rorp, dsrp):
- filename = dsrp and dsrp.path or os.path.join(*diff_rorp.index)
- log.Log("Error: %s processing file %s" % (exc, filename), 2)
-
- for indexed_tuple in collated:
- log.Log(lambda: "Processing %s" % str(indexed_tuple), 7)
- diff_rorp, dsrp = indexed_tuple
- dsrp = robust.check_common_error(error_handler, patch,
- [diff_rorp, dsrp])
- #finalizer(dsrp.index, dsrp)
- #finalizer.Finish()
-
- def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath):
- """Apply diffs and finalize, with checkpointing and statistics"""
- collated = rorpiter.CollateIterators(diffs, cls.initial_dsiter2)
- #finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
- finalizer, ITR = None, cls.get_MirrorITR(inc_rpath)
- MiscStats.open_dir_stats_file()
- dsrp, finished_dsrp = None, None
+ if use_metadata:
+ metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir,
+ Time.curtime)
+ if metadata_iter: return metadata_iter
+ log.Log("Warning: Metadata file not found.\n"
+ "Metadata will be read from filesystem.", 2)
+
+ sel = selection.Select(rpath)
+ sel.parse_rbdir_exclude()
+ return sel.set_iter()
+
+ def dest_iter_filter(cls, dest_iter):
+ """Destination rorps pass through this - record stats"""
+ for dest_rorp in dest_iter:
+ # XXX Statistics process
+ Hardlink.add_rorp(dest_rorp, source = 0)
+ yield dest_rorp
+
+ def src_iter_filter(cls, source_iter):
+ """Source rorps pass through this - record stats, write metadata"""
+ metadata.OpenMetadata()
+ for src_rorp in source_iter:
+ Hardlink.add_rorp(src_rorp, source = 1)
+ metadata.WriteMetadata(src_rorp)
+ #XXXX Statistics process
+ yield src_rorp
+ metadata.CloseMetadata()
+
+ def process_source_get_sigs(cls, baserp, source_iter, for_increment):
+ """Process the source rorpiter and return signatures of dest dir
+
+ Write all metadata to file, then return signatures of any
+ destination files that have changed. for_increment should be
+ true if we are mirror+incrementing, and false if we are just
+ mirroring.
- try:
- for indexed_tuple in collated:
- log.Log(lambda: "Processing %s" % str(indexed_tuple), 7)
- diff_rorp, dsrp = indexed_tuple
- if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
- if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
- ITR(dsrp.index, diff_rorp, dsrp)
- #finalizer(dsrp.index, dsrp)
- finished_dsrp = dsrp
- ITR.Finish()
- #finalizer.Finish()
- except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
+ """
+ source_iter = cls.src_iter_filter(source_iter)
+ dest_iter = cls.dest_iter_filter(cls.get_dest_select(baserp,
+ for_increment))
+ for index in rorpiter.get_dissimilar_indicies(source_iter, dest_iter):
+ dest_rp = baserp.new_index(index)
+ dest_sig = dest_rp.getRORPath()
+ if Globals.preserve_hardlinks and Hardlink.islinked(dest_rp):
+ dest_sig.flaglinked()
+ elif dest_rp.isreg():
+ dest_sig.setfile(Rdiff.get_signature(dest_rp))
+ yield dest_sig
+
+ def patch(cls, dest_rpath, source_diffiter):
+ """Patch dest_rpath with an rorpiter of diffs"""
+ ITR = rorpiter.IterTreeReducer(increment.PatchITRB, [dest_rpath])
+ for diff in rorpiter.FillInIter(source_diffiter, dest_rpath):
+ ITR(diff.index, diff)
+ ITR.Finish()
+ dest_rpath.setdata()
- if Globals.preserve_hardlinks: Hardlink.final_writedata()
- MiscStats.close_dir_stats_file()
- MiscStats.write_session_statistics(ITR.root_branch)
+ def patch_and_increment(cls, dest_rpath, source_diffiter, inc_rpath):
+ """Patch dest_rpath with rorpiter of diffs and write increments"""
+ ITR = rorpiter.IterTreeReducer(increment.IncrementITRB,
+ [dest_rpath, inc_rpath])
+ for diff in rorpiter.FillInIter(source_diffiter, dest_rpath):
+ ITR(diff.index, diff)
+ ITR.Finish()
+ dest_rpath.setdata()
def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
"""Apply diffs, write increment if necessary, and finalize"""
diff --git a/rdiff-backup/rdiff_backup/increment.py b/rdiff-backup/rdiff_backup/increment.py
index 46afd42..a11dd6a 100644
--- a/rdiff-backup/rdiff_backup/increment.py
+++ b/rdiff-backup/rdiff_backup/increment.py
@@ -285,10 +285,128 @@ class IncrementITRB(statistics.ITRB):
self.add_file_stats(branch)
+class PatchITRB(statistics.ITRB):
+ """Patch an rpath with the given diff iters (use with IterTreeReducer)
+
+ The main complication here involves directories. We have to
+ finish processing the directory after what's in the directory, as
+ the directory may have inappropriate permissions to alter the
+ contents or the dir's mtime could change as we change the
+ contents.
+
+ """
+ def __init__(self, basis_root_rp):
+ """Set basis_root_rp, the base of the tree to be incremented"""
+ self.basis_root_rp = basis_root_rp
+ assert basis_root_rp.conn is Globals.local_connection
+ #statistics.ITRB.__init__(self)
+ self.dir_replacement, self.dir_update = None, None
+ self.cached_rp = None
+
+ def get_rp_from_root(self, index):
+ """Return RPath by adding index to self.basis_root_rp"""
+ if not self.cached_rp or self.cached_rp.index != index:
+ self.cached_rp = self.basis_root_rp.new_index(index)
+ return self.cached_rp
+
+ def can_fast_process(self, index, diff_rorp):
+ """True if diff_rorp and mirror are not directories"""
+ rp = self.get_rp_from_root(index)
+ return not diff_rorp.isdir() and not rp.isdir()
+
+ def fast_process(self, index, diff_rorp):
+ """Patch base_rp with diff_rorp (case where neither is directory)"""
+ rp = self.get_rp_from_root(index)
+ tf = TempFile.new(rp)
+ self.patch_to_temp(rp, diff_rorp, tf)
+ tf.rename(rp)
+
+ def patch_to_temp(self, basis_rp, diff_rorp, new):
+ """Patch basis_rp, writing output in new, which doesn't exist yet"""
+ if diff_rorp.isflaglinked():
+ Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
+ elif diff_rorp.get_attached_filetype() == 'snapshot':
+ rpath.copy(diff_rorp, new)
+ else:
+ assert diff_rorp.get_attached_filetype() == 'diff'
+ Rdiff.patch_local(basis_rp, diff_rorp, new)
+ if new.lstat(): rpath.copy_attribs(diff_rorp, new)
+
+ def start_process(self, index, diff_rorp):
+ """Start processing directory - record information for later"""
+ base_rp = self.base_rp = self.get_rp_from_root(index)
+ assert diff_rorp.isdir() or base_rp.isdir()
+ if diff_rorp.isdir(): self.prepare_dir(diff_rorp, base_rp)
+ else: self.set_dir_replacement(diff_rorp, base_rp)
+
+ def set_dir_replacement(self, diff_rorp, base_rp):
+ """Set self.dir_replacement, which holds data until done with dir
+
+ This is used when base_rp is a dir, and diff_rorp is not.
+
+ """
+ assert diff_rorp.get_attached_filetype() == 'snapshot'
+ self.dir_replacement = TempFile.new(base_rp)
+ rpath.copy_with_attribs(diff_rorp, self.dir_replacement)
+
+ def prepare_dir(self, diff_rorp, base_rp):
+ """Prepare base_rp to turn into a directory"""
+ self.dir_update = diff_rorp.getRORPath() # make copy in case changes
+ if not base_rp.isdir():
+ if base_rp.lstat(): base_rp.delete()
+ base_rp.mkdir()
+ base_rp.chmod(0700)
+
+ def end_process(self):
+ """Finish processing directory"""
+ if self.dir_update:
+ assert self.base_rp.isdir()
+ rpath.copy_attribs(self.dir_update, self.base_rp)
+ else:
+ assert self.dir_replacement and self.base_rp.isdir()
+ self.base_rp.rmdir()
+ self.dir_replacement.rename(self.base_rp)
+
+
+class IncrementITRB(PatchITRB):
+ """Patch an rpath with the given diff iters and write increments
+
+ Like PatchITRB, but this time also write increments.
+
+ """
+ def __init__(self, basis_root_rp, inc_root_rp):
+ self.inc_root_rp = inc_root_rp
+ self.cached_incrp = None
+ PatchITRB.__init__(self, basis_root_rp)
+
+ def get_incrp(self, index):
+ """Return inc RPath by adding index to self.basis_root_rp"""
+ if not self.cached_incrp or self.cached_incrp.index != index:
+ self.cached_incrp = self.inc_root_rp.new_index(index)
+ return self.cached_incrp
+
+ def fast_process(self, index, diff_rorp):
+ """Patch base_rp with diff_rorp and write increment (neither is dir)"""
+ rp = self.get_rp_from_root(index)
+ tf = TempFile.new(rp)
+ self.patch_to_temp(rp, diff_rorp, tf)
+ Increment(tf, rp, self.get_incrp(index))
+ tf.rename(rp)
+
+ def start_process(self, index, diff_rorp):
+ """Start processing directory"""
+ base_rp = self.base_rp = self.get_rp_from_root(index)
+ assert diff_rorp.isdir() or base_rp.isdir()
+ if diff_rorp.isdir():
+ Increment(diff_rorp, base_rp, self.get_incrp(index))
+ self.prepare_dir(diff_rorp, base_rp)
+ else:
+ self.set_dir_replacement(diff_rorp, base_rp)
+ Increment(self.dir_replacement, base_rp, self.get_incrp(index))
+
+
class MirrorITRB(statistics.ITRB):
"""Like IncrementITR, but only patch mirror directory, don't increment"""
- # This is always None since no increments will be created
- incrp = None
def __init__(self, inc_rpath):
"""Set inc_rpath, an rpath of the base of the inc tree"""
self.inc_rpath = inc_rpath
@@ -330,5 +448,3 @@ class MirrorITRB(statistics.ITRB):
if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio)
self.add_file_stats(branch)
-
-
diff --git a/rdiff-backup/rdiff_backup/metadata.py b/rdiff-backup/rdiff_backup/metadata.py
index d8e7216..de66588 100644
--- a/rdiff-backup/rdiff_backup/metadata.py
+++ b/rdiff-backup/rdiff_backup/metadata.py
@@ -56,7 +56,7 @@ field names and values.
from __future__ import generators
import re, gzip
-import log, Globals, rpath, Time
+import log, Globals, rpath, Time, robust
class ParsingError(Exception):
"""This is raised when bad or unparsable data is received"""
@@ -259,7 +259,7 @@ class rorp_extractor:
metadata_rp = None
metadata_fileobj = None
def OpenMetadata(rp = None, compress = 1):
- """Open the Metadata file for writing"""
+ """Open the Metadata file for writing, return metadata fileobj"""
global metadata_rp, metadata_fileobj
assert not metadata_fileobj, "Metadata file already open"
if rp: metadata_rp = rp
@@ -276,13 +276,13 @@ def WriteMetadata(rorp):
def CloseMetadata():
"""Close the metadata file"""
- global metadata_fileobj
+ global metadata_rp, metadata_fileobj
result = metadata_fileobj.close()
metadata_fileobj = None
metadata_rp.setdata()
return result
-def GetMetadata(rp = None, restrict_index = None, compressed = None):
+def GetMetadata(rp, restrict_index = None, compressed = None):
"""Return iterator of metadata from given metadata file rp"""
if compressed is None:
if rp.isincfile():
@@ -294,16 +294,17 @@ def GetMetadata(rp = None, restrict_index = None, compressed = None):
if restrict_index is None: return rorp_extractor(fileobj).iterate()
else: return rorp_extractor(fileobj).iterate_starting_with(restrict_index)
-def GetMetadata_at_time(rpdir, time, restrict_index = None, rplist = None):
- """Scan through rpdir, finding metadata file at given time, iterate
+def GetMetadata_at_time(rbdir, time, restrict_index = None, rblist = None):
+ """Scan through rbdir, finding metadata file at given time, iterate
- If rplist is given, use that instead of listing rpdir. Time here
+ If rdlist is given, use that instead of listing rddir. Time here
is exact, we don't take the next one older or anything. Returns
None if no matching metadata found.
"""
- if rplist is None: rplist = map(lambda x: rpdir.append(x), rpdir.listdir())
- for rp in rplist:
+ if rblist is None: rblist = map(lambda x: rbdir.append(x),
+ robust.listrp(rbdir))
+ for rp in rblist:
if (rp.isincfile() and rp.getinctype() == "data" and
rp.getincbase_str() == "mirror_metadata"):
if Time.stringtotime(rp.getinctime()) == time:
diff --git a/rdiff-backup/rdiff_backup/restore.py b/rdiff-backup/rdiff_backup/restore.py
index 40720a4..c3a2f6c 100644
--- a/rdiff-backup/rdiff_backup/restore.py
+++ b/rdiff-backup/rdiff_backup/restore.py
@@ -26,6 +26,11 @@ import Globals, Time, Rdiff, Hardlink, FilenameMapping, SetConnections, \
rorpiter, selection, destructive_stepping, rpath, lazy
+# This should be set to selection.Select objects over the source and
+# mirror directories respectively.
+_select_source = None
+_select_mirror = None
+
class RestoreError(Exception): pass
def Restore(inc_rpath, mirror, target, rest_time):
@@ -151,10 +156,10 @@ def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time):
the source directory.
"""
- select_result = Globals.select_mirror.Select(target)
+ select_result = _select_mirror.Select(target)
if select_result == 0: return
- if mirrorrp and not Globals.select_source.Select(mirrorrp):
+ if mirrorrp and not _select_source.Select(mirrorrp):
mirrorrp = None
rcd = RestoreCombinedData(rid, mirrorrp, target)
diff --git a/rdiff-backup/rdiff_backup/rorpiter.py b/rdiff-backup/rdiff_backup/rorpiter.py
index 875ab1e..f8ba83c 100644
--- a/rdiff-backup/rdiff_backup/rorpiter.py
+++ b/rdiff-backup/rdiff_backup/rorpiter.py
@@ -29,8 +29,9 @@ files), where files is the number of files attached (usually 1 or
"""
from __future__ import generators
-import tempfile, UserList, types, librsync, Globals, Rdiff, \
- Hardlink, robust, log, static, rpath, iterfile, TempFile
+import os, tempfile, UserList, types
+import librsync, Globals, Rdiff, Hardlink, robust, log, static, \
+ rpath, iterfile, TempFile
class RORPIterException(Exception): pass
@@ -60,39 +61,6 @@ def FromFile(fileobj):
"""Recover rorp iterator from file interface"""
return FromRaw(iterfile.IterWrappingFile(fileobj))
-def IterateRPaths(base_rp):
- """Return an iterator yielding RPaths with given base rp"""
- yield base_rp
- if base_rp.isdir():
- dirlisting = base_rp.listdir()
- dirlisting.sort()
- for filename in dirlisting:
- for rp in IterateRPaths(base_rp.append(filename)):
- yield rp
-
-def Signatures(rp_iter):
- """Yield signatures of rpaths in given rp_iter"""
- def error_handler(exc, rp):
- log.Log("Error generating signature for %s" % rp.path)
- return None
-
- for rp in rp_iter:
- if rp.isplaceholder(): yield rp
- else:
- rorp = rp.getRORPath()
- if rp.isreg():
- if rp.isflaglinked(): rorp.flaglinked()
- else:
- fp = robust.check_common_error(
- error_handler, Rdiff.get_signature, (rp,))
- if fp: rorp.setfile(fp)
- else: continue
- yield rorp
-
-def GetSignatureIter(base_rp):
- """Return a signature iterator recurring over the base_rp"""
- return Signatures(IterateRPaths(base_rp))
-
def CollateIterators(*rorp_iters):
"""Collate RORPath iterators by index
@@ -151,28 +119,28 @@ def Collate2Iters(riter1, riter2):
if not relem1:
try: relem1 = riter1.next()
except StopIteration:
- if relem2: yield IndexedTuple(index2, (None, relem2))
+ if relem2: yield (None, relem2)
for relem2 in riter2:
- yield IndexedTuple(relem2.index, (None, relem2))
+ yield (None, relem2)
break
index1 = relem1.index
if not relem2:
try: relem2 = riter2.next()
except StopIteration:
- if relem1: yield IndexedTuple(index1, (relem1, None))
+ if relem1: yield (relem1, None)
for relem1 in riter1:
- yield IndexedTuple(relem1.index, (relem1, None))
+ yield (relem1, None)
break
index2 = relem2.index
if index1 < index2:
- yield IndexedTuple(index1, (relem1, None))
+ yield (relem1, None)
relem1 = None
elif index1 == index2:
- yield IndexedTuple(index1, (relem1, relem2))
+ yield (relem1, relem2)
relem1, relem2 = None, None
else: # index2 is less
- yield IndexedTuple(index2, (None, relem2))
+ yield (None, relem2)
relem2 = None
def getnext(iter):
@@ -181,6 +149,21 @@ def getnext(iter):
except StopIteration: raise RORPIterException("Unexpected end to iter")
return next
+def get_dissimilar_indicies(src_init_iter, dest_init_iter):
+ """Get dissimilar indicies given two rorpiters
+
+ Returns an iterator which enumerates the indicies of the rorps
+ which are different on the source and destination ends.
+
+ """
+ collated = Collate2Iters(src_init_iter, dest_init_iter)
+ for src_rorp, dest_rorp in collated:
+ if not src_rorp: yield dest_rorp.index
+ elif not dest_rorp: yield src_rorp.index
+ elif not src_rorp == dest_rorp: yield dest_rorp.index
+ elif (Globals.preserve_hardlinks and not
+ Hardlink.rorp_eq(src_rorp, dest_rorp)): yield dest_rorp.index
+
def GetDiffIter(sig_iter, new_iter):
"""Return delta iterator from sig_iter to new_iter
@@ -225,13 +208,6 @@ def diffonce(sig_rorp, new_rp):
return diff_rorp
else: return new_rp.getRORPath()
-def PatchIter(base_rp, diff_iter):
- """Patch the appropriate rps in basis_iter using diff_iter"""
- basis_iter = IterateRPaths(base_rp)
- collated_iter = CollateIterators(basis_iter, diff_iter)
- for basisrp, diff_rorp in collated_iter:
- patchonce_action(base_rp, basisrp, diff_rorp).execute()
-
def patchonce_action(base_rp, basisrp, diff_rorp):
"""Return action patching basisrp using diff_rorp"""
assert diff_rorp, "Missing diff index %s" % basisrp.index
@@ -293,91 +269,6 @@ class IndexedTuple(UserList.UserList):
return "(%s).%s" % (", ".join(map(str, self.data)), self.index)
-class DirHandler:
- """Handle directories when entering and exiting in mirror
-
- The problem is that we may need to write to a directory that may
- have only read and exec permissions. Also, when leaving a
- directory tree, we may have modified the directory and thus
- changed the mod and access times. These need to be updated when
- leaving.
-
- """
- def __init__(self, rootrp):
- """DirHandler initializer - call with root rpath of mirror dir"""
- self.rootrp = rootrp
- assert rootrp.index == ()
- self.cur_dir_index = None # Current directory we have descended into
- self.last_index = None # last index processed
-
- # This dictionary maps indicies to (rpath, (atime, mtime),
- # perms) triples. Either or both of the time pair and perms
- # can be None, which means not to update the times or the
- # perms when leaving. We don't have to update the perms if we
- # didn't have to change them in the first place. If a
- # directory is explicitly given, then we don't have to update
- # anything because it will be done by the normal process.
- self.index_dict = {}
-
- def process_old_directories(self, new_dir_index):
- """Update times/permissions for directories we are leaving
-
- Returns greatest index of the current index that has been seen
- before (i.e. no need to process up to then as new dir).
-
- """
- if self.cur_dir_index is None: return -1 # no previous directory
-
- i = len(self.cur_dir_index)
- while 1:
- if new_dir_index[:i] == self.cur_dir_index[:i]:
- return i
- self.process_old_dir(self.cur_dir_index[:i])
- i-=1
-
- def process_old_dir(self, dir_index):
- """Process outstanding changes for given dir index"""
- rpath, times, perms = self.index_dict[dir_index]
- if times: apply(rpath.settime, times)
- if perms: rpath.chmod(perms)
-
- def init_new_dirs(self, rpath, new_dir_index, common_dir_index):
- """Initialize any new directories
-
- Record the time, and change permissions if no write access.
- Use rpath if it is given to access permissions and times.
-
- """
- for i in range(common_dir_index, len(new_dir_index)):
- process_index = new_dir_index[:i]
- if rpath.index == process_index:
- self.index_dict[process_index] = (None, None, None)
- else:
- new_rpath = self.rootrp.new_index(process_index)
- if new_rpath.hasfullperms(): perms = None
- else: perms = new_rpath.getperms()
- times = (new_rpath.getatime(), new_rpath.getmtime())
- self.index_dict[process_index] = new_rpath, times, perms
-
- def __call__(self, rpath):
- """Given rpath, process containing directories"""
- if rpath.isdir(): new_dir_index = rpath.index
- elif not rpath.index: return # no directory contains root
- else: new_dir_index = rpath.index[:-1]
-
- common_dir_index = self.process_old_directories(new_dir_index)
- self.init_new_dirs(rpath, new_dir_index, common_dir_index)
- self.cur_dir_index = new_dir_index
-
- def Finish(self):
- """Process any remaining directories"""
- indicies = self.index_dict.keys()
- indicies.sort()
- assert len(indicies) >= 1, indicies
- indicies.reverse()
- map(self.process_old_dir, indicies)
-
-
def FillInIter(rpiter, rootrp):
"""Given ordered rpiter and rootrp, fill in missing indicies with rpaths
@@ -396,13 +287,15 @@ def FillInIter(rpiter, rootrp):
del first_rp
old_index = cur_index
- # Now do the others (1,2,3) (1,4,5)
+ # Now do all the other elements
for rp in rpiter:
cur_index = rp.index
if not cur_index[:-1] == old_index[:-1]: # Handle special case quickly
for i in range(1, len(cur_index)): # i==0 case already handled
if cur_index[:i] != old_index[:i]:
- yield rootrp.new_index(cur_index[:i])
+ filler_rp = rootrp.new_index(cur_index[:i])
+ assert filler_rp.isdir(), "This shouldn't be possible"
+ yield filler_rp
yield rp
old_index = cur_index
@@ -514,9 +407,6 @@ class ITRBranch:
subclasses this one will probably fill in these functions to do
more.
- It is important that this class be pickable, so keep that in mind
- when subclassing (this is used to resume failed sessions).
-
"""
base_index = index = None
finished = None
@@ -566,25 +456,3 @@ class ITRBranch:
(os.path.join(*index),), 2)
-class DestructiveSteppingFinalizer(ITRBranch):
- """Finalizer that can work on an iterator of dsrpaths
-
- The reason we have to use an IterTreeReducer is that some files
- should be updated immediately, but for directories we sometimes
- need to update all the files in the directory before finally
- coming back to it.
-
- """
- dsrpath = None
- def start_process(self, index, dsrpath):
- self.dsrpath = dsrpath
-
- def end_process(self):
- if self.dsrpath: self.dsrpath.write_changes()
-
- def can_fast_process(self, index, dsrpath):
- return not self.dsrpath.isdir()
-
- def fast_process(self, index, dsrpath):
- if self.dsrpath: self.dsrpath.write_changes()
-
diff --git a/rdiff-backup/rdiff_backup/rpath.py b/rdiff-backup/rdiff_backup/rpath.py
index 98914ec..d0b1156 100644
--- a/rdiff-backup/rdiff_backup/rpath.py
+++ b/rdiff-backup/rdiff_backup/rpath.py
@@ -71,8 +71,8 @@ def cmpfileobj(fp1, fp2):
def check_for_files(*rps):
"""Make sure that all the rps exist, raise error if not"""
for rp in rps:
- if not rp.lstat():
- raise RPathException("File %s does not exist" % rp.path)
+ if not rp.lstat(): raise RPathException("File %s does not exist"
+ % rp.get_indexpath())
def move(rpin, rpout):
"""Move rpin to rpout, renaming if possible"""
@@ -85,7 +85,8 @@ def copy(rpin, rpout):
"""Copy RPath rpin to rpout. Works for symlinks, dirs, etc."""
log.Log("Regular copying %s to %s" % (rpin.index, rpout.path), 6)
if not rpin.lstat():
- raise RPathException, ("File %s does not exist" % rpin.index)
+ if rpout.lstat(): rpout.delete()
+ return
if rpout.lstat():
if rpin.isreg() or not cmp(rpin, rpout):
@@ -181,7 +182,7 @@ def cmp_attribs(rp1, rp2):
def copy_with_attribs(rpin, rpout):
"""Copy file and then copy over attributes"""
copy(rpin, rpout)
- copy_attribs(rpin, rpout)
+ if rpin.lstat(): copy_attribs(rpin, rpout)
def quick_cmp_with_attribs(rp1, rp2):
"""Quicker version of cmp_with_attribs
@@ -204,9 +205,11 @@ def rename(rp_source, rp_dest):
assert rp_source.conn is rp_dest.conn
log.Log(lambda: "Renaming %s to %s" %
(rp_source.path, rp_dest.path), 7)
- rp_source.conn.os.rename(rp_source.path, rp_dest.path)
- rp_dest.data = rp_source.data
- rp_source.data = {'type': None}
+ if not rp_source.lstat(): rp_dest.delete()
+ else:
+ rp_source.conn.os.rename(rp_source.path, rp_dest.path)
+ rp_dest.data = rp_source.data
+ rp_source.data = {'type': None}
def tupled_lstat(filename):
"""Like os.lstat, but return only a tuple, or None if os.error
@@ -286,8 +289,6 @@ class RORPath:
(not Globals.change_ownership or self.issym())):
# Don't compare gid/uid for symlinks or if not change_ownership
pass
- elif key == 'mtime':
- log.Log("%s differs only in mtime, skipping" % (self.path,), 2)
elif key == 'atime' and not Globals.preserve_atime: pass
elif key == 'devloc' or key == 'inode' or key == 'nlink': pass
elif key == 'size' and not self.isreg(): pass
@@ -319,28 +320,10 @@ class RORPath:
"""Reproduce RORPath from __getstate__ output"""
self.index, self.data = rorp_state
- def get_rorpath(self):
+ def getRORPath(self):
"""Return new rorpath based on self"""
return RORPath(self.index, self.data.copy())
- def make_placeholder(self):
- """Make rorp into a placeholder
-
- This object doesn't contain any information about the file,
- but, when passed along, may show where the previous stages are
- in their processing. It is the RORPath equivalent of fiber.
- This placeholder size, in conjunction with the placeholder
- threshold in Highlevel .. generate_dissimilar seem to yield an
- OK tradeoff between unnecessary placeholders and lots of
- memory usage, but I'm not sure exactly why.
-
- """
- self.data = {'placeholder': " "*500}
-
- def isplaceholder(self):
- """True if the object is a placeholder"""
- return self.data.has_key('placeholder')
-
def lstat(self):
"""Returns type of file
@@ -863,12 +846,6 @@ class RPath(RORPath):
else: raise RPathException
self.setdata()
- def getRORPath(self, include_contents = None):
- """Return read only version of self"""
- rorp = RORPath(self.index, self.data)
- if include_contents: rorp.setfile(self.open("rb"))
- return rorp
-
class RPathFileHook:
"""Look like a file, but add closing hook"""