summaryrefslogtreecommitdiff
path: root/rdiff-backup
diff options
context:
space:
mode:
authorben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109>2002-12-03 01:32:09 +0000
committerben <ben@2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109>2002-12-03 01:32:09 +0000
commitb99e3e4547ba36f2c11c0d9269fab12a61d4bd89 (patch)
tree18c005d050088aa9bc55deefaeff82c11acef0d6 /rdiff-backup
parent90b3185d36976bb1c6404332772ae15beac8cb77 (diff)
downloadrdiff-backup-b99e3e4547ba36f2c11c0d9269fab12a61d4bd89.tar.gz
Removed all files from src/ directory, added to rdiff_backup/
git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup/trunk@237 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
Diffstat (limited to 'rdiff-backup')
-rw-r--r--rdiff-backup/src/FilenameMapping.py112
-rw-r--r--rdiff-backup/src/Globals.py287
-rw-r--r--rdiff-backup/src/Hardlink.py277
-rw-r--r--rdiff-backup/src/Main.py573
-rwxr-xr-xrdiff-backup/src/Make.old38
-rw-r--r--rdiff-backup/src/MiscStats.py93
-rw-r--r--rdiff-backup/src/Rdiff.py127
-rw-r--r--rdiff-backup/src/Security.py187
-rw-r--r--rdiff-backup/src/SetConnections.py243
-rw-r--r--rdiff-backup/src/Time.py234
-rw-r--r--rdiff-backup/src/__init__.py0
-rw-r--r--rdiff-backup/src/_librsyncmodule.c468
-rw-r--r--rdiff-backup/src/cmodule.c217
-rwxr-xr-xrdiff-backup/src/compilec.py18
-rw-r--r--rdiff-backup/src/connection.py563
-rw-r--r--rdiff-backup/src/destructive_stepping.py236
-rw-r--r--rdiff-backup/src/filelist.py106
-rw-r--r--rdiff-backup/src/header.py19
-rw-r--r--rdiff-backup/src/highlevel.py333
-rw-r--r--rdiff-backup/src/increment.py333
-rw-r--r--rdiff-backup/src/iterfile.py257
-rw-r--r--rdiff-backup/src/lazy.py368
-rw-r--r--rdiff-backup/src/librsync.py195
-rw-r--r--rdiff-backup/src/librsync_memoryleak2.py49
-rw-r--r--rdiff-backup/src/log.py183
-rw-r--r--rdiff-backup/src/manage.py136
-rw-r--r--rdiff-backup/src/memoryleak.c23
-rwxr-xr-xrdiff-backup/src/myrdiff.py51
-rwxr-xr-xrdiff-backup/src/profiled_rdb.py19
-rw-r--r--rdiff-backup/src/restore.py392
-rw-r--r--rdiff-backup/src/rlist.py240
-rw-r--r--rdiff-backup/src/robust.py698
-rw-r--r--rdiff-backup/src/rorpiter.py301
-rw-r--r--rdiff-backup/src/rpath.py875
-rw-r--r--rdiff-backup/src/selection.py650
-rw-r--r--rdiff-backup/src/static.py44
-rw-r--r--rdiff-backup/src/statistics.py345
-rw-r--r--rdiff-backup/src/temp-stats241
38 files changed, 0 insertions, 9531 deletions
diff --git a/rdiff-backup/src/FilenameMapping.py b/rdiff-backup/src/FilenameMapping.py
deleted file mode 100644
index c160bed..0000000
--- a/rdiff-backup/src/FilenameMapping.py
+++ /dev/null
@@ -1,112 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Coordinate corresponding files with different names
-
-For instance, some source filenames may contain characters not allowed
-on the mirror end. Also, if a source filename is very long (say 240
-characters), the extra characters added to related increments may put
-them over the usual 255 character limit.
-
-"""
-
-import re
-from log import *
-from robust import *
-import Globals
-
-max_filename_length = 255
-
-# If true, enable character quoting, and set characters making
-# regex-style range.
-chars_to_quote = None
-
-# These compiled regular expressions are used in quoting and unquoting
-chars_to_quote_regexp = None
-unquoting_regexp = None
-
-# Use given char to quote. Default is set in Globals.
-quoting_char = None
-
-
-def set_init_quote_vals():
- """Set quoting value from Globals on all conns"""
- for conn in Globals.connections:
- conn.FilenameMapping.set_init_quote_vals_local()
-
-def set_init_quote_vals_local():
- """Set value on local connection, initialize regexps"""
- global chars_to_quote, quoting_char
- chars_to_quote = Globals.chars_to_quote
- if len(Globals.quoting_char) != 1:
- Log.FatalError("Expected single character for quoting char,"
- "got '%s' instead" % (Globals.quoting_char,))
- quoting_char = Globals.quoting_char
- init_quoting_regexps()
-
-def init_quoting_regexps():
- """Compile quoting regular expressions"""
- global chars_to_quote_regexp, unquoting_regexp
- try:
- chars_to_quote_regexp = \
- re.compile("[%s%s]" % (chars_to_quote, quoting_char), re.S)
- unquoting_regexp = re.compile("%s[0-9]{3}" % quoting_char, re.S)
- except re.error:
- Log.FatalError("Error '%s' when processing char quote list %s" %
- (re.error, chars_to_quote))
-
-def quote(path):
- """Return quoted version of given path
-
- Any characters quoted will be replaced by the quoting char and
- the ascii number of the character. For instance, "10:11:12"
- would go to "10;05811;05812" if ":" were quoted and ";" were
- the quoting character.
-
- """
- return chars_to_quote_regexp.sub(quote_single, path)
-
-def quote_single(match):
- """Return replacement for a single character"""
- return "%s%03d" % (quoting_char, ord(match.group()))
-
-def unquote(path):
- """Return original version of quoted filename"""
- return unquoting_regexp.sub(unquote_single, path)
-
-def unquote_single(match):
- """Unquote a single quoted character"""
- assert len(match.group()) == 4
- return chr(int(match.group()[1:]))
-
-def get_quoted_dir_children(rpath):
- """For rpath directory, return list of quoted children in dir"""
- if not rpath.isdir(): return []
- dir_pairs = [(unquote(filename), filename)
- for filename in Robust.listrp(rpath)]
- dir_pairs.sort() # sort by real index, not quoted part
- child_list = []
- for unquoted, filename in dir_pairs:
- childrp = rpath.append(unquoted)
- childrp.quote_path()
- child_list.append(childrp)
- return child_list
-
-
-
diff --git a/rdiff-backup/src/Globals.py b/rdiff-backup/src/Globals.py
deleted file mode 100644
index 49d84db..0000000
--- a/rdiff-backup/src/Globals.py
+++ /dev/null
@@ -1,287 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Hold a variety of constants usually set at initialization."""
-
-import re, os
-
-
-# The current version of rdiff-backup
-version = "$version"
-
-# If this is set, use this value in seconds as the current time
-# instead of reading it from the clock.
-current_time = None
-
-# This determines how many bytes to read at a time when copying
-blocksize = 32768
-
-# This is used by the BufferedRead class to determine how many
-# bytes to request from the underlying file per read(). Larger
-# values may save on connection overhead and latency.
-conn_bufsize = 98304
-
-# True if script is running as a server
-server = None
-
-# uid and gid of the owner of the rdiff-backup process. This can
-# vary depending on the connection.
-process_uid = os.getuid()
-process_gid = os.getgid()
-
-# If true, when copying attributes, also change target's uid/gid
-change_ownership = None
-
-# If true, change the permissions of unwriteable mirror files
-# (such as directories) so that they can be written, and then
-# change them back. This defaults to 1 just in case the process
-# is not running as root (root doesn't need to change
-# permissions).
-change_mirror_perms = (process_uid != 0)
-
-# If true, temporarily change permissions of unreadable files in
-# the source directory to make sure we can read all files.
-change_source_perms = None
-
-# If true, try to reset the atimes of the source partition.
-preserve_atime = None
-
-# This will be set as soon as the LocalConnection class loads
-local_connection = None
-
-# All connections should be added to the following list, so
-# further global changes can be propagated to the remote systems.
-# The first element should be Globals.local_connection. For a
-# server, the second is the connection to the client.
-connections = []
-
-# Each process should have a connection number unique to the
-# session. The client has connection number 0.
-connection_number = 0
-
-# Dictionary pairing connection numbers with connections. Set in
-# SetConnections for all connections.
-connection_dict = {}
-
-# True if the script is the end that reads the source directory
-# for backups. It is true for purely local sessions.
-isbackup_reader = None
-
-# Connection of the real backup reader (for which isbackup_reader
-# is true)
-backup_reader = None
-
-# True if the script is the end that writes to the increment and
-# mirror directories. True for purely local sessions.
-isbackup_writer = None
-
-# Connection of the backup writer
-backup_writer = None
-
-# Connection of the client
-client_conn = None
-
-# This list is used by the set function below. When a new
-# connection is created with init_connection, its Globals class
-# will match this one for all the variables mentioned in this
-# list.
-changed_settings = []
-
-# rdiff-backup will try to checkpoint its state every
-# checkpoint_interval seconds. Then when resuming, at most this
-# amount of time is lost.
-checkpoint_interval = 20
-
-# The RPath of the rdiff-backup-data directory.
-rbdir = None
-
-# Indicates if a resume or a lack of resume is forced. This
-# should be None for the default. 0 means don't resume, and 1
-# means resume.
-resume = None
-
-# If there has been an aborted backup fewer than this many seconds
-# ago, attempt to resume it where it left off instead of starting
-# a new one.
-resume_window = 7200
-
-# This string is used when recognizing and creating time strings.
-# If the time_separator is ":", then W3 datetime strings like
-# 2001-12-07T04:22:01-07:00 are produced. It can be set to "_" to
-# make filenames that don't contain colons, which aren't allowed
-# under MS windows NT.
-time_separator = ":"
-
-# quoting_enabled is true if we should quote certain characters in
-# filenames on the source side (see FilenameMapping for more
-# info). chars_to_quote is a string whose characters should be
-# quoted, and quoting_char is the character to quote with.
-quoting_enabled = None
-chars_to_quote = ""
-quoting_char = ';'
-
-# If true, emit output intended to be easily readable by a
-# computer. False means output is intended for humans.
-parsable_output = None
-
-# If true, then hardlinks will be preserved to mirror and recorded
-# in the increments directory. There is also a difference here
-# between None and 0. When restoring, None or 1 means to preserve
-# hardlinks iff can find a hardlink dictionary. 0 means ignore
-# hardlink information regardless.
-preserve_hardlinks = 1
-
-# If this is false, then rdiff-backup will not compress any
-# increments. Default is to compress based on regexp below.
-compression = 1
-
-# Increments based on files whose names match this
-# case-insensitive regular expression won't be compressed (applies
-# to .snapshots and .diffs). The second below will be the
-# compiled version of the first.
-no_compression_regexp_string = "(?i).*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|" \
- "jpg|gif|png|jp2|mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$"
-no_compression_regexp = None
-
-# If true, filelists and directory statistics will be split on
-# nulls instead of newlines.
-null_separator = None
-
-# Determines whether or not ssh will be run with the -C switch
-ssh_compression = 1
-
-# If true, print statistics after successful backup
-print_statistics = None
-
-# On the reader and writer connections, the following will be
-# replaced by the source and mirror Select objects respectively.
-select_source, select_mirror = None, None
-
-# On the backup writer connection, holds the root incrementing branch
-# object. Access is provided to increment error counts.
-ITRB = None
-
-# Percentage of time to spend sleeping. None means never sleep.
-sleep_ratio = None
-
-# security_level has 4 values and controls which requests from remote
-# systems will be honored. "all" means anything goes. "read-only"
-# means that the requests must not write to disk. "update-only" means
-# that requests shouldn't destructively update the disk (but normal
-# incremental updates are OK). "minimal" means only listen to a few
-# basic requests.
-security_level = "all"
-
-# If this is set, it indicates that the remote connection should only
-# deal with paths inside of restrict_path.
-restrict_path = None
-
-
-def get(name):
- """Return the value of something in this module"""
- return globals()[name]
-
-def is_not_None(name):
- """Returns true if value is not None"""
- return globals()[name] is not None
-
-def set(name, val):
- """Set the value of something in this module
-
- Use this instead of writing the values directly if the setting
- matters to remote sides. This function updates the
- changed_settings list, so other connections know to copy the
- changes.
-
- """
- changed_settings.append(name)
- globals()[name] = val
-
-def set_integer(name, val):
- """Like set, but make sure val is an integer"""
- try: intval = int(val)
- except ValueError:
- Log.FatalError("Variable %s must be set to an integer -\n"
- "received %s instead." % (name, val))
- set(name, intval)
-
-def set_float(name, val, min = None, max = None, inclusive = 1):
- """Like set, but make sure val is float within given bounds"""
- def error():
- s = "Variable %s must be set to a float" % (name,)
- if min is not None and max is not None:
- s += " between %s and %s " % (min, max)
- if inclusive: s += "inclusive"
- else: s += "not inclusive"
- elif min is not None or max is not None:
- if inclusive: inclusive_string = "or equal to "
- else: inclusive_string = ""
- if min is not None:
- s += " greater than %s%s" % (inclusive_string, min)
- else: s+= " less than %s%s" % (inclusive_string, max)
- Log.FatalError(s)
-
- try: f = float(val)
- except ValueError: error()
- if min is not None:
- if inclusive and f < min: error()
- elif not inclusive and f <= min: error()
- if max is not None:
- if inclusive and f > max: error()
- elif not inclusive and f >= max: error()
- set(name, f)
-
-def get_dict_val(name, key):
- """Return val from dictionary in this class"""
- return globals()[name][key]
-
-def set_dict_val(name, key, val):
- """Set value for dictionary in this class"""
- globals()[name][key] = val
-
-def postset_regexp(name, re_string, flags = None):
- """Compile re_string on all existing connections, set to name"""
- for conn in connections:
- conn.Globals.postset_regexp_local(name, re_string, flags)
-
-def postset_regexp_local(name, re_string, flags):
- """Set name to compiled re_string locally"""
- if flags: globals()[name] = re.compile(re_string, flags)
- else: globals()[name] = re.compile(re_string)
-
-def set_select(dsrpath, tuplelist, quote_mode, *filelists):
- """Initialize select object using tuplelist
-
- Note that each list in filelists must each be passed as
- separate arguments, so each is recognized as a file by the
- connection. Otherwise we will get an error because a list
- containing files can't be pickled.
-
- """
- global select_source, select_mirror
- if dsrpath.source:
- select_source = Select(dsrpath, quote_mode)
- select_source.ParseArgs(tuplelist, filelists)
- else:
- select_mirror = Select(dsrpath, quote_mode)
- select_mirror.ParseArgs(tuplelist, filelists)
-
-
-from rpath import * # kludge to avoid circularity - not needed in this module
-from selection import *
diff --git a/rdiff-backup/src/Hardlink.py b/rdiff-backup/src/Hardlink.py
deleted file mode 100644
index 38dcad8..0000000
--- a/rdiff-backup/src/Hardlink.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Preserve and restore hard links
-
-If the preserve_hardlinks option is selected, linked files in the
-source directory will be linked in the mirror directory. Linked files
-are treated like any other with respect to incrementing, but a
-database of all links will be recorded at each session, so linked
-files can still be restored from the increments.
-
-All these functions are meant to be executed on the destination
-side. The source side should only transmit inode information.
-
-"""
-
-from __future__ import generators
-import cPickle
-
-
-# In all of these lists of indicies are the values. The keys in
-# _inode_ ones are (inode, devloc) pairs.
-_src_inode_indicies = {}
-_dest_inode_indicies = {}
-
-# The keys for these two are just indicies. They share values
-# with the earlier dictionaries.
-_src_index_indicies = {}
-_dest_index_indicies = {}
-
-# When a linked file is restored, its path is added to this dict,
-# so it can be found when later paths being restored are linked to
-# it.
-_restore_index_path = {}
-
-def get_inode_key(rorp):
- """Return rorp's key for _inode_ dictionaries"""
- return (rorp.getinode(), rorp.getdevloc())
-
-def get_indicies(rorp, source):
- """Return a list of similarly linked indicies, using rorp's index"""
- if source: dict = _src_index_indicies
- else: dict = _dest_index_indicies
- try: return dict[rorp.index]
- except KeyError: return []
-
-def add_rorp(rorp, source):
- """Process new rorp and update hard link dictionaries
-
- First enter it into src_inode_indicies. If we have already
- seen all the hard links, then we can delete the entry.
- Everything must stay recorded in src_index_indicies though.
-
- """
- if not rorp.isreg() or rorp.getnumlinks() < 2: return
-
- if source:
- inode_dict, index_dict = _src_inode_indicies, _src_index_indicies
- else: inode_dict, index_dict = _dest_inode_indicies, _dest_index_indicies
-
- rp_inode_key = get_inode_key(rorp)
- if inode_dict.has_key(rp_inode_key):
- index_list = inode_dict[rp_inode_key]
- index_list.append(rorp.index)
- if len(index_list) == rorp.getnumlinks():
- del inode_dict[rp_inode_key]
- else: # make new entry in both src dicts
- index_list = [rorp.index]
- inode_dict[rp_inode_key] = index_list
- index_dict[rorp.index] = index_list
-
-def add_rorp_iter(iter, source):
- """Return new rorp iterator like iter that add_rorp's first"""
- for rorp in iter:
- add_rorp(rorp, source)
- yield rorp
-
-def rorp_eq(src_rorp, dest_rorp):
- """Compare hardlinked for equality
-
- Two files may otherwise seem equal but be hardlinked in
- different ways. This function considers them equal enough if
- they have been hardlinked correctly to the previously seen
- indicies.
-
- """
- if not src_rorp.index == dest_rorp.index: return None
- if (not src_rorp.isreg() or not dest_rorp.isreg() or
- src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1):
- return 1 # Hard links don't apply
-
- src_index_list = get_indicies(src_rorp, 1)
- dest_index_list = get_indicies(dest_rorp, None)
-
- # If a list only has one element, then it is only hardlinked
- # to itself so far, so that is not a genuine difference yet.
- if not src_index_list or len(src_index_list) == 1:
- return not dest_index_list or len(dest_index_list) == 1
- if not dest_index_list or len(dest_index_list) == 1: return None
-
- # Both index lists exist and are non-empty
- return src_index_list == dest_index_list # they are always sorted
-
-def islinked(rorp):
- """True if rorp's index is already linked to something on src side"""
- return len(get_indicies(rorp, 1)) >= 2
-
-def restore_link(index, rpath):
- """Restores a linked file by linking it
-
- When restoring, all the hardlink data is already present, and
- we can only link to something already written. In either
- case, add to the _restore_index_path dict, so we know later
- that the file is available for hard
- linking.
-
- Returns true if succeeded in creating rpath, false if must
- restore rpath normally.
-
- """
- if index not in _src_index_indicies: return None
- for linked_index in _src_index_indicies[index]:
- if linked_index in _restore_index_path:
- srcpath = _restore_index_path[linked_index]
- Log("Restoring %s by hard linking to %s" %
- (rpath.path, srcpath), 6)
- rpath.hardlink(srcpath)
- return 1
- _restore_index_path[index] = rpath.path
- return None
-
-def link_rp(src_rorp, dest_rpath, dest_root = None):
- """Make dest_rpath into a link analogous to that of src_rorp"""
- if not dest_root: dest_root = dest_rpath # use base of dest_rpath
- dest_link_rpath = RPath(dest_root.conn, dest_root.base,
- get_indicies(src_rorp, 1)[0])
- dest_rpath.hardlink(dest_link_rpath.path)
-
-def write_linkdict(rpath, dict, compress = None):
- """Write link data to the rbdata dir
-
- It is stored as the a big pickled dictionary dated to match
- the current hardlinks.
-
- """
- assert (Globals.isbackup_writer and
- rpath.conn is Globals.local_connection)
- tf = TempFileManager.new(rpath)
- def init():
- fp = tf.open("wb", compress)
- cPickle.dump(dict, fp)
- assert not fp.close()
- tf.setdata()
- Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()
-
-def get_linkrp(data_rpath, time, prefix):
- """Return RPath of linkdata, or None if cannot find"""
- for rp in map(data_rpath.append, data_rpath.listdir()):
- if (rp.isincfile() and rp.getincbase_str() == prefix and
- (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')
- and Time.stringtotime(rp.getinctime()) == time):
- return rp
- return None
-
-def get_linkdata(data_rpath, time, prefix = 'hardlink_data'):
- """Return index dictionary written by write_linkdata at time"""
- rp = get_linkrp(data_rpath, time, prefix)
- if not rp: return None
- fp = rp.open("rb", rp.isinccompressed())
- index_dict = cPickle.load(fp)
- assert not fp.close()
- return index_dict
-
-def final_writedata():
- """Write final checkpoint data to rbdir after successful backup"""
- global final_inc
- if _src_index_indicies:
- Log("Writing hard link data", 6)
- if Globals.compression:
- final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" %
- Time.curtimestr)
- else: final_inc = Globals.rbdir.append("hardlink_data.%s.data" %
- Time.curtimestr)
- write_linkdict(final_inc, _src_index_indicies, Globals.compression)
- else: # no hardlinks, so writing unnecessary
- final_inc = None
-
-def retrieve_final(time):
- """Set source index dictionary from hardlink_data file if avail"""
- global _src_index_indicies
- hd = get_linkdata(Globals.rbdir, time)
- if hd is None: return None
- _src_index_indicies = hd
- return 1
-
-def final_checkpoint(data_rpath):
- """Write contents of the four dictionaries to the data dir
-
- If rdiff-backup receives a fatal error, it may still be able
- to save the contents of the four hard link dictionaries.
- Because these dictionaries may be big, they are not saved
- after every 20 seconds or whatever, but just at the end.
-
- """
- Log("Writing intermediate hard link data to disk", 2)
- src_inode_rp = data_rpath.append("hardlink_source_inode_checkpoint."
- "%s.data" % Time.curtimestr)
- src_index_rp = data_rpath.append("hardlink_source_index_checkpoint."
- "%s.data" % Time.curtimestr)
- dest_inode_rp = data_rpath.append("hardlink_dest_inode_checkpoint."
- "%s.data" % Time.curtimestr)
- dest_index_rp = data_rpath.append("hardlink_dest_index_checkpoint."
- "%s.data" % Time.curtimestr)
- for (rp, dict) in ((src_inode_rp, _src_inode_indicies),
- (src_index_rp, _src_index_indicies),
- (dest_inode_rp, _dest_inode_indicies),
- (dest_index_rp, _dest_index_indicies)):
- write_linkdict(rp, dict)
-
-def retrieve_checkpoint(data_rpath, time):
- """Retrieve hardlink data from final checkpoint
-
- Return true if the retrieval worked, false otherwise.
-
- """
- global _src_inode_indicies, _src_index_indicies
- global _dest_inode_indicies, _dest_index_indicies
- try:
- src_inode = get_linkdata(data_rpath, time,
- "hardlink_source_inode_checkpoint")
- src_index = get_linkdata(data_rpath, time,
- "hardlink_source_index_checkpoint")
- dest_inode = get_linkdata(data_rpath, time,
- "hardlink_dest_inode_checkpoint")
- dest_index = get_linkdata(data_rpath, time,
- "hardlink_dest_index_checkpoint")
- except cPickle.UnpicklingError:
- Log("Unpickling Error", 2)
- return None
- if (src_inode is None or src_index is None or
- dest_inode is None or dest_index is None): return None
- _src_inode_indicies, _src_index_indicies = src_inode, src_index
- _dest_inode_indicies, _dest_index_indicies = dest_inode, dest_index
- return 1
-
-def remove_all_checkpoints():
- """Remove all hardlink checkpoint information from directory"""
- prefix_list = ["hardlink_source_inode_checkpoint",
- "hardlink_source_index_checkpoint",
- "hardlink_dest_inode_checkpoint",
- "hardlink_dest_index_checkpoint"]
- for rp in map(Globals.rbdir.append, Globals.rbdir.listdir()):
- if (rp.isincfile() and rp.getincbase_str() in prefix_list and
- (rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')):
- rp.delete()
-
-
-from log import *
-from robust import *
-from rpath import *
-import Globals, Time
diff --git a/rdiff-backup/src/Main.py b/rdiff-backup/src/Main.py
deleted file mode 100644
index b450876..0000000
--- a/rdiff-backup/src/Main.py
+++ /dev/null
@@ -1,573 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Start (and end) here - read arguments, set global settings, etc."""
-
-from __future__ import generators
-import getopt, sys, re
-from log import *
-from lazy import *
-from connection import *
-from rpath import *
-from destructive_stepping import *
-from robust import *
-from restore import *
-from highlevel import *
-from manage import *
-import Globals, Time, SetConnections
-
-
-action = None
-remote_cmd, remote_schema = None, None
-force = None
-select_opts, select_mirror_opts = [], []
-select_files = []
-
-def parse_cmdlineoptions(arglist):
- """Parse argument list and set global preferences"""
- global args, action, force, restore_timestr, remote_cmd, remote_schema
- global remove_older_than_string
- def sel_fl(filename):
- """Helper function for including/excluding filelists below"""
- try: return open(filename, "r")
- except IOError: Log.FatalError("Error opening file %s" % filename)
-
- try: optlist, args = getopt.getopt(arglist, "blmr:sv:V",
- ["backup-mode", "calculate-average",
- "change-source-perms", "chars-to-quote=",
- "checkpoint-interval=", "current-time=", "exclude=",
- "exclude-device-files", "exclude-filelist=",
- "exclude-filelist-stdin", "exclude-globbing-filelist=",
- "exclude-mirror=", "exclude-other-filesystems",
- "exclude-regexp=", "exclude-special-files", "force",
- "include=", "include-filelist=", "include-filelist-stdin",
- "include-globbing-filelist=", "include-regexp=",
- "list-changed-since=", "list-increments", "mirror-only",
- "no-compression", "no-compression-regexp=", "no-hard-links",
- "no-resume", "null-separator", "parsable-output",
- "print-statistics", "quoting-char=", "remote-cmd=",
- "remote-schema=", "remove-older-than=", "restore-as-of=",
- "restrict=", "restrict-read-only=", "restrict-update-only=",
- "resume", "resume-window=", "server", "sleep-ratio=",
- "ssh-no-compression", "terminal-verbosity=", "test-server",
- "verbosity=", "version", "windows-mode",
- "windows-time-format"])
- except getopt.error, e:
- commandline_error("Bad commandline options: %s" % str(e))
-
- for opt, arg in optlist:
- if opt == "-b" or opt == "--backup-mode": action = "backup"
- elif opt == "--calculate-average": action = "calculate-average"
- elif opt == "--change-source-perms":
- Globals.set('change_source_perms', 1)
- elif opt == "--chars-to-quote":
- Globals.set('chars_to_quote', arg)
- Globals.set('quoting_enabled', 1)
- elif opt == "--checkpoint-interval":
- Globals.set_integer('checkpoint_interval', arg)
- elif opt == "--current-time":
- Globals.set_integer('current_time', arg)
- elif opt == "--exclude": select_opts.append((opt, arg))
- elif opt == "--exclude-device-files": select_opts.append((opt, arg))
- elif opt == "--exclude-filelist":
- select_opts.append((opt, arg))
- select_files.append(sel_fl(arg))
- elif opt == "--exclude-filelist-stdin":
- select_opts.append(("--exclude-filelist", "standard input"))
- select_files.append(sys.stdin)
- elif opt == "--exclude-globbing-filelist":
- select_opts.append((opt, arg))
- select_files.append(sel_fl(arg))
- elif opt == "--exclude-mirror":
- select_mirror_opts.append(("--exclude", arg))
- elif (opt == "--exclude-other-filesystems" or
- opt == "--exclude-regexp" or
- opt == "--exclude-special-files"): select_opts.append((opt, arg))
- elif opt == "--force": force = 1
- elif opt == "--include": select_opts.append((opt, arg))
- elif opt == "--include-filelist":
- select_opts.append((opt, arg))
- select_files.append(sel_fl(arg))
- elif opt == "--include-filelist-stdin":
- select_opts.append(("--include-filelist", "standard input"))
- select_files.append(sys.stdin)
- elif opt == "--include-globbing-filelist":
- select_opts.append((opt, arg))
- select_files.append(sel_fl(arg))
- elif opt == "--include-regexp": select_opts.append((opt, arg))
- elif opt == "--list-changed-since":
- restore_timestr, action = arg, "list-changed-since"
- elif opt == "-l" or opt == "--list-increments":
- action = "list-increments"
- elif opt == "-m" or opt == "--mirror-only": action = "mirror"
- elif opt == "--no-compression": Globals.set("compression", None)
- elif opt == "--no-compression-regexp":
- Globals.set("no_compression_regexp_string", arg)
- elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
- elif opt == '--no-resume': Globals.resume = 0
- elif opt == "--null-separator": Globals.set("null_separator", 1)
- elif opt == "--parsable-output": Globals.set('parsable_output', 1)
- elif opt == "--print-statistics":
- Globals.set('print_statistics', 1)
- elif opt == "--quoting-char":
- Globals.set('quoting_char', arg)
- Globals.set('quoting_enabled', 1)
- elif opt == "-r" or opt == "--restore-as-of":
- restore_timestr, action = arg, "restore-as-of"
- elif opt == "--remote-cmd": remote_cmd = arg
- elif opt == "--remote-schema": remote_schema = arg
- elif opt == "--remove-older-than":
- remove_older_than_string = arg
- action = "remove-older-than"
- elif opt == "--restrict": Globals.restrict_path = arg
- elif opt == "--restrict-read-only":
- Globals.security_level = "read-only"
- Globals.restrict_path = arg
- elif opt == "--restrict-update-only":
- Globals.security_level = "update-only"
- Globals.restrict_path = arg
- elif opt == '--resume': Globals.resume = 1
- elif opt == '--resume-window':
- Globals.set_integer('resume_window', arg)
- elif opt == "-s" or opt == "--server":
- action = "server"
- Globals.server = 1
- elif opt == "--sleep-ratio":
- Globals.set_float("sleep_ratio", arg, 0, 1, inclusive=0)
- elif opt == "--ssh-no-compression":
- Globals.set('ssh_compression', None)
- elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
- elif opt == "--test-server": action = "test-server"
- elif opt == "-V" or opt == "--version":
- print "rdiff-backup " + Globals.version
- sys.exit(0)
- elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
- elif opt == "--windows-mode":
- Globals.set('time_separator', "_")
- Globals.set('chars_to_quote', "A-Z:")
- Globals.set('quoting_enabled', 1)
- Globals.set('preserve_hardlinks', 0)
- select_opts.append(("--exclude-special-files", None))
- elif opt == '--windows-time-format':
- Globals.set('time_separator', "_")
- else: Log.FatalError("Unknown option %s" % opt)
-
-def set_action():
- """Check arguments and try to set action"""
- global action
- l = len(args)
- if not action:
- if l == 0: commandline_error("No arguments given")
- elif l == 1: action = "restore"
- elif l == 2:
- if RPath(Globals.local_connection, args[0]).isincfile():
- action = "restore"
- else: action = "backup"
- else: commandline_error("Too many arguments given")
-
- if l == 0 and action != "server":
- commandline_error("No arguments given")
- if l > 0 and action == "server":
- commandline_error("Too many arguments given")
- if l < 2 and (action == "backup" or action == "mirror" or
- action == "restore-as-of"):
- commandline_error("Two arguments are required (source, destination).")
- if l == 2 and (action == "list-increments" or
- action == "remove-older-than" or
- action == "list-changed-since"):
- commandline_error("Only use one argument, "
- "the root of the backup directory")
- if l > 2 and action != "calculate-average":
- commandline_error("Too many arguments given")
-
-def commandline_error(message):
- sys.stderr.write("Error: %s\n" % message)
- sys.stderr.write("See the rdiff-backup manual page for instructions\n")
- sys.exit(1)
-
-def misc_setup(rps):
- """Set default change ownership flag, umask, relay regexps"""
- if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or
- (len(rps) < 2 and os.getuid() == 0)):
- # Allow change_ownership if destination connection is root
- for conn in Globals.connections:
- conn.Globals.set('change_ownership', 1)
- for rp in rps: rp.setdata() # Update with userinfo
-
- os.umask(077)
- Time.setcurtime(Globals.current_time)
- FilenameMapping.set_init_quote_vals()
- SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
-
- # This is because I originally didn't think compiled regexps
- # could be pickled, and so must be compiled on remote side.
- Globals.postset_regexp('no_compression_regexp',
- Globals.no_compression_regexp_string)
-
- for conn in Globals.connections: Robust.install_signal_handlers()
-
-def take_action(rps):
- """Do whatever action says"""
- if action == "server": PipeConnection(sys.stdin, sys.stdout).Server()
- elif action == "backup": Backup(rps[0], rps[1])
- elif action == "restore": restore(*rps)
- elif action == "restore-as-of": RestoreAsOf(rps[0], rps[1])
- elif action == "mirror": Mirror(rps[0], rps[1])
- elif action == "test-server": SetConnections.TestConnections()
- elif action == "list-changed-since": ListChangedSince(rps[0])
- elif action == "list-increments": ListIncrements(rps[0])
- elif action == "remove-older-than": RemoveOlderThan(rps[0])
- elif action == "calculate-average": CalculateAverage(rps)
- else: raise AssertionError("Unknown action " + action)
-
-def cleanup():
- """Do any last minute cleaning before exiting"""
- Log("Cleaning up", 6)
- Log.close_logfile()
- if not Globals.server: SetConnections.CloseConnections()
-
-def Main(arglist):
- """Start everything up!"""
- parse_cmdlineoptions(arglist)
- set_action()
- cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd)
- Security.initialize(action, cmdpairs)
- rps = map(SetConnections.cmdpair2rp, cmdpairs)
- misc_setup(rps)
- take_action(rps)
- cleanup()
-
-
-def Mirror(src_rp, dest_rp):
- """Turn dest_path into a copy of src_path"""
- Log("Mirroring %s to %s" % (src_rp.path, dest_rp.path), 5)
- mirror_check_paths(src_rp, dest_rp)
- # Since no "rdiff-backup-data" dir, use root of destination.
- SetConnections.UpdateGlobal('rbdir', dest_rp)
- SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn)
- backup_init_select(src_rp, dest_rp)
- HighLevel.Mirror(src_rp, dest_rp)
-
-def mirror_check_paths(rpin, rpout):
- """Check paths and return rpin, rpout"""
- if not rpin.lstat():
- Log.FatalError("Source directory %s does not exist" % rpin.path)
- if rpout.lstat() and not force: Log.FatalError(
-"""Destination %s exists so continuing could mess it up. Run
-rdiff-backup with the --force option if you want to mirror anyway.""" %
- rpout.path)
-
-
-def Backup(rpin, rpout):
- """Backup, possibly incrementally, src_path to dest_path."""
- SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
- backup_init_select(rpin, rpout)
- backup_init_dirs(rpin, rpout)
- RSI = Globals.backup_writer.Resume.ResumeCheck()
- SaveState.init_filenames()
- if prevtime:
- Time.setprevtime(prevtime)
- HighLevel.Mirror_and_increment(rpin, rpout, incdir, RSI)
- else: HighLevel.Mirror(rpin, rpout, incdir, RSI)
- rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout)
-
-def backup_init_select(rpin, rpout):
- """Create Select objects on source and dest connections"""
- rpin.conn.Globals.set_select(DSRPath(1, rpin), select_opts,
- None, *select_files)
- rpout.conn.Globals.set_select(DSRPath(None, rpout), select_mirror_opts, 1)
-
-def backup_init_dirs(rpin, rpout):
- """Make sure rpin and rpout are valid, init data dir and logging"""
- global datadir, incdir, prevtime
- if rpout.lstat() and not rpout.isdir():
- if not force: Log.FatalError("Destination %s exists and is not a "
- "directory" % rpout.path)
- else:
- Log("Deleting %s" % rpout.path, 3)
- rpout.delete()
-
- if not rpin.lstat():
- Log.FatalError("Source directory %s does not exist" % rpin.path)
- elif not rpin.isdir():
- Log.FatalError("Source %s is not a directory" % rpin.path)
-
- datadir = rpout.append("rdiff-backup-data")
- SetConnections.UpdateGlobal('rbdir', datadir)
- incdir = RPath(rpout.conn, os.path.join(datadir.path, "increments"))
- prevtime = backup_get_mirrortime()
-
- if rpout.lstat():
- if rpout.isdir() and not rpout.listdir(): # rpout is empty dir
- rpout.chmod(0700) # just make sure permissions aren't too lax
- elif not datadir.lstat() and not force: Log.FatalError(
-"""Destination directory %s exists, but does not look like a
-rdiff-backup directory. Running rdiff-backup like this could mess up
-what is currently in it. If you want to update or overwrite it, run
-rdiff-backup with the --force option.""" % rpout.path)
-
- if not rpout.lstat():
- try: rpout.mkdir()
- except os.error:
- Log.FatalError("Unable to create directory %s" % rpout.path)
- if not datadir.lstat(): datadir.mkdir()
- if Log.verbosity > 0:
- Log.open_logfile(datadir.append("backup.log"))
- backup_warn_if_infinite_regress(rpin, rpout)
-
-def backup_warn_if_infinite_regress(rpin, rpout):
- """Warn user if destination area contained in source area"""
- if rpout.conn is rpin.conn: # it's meaningful to compare paths
- if ((len(rpout.path) > len(rpin.path)+1 and
- rpout.path[:len(rpin.path)] == rpin.path and
- rpout.path[len(rpin.path)] == '/') or
- (rpin.path == "." and rpout.path[0] != '/' and
- rpout.path[:2] != '..')):
- # Just a few heuristics, we don't have to get every case
- if Globals.backup_reader.Globals.select_source.Select(rpout): Log(
-"""Warning: The destination directory '%s' may be contained in the
-source directory '%s'. This could cause an infinite regress. You
-may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)
-
-def backup_get_mirrorrps():
- """Return list of current_mirror rps"""
- datadir = Globals.rbdir
- if not datadir.isdir(): return []
- mirrorrps = [datadir.append(fn) for fn in datadir.listdir()
- if fn.startswith("current_mirror.")]
- return filter(lambda rp: rp.isincfile(), mirrorrps)
-
-def backup_get_mirrortime():
- """Return time in seconds of previous mirror, or None if cannot"""
- mirrorrps = backup_get_mirrorrps()
- if not mirrorrps: return None
- if len(mirrorrps) > 1:
- Log(
-"""Warning: duplicate current_mirror files found. Perhaps something
-went wrong during your last backup? Using """ + mirrorrps[-1].path, 2)
-
- timestr = mirrorrps[-1].getinctime()
- return Time.stringtotime(timestr)
-
-def backup_touch_curmirror_local(rpin, rpout):
- """Make a file like current_mirror.time.data to record time
-
- Also updates rpout so mod times don't get messed up. This should
- be run on the destination connection.
-
- """
- datadir = Globals.rbdir
- map(RPath.delete, backup_get_mirrorrps())
- mirrorrp = datadir.append("current_mirror.%s.%s" % (Time.curtimestr,
- "data"))
- Log("Touching mirror marker %s" % mirrorrp.path, 6)
- mirrorrp.touch()
- RPath.copy_attribs(rpin, rpout)
-
-def restore(src_rp, dest_rp = None):
- """Main restoring function
-
- Here src_rp should be an increment file, and if dest_rp is
- missing it defaults to the base of the increment.
-
- """
- rpin, rpout = restore_check_paths(src_rp, dest_rp)
- time = Time.stringtotime(rpin.getinctime())
- restore_common(rpin, rpout, time)
-
-def RestoreAsOf(rpin, target):
- """Secondary syntax for restore operation
-
- rpin - RPath of mirror file to restore (not nec. with correct index)
- target - RPath of place to put restored file
-
- """
- restore_check_paths(rpin, target, 1)
- try: time = Time.genstrtotime(restore_timestr)
- except Time.TimeException, exc: Log.FatalError(str(exc))
- restore_common(rpin, target, time)
-
-def restore_common(rpin, target, time):
- """Restore operation common to Restore and RestoreAsOf"""
- mirror_root, index = restore_get_root(rpin)
- mirror = mirror_root.new_index(index)
- inc_rpath = datadir.append_path('increments', index)
- restore_init_select(mirror_root, target)
- restore_start_log(rpin, target, time)
- Restore.Restore(inc_rpath, mirror, target, time)
- Log("Restore ended", 4)
-
-def restore_start_log(rpin, target, time):
- """Open restore log file, log initial message"""
- try: Log.open_logfile(datadir.append("restore.log"))
- except LoggerError, e: Log("Warning, " + str(e), 2)
-
- # Log following message at file verbosity 3, but term verbosity 4
- log_message = ("Starting restore of %s to %s as it was as of %s." %
- (rpin.path, target.path, Time.timetopretty(time)))
- if Log.term_verbosity >= 4: Log.log_to_term(log_message, 4)
- if Log.verbosity >= 3: Log.log_to_file(log_message)
-
-def restore_check_paths(rpin, rpout, restoreasof = None):
- """Check paths and return pair of corresponding rps"""
- if not restoreasof:
- if not rpin.lstat():
- Log.FatalError("Source file %s does not exist" % rpin.path)
- elif not rpin.isincfile():
- Log.FatalError("""File %s does not look like an increment file.
-
-Try restoring from an increment file (the filenames look like
-"foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path)
-
- if not rpout: rpout = RPath(Globals.local_connection,
- rpin.getincbase_str())
- if rpout.lstat():
- Log.FatalError("Restore target %s already exists, "
- "and will not be overwritten." % rpout.path)
- return rpin, rpout
-
-def restore_init_select(rpin, rpout):
- """Initialize Select
-
- Unlike the backup selections, here they are on the local
- connection, because the backup operation is pipelined in a way
- the restore operation isn't.
-
- """
- Globals.set_select(DSRPath(1, rpin), select_mirror_opts, None)
- Globals.set_select(DSRPath(None, rpout), select_opts, None, *select_files)
-
-def restore_get_root(rpin):
- """Return (mirror root, index) and set the data dir
-
- The idea here is to keep backing up on the path until we find
- a directory that contains "rdiff-backup-data". That is the
- mirror root. If the path from there starts
- "rdiff-backup-data/increments*", then the index is the
- remainder minus that. Otherwise the index is just the path
- minus the root.
-
- All this could fail if the increment file is pointed to in a
- funny way, using symlinks or somesuch.
-
- """
- global datadir
- if rpin.isincfile(): relpath = rpin.getincbase().path
- else: relpath = rpin.path
- pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/")
- assert len(pathcomps) >= 2 # path should be relative to /
-
- i = len(pathcomps)
- while i >= 2:
- parent_dir = RPath(rpin.conn, "/".join(pathcomps[:i]))
- if (parent_dir.isdir() and
- "rdiff-backup-data" in parent_dir.listdir()): break
- i = i-1
- else: Log.FatalError("Unable to find rdiff-backup-data directory")
-
- rootrp = parent_dir
- Log("Using mirror root directory %s" % rootrp.path, 6)
-
- datadir = rootrp.append_path("rdiff-backup-data")
- SetConnections.UpdateGlobal('rbdir', datadir)
- if not datadir.isdir():
- Log.FatalError("Unable to read rdiff-backup-data directory %s" %
- datadir.path)
-
- from_datadir = tuple(pathcomps[i:])
- if not from_datadir or from_datadir[0] != "rdiff-backup-data":
- return (rootrp, from_datadir) # in mirror, not increments
- assert from_datadir[1] == "increments"
- return (rootrp, from_datadir[2:])
-
-
-def ListIncrements(rp):
- """Print out a summary of the increments and their times"""
- mirror_root, index = restore_get_root(rp)
- Globals.rbdir = datadir = \
- mirror_root.append_path("rdiff-backup-data")
- mirrorrp = mirror_root.new_index(index)
- inc_rpath = datadir.append_path('increments', index)
- incs = Restore.get_inclist(inc_rpath)
- mirror_time = Restore.get_mirror_time()
- if Globals.parsable_output:
- print Manage.describe_incs_parsable(incs, mirror_time, mirrorrp)
- else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp)
-
-
-def CalculateAverage(rps):
- """Print out the average of the given statistics files"""
- statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
- average_stats = StatsObj().set_to_average(statobjs)
- print average_stats.get_stats_logstring(
- "Average of %d stat files" % len(rps))
-
-
-def RemoveOlderThan(rootrp):
- """Remove all increment files older than a certain time"""
- datadir = rootrp.append("rdiff-backup-data")
- if not datadir.lstat() or not datadir.isdir():
- Log.FatalError("Unable to open rdiff-backup-data dir %s" %
- (datadir.path,))
-
- try: time = Time.genstrtotime(remove_older_than_string)
- except Time.TimeException, exc: Log.FatalError(str(exc))
- timep = Time.timetopretty(time)
- Log("Deleting increment(s) before %s" % timep, 4)
-
- times_in_secs = map(lambda inc: Time.stringtotime(inc.getinctime()),
- Restore.get_inclist(datadir.append("increments")))
- times_in_secs = filter(lambda t: t < time, times_in_secs)
- if not times_in_secs:
- Log.FatalError("No increments older than %s found" % timep)
-
- times_in_secs.sort()
- inc_pretty_time = "\n".join(map(Time.timetopretty, times_in_secs))
- if len(times_in_secs) > 1 and not force:
- Log.FatalError("Found %d relevant increments, dated:\n%s"
- "\nIf you want to delete multiple increments in this way, "
- "use the --force." % (len(times_in_secs), inc_pretty_time))
-
- if len(times_in_secs) == 1:
- Log("Deleting increment at time:\n" + inc_pretty_time, 3)
- else: Log("Deleting increments at times:\n" + inc_pretty_time, 3)
- Manage.delete_earlier_than(datadir, time)
-
-
-def ListChangedSince(rp):
- """List all the files under rp that have changed since restoretime"""
- try: rest_time = Time.genstrtotime(restore_timestr)
- except Time.TimeException, exc: Log.FatalError(str(exc))
- mirror_root, index = restore_get_root(rp)
- Globals.rbdir = datadir = mirror_root.append_path("rdiff-backup-data")
- mirror_time = Restore.get_mirror_time()
-
- def get_rids_recursive(rid):
- """Yield all the rids under rid that have inc newer than rest_time"""
- yield rid
- for sub_rid in Restore.yield_rids(rid, rest_time, mirror_time):
- for sub_sub_rid in get_rids_recursive(sub_rid): yield sub_sub_rid
-
- inc_rpath = datadir.append_path('increments', index)
- inc_list = Restore.get_inclist(inc_rpath)
- root_rid = RestoreIncrementData(index, inc_rpath, inc_list)
- for rid in get_rids_recursive(root_rid):
- if rid.inc_list: print "/".join(rid.index)
-
-
diff --git a/rdiff-backup/src/Make.old b/rdiff-backup/src/Make.old
deleted file mode 100755
index 2b79ffe..0000000
--- a/rdiff-backup/src/Make.old
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-"""Read component files of rdiff-backup, and glue them together after
-removing unnecessary bits."""
-
-import os
-
-def mystrip(filename):
- """Open filename, read input, strip appropriately, and return contents"""
- fp = open(filename, "r")
- lines = fp.readlines()
- fp.close()
-
- i = 0
- while(lines[i][:60] !=
- "############################################################"):
- i = i+1
-
- return "".join(lines[i:]).strip() + "\n\n\n"
-
-
-
-files = ["globals.py", "static.py", "lazy.py", "log.py", "ttime.py",
- "iterfile.py", "rdiff.py", "connection.py", "rpath.py",
- "hardlink.py", "robust.py", "rorpiter.py",
- "destructive_stepping.py", "selection.py",
- "filename_mapping.py", "statistics.py", "increment.py",
- "restore.py", "manage.py", "highlevel.py",
- "setconnections.py", "main.py"]
-
-os.system("cp header.py rdiff-backup")
-
-outfp = open("rdiff-backup", "a")
-for file in files:
- outfp.write(mystrip(file))
-outfp.close()
-
-os.system("chmod 755 rdiff-backup")
diff --git a/rdiff-backup/src/MiscStats.py b/rdiff-backup/src/MiscStats.py
deleted file mode 100644
index ff02ff3..0000000
--- a/rdiff-backup/src/MiscStats.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Misc statistics methods, pertaining to dir and session stat files"""
-
-from statistics import *
-
-
-# This is the RPath of the directory statistics file, and the
-# associated open file. It will hold a line of statistics for
-# each directory that is backed up.
-_dir_stats_rp = None
-_dir_stats_fp = None
-
-# This goes at the beginning of the directory statistics file and
-# explains the format.
-_dir_stats_header = """# rdiff-backup directory statistics file
-#
-# Each line is in the following format:
-# RelativeDirName %s
-""" % " ".join(StatsObj.stat_file_attrs)
-
-def open_dir_stats_file():
- """Open directory statistics file, write header"""
- global _dir_stats_fp, _dir_stats_rp
- assert not _dir_stats_fp, "Directory file already open"
-
- if Globals.compression: suffix = "data.gz"
- else: suffix = "data"
- _dir_stats_rp = Inc.get_inc(Globals.rbdir.append("directory_statistics"),
- Time.curtime, suffix)
-
- if _dir_stats_rp.lstat():
- Log("Warning, statistics file %s already exists, appending" %
- _dir_stats_rp.path, 2)
- _dir_stats_fp = _dir_stats_rp.open("ab", Globals.compression)
- else: _dir_stats_fp = _dir_stats_rp.open("wb", Globals.compression)
- _dir_stats_fp.write(_dir_stats_header)
-
-def write_dir_stats_line(statobj, index):
- """Write info from statobj about rpath to statistics file"""
- if Globals.null_separator:
- _dir_stats_fp.write(statobj.get_stats_line(index, None) + "\0")
- else: _dir_stats_fp.write(statobj.get_stats_line(index) + "\n")
-
-def close_dir_stats_file():
- """Close directory statistics file if its open"""
- global _dir_stats_fp
- if _dir_stats_fp:
- _dir_stats_fp.close()
- _dir_stats_fp = None
-
-def write_session_statistics(statobj):
- """Write session statistics into file, log"""
- stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
- Time.curtime, "data")
- statobj.StartTime = Time.curtime
- statobj.EndTime = time.time()
-
- # include hardlink data and dir stats in size of increments
- if Globals.preserve_hardlinks and Hardlink.final_inc:
- # include hardlink data in size of increments
- statobj.IncrementFiles += 1
- statobj.IncrementFileSize += Hardlink.final_inc.getsize()
- if _dir_stats_rp and _dir_stats_rp.lstat():
- statobj.IncrementFiles += 1
- statobj.IncrementFileSize += _dir_stats_rp.getsize()
-
- statobj.write_stats_to_rp(stat_inc)
- if Globals.print_statistics:
- message = statobj.get_stats_logstring("Session statistics")
- Log.log_to_file(message)
- Globals.client_conn.sys.stdout.write(message)
-
-
-from increment import *
-import Hardlink
diff --git a/rdiff-backup/src/Rdiff.py b/rdiff-backup/src/Rdiff.py
deleted file mode 100644
index cc12cfc..0000000
--- a/rdiff-backup/src/Rdiff.py
+++ /dev/null
@@ -1,127 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Invoke rdiff utility to make signatures, deltas, or patch
-
-All these operations should be done in a relatively safe manner using
-RobustAction and the like.
-
-"""
-
-import os, librsync
-
-
-class RdiffException(Exception): pass
-
-def get_signature(rp):
- """Take signature of rpin file and return in file object"""
- Log("Getting signature of %s" % rp.path, 7)
- return librsync.SigFile(rp.open("rb"))
-
-def get_delta_sigfileobj(sig_fileobj, rp_new):
- """Like get_delta but signature is in a file object"""
- Log("Getting delta of %s with signature stream" % (rp_new.path,), 7)
- return librsync.DeltaFile(sig_fileobj, rp_new.open("rb"))
-
-def get_delta_sigrp(rp_signature, rp_new):
- """Take signature rp and new rp, return delta file object"""
- Log("Getting delta of %s with signature %s" %
- (rp_new.path, rp_signature.path), 7)
- return librsync.DeltaFile(rp_signature.open("rb"), rp_new.open("rb"))
-
-def write_delta_action(basis, new, delta, compress = None):
- """Return action writing delta which brings basis to new
-
- If compress is true, the output of rdiff will be gzipped
- before written to delta.
-
- """
- delta_tf = TempFileManager.new(delta)
- def init(): write_delta(basis, new, delta_tf, compress)
- return Robust.make_tf_robustaction(init, delta_tf, delta)
-
-def write_delta(basis, new, delta, compress = None):
- """Write rdiff delta which brings basis to new"""
- Log("Writing delta %s from %s -> %s" %
- (basis.path, new.path, delta.path), 7)
- sigfile = librsync.SigFile(basis.open("rb"))
- deltafile = librsync.DeltaFile(sigfile, new.open("rb"))
- delta.write_from_fileobj(deltafile, compress)
-
-def patch_action(rp_basis, rp_delta, rp_out = None, out_tf = None,
- delta_compressed = None):
- """Return RobustAction which patches rp_basis with rp_delta
-
- If rp_out is None, put output in rp_basis. Will use TempFile
- out_tf it is specified. If delta_compressed is true, the
- delta file will be decompressed before processing with rdiff.
-
- """
- if not rp_out: rp_out = rp_basis
- if not out_tf: out_tf = TempFileManager.new(rp_out)
- def init():
- rp_basis.conn.Rdiff.patch_local(rp_basis, rp_delta,
- out_tf, delta_compressed)
- out_tf.setdata()
- return Robust.make_tf_robustaction(init, out_tf, rp_out)
-
-def patch_local(rp_basis, rp_delta, outrp, delta_compressed = None):
- """Patch routine that must be run on rp_basis.conn
-
- This is because librsync may need to seek() around in rp_basis,
- and so needs a real file. Other rpaths can be remote.
-
- """
- assert rp_basis.conn is Globals.local_connection
- if delta_compressed: deltafile = rp_delta.open("rb", 1)
- else: deltafile = rp_delta.open("rb")
-
- sigfile = librsync.SigFile(rp_basis.open("rb"))
- patchfile = librsync.PatchedFile(rp_basis.open("rb"), deltafile)
- outrp.write_from_fileobj(patchfile)
-
-def patch_with_attribs_action(rp_basis, rp_delta, rp_out = None):
- """Like patch_action, but also transfers attributs from rp_delta"""
- if not rp_out: rp_out = rp_basis
- tf = TempFileManager.new(rp_out)
- return Robust.chain_nested(patch_action(rp_basis, rp_delta, rp_out, tf),
- Robust.copy_attribs_action(rp_delta, tf))
-
-def copy_action(rpin, rpout):
- """Use rdiff to copy rpin to rpout, conserving bandwidth"""
- if not rpin.isreg() or not rpout.isreg() or rpin.conn is rpout.conn:
- # rdiff not applicable, fallback to regular copying
- return Robust.copy_action(rpin, rpout)
-
- Log("Rdiff copying %s to %s" % (rpin.path, rpout.path), 6)
- out_tf = TempFileManager.new(rpout)
- def init(): rpout.conn.Rdiff.copy_local(rpin, rpout, out_tf)
- return Robust.make_tf_robustaction(init, out_tf, rpout)
-
-def copy_local(rpin, rpout, rpnew):
- """Write rpnew == rpin using rpout as basis. rpout and rpnew local"""
- assert rpnew.conn is rpout.conn is Globals.local_connection
- sigfile = librsync.SigFile(rpout.open("rb"))
- deltafile = rpin.conn.librsync.DeltaFile(sigfile, rpin.open("rb"))
- rpnew.write_from_fileobj(librsync.PatchedFile(rpout.open("rb"), deltafile))
-
-
-from log import *
-from robust import *
-
diff --git a/rdiff-backup/src/Security.py b/rdiff-backup/src/Security.py
deleted file mode 100644
index 24923ef..0000000
--- a/rdiff-backup/src/Security.py
+++ /dev/null
@@ -1,187 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Functions to make sure remote requests are kosher"""
-
-import sys, tempfile
-import Globals, Main
-from rpath import *
-
-class Violation(Exception):
- """Exception that indicates an improper request has been received"""
- pass
-
-
-# This will store the list of functions that will be honored from
-# remote connections.
-allowed_requests = None
-
-# This stores the list of global variables that the client can not
-# set on the server.
-disallowed_server_globals = ["server", "security_level", "restrict_path"]
-
-def initialize(action, cmdpairs):
- """Initialize allowable request list and chroot"""
- global allowed_requests
- set_security_level(action, cmdpairs)
- set_allowed_requests(Globals.security_level)
-
-def set_security_level(action, cmdpairs):
- """If running client, set security level and restrict_path
-
- To find these settings, we must look at the action to see what is
- supposed to happen, and then look at the cmdpairs to see what end
- the client is on.
-
- """
- def islocal(cmdpair): return not cmdpair[0]
- def bothlocal(cp1, cp2): return islocal(cp1) and islocal(cp2)
- def bothremote(cp1, cp2): return not islocal(cp1) and not islocal(cp2)
- def getpath(cmdpair): return cmdpair[1]
-
- if Globals.server: return
- cp1 = cmdpairs[0]
- if len(cmdpairs) > 1: cp2 = cmdpairs[1]
-
- if action == "backup":
- if bothlocal(cp1, cp2) or bothremote(cp1, cp2):
- sec_level = "minimal"
- rdir = tempfile.gettempdir()
- elif islocal(cp1):
- sec_level = "read-only"
- rdir = getpath(cp1)
- else:
- assert islocal(cp2)
- sec_level = "update-only"
- rdir = getpath(cp2)
- elif action == "restore" or action == "restore-as-of":
- if len(cmdpairs) == 1 or bothlocal(cp1, cp2) or bothremote(cp1, cp2):
- sec_level = "minimal"
- rdir = tempfile.gettempdir()
- elif islocal(cp1):
- sec_level = "read-only"
- rdir = Main.restore_get_root(RPath(Globals.local_connection,
- getpath(cp1)))[0].path
- else:
- assert islocal(cp2)
- sec_level = "all"
- rdir = getpath(cp2)
- elif action == "mirror":
- if bothlocal(cp1, cp2) or bothremote(cp1, cp2):
- sec_level = "minimal"
- rdir = tempfile.gettempdir()
- elif islocal(cp1):
- sec_level = "read-only"
- rdir = getpath(cp1)
- else:
- assert islocal(cp2)
- sec_level = "all"
- rdir = getpath(cp2)
- elif (action == "test-server" or action == "list-increments" or
- action == "list-changed-since" or action ==
- "calculate-average" or action == "remove-older-than"):
- sec_level = "minimal"
- rdir = tempfile.gettempdir()
- else: assert 0, "Unknown action %s" % action
-
- Globals.security_level = sec_level
- Globals.restrict_path = RPath(Globals.local_connection,
- rdir).normalize().path
-
-def set_allowed_requests(sec_level):
- """Set the allowed requests list using the security level"""
- global allowed_requests
- if sec_level == "all": return
- allowed_requests = ["VirtualFile.readfromid", "VirtualFile.closebyid",
- "Globals.get", "Globals.is_not_None",
- "Globals.get_dict_val",
- "Log.open_logfile_allconn",
- "Log.close_logfile_allconn",
- "SetConnections.add_redirected_conn",
- "RedirectedRun",
- "sys.stdout.write"]
- if sec_level == "minimal": pass
- elif sec_level == "read-only" or sec_level == "update-only":
- allowed_requests.extend(["C.make_file_dict",
- "os.getuid",
- "os.listdir",
- "Time.setcurtime_local",
- "Resume.ResumeCheck",
- "HLSourceStruct.split_initial_dsiter",
- "HLSourceStruct.get_diffs_and_finalize",
- "RPathStatic.gzip_open_local_read",
- "RPathStatic.open_local_read"])
- if sec_level == "update-only":
- allowed_requests. \
- extend(["Log.open_logfile_local", "Log.close_logfile_local",
- "Log.close_logfile_allconn", "Log.log_to_file",
- "SaveState.init_filenames",
- "SaveState.touch_last_file",
- "HLDestinationStruct.get_sigs",
- "HLDestinationStruct.patch_w_datadir_writes",
- "HLDestinationStruct.patch_and_finalize",
- "HLDestinationStruct.patch_increment_and_finalize",
- "Main.backup_touch_curmirror_local",
- "Globals.ITRB.increment_stat"])
- if Globals.server:
- allowed_requests.extend(["SetConnections.init_connection_remote",
- "Log.setverbosity",
- "Log.setterm_verbosity",
- "Time.setprevtime_local",
- "FilenameMapping.set_init_quote_vals_local",
- "Globals.postset_regexp_local",
- "Globals.set_select",
- "HLSourceStruct.set_session_info",
- "HLDestinationStruct.set_session_info"])
-
-def vet_request(request, arglist):
- """Examine request for security violations"""
- #if Globals.server: sys.stderr.write(str(request) + "\n")
- security_level = Globals.security_level
- if Globals.restrict_path:
- for arg in arglist:
- if isinstance(arg, RPath): vet_rpath(arg)
- if security_level == "all": return
- if request.function_string in allowed_requests: return
- if request.function_string == "Globals.set":
- if Globals.server and arglist[0] not in disallowed_server_globals:
- return
- raise Violation("\nWarning Security Violation!\n"
- "Bad request for function: %s\n"
- "with arguments: %s\n" % (request.function_string,
- arglist))
-
-def vet_rpath(rpath):
- """Require rpath not to step outside retricted directory"""
- if Globals.restrict_path and rpath.conn is Globals.local_connection:
- normalized, restrict = rpath.normalize().path, Globals.restrict_path
- components = normalized.split("/")
- # 3 cases for restricted dir /usr/foo: /var, /usr/foobar, /usr/foo/..
- if (not normalized.startswith(restrict) or
- (len(normalized) > len(restrict) and
- normalized[len(restrict)] != "/") or
- ".." in components):
- raise Violation("\nWarning Security Violation!\n"
- "Request to handle path %s\n"
- "which doesn't appear to be within "
- "restrict path %s.\n" % (normalized, restrict))
-
-
-
-
diff --git a/rdiff-backup/src/SetConnections.py b/rdiff-backup/src/SetConnections.py
deleted file mode 100644
index f0e413a..0000000
--- a/rdiff-backup/src/SetConnections.py
+++ /dev/null
@@ -1,243 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Parse args and setup connections
-
-The functions in this module are used once by Main to parse file
-descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to set up
-the related connections.
-
-"""
-
-# This is the schema that determines how rdiff-backup will open a
-# pipe to the remote system. If the file is given as A::B, %s will
-# be substituted with A in the schema.
-__cmd_schema = 'ssh -C %s rdiff-backup --server'
-__cmd_schema_no_compress = 'ssh %s rdiff-backup --server'
-
-# This is a list of remote commands used to start the connections.
-# The first is None because it is the local connection.
-__conn_remote_cmds = [None]
-
-class SetConnectionsException(Exception): pass
-
-def get_cmd_pairs(arglist, remote_schema = None, remote_cmd = None):
- """Map the given file descriptions into command pairs
-
- Command pairs are tuples cmdpair with length 2. cmdpair[0] is
- None iff it describes a local path, and cmdpair[1] is the path.
-
- """
- global __cmd_schema
- if remote_schema: __cmd_schema = remote_schema
- elif not Globals.ssh_compression: __cmd_schema = __cmd_schema_no_compress
-
- if not arglist: return []
- desc_pairs = map(parse_file_desc, arglist)
-
- if filter(lambda x: x[0], desc_pairs): # True if any host_info found
- if remote_cmd:
- Log.FatalError("The --remote-cmd flag is not compatible "
- "with remote file descriptions.")
- elif remote_schema:
- Log("Remote schema option ignored - no remote file "
- "descriptions.", 2)
- cmdpairs = map(desc2cmd_pairs, desc_pairs)
- if remote_cmd: # last file description gets remote_cmd
- cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1])
- return cmdpairs
-
-def cmdpair2rp(cmd_pair):
- """Return normalized RPath from cmd_pair (remote_cmd, filename)"""
- cmd, filename = cmd_pair
- if cmd: conn = init_connection(cmd)
- else: conn = Globals.local_connection
- return RPath(conn, filename).normalize()
-
-def desc2cmd_pairs(desc_pair):
- """Return pair (remote_cmd, filename) from desc_pair"""
- host_info, filename = desc_pair
- if not host_info: return (None, filename)
- else: return (fill_schema(host_info), filename)
-
-def parse_file_desc(file_desc):
- """Parse file description returning pair (host_info, filename)
-
- In other words, bescoto@folly.stanford.edu::/usr/bin/ls =>
- ("bescoto@folly.stanford.edu", "/usr/bin/ls"). The
- complication is to allow for quoting of : by a \. If the
- string is not separated by :, then the host_info is None.
-
- """
- def check_len(i):
- if i >= len(file_desc):
- raise SetConnectionsException(
- "Unexpected end to file description %s" % file_desc)
-
- host_info_list, i, last_was_quoted = [], 0, None
- while 1:
- if i == len(file_desc):
- return (None, file_desc)
-
- if file_desc[i] == '\\':
- i = i+1
- check_len(i)
- last_was_quoted = 1
- elif (file_desc[i] == ":" and i > 0 and file_desc[i-1] == ":"
- and not last_was_quoted):
- host_info_list.pop() # Remove last colon from name
- break
- else: last_was_quoted = None
- host_info_list.append(file_desc[i])
- i = i+1
-
- check_len(i+1)
- return ("".join(host_info_list), file_desc[i+1:])
-
-def fill_schema(host_info):
- """Fills host_info into the schema and returns remote command"""
- return __cmd_schema % host_info
-
-def init_connection(remote_cmd):
- """Run remote_cmd, register connection, and then return it
-
- If remote_cmd is None, then the local connection will be
- returned. This also updates some settings on the remote side,
- like global settings, its connection number, and verbosity.
-
- """
- if not remote_cmd: return Globals.local_connection
-
- Log("Executing " + remote_cmd, 4)
- stdin, stdout = os.popen2(remote_cmd)
- conn_number = len(Globals.connections)
- conn = PipeConnection(stdout, stdin, conn_number)
-
- check_connection_version(conn, remote_cmd)
- Log("Registering connection %d" % conn_number, 7)
- init_connection_routing(conn, conn_number, remote_cmd)
- init_connection_settings(conn)
- return conn
-
-def check_connection_version(conn, remote_cmd):
- """Log warning if connection has different version"""
- try: remote_version = conn.Globals.get('version')
- except ConnectionReadError, exception:
- Log.FatalError("""%s
-
-Couldn't start up the remote connection by executing
-
- %s
-
-Remember that, under the default settings, rdiff-backup must be
-installed in the PATH on the remote system. See the man page for more
-information.""" % (exception, remote_cmd))
-
- if remote_version != Globals.version:
- Log("Warning: Local version %s does not match remote version %s."
- % (Globals.version, remote_version), 2)
-
-def init_connection_routing(conn, conn_number, remote_cmd):
- """Called by init_connection, establish routing, conn dict"""
- Globals.connection_dict[conn_number] = conn
-
- conn.SetConnections.init_connection_remote(conn_number)
- for other_remote_conn in Globals.connections[1:]:
- conn.SetConnections.add_redirected_conn(
- other_remote_conn.conn_number)
- other_remote_conn.SetConnections.add_redirected_conn(conn_number)
-
- Globals.connections.append(conn)
- __conn_remote_cmds.append(remote_cmd)
-
-def init_connection_settings(conn):
- """Tell new conn about log settings and updated globals"""
- conn.Log.setverbosity(Log.verbosity)
- conn.Log.setterm_verbosity(Log.term_verbosity)
- for setting_name in Globals.changed_settings:
- conn.Globals.set(setting_name, Globals.get(setting_name))
- FilenameMapping.set_init_quote_vals()
-
-def init_connection_remote(conn_number):
- """Run on server side to tell self that have given conn_number"""
- Globals.connection_number = conn_number
- Globals.local_connection.conn_number = conn_number
- Globals.connection_dict[0] = Globals.connections[1]
- Globals.connection_dict[conn_number] = Globals.local_connection
-
-def add_redirected_conn(conn_number):
- """Run on server side - tell about redirected connection"""
- Globals.connection_dict[conn_number] = \
- RedirectedConnection(conn_number)
-
-def UpdateGlobal(setting_name, val):
- """Update value of global variable across all connections"""
- for conn in Globals.connections:
- conn.Globals.set(setting_name, val)
-
-def BackupInitConnections(reading_conn, writing_conn):
- """Backup specific connection initialization"""
- reading_conn.Globals.set("isbackup_reader", 1)
- writing_conn.Globals.set("isbackup_writer", 1)
- UpdateGlobal("backup_reader", reading_conn)
- UpdateGlobal("backup_writer", writing_conn)
- if (Globals.change_source_perms and
- reading_conn.Globals.get("process_uid") == 0):
- Log("Warning: --change_source_perms should usually not be used when\n"
- "the reading connection is running as root, because root can\n"
- "read all files regardless of their permissions.", 2)
-
-def CloseConnections():
- """Close all connections. Run by client"""
- assert not Globals.server
- for conn in Globals.connections: conn.quit()
- del Globals.connections[1:] # Only leave local connection
- Globals.connection_dict = {0: Globals.local_connection}
- Globals.backup_reader = Globals.isbackup_reader = \
- Globals.backup_writer = Globals.isbackup_writer = None
-
-def TestConnections():
- """Test connections, printing results"""
- if len(Globals.connections) == 1: print "No remote connections specified"
- else:
- for i in range(1, len(Globals.connections)): test_connection(i)
-
-def test_connection(conn_number):
- """Test connection. conn_number 0 is the local connection"""
- print "Testing server started by: ", __conn_remote_cmds[conn_number]
- conn = Globals.connections[conn_number]
- try:
- assert conn.pow(2,3) == 8
- assert conn.os.path.join("a", "b") == "a/b"
- version = conn.reval("lambda: Globals.version")
- except:
- sys.stderr.write("Server tests failed\n")
- raise
- if not version == Globals.version:
- print """Server may work, but there is a version mismatch:
-Local version: %s
-Remote version: %s""" % (Globals.version, version)
- else: print "Server OK"
-
-
-from log import *
-from rpath import *
-from connection import *
-import Globals, FilenameMapping
-
diff --git a/rdiff-backup/src/Time.py b/rdiff-backup/src/Time.py
deleted file mode 100644
index bc8fb1b..0000000
--- a/rdiff-backup/src/Time.py
+++ /dev/null
@@ -1,234 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Provide time related exceptions and functions"""
-
-import time, types, re
-import Globals
-
-
-class TimeException(Exception): pass
-
-_interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400,
- "W": 7*86400, "M": 30*86400, "Y": 365*86400}
-_integer_regexp = re.compile("^[0-9]+$")
-_interval_regexp = re.compile("^([0-9]+)([smhDWMY])")
-_genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
- "(?P<month>[0-9]{1,2})[-/](?P<day>[0-9]{1,2})$")
-_genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]"
- "(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$")
-curtime = curtimestr = None
-been_awake_since = None # stores last time sleep() was run
-
-def setcurtime(curtime = None):
- """Sets the current time in curtime and curtimestr on all systems"""
- t = curtime or time.time()
- for conn in Globals.connections:
- conn.Time.setcurtime_local(t)
-
-def setcurtime_local(timeinseconds):
- """Only set the current time locally"""
- global curtime, curtimestr
- curtime, curtimestr = timeinseconds, timetostring(timeinseconds)
-
-def setprevtime(timeinseconds):
- """Sets the previous inc time in prevtime and prevtimestr"""
- assert timeinseconds > 0, timeinseconds
- timestr = timetostring(timeinseconds)
- for conn in Globals.connections:
- conn.Time.setprevtime_local(timeinseconds, timestr)
-
-def setprevtime_local(timeinseconds, timestr):
- """Like setprevtime but only set the local version"""
- global prevtime, prevtimestr
- prevtime, prevtimestr = timeinseconds, timestr
-
-def timetostring(timeinseconds):
- """Return w3 datetime compliant listing of timeinseconds"""
- return time.strftime("%Y-%m-%dT%H" + Globals.time_separator +
- "%M" + Globals.time_separator + "%S",
- time.localtime(timeinseconds)) + gettzd()
-
-def stringtotime(timestring):
- """Return time in seconds from w3 timestring
-
- If there is an error parsing the string, or it doesn't look
- like a w3 datetime string, return None.
-
- """
- try:
- date, daytime = timestring[:19].split("T")
- year, month, day = map(int, date.split("-"))
- hour, minute, second = map(int,
- daytime.split(Globals.time_separator))
- assert 1900 < year < 2100, year
- assert 1 <= month <= 12
- assert 1 <= day <= 31
- assert 0 <= hour <= 23
- assert 0 <= minute <= 59
- assert 0 <= second <= 61 # leap seconds
- timetuple = (year, month, day, hour, minute, second, -1, -1, -1)
- if time.daylight:
- utc_in_secs = time.mktime(timetuple) - time.altzone
- else: utc_in_secs = time.mktime(timetuple) - time.timezone
-
- return long(utc_in_secs) + tzdtoseconds(timestring[19:])
- except (TypeError, ValueError, AssertionError): return None
-
-def timetopretty(timeinseconds):
- """Return pretty version of time"""
- return time.asctime(time.localtime(timeinseconds))
-
-def stringtopretty(timestring):
- """Return pretty version of time given w3 time string"""
- return timetopretty(stringtotime(timestring))
-
-def inttopretty(seconds):
- """Convert num of seconds to readable string like "2 hours"."""
- partlist = []
- hours, seconds = divmod(seconds, 3600)
- if hours > 1: partlist.append("%d hours" % hours)
- elif hours == 1: partlist.append("1 hour")
-
- minutes, seconds = divmod(seconds, 60)
- if minutes > 1: partlist.append("%d minutes" % minutes)
- elif minutes == 1: partlist.append("1 minute")
-
- if seconds == 1: partlist.append("1 second")
- elif not partlist or seconds > 1:
- if isinstance(seconds, int) or isinstance(seconds, long):
- partlist.append("%s seconds" % seconds)
- else: partlist.append("%.2f seconds" % seconds)
- return " ".join(partlist)
-
-def intstringtoseconds(interval_string):
- """Convert a string expressing an interval (e.g. "4D2s") to seconds"""
- def error():
- raise TimeException("""Bad interval string "%s"
-
-Intervals are specified like 2Y (2 years) or 2h30m (2.5 hours). The
-allowed special characters are s, m, h, D, W, M, and Y. See the man
-page for more information.
-""" % interval_string)
- if len(interval_string) < 2: error()
-
- total = 0
- while interval_string:
- match = _interval_regexp.match(interval_string)
- if not match: error()
- num, ext = int(match.group(1)), match.group(2)
- if not ext in _interval_conv_dict or num < 0: error()
- total += num*_interval_conv_dict[ext]
- interval_string = interval_string[match.end(0):]
- return total
-
-def gettzd():
- """Return w3's timezone identification string.
-
- Expresed as [+/-]hh:mm. For instance, PST is -08:00. Zone is
- coincides with what localtime(), etc., use.
-
- """
- if time.daylight: offset = -1 * time.altzone/60
- else: offset = -1 * time.timezone/60
- if offset > 0: prefix = "+"
- elif offset < 0: prefix = "-"
- else: return "Z" # time is already in UTC
-
- hours, minutes = map(abs, divmod(offset, 60))
- assert 0 <= hours <= 23
- assert 0 <= minutes <= 59
- return "%s%02d%s%02d" % (prefix, hours,
- Globals.time_separator, minutes)
-
-def tzdtoseconds(tzd):
- """Given w3 compliant TZD, return how far ahead UTC is"""
- if tzd == "Z": return 0
- assert len(tzd) == 6 # only accept forms like +08:00 for now
- assert (tzd[0] == "-" or tzd[0] == "+") and \
- tzd[3] == Globals.time_separator
- return -60 * (60 * int(tzd[:3]) + int(tzd[4:]))
-
-def cmp(time1, time2):
- """Compare time1 and time2 and return -1, 0, or 1"""
- if type(time1) is types.StringType:
- time1 = stringtotime(time1)
- assert time1 is not None
- if type(time2) is types.StringType:
- time2 = stringtotime(time2)
- assert time2 is not None
-
- if time1 < time2: return -1
- elif time1 == time2: return 0
- else: return 1
-
-
-def sleep(sleep_ratio):
- """Sleep for period to maintain given sleep_ratio
-
- On my system sleeping for periods less than 1/20th of a second
- doesn't seem to work very accurately, so accumulate at least that
- much time before sleeping.
-
- """
- global been_awake_since
- if been_awake_since is None: # first running
- been_awake_since = time.time()
- else:
- elapsed_time = time.time() - been_awake_since
- sleep_time = elapsed_time * (sleep_ratio/(1-sleep_ratio))
- if sleep_time >= 0.05:
- time.sleep(sleep_time)
- been_awake_since = time.time()
-
-def genstrtotime(timestr, curtime = None):
- """Convert a generic time string to a time in seconds"""
- if curtime is None: curtime = globals()['curtime']
- if timestr == "now": return curtime
-
- def error():
- raise TimeException("""Bad time string "%s"
-
-The acceptible time strings are intervals (like "3D64s"), w3-datetime
-strings, like "2002-04-26T04:22:01-07:00" (strings like
-"2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the
-current time zone), or ordinary dates like 2/4/1997 or 2001-04-23
-(various combinations are acceptable, but the month always precedes
-the day).""" % timestr)
-
- # Test for straight integer
- if _integer_regexp.search(timestr): return int(timestr)
-
- # Test for w3-datetime format, possibly missing tzd
- t = stringtotime(timestr) or stringtotime(timestr+gettzd())
- if t: return t
-
- try: # test for an interval, like "2 days ago"
- return curtime - intstringtoseconds(timestr)
- except TimeException: pass
-
- # Now check for dates like 2001/3/23
- match = _genstr_date_regexp1.search(timestr) or \
- _genstr_date_regexp2.search(timestr)
- if not match: error()
- timestr = "%s-%02d-%02dT00:00:00%s" % (match.group('year'),
- int(match.group('month')), int(match.group('day')), gettzd())
- t = stringtotime(timestr)
- if t: return t
- else: error()
diff --git a/rdiff-backup/src/__init__.py b/rdiff-backup/src/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/rdiff-backup/src/__init__.py
+++ /dev/null
diff --git a/rdiff-backup/src/_librsyncmodule.c b/rdiff-backup/src/_librsyncmodule.c
deleted file mode 100644
index 6fca4cc..0000000
--- a/rdiff-backup/src/_librsyncmodule.c
+++ /dev/null
@@ -1,468 +0,0 @@
-/* ----------------------------------------------------------------------- *
- *
- * Copyright 2002 Ben Escoto
- *
- * This file is part of rdiff-backup.
- *
- * rdiff-backup is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation, Inc., 675 Mass Ave,
- * Cambridge MA 02139, USA; either version 2 of the License, or (at
- * your option) any later version; incorporated herein by reference.
- *
- * ----------------------------------------------------------------------- */
-
-#include <Python.h>
-#include <rsync.h>
-#define RS_JOB_BLOCKSIZE 65536
-
-static PyObject *librsyncError;
-
-/* Sets python error string from result */
-static void
-_librsync_seterror(rs_result result, char *location)
-{
- char error_string[200];
- sprintf(error_string, "librsync error %d while in %s", result, location);
- PyErr_SetString(librsyncError, error_string);
-}
-
-
-/* --------------- SigMaker Object for incremental signatures */
-staticforward PyTypeObject _librsync_SigMakerType;
-
-typedef struct {
- PyObject_HEAD
- PyObject *x_attr;
- rs_job_t *sig_job;
-} _librsync_SigMakerObject;
-
-static PyObject*
-_librsync_new_sigmaker(PyObject* self, PyObject* args)
-{
- _librsync_SigMakerObject* sm;
-
- if (!PyArg_ParseTuple(args,":new_sigmaker"))
- return NULL;
-
- sm = PyObject_New(_librsync_SigMakerObject, &_librsync_SigMakerType);
- if (sm == NULL) return NULL;
- sm->x_attr = NULL;
-
- sm->sig_job = rs_sig_begin((size_t)RS_DEFAULT_BLOCK_LEN,
- (size_t)RS_DEFAULT_STRONG_LEN);
- return (PyObject*)sm;
-}
-
-static void
-_librsync_sigmaker_dealloc(PyObject* self)
-{
- rs_job_free(((_librsync_SigMakerObject *)self)->sig_job);
- PyObject_Del(self);
-}
-
-/* Take an input string, and generate a signature from it. The output
- will be a triple (done, bytes_used, signature_string), where done
- is true iff there is no more data coming and bytes_used is the
- number of bytes of the input string processed.
-*/
-static PyObject *
-_librsync_sigmaker_cycle(_librsync_SigMakerObject *self, PyObject *args)
-{
- char *inbuf, outbuf[RS_JOB_BLOCKSIZE];
- long inbuf_length;
- rs_buffers_t buf;
- rs_result result;
-
- if (!PyArg_ParseTuple(args, "s#:cycle", &inbuf, &inbuf_length))
- return NULL;
-
- buf.next_in = inbuf;
- buf.avail_in = (size_t)inbuf_length;
- buf.next_out = outbuf;
- buf.avail_out = (size_t)RS_JOB_BLOCKSIZE;
- buf.eof_in = (inbuf_length == 0);
-
- result = rs_job_iter(self->sig_job, &buf);
-
- if (result != RS_DONE && result != RS_BLOCKED) {
- _librsync_seterror(result, "signature cycle");
- return NULL;
- }
-
- return Py_BuildValue("(ils#)", (result == RS_DONE),
- inbuf_length - (long)buf.avail_in,
- outbuf, RS_JOB_BLOCKSIZE - (long)buf.avail_out);
-}
-
-static PyMethodDef _librsync_sigmaker_methods[] = {
- {"cycle", (PyCFunction)_librsync_sigmaker_cycle, METH_VARARGS},
- {NULL, NULL, 0, NULL} /* sentinel */
-};
-
-static PyObject *
-_librsync_sigmaker_getattr(_librsync_SigMakerObject *sm,
- char *name)
-{
- if (sm->x_attr != NULL) {
- PyObject *v = PyDict_GetItemString(sm->x_attr, name);
- if (v != NULL) {
- Py_INCREF(v);
- return v;
- }
- }
- return Py_FindMethod(_librsync_sigmaker_methods, (PyObject *)sm, name);
-}
-
-static int
-_librsync_sigmaker_setattr(_librsync_SigMakerObject *sm,
- char *name, PyObject *v)
-{
- if (sm->x_attr == NULL) {
- sm->x_attr = PyDict_New();
- if (sm->x_attr == NULL) return -1;
- }
- if (v == NULL) {
- int rv = PyDict_DelItemString(sm->x_attr, name);
- if (rv < 0)
- PyErr_SetString(PyExc_AttributeError,
- "delete non-existing sigmaker attribute");
- return rv;
- }
- else return PyDict_SetItemString(sm->x_attr, name, v);
-}
-
-static PyTypeObject _librsync_SigMakerType = {
- PyObject_HEAD_INIT(NULL)
- 0,
- "sigmaker",
- sizeof(_librsync_SigMakerObject),
- 0,
- _librsync_sigmaker_dealloc, /*tp_dealloc*/
- 0, /*tp_print*/
- (getattrfunc)_librsync_sigmaker_getattr, /*tp_getattr*/
- (setattrfunc)_librsync_sigmaker_setattr, /*tp_setattr*/
- 0, /*tp_compare*/
- 0, /*tp_repr*/
- 0, /*tp_as_number*/
- 0, /*tp_as_sequence*/
- 0, /*tp_as_mapping*/
- 0, /*tp_hash */
-};
-
-
-/* --------------- DeltaMaker Object for incremental deltas */
-
-staticforward PyTypeObject _librsync_DeltaMakerType;
-
-typedef struct {
- PyObject_HEAD
- PyObject *x_attr;
- rs_job_t *delta_job;
- rs_signature_t *sig_ptr;
-} _librsync_DeltaMakerObject;
-
-/* Call with the entire signature loaded into one big string */
-static PyObject*
-_librsync_new_deltamaker(PyObject* self, PyObject* args)
-{
- _librsync_DeltaMakerObject* dm;
- char *sig_string, outbuf[RS_JOB_BLOCKSIZE];
- long sig_length;
- rs_job_t *sig_loader;
- rs_signature_t *sig_ptr;
- rs_buffers_t buf;
- rs_result result;
-
- if (!PyArg_ParseTuple(args,"s#:new_deltamaker", &sig_string, &sig_length))
- return NULL;
-
- dm = PyObject_New(_librsync_DeltaMakerObject, &_librsync_DeltaMakerType);
- if (dm == NULL) return NULL;
- dm->x_attr = NULL;
-
- /* Put signature at sig_ptr and build hash */
- sig_loader = rs_loadsig_begin(&sig_ptr);
- buf.next_in = sig_string;
- buf.avail_in = (size_t)sig_length;
- buf.next_out = outbuf;
- buf.avail_out = (size_t)RS_JOB_BLOCKSIZE;
- buf.eof_in = 1;
- result = rs_job_iter(sig_loader, &buf);
- rs_job_free(sig_loader);
- if (result != RS_DONE) {
- _librsync_seterror(result, "delta rs_signature_t builder");
- return NULL;
- }
- if ((result = rs_build_hash_table(sig_ptr)) != RS_DONE) {
- _librsync_seterror(result, "delta rs_build_hash_table");
- return NULL;
- }
-
- dm->sig_ptr = sig_ptr;
- dm->delta_job = rs_delta_begin(sig_ptr);
- return (PyObject*)dm;
-}
-
-static void
-_librsync_deltamaker_dealloc(PyObject* self)
-{
- _librsync_DeltaMakerObject *dm = (_librsync_DeltaMakerObject *)self;
- rs_signature_t *sig_ptr = dm->sig_ptr;
-
- rs_free_sumset(sig_ptr);
- rs_job_free(dm->delta_job);
- PyObject_Del(self);
-}
-
-/* Take a chunk of the new file in an input string, and return a
- triple (done bytes_used, delta_string), where done is true iff no
- more data is coming and bytes_used is the number of bytes of the
- input string processed.
-*/
-static PyObject *
-_librsync_deltamaker_cycle(_librsync_DeltaMakerObject *self, PyObject *args)
-{
- char *inbuf, outbuf[RS_JOB_BLOCKSIZE];
- long inbuf_length;
- rs_buffers_t buf;
- rs_result result;
-
- if (!PyArg_ParseTuple(args, "s#:cycle", &inbuf, &inbuf_length))
- return NULL;
-
- buf.next_in = inbuf;
- buf.avail_in = (size_t)inbuf_length;
- buf.next_out = outbuf;
- buf.avail_out = (size_t)RS_JOB_BLOCKSIZE;
- buf.eof_in = (inbuf_length == 0);
-
- result = rs_job_iter(self->delta_job, &buf);
- if (result != RS_DONE && result != RS_BLOCKED) {
- _librsync_seterror(result, "delta cycle");
- return NULL;
- }
-
- return Py_BuildValue("(ils#)", (result == RS_DONE),
- inbuf_length - (long)buf.avail_in,
- outbuf, RS_JOB_BLOCKSIZE - (long)buf.avail_out);
-}
-
-static PyMethodDef _librsync_deltamaker_methods[] = {
- {"cycle", (PyCFunction)_librsync_deltamaker_cycle, METH_VARARGS},
- {NULL, NULL, 0, NULL} /* sentinel */
-};
-
-static PyObject *
-_librsync_deltamaker_getattr(_librsync_DeltaMakerObject *dm, char *name)
-{
- if (dm->x_attr != NULL) {
- PyObject *v = PyDict_GetItemString(dm->x_attr, name);
- if (v != NULL) {
- Py_INCREF(v);
- return v;
- }
- }
- return Py_FindMethod(_librsync_deltamaker_methods, (PyObject *)dm, name);
-}
-
-static int
-_librsync_deltamaker_setattr(_librsync_DeltaMakerObject *dm,
- char *name, PyObject *v)
-{
- if (dm->x_attr == NULL) {
- dm->x_attr = PyDict_New();
- if (dm->x_attr == NULL) return -1;
- }
- if (v == NULL) {
- int rv = PyDict_DelItemString(dm->x_attr, name);
- if (rv < 0)
- PyErr_SetString(PyExc_AttributeError,
- "delete non-existing deltamaker attribute");
- return rv;
- }
- else return PyDict_SetItemString(dm->x_attr, name, v);
-}
-
-static PyTypeObject _librsync_DeltaMakerType = {
- PyObject_HEAD_INIT(NULL)
- 0,
- "deltamaker",
- sizeof(_librsync_DeltaMakerObject),
- 0,
- _librsync_deltamaker_dealloc, /*tp_dealloc*/
- 0, /*tp_print*/
- (getattrfunc)_librsync_deltamaker_getattr, /*tp_getattr*/
- (setattrfunc)_librsync_deltamaker_setattr, /*tp_setattr*/
- 0, /*tp_compare*/
- 0, /*tp_repr*/
- 0, /*tp_as_number*/
- 0, /*tp_as_sequence*/
- 0, /*tp_as_mapping*/
- 0, /*tp_hash */
-};
-
-
-/* --------------- PatchMaker Object for incremental patching */
-
-
-staticforward PyTypeObject _librsync_PatchMakerType;
-
-typedef struct {
- PyObject_HEAD
- PyObject *x_attr;
- rs_job_t *patch_job;
- PyObject *basis_file;
-} _librsync_PatchMakerObject;
-
-/* Call with the basis file */
-static PyObject*
-_librsync_new_patchmaker(PyObject* self, PyObject* args)
-{
- _librsync_PatchMakerObject* pm;
- PyObject *python_file;
- FILE *cfile;
-
- if (!PyArg_ParseTuple(args, "O:new_patchmaker", &python_file))
- return NULL;
- if (!PyFile_Check(python_file)) {
- PyErr_SetString(PyExc_TypeError, "Need true file object");
- return NULL;
- }
- Py_INCREF(python_file);
-
- pm = PyObject_New(_librsync_PatchMakerObject, &_librsync_PatchMakerType);
- if (pm == NULL) return NULL;
- pm->x_attr = NULL;
-
- pm->basis_file = python_file;
- cfile = PyFile_AsFile(python_file);
- pm->patch_job = rs_patch_begin(rs_file_copy_cb, cfile);
-
- return (PyObject*)pm;
-}
-
-static void
-_librsync_patchmaker_dealloc(PyObject* self)
-{
- _librsync_PatchMakerObject *pm = (_librsync_PatchMakerObject *)self;
- Py_DECREF(pm->basis_file);
- rs_job_free(pm->patch_job);
- PyObject_Del(self);
-}
-
-/* Take a chunk of the delta file in an input string, and return a
- triple (done, bytes_used, patched_string), where done is true iff
- there is no more data coming out and bytes_used is the number of
- bytes of the input string processed.
-*/
-static PyObject *
-_librsync_patchmaker_cycle(_librsync_PatchMakerObject *self, PyObject *args)
-{
- char *inbuf, outbuf[RS_JOB_BLOCKSIZE];
- long inbuf_length;
- rs_buffers_t buf;
- rs_result result;
-
- if (!PyArg_ParseTuple(args, "s#:cycle", &inbuf, &inbuf_length))
- return NULL;
-
- buf.next_in = inbuf;
- buf.avail_in = (size_t)inbuf_length;
- buf.next_out = outbuf;
- buf.avail_out = (size_t)RS_JOB_BLOCKSIZE;
- buf.eof_in = (inbuf_length == 0);
-
- result = rs_job_iter(self->patch_job, &buf);
- if (result != RS_DONE && result != RS_BLOCKED) {
- _librsync_seterror(result, "patch cycle");
- return NULL;
- }
-
- return Py_BuildValue("(ils#)", (result == RS_DONE),
- inbuf_length - (long)buf.avail_in,
- outbuf, RS_JOB_BLOCKSIZE - (long)buf.avail_out);
-}
-
-static PyMethodDef _librsync_patchmaker_methods[] = {
- {"cycle", (PyCFunction)_librsync_patchmaker_cycle, METH_VARARGS},
- {NULL, NULL, 0, NULL} /* sentinel */
-};
-
-static PyObject *
-_librsync_patchmaker_getattr(_librsync_PatchMakerObject *pm, char *name)
-{
- if (pm->x_attr != NULL) {
- PyObject *v = PyDict_GetItemString(pm->x_attr, name);
- if (v != NULL) {
- Py_INCREF(v);
- return v;
- }
- }
- return Py_FindMethod(_librsync_patchmaker_methods, (PyObject *)pm, name);
-}
-
-static int
-_librsync_patchmaker_setattr(_librsync_PatchMakerObject *pm,
- char *name, PyObject *v)
-{
- if (pm->x_attr == NULL) {
- pm->x_attr = PyDict_New();
- if (pm->x_attr == NULL) return -1;
- }
- if (v == NULL) {
- int rv = PyDict_DelItemString(pm->x_attr, name);
- if (rv < 0)
- PyErr_SetString(PyExc_AttributeError,
- "delete non-existing patchmaker attribute");
- return rv;
- }
- else return PyDict_SetItemString(pm->x_attr, name, v);
-}
-
-static PyTypeObject _librsync_PatchMakerType = {
- PyObject_HEAD_INIT(NULL)
- 0,
- "patchmaker",
- sizeof(_librsync_PatchMakerObject),
- 0,
- _librsync_patchmaker_dealloc, /*tp_dealloc*/
- 0, /*tp_print*/
- (getattrfunc)_librsync_patchmaker_getattr, /*tp_getattr*/
- (setattrfunc)_librsync_patchmaker_setattr, /*tp_setattr*/
- 0, /*tp_compare*/
- 0, /*tp_repr*/
- 0, /*tp_as_number*/
- 0, /*tp_as_sequence*/
- 0, /*tp_as_mapping*/
- 0, /*tp_hash */
-};
-
-
-/* --------------- _librsync module definition */
-
-static PyMethodDef _librsyncMethods[] = {
- {"new_sigmaker", _librsync_new_sigmaker, METH_VARARGS,
- "Return a sigmaker object, for finding the signature of an object"},
- {"new_deltamaker", _librsync_new_deltamaker, METH_VARARGS,
- "Return a deltamaker object, for computing deltas"},
- {"new_patchmaker", _librsync_new_patchmaker, METH_VARARGS,
- "Return a patchmaker object, for patching basis files"},
- {NULL, NULL, 0, NULL}
-};
-
-void init_librsync(void)
-{
- PyObject *m, *d;
-
- _librsync_SigMakerType.ob_type = &PyType_Type;
- _librsync_DeltaMakerType.ob_type = &PyType_Type;
- m = Py_InitModule("_librsync", _librsyncMethods);
- d = PyModule_GetDict(m);
- librsyncError = PyErr_NewException("_librsync.librsyncError", NULL, NULL);
- PyDict_SetItemString(d, "librsyncError", librsyncError);
- PyDict_SetItemString(d, "RS_JOB_BLOCKSIZE",
- Py_BuildValue("l", (long)RS_JOB_BLOCKSIZE));
- PyDict_SetItemString(d, "RS_DEFAULT_BLOCK_LEN",
- Py_BuildValue("l", (long)RS_DEFAULT_BLOCK_LEN));
-}
diff --git a/rdiff-backup/src/cmodule.c b/rdiff-backup/src/cmodule.c
deleted file mode 100644
index b9e3e3e..0000000
--- a/rdiff-backup/src/cmodule.c
+++ /dev/null
@@ -1,217 +0,0 @@
-/* ----------------------------------------------------------------------- *
- *
- * Copyright 2002 Ben Escoto
- *
- * This file is part of rdiff-backup.
- *
- * rdiff-backup is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation, Inc., 675 Mass Ave,
- * Cambridge MA 02139, USA; either version 2 of the License, or (at
- * your option) any later version; incorporated herein by reference.
- *
- * ----------------------------------------------------------------------- */
-
-
-#include <Python.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <errno.h>
-
-/* choose the appropriate stat and fstat functions and return structs */
-/* This code taken from Python's posixmodule.c */
-#undef STAT
-#if defined(MS_WIN64) || defined(MS_WIN32)
-# define STAT _stati64
-# define FSTAT _fstati64
-# define STRUCT_STAT struct _stati64
-#else
-# define STAT stat
-# define FSTAT fstat
-# define STRUCT_STAT struct stat
-#endif
-
-static PyObject *UnknownFileTypeError;
-static PyObject *c_make_file_dict(PyObject *self, PyObject *args);
-static PyObject *long2str(PyObject *self, PyObject *args);
-static PyObject *str2long(PyObject *self, PyObject *args);
-
-
-/* Turn a stat structure into a python dictionary. The preprocessor
- stuff taken from Python's posixmodule.c */
-static PyObject *c_make_file_dict(self, args)
- PyObject *self;
- PyObject *args;
-{
- PyObject *size, *inode, *mtime, *atime, *devloc, *return_val;
- char *filename, filetype[5];
- STRUCT_STAT sbuf;
- long int mode, perms;
- int res;
-
- if (!PyArg_ParseTuple(args, "s", &filename)) return NULL;
-
- Py_BEGIN_ALLOW_THREADS
- res = lstat(filename, &sbuf);
- Py_END_ALLOW_THREADS
-
- if (res != 0) {
- if (errno == ENOENT || errno == ENOTDIR)
- return Py_BuildValue("{s:s}", "type", NULL);
- else {
- PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
- return NULL;
- }
- }
-#ifdef HAVE_LARGEFILE_SUPPORT
- size = PyLong_FromLongLong((LONG_LONG)sbuf.st_size);
- inode = PyLong_FromLongLong((LONG_LONG)sbuf.st_ino);
-#else
- size = PyInt_FromLong(sbuf.st_size);
- inode = PyInt_FromLong((long)sbuf.st_ino);
-#endif
- mode = (long)sbuf.st_mode;
- perms = mode & 07777;
-#if defined(HAVE_LONG_LONG) && !defined(MS_WINDOWS)
- devloc = PyLong_FromLongLong((LONG_LONG)sbuf.st_dev);
-#else
- devloc = PyInt_FromLong((long)sbuf.st_dev);
-#endif
-#if SIZEOF_TIME_T > SIZEOF_LONG
- mtime = PyLong_FromLongLong((LONG_LONG)sbuf.st_mtime);
- atime = PyLong_FromLongLong((LONG_LONG)sbuf.st_atime);
-#else
- mtime = PyInt_FromLong((long)sbuf.st_mtime);
- atime = PyInt_FromLong((long)sbuf.st_atime);
-#endif
-
- /* Build return dictionary from stat struct */
- if (S_ISREG(mode) || S_ISDIR(mode) || S_ISSOCK(mode) || S_ISFIFO(mode)) {
- /* Regular files, directories, sockets, and fifos */
- if S_ISREG(mode) strcpy(filetype, "reg");
- else if S_ISDIR(mode) strcpy(filetype, "dir");
- else if S_ISSOCK(mode) strcpy(filetype, "sock");
- else strcpy(filetype, "fifo");
- return_val = Py_BuildValue("{s:s,s:O,s:l,s:l,s:l,s:O,s:O,s:l,s:O,s:O}",
- "type", filetype,
- "size", size,
- "perms", perms,
- "uid", (long)sbuf.st_uid,
- "gid", (long)sbuf.st_gid,
- "inode", inode,
- "devloc", devloc,
- "nlink", (long)sbuf.st_nlink,
- "mtime", mtime,
- "atime", atime);
- } else if S_ISLNK(mode) {
- /* Symbolic links */
- char linkname[1024];
- int len_link = readlink(filename, linkname, 1023);
- if (len_link < 0) {
- PyErr_SetFromErrno(PyExc_OSError);
- return_val = NULL;
- } else {
- linkname[len_link] = '\0';
- return_val = Py_BuildValue("{s:s,s:O,s:l,s:l,s:l,s:O,s:O,s:l,s:s}",
- "type", "sym",
- "size", size,
- "perms", perms,
- "uid", (long)sbuf.st_uid,
- "gid", (long)sbuf.st_gid,
- "inode", inode,
- "devloc", devloc,
- "nlink", (long)sbuf.st_nlink,
- "linkname", linkname);
- }
- } else if (S_ISCHR(mode) || S_ISBLK(mode)) {
- /* Device files */
- char devtype[2];
-#if defined(HAVE_LONG_LONG) && !defined(MS_WINDOWS)
- LONG_LONG devnums = (LONG_LONG)sbuf.st_rdev;
- PyObject *major_num = PyLong_FromLongLong(major(devnums));
-#else
- long int devnums = (long)sbuf.st_dev;
- PyObject *major_num = PyInt_FromLong(devnums >> 8);
-#endif
- int minor_num = (int)(minor(devnums));
- if S_ISCHR(mode) strcpy(devtype, "c");
- else strcpy(devtype, "b");
- return_val = Py_BuildValue("{s:s,s:O,s:l,s:l,s:l,s:O,s:O,s:l,s:N}",
- "type", "dev",
- "size", size,
- "perms", perms,
- "uid", (long)sbuf.st_uid,
- "gid", (long)sbuf.st_gid,
- "inode", inode,
- "devloc", devloc,
- "nlink", (long)sbuf.st_nlink,
- "devnums", Py_BuildValue("(s,O,i)", devtype,
- major_num, minor_num));
- Py_DECREF(major_num);
- } else {
- /* Unrecognized file type - raise exception */
- PyErr_SetString(UnknownFileTypeError, filename);
- return_val = NULL;
- }
- Py_DECREF(size);
- Py_DECREF(inode);
- Py_DECREF(devloc);
- Py_DECREF(mtime);
- Py_DECREF(atime);
- return return_val;
-}
-
-
-/* Convert python long into 7 byte string */
-static PyObject *long2str(self, args)
- PyObject *self;
- PyObject *args;
-{
- unsigned char s[7];
- PyLongObject *pylong;
- PyObject *return_val;
-
- if (!PyArg_ParseTuple(args, "O!", &PyLong_Type, &pylong)) return NULL;
- if (_PyLong_AsByteArray(pylong, s, 7, 0, 0) != 0) return NULL;
- else return Py_BuildValue("s#", s, 7);
- return return_val;
-}
-
-
-/* Reverse of above; convert 7 byte string into python long */
-static PyObject *str2long(self, args)
- PyObject *self;
- PyObject *args;
-{
- unsigned char *s;
- int ssize;
-
- if (!PyArg_ParseTuple(args, "s#", &s, &ssize)) return NULL;
- if (ssize != 7) {
- PyErr_SetString(PyExc_TypeError, "Single argument must be 7 char string");
- return NULL;
- }
- return _PyLong_FromByteArray(s, 7, 0, 0);
-}
-
-
-static PyMethodDef CMethods[] = {
- {"make_file_dict", c_make_file_dict, METH_VARARGS,
- "Make dictionary from file stat"},
- {"long2str", long2str, METH_VARARGS, "Convert python long to 7 byte string"},
- {"str2long", str2long, METH_VARARGS, "Convert 7 byte string to python long"},
- {NULL, NULL, 0, NULL}
-};
-
-void initC(void)
-{
- PyObject *m, *d;
-
- m = Py_InitModule("C", CMethods);
- d = PyModule_GetDict(m);
- UnknownFileTypeError = PyErr_NewException("C.UnknownFileTypeError",
- NULL, NULL);
- PyDict_SetItemString(d, "UnknownFileTypeError", UnknownFileTypeError);
-}
-
diff --git a/rdiff-backup/src/compilec.py b/rdiff-backup/src/compilec.py
deleted file mode 100755
index 17f6ea0..0000000
--- a/rdiff-backup/src/compilec.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-
-import sys, os
-from distutils.core import setup, Extension
-
-assert len(sys.argv) == 1
-sys.argv.append("build")
-
-setup(name="CModule",
- version="0.9.0",
- description="rdiff-backup's C component",
- ext_modules=[Extension("C", ["cmodule.c"]),
- Extension("_librsync", ["_librsyncmodule.c"],
- libraries=["rsync"])])
-
-assert not os.system("mv build/lib.linux-i686-2.2/C.so .")
-assert not os.system("mv build/lib.linux-i686-2.2/_librsync.so .")
-assert not os.system("rm -rf build")
diff --git a/rdiff-backup/src/connection.py b/rdiff-backup/src/connection.py
deleted file mode 100644
index 9d139af..0000000
--- a/rdiff-backup/src/connection.py
+++ /dev/null
@@ -1,563 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Support code for remote execution and data transfer"""
-
-from __future__ import generators
-import types, os, tempfile, cPickle, shutil, traceback, pickle, socket
-
-
-class ConnectionError(Exception): pass
-class ConnectionReadError(ConnectionError): pass
-class ConnectionQuit(Exception): pass
-
-
-class Connection:
- """Connection class - represent remote execution
-
- The idea is that, if c is an instance of this class, c.foo will
- return the object on the remote side. For functions, c.foo will
- return a function that, when called, executes foo on the remote
- side, sending over the arguments and sending back the result.
-
- """
- def __repr__(self): return self.__str__()
- def __str__(self): return "Simple Connection" # override later
-
-class LocalConnection(Connection):
- """Local connection
-
- This is a dummy connection class, so that LC.foo just evaluates to
- foo using global scope.
-
- """
- def __init__(self):
- """This prevents two instances of LocalConnection"""
- assert not Globals.local_connection
- self.conn_number = 0 # changed by SetConnections for server
-
- def __getattr__(self, name):
- if name in globals(): return globals()[name]
- elif isinstance(__builtins__, dict): return __builtins__[name]
- else: return __builtins__.__dict__[name]
-
- def __setattr__(self, name, value): globals()[name] = value
-
- def __delattr__(self, name): del globals()[name]
-
- def __str__(self): return "LocalConnection"
-
- def reval(self, function_string, *args):
- return apply(eval(function_string), args)
-
- def quit(self): pass
-
-
-class ConnectionRequest:
- """Simple wrapper around a PipeConnection request"""
- def __init__(self, function_string, num_args):
- self.function_string = function_string
- self.num_args = num_args
-
- def __str__(self):
- return "ConnectionRequest: %s with %d arguments" % \
- (self.function_string, self.num_args)
-
-
-class LowLevelPipeConnection(Connection):
- """Routines for just sending objects from one side of pipe to another
-
- Each thing sent down the pipe is paired with a request number,
- currently limited to be between 0 and 255. The size of each thing
- should be less than 2^56.
-
- Each thing also has a type, indicated by one of the following
- characters:
-
- o - generic object
- i - iterator/generator of RORPs
- f - file object
- b - string
- q - quit signal
- t - TempFile
- d - DSRPath
- R - RPath
- r - RORPath only
- c - PipeConnection object
-
- """
- def __init__(self, inpipe, outpipe):
- """inpipe is a file-type open for reading, outpipe for writing"""
- self.inpipe = inpipe
- self.outpipe = outpipe
-
- def __str__(self):
- """Return string version
-
- This is actually an important function, because otherwise
- requests to represent this object would result in "__str__"
- being executed on the other side of the connection.
-
- """
- return "LowLevelPipeConnection"
-
- def _put(self, obj, req_num):
- """Put an object into the pipe (will send raw if string)"""
- Log.conn("sending", obj, req_num)
- if type(obj) is types.StringType: self._putbuf(obj, req_num)
- elif isinstance(obj, Connection): self._putconn(obj, req_num)
- elif isinstance(obj, TempFile): self._puttempfile(obj, req_num)
- elif isinstance(obj, DSRPath): self._putdsrpath(obj, req_num)
- elif isinstance(obj, RPath): self._putrpath(obj, req_num)
- elif isinstance(obj, RORPath): self._putrorpath(obj, req_num)
- elif ((hasattr(obj, "read") or hasattr(obj, "write"))
- and hasattr(obj, "close")): self._putfile(obj, req_num)
- elif hasattr(obj, "next"): self._putiter(obj, req_num)
- else: self._putobj(obj, req_num)
-
- def _putobj(self, obj, req_num):
- """Send a generic python obj down the outpipe"""
- # for some reason there is an error when cPickle is used below..
- self._write("o", pickle.dumps(obj, 1), req_num)
-
- def _putbuf(self, buf, req_num):
- """Send buffer buf down the outpipe"""
- self._write("b", buf, req_num)
-
- def _putfile(self, fp, req_num):
- """Send a file to the client using virtual files"""
- self._write("f", str(VirtualFile.new(fp)), req_num)
-
- def _putiter(self, iterator, req_num):
- """Put an iterator through the pipe"""
- self._write("i", str(VirtualFile.new(RORPIter.ToFile(iterator))),
- req_num)
-
- def _puttempfile(self, tempfile, req_num):
- """Put a tempfile into pipe. See _putrpath"""
- tf_repr = (tempfile.conn.conn_number, tempfile.base,
- tempfile.index, tempfile.data)
- self._write("t", cPickle.dumps(tf_repr, 1), req_num)
-
- def _putdsrpath(self, dsrpath, req_num):
- """Put DSRPath into pipe. See _putrpath"""
- dsrpath_repr = (dsrpath.conn.conn_number, dsrpath.getstatedict())
- self._write("d", cPickle.dumps(dsrpath_repr, 1), req_num)
-
- def _putrpath(self, rpath, req_num):
- """Put an rpath into the pipe
-
- The rpath's connection will be encoded as its conn_number. It
- and the other information is put in a tuple.
-
- """
- rpath_repr = (rpath.conn.conn_number, rpath.base,
- rpath.index, rpath.data)
- self._write("R", cPickle.dumps(rpath_repr, 1), req_num)
-
- def _putrorpath(self, rorpath, req_num):
- """Put an rorpath into the pipe
-
- This is only necessary because if there is a .file attached,
- it must be excluded from the pickling
-
- """
- rorpath_repr = (rorpath.index, rorpath.data)
- self._write("r", cPickle.dumps(rorpath_repr, 1), req_num)
-
- def _putconn(self, pipeconn, req_num):
- """Put a connection into the pipe
-
- A pipe connection is represented just as the integer (in
- string form) of its connection number it is *connected to*.
-
- """
- self._write("c", str(pipeconn.conn_number), req_num)
-
- def _putquit(self):
- """Send a string that takes down server"""
- self._write("q", "", 255)
-
- def _write(self, headerchar, data, req_num):
- """Write header and then data to the pipe"""
- self.outpipe.write(headerchar + chr(req_num) +
- C.long2str(long(len(data))))
- self.outpipe.write(data)
- self.outpipe.flush()
-
- def _read(self, length):
- """Read length bytes from inpipe, returning result"""
- return self.inpipe.read(length)
-
- def _s2l_old(self, s):
- """Convert string to long int"""
- assert len(s) == 7
- l = 0L
- for i in range(7): l = l*256 + ord(s[i])
- return l
-
- def _l2s_old(self, l):
- """Convert long int to string"""
- s = ""
- for i in range(7):
- l, remainder = divmod(l, 256)
- s = chr(remainder) + s
- assert remainder == 0
- return s
-
- def _get(self):
- """Read an object from the pipe and return (req_num, value)"""
- header_string = self.inpipe.read(9)
- if not len(header_string) == 9:
- raise ConnectionReadError("Truncated header string (problem "
- "probably originated remotely)")
- try:
- format_string, req_num, length = (header_string[0],
- ord(header_string[1]),
- C.str2long(header_string[2:]))
- except IndexError: raise ConnectionError()
- if format_string == "q": raise ConnectionQuit("Received quit signal")
-
- data = self._read(length)
- if format_string == "o": result = cPickle.loads(data)
- elif format_string == "b": result = data
- elif format_string == "f": result = VirtualFile(self, int(data))
- elif format_string == "i":
- result = RORPIter.FromFile(BufferedRead(VirtualFile(self,
- int(data))))
- elif format_string == "t": result = self._gettempfile(data)
- elif format_string == "r": result = self._getrorpath(data)
- elif format_string == "R": result = self._getrpath(data)
- elif format_string == "d": result = self._getdsrpath(data)
- else:
- assert format_string == "c", header_string
- result = Globals.connection_dict[int(data)]
- Log.conn("received", result, req_num)
- return (req_num, result)
-
- def _getrorpath(self, raw_rorpath_buf):
- """Reconstruct RORPath object from raw data"""
- index, data = cPickle.loads(raw_rorpath_buf)
- return RORPath(index, data)
-
- def _gettempfile(self, raw_tf_buf):
- """Return TempFile object indicated by raw_tf_buf"""
- conn_number, base, index, data = cPickle.loads(raw_tf_buf)
- return TempFile(Globals.connection_dict[conn_number],
- base, index, data)
-
- def _getrpath(self, raw_rpath_buf):
- """Return RPath object indicated by raw_rpath_buf"""
- conn_number, base, index, data = cPickle.loads(raw_rpath_buf)
- return RPath(Globals.connection_dict[conn_number], base, index, data)
-
- def _getdsrpath(self, raw_dsrpath_buf):
- """Return DSRPath object indicated by buf"""
- conn_number, state_dict = cPickle.loads(raw_dsrpath_buf)
- empty_dsrp = DSRPath("bypass", Globals.local_connection, None)
- empty_dsrp.__setstate__(state_dict)
- empty_dsrp.conn = Globals.connection_dict[conn_number]
- empty_dsrp.file = None
- return empty_dsrp
-
- def _close(self):
- """Close the pipes associated with the connection"""
- self.outpipe.close()
- self.inpipe.close()
-
-
-class PipeConnection(LowLevelPipeConnection):
- """Provide server and client functions for a Pipe Connection
-
- Both sides act as modules that allows for remote execution. For
- instance, self.conn.pow(2,8) will execute the operation on the
- server side.
-
- The only difference between the client and server is that the
- client makes the first request, and the server listens first.
-
- """
- def __init__(self, inpipe, outpipe, conn_number = 0):
- """Init PipeConnection
-
- conn_number should be a unique (to the session) integer to
- identify the connection. For instance, all connections to the
- client have conn_number 0. Other connections can use this
- number to route commands to the correct process.
-
- """
- LowLevelPipeConnection.__init__(self, inpipe, outpipe)
- self.conn_number = conn_number
- self.unused_request_numbers = {}
- for i in range(256): self.unused_request_numbers[i] = None
-
- def __str__(self): return "PipeConnection %d" % self.conn_number
-
- def get_response(self, desired_req_num):
- """Read from pipe, responding to requests until req_num.
-
- Sometimes after a request is sent, the other side will make
- another request before responding to the original one. In
- that case, respond to the request. But return once the right
- response is given.
-
- """
- while 1:
- try: req_num, object = self._get()
- except ConnectionQuit:
- self._put("quitting", self.get_new_req_num())
- self._close()
- return
- if req_num == desired_req_num: return object
- else:
- assert isinstance(object, ConnectionRequest)
- self.answer_request(object, req_num)
-
- def answer_request(self, request, req_num):
- """Put the object requested by request down the pipe"""
- del self.unused_request_numbers[req_num]
- argument_list = []
- for i in range(request.num_args):
- arg_req_num, arg = self._get()
- assert arg_req_num == req_num
- argument_list.append(arg)
- try:
- Security.vet_request(request, argument_list)
- result = apply(eval(request.function_string), argument_list)
- except: result = self.extract_exception()
- self._put(result, req_num)
- self.unused_request_numbers[req_num] = None
-
- def extract_exception(self):
- """Return active exception"""
- if Log.verbosity >= 5 or Log.term_verbosity >= 5:
- Log("Sending back exception %s of type %s: \n%s" %
- (sys.exc_info()[1], sys.exc_info()[0],
- "".join(traceback.format_tb(sys.exc_info()[2]))), 5)
- return sys.exc_info()[1]
-
- def Server(self):
- """Start server's read eval return loop"""
- Globals.server = 1
- Globals.connections.append(self)
- Log("Starting server", 6)
- self.get_response(-1)
-
- def reval(self, function_string, *args):
- """Execute command on remote side
-
- The first argument should be a string that evaluates to a
- function, like "pow", and the remaining are arguments to that
- function.
-
- """
- req_num = self.get_new_req_num()
- self._put(ConnectionRequest(function_string, len(args)), req_num)
- for arg in args: self._put(arg, req_num)
- result = self.get_response(req_num)
- self.unused_request_numbers[req_num] = None
- if isinstance(result, Exception): raise result
- else: return result
-
- def get_new_req_num(self):
- """Allot a new request number and return it"""
- if not self.unused_request_numbers:
- raise ConnectionError("Exhaused possible connection numbers")
- req_num = self.unused_request_numbers.keys()[0]
- del self.unused_request_numbers[req_num]
- return req_num
-
- def quit(self):
- """Close the associated pipes and tell server side to quit"""
- assert not Globals.server
- self._putquit()
- self._get()
- self._close()
-
- def __getattr__(self, name):
- """Intercept attributes to allow for . invocation"""
- return EmulateCallable(self, name)
-
-
-class RedirectedConnection(Connection):
- """Represent a connection more than one move away
-
- For instance, suppose things are connected like this: S1---C---S2.
- If Server1 wants something done by Server2, it will have to go
- through the Client. So on S1's side, S2 will be represented by a
- RedirectedConnection.
-
- """
- def __init__(self, conn_number, routing_number = 0):
- """RedirectedConnection initializer
-
- Returns a RedirectedConnection object for the given
- conn_number, where commands are routed through the connection
- with the given routing_number. 0 is the client, so the
- default shouldn't have to be changed.
-
- """
- self.conn_number = conn_number
- self.routing_number = routing_number
- self.routing_conn = Globals.connection_dict[routing_number]
-
- def reval(self, function_string, *args):
- """Evalution function_string on args on remote connection"""
- return self.routing_conn.reval("RedirectedRun", self.conn_number,
- function_string, *args)
-
- def __str__(self):
- return "RedirectedConnection %d,%d" % (self.conn_number,
- self.routing_number)
-
- def __getattr__(self, name):
- return EmulateCallableRedirected(self.conn_number, self.routing_conn,
- name)
-
-def RedirectedRun(conn_number, func, *args):
- """Run func with args on connection with conn number conn_number
-
- This function is meant to redirect requests from one connection to
- another, so conn_number must not be the local connection (and also
- for security reasons since this function is always made
- available).
-
- """
- conn = Globals.connection_dict[conn_number]
- assert conn is not Globals.local_connection, conn
- return conn.reval(func, *args)
-
-
-class EmulateCallable:
- """This is used by PipeConnection in calls like conn.os.chmod(foo)"""
- def __init__(self, connection, name):
- self.connection = connection
- self.name = name
- def __call__(self, *args):
- return apply(self.connection.reval, (self.name,) + args)
- def __getattr__(self, attr_name):
- return EmulateCallable(self.connection,
- "%s.%s" % (self.name, attr_name))
-
-class EmulateCallableRedirected:
- """Used by RedirectedConnection in calls like conn.os.chmod(foo)"""
- def __init__(self, conn_number, routing_conn, name):
- self.conn_number, self.routing_conn = conn_number, routing_conn
- self.name = name
- def __call__(self, *args):
- return apply(self.routing_conn.reval,
- ("RedirectedRun", self.conn_number, self.name) + args)
- def __getattr__(self, attr_name):
- return EmulateCallableRedirected(self.conn_number, self.routing_conn,
- "%s.%s" % (self.name, attr_name))
-
-
-class VirtualFile:
- """When the client asks for a file over the connection, it gets this
-
- The returned instance then forwards requests over the connection.
- The class's dictionary is used by the server to associate each
- with a unique file number.
-
- """
- #### The following are used by the server
- vfiles = {}
- counter = 0
-
- def getbyid(cls, id):
- return cls.vfiles[id]
- getbyid = classmethod(getbyid)
-
- def readfromid(cls, id, length):
- return cls.vfiles[id].read(length)
- readfromid = classmethod(readfromid)
-
- def readlinefromid(cls, id):
- return cls.vfiles[id].readline()
- readlinefromid = classmethod(readlinefromid)
-
- def writetoid(cls, id, buffer):
- return cls.vfiles[id].write(buffer)
- writetoid = classmethod(writetoid)
-
- def closebyid(cls, id):
- fp = cls.vfiles[id]
- del cls.vfiles[id]
- return fp.close()
- closebyid = classmethod(closebyid)
-
- def new(cls, fileobj):
- """Associate a new VirtualFile with a read fileobject, return id"""
- count = cls.counter
- cls.vfiles[count] = fileobj
- cls.counter = count + 1
- return count
- new = classmethod(new)
-
-
- #### And these are used by the client
- def __init__(self, connection, id):
- self.connection = connection
- self.id = id
-
- def read(self, length = -1):
- return self.connection.VirtualFile.readfromid(self.id, length)
-
- def readline(self):
- return self.connection.VirtualFile.readlinefromid(self.id)
-
- def write(self, buf):
- return self.connection.VirtualFile.writetoid(self.id, buf)
-
- def close(self):
- return self.connection.VirtualFile.closebyid(self.id)
-
- def __iter__(self):
- """Iterates lines in file, like normal iter(file) behavior"""
- while 1:
- line = self.readline()
- if not line: break
- yield line
-
-
-# everything has to be available here for remote connection's use, but
-# put at bottom to reduce circularities.
-import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, Main
-from static import *
-from lazy import *
-from log import *
-from iterfile import *
-from connection import *
-from rpath import *
-from robust import *
-from rorpiter import *
-from destructive_stepping import *
-from selection import *
-from statistics import *
-from increment import *
-from restore import *
-from manage import *
-from highlevel import *
-
-
-Globals.local_connection = LocalConnection()
-Globals.connections.append(Globals.local_connection)
-# Following changed by server in SetConnections
-Globals.connection_dict[0] = Globals.local_connection
-
diff --git a/rdiff-backup/src/destructive_stepping.py b/rdiff-backup/src/destructive_stepping.py
deleted file mode 100644
index fdce815..0000000
--- a/rdiff-backup/src/destructive_stepping.py
+++ /dev/null
@@ -1,236 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Deal with side effects from traversing trees"""
-
-from __future__ import generators
-import types
-from rpath import *
-from lazy import *
-
-class DSRPPermError(Exception):
- """Exception used when a DSRPath can't get sufficient permissions"""
- pass
-
-class DSRPath(RPath):
- """Destructive Stepping RPath
-
- Sometimes when we traverse the directory tree, even when we just
- want to read files, we have to change things, like the permissions
- of a file or directory in order to read it, or the file's access
- times. This class is like an RPath, but the permission and time
- modifications are delayed, so that they can be done at the very
- end when they won't be disturbed later.
-
- Here are the new class variables:
- delay_perms - true iff future perm changes should be delayed
- newperms - holds the perm values while they are delayed
- delay_atime - true iff some atime change are being delayed
- newatime - holds the new atime
- delay_mtime - true if some mtime change is being delayed
- newmtime - holds the new mtime
-
- """
- def __init__(self, source, conn_or_rp, base = 0, index = ()):
- """Initialize DSRP
-
- Source should be true iff the DSRPath is taken from the
- "source" partition and thus settings like
- Globals.change_source_perms should be paid attention to.
-
- If args is [rpath], return the dsrpath equivalent of rpath,
- otherwise use the same arguments as the RPath initializer.
-
- """
- if base == 0:
- assert isinstance(conn_or_rp, RPath)
- RPath.__init__(self, conn_or_rp.conn,
- conn_or_rp.base, conn_or_rp.index)
- self.path = conn_or_rp.path # conn_or_rp may be quoted
- else: RPath.__init__(self, conn_or_rp, base, index)
-
- if source != "bypass":
- # "bypass" val is used when unpackaging over connection
- assert source is None or source is 1
- self.source = source
- self.set_delays(source)
- self.set_init_perms(source)
-
- def set_delays(self, source):
- """Delay writing permissions and times where appropriate"""
- if not source or Globals.change_source_perms:
- self.delay_perms, self.newperms = 1, None
- else: self.delay_perms = None
-
- if Globals.preserve_atime:
- self.delay_atime = 1
- # Now get atime right away if possible
- if self.data.has_key('atime'): self.newatime = self.data['atime']
- else: self.newatime = None
- else: self.delay_atime = None
-
- if source:
- self.delay_mtime = None # we'll never change mtime of source file
- else:
- self.delay_mtime = 1
- # Save mtime now for a dir, because it might inadvertantly change
- if self.isdir(): self.newmtime = self.data['mtime']
- else: self.newmtime = None
-
- def set_init_perms(self, source):
- """If necessary, change permissions to ensure access"""
- if self.isreg() and not self.readable():
- if (source and Globals.change_source_perms or
- not source and Globals.change_mirror_perms):
- self.chmod_bypass(0400)
- elif self.isdir():
- if source and Globals.change_source_perms:
- if not self.readable() or not self.executable():
- self.chmod_bypass(0500)
- elif not source and Globals.change_mirror_perms:
- if not self.hasfullperms(): self.chmod_bypass(0700)
-
- def warn(self, err):
- Log("Received error '%s' when dealing with file %s, skipping..."
- % (err, self.path), 1)
- raise DSRPPermError(self.path)
-
- def __getstate__(self):
- """Return picklable state. See RPath __getstate__."""
- assert self.conn is Globals.local_connection # Can't pickle a conn
- return self.getstatedict()
-
- def getstatedict(self):
- """Return dictionary containing the attributes we can save"""
- pickle_dict = {}
- for attrib in ['index', 'data', 'delay_perms', 'newperms',
- 'delay_atime', 'newatime',
- 'delay_mtime', 'newmtime',
- 'path', 'base', 'source']:
- if self.__dict__.has_key(attrib):
- pickle_dict[attrib] = self.__dict__[attrib]
- return pickle_dict
-
- def __setstate__(self, pickle_dict):
- """Set state from object produced by getstate"""
- self.conn = Globals.local_connection
- for attrib in pickle_dict.keys():
- self.__dict__[attrib] = pickle_dict[attrib]
-
- def chmod(self, permissions):
- """Change permissions, delaying if self.perms_delayed is set"""
- if self.delay_perms: self.newperms = self.data['perms'] = permissions
- else: RPath.chmod(self, permissions)
-
- def getperms(self):
- """Return dsrp's intended permissions"""
- if self.delay_perms and self.newperms is not None:
- return self.newperms
- else: return self.data['perms']
-
- def chmod_bypass(self, permissions):
- """Change permissions without updating the data dictionary"""
- self.delay_perms = 1
- if self.newperms is None: self.newperms = self.getperms()
- Log("DSRP: Perm bypass %s to %o" % (self.path, permissions), 8)
- self.conn.os.chmod(self.path, permissions)
-
- def settime(self, accesstime, modtime):
- """Change times, delaying if self.times_delayed is set"""
- if self.delay_atime: self.newatime = self.data['atime'] = accesstime
- if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime
-
- if not self.delay_atime or not self.delay_mtime:
- RPath.settime(self, accesstime, modtime)
-
- def setmtime(self, modtime):
- """Change mtime, delaying if self.times_delayed is set"""
- if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime
- else: RPath.setmtime(self, modtime)
-
- def getmtime(self):
- """Return dsrp's intended modification time"""
- if self.delay_mtime and self.newmtime is not None:
- return self.newmtime
- else: return self.data['mtime']
-
- def getatime(self):
- """Return dsrp's intended access time"""
- if self.delay_atime and self.newatime is not None:
- return self.newatime
- else: return self.data['atime']
-
- def write_changes(self):
- """Write saved up permission/time changes"""
- if not self.lstat(): return # File has been deleted in meantime
-
- if self.delay_perms and self.newperms is not None:
- Log("Finalizing permissions of dsrp %s to %s" %
- (self.path, self.newperms), 8)
- RPath.chmod(self, self.newperms)
-
- do_atime = self.delay_atime and self.newatime is not None
- do_mtime = self.delay_mtime and self.newmtime is not None
- if do_atime and do_mtime:
- RPath.settime(self, self.newatime, self.newmtime)
- elif do_atime and not do_mtime:
- RPath.settime(self, self.newatime, self.getmtime())
- elif not do_atime and do_mtime:
- RPath.setmtime(self, self.newmtime)
-
- def newpath(self, newpath, index = ()):
- """Return similar DSRPath but with new path"""
- return self.__class__(self.source, self.conn, newpath, index)
-
- def append(self, ext):
- """Return similar DSRPath with new extension"""
- return self.__class__(self.source, self.conn, self.base,
- self.index + (ext,))
-
- def new_index(self, index):
- """Return similar DSRPath with new index"""
- return self.__class__(self.source, self.conn, self.base, index)
-
-
-class DestructiveSteppingFinalizer(ITRBranch):
- """Finalizer that can work on an iterator of dsrpaths
-
- The reason we have to use an IterTreeReducer is that some files
- should be updated immediately, but for directories we sometimes
- need to update all the files in the directory before finally
- coming back to it.
-
- """
- dsrpath = None
- def start_process(self, index, dsrpath):
- self.dsrpath = dsrpath
-
- def end_process(self):
- if self.dsrpath: self.dsrpath.write_changes()
-
- def can_fast_process(self, index, dsrpath):
- return not self.dsrpath.isdir()
-
- def fast_process(self, index, dsrpath):
- if self.dsrpath: self.dsrpath.write_changes()
-
-
-from log import *
-from robust import *
-import Globals
diff --git a/rdiff-backup/src/filelist.py b/rdiff-backup/src/filelist.py
deleted file mode 100644
index 7a660c3..0000000
--- a/rdiff-backup/src/filelist.py
+++ /dev/null
@@ -1,106 +0,0 @@
-from __future__ import generators
-execfile("manage.py")
-
-#######################################################################
-#
-# filelist - Some routines that help with operations over files listed
-# in standard input instead of over whole directories.
-#
-
-class FilelistError(Exception): pass
-
-class Filelist:
- """Many of these methods have analogs in highlevel.py"""
- def File2Iter(fp, baserp):
- """Convert file obj with one pathname per line into rpiter
-
- Closes fp when done. Given files are added to baserp.
-
- """
- while 1:
- line = fp.readline()
- if not line: break
- if line[-1] == "\n": line = line[:-1] # strip trailing newline
- if not line: continue # skip blank lines
- elif line[0] == "/": raise FilelistError(
- "Read in absolute file name %s." % line)
- yield baserp.append(line)
- assert not fp.close(), "Error closing filelist fp"
-
- def Mirror(src_rpath, dest_rpath, rpiter):
- """Copy files in fileiter from src_rpath to dest_rpath"""
- sigiter = dest_rpath.conn.Filelist.get_sigs(dest_rpath, rpiter)
- diffiter = Filelist.get_diffs(src_rpath, sigiter)
- dest_rpath.conn.Filelist.patch(dest_rpath, diffiter)
- dest_rpath.setdata()
-
- def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath):
- """Mirror + put increment in tree based at inc_rpath"""
- sigiter = dest_rpath.conn.Filelist.get_sigs(dest_rpath, rpiter)
- diffiter = Filelist.get_diffs(src_rpath, sigiter)
- dest_rpath.conn.Filelist.patch_and_increment(dest_rpath, diffiter,
- inc_rpath)
- dest_rpath.setdata()
-
- def get_sigs(dest_rpbase, rpiter):
- """Get signatures of file analogs in rpiter
-
- This is meant to be run on the destination side. Only the
- extention part of the rps in rpiter will be used; the base is
- ignored.
-
- """
- def dest_iter(src_iter):
- for src_rp in src_iter: yield dest_rpbase.new_index(src_rp.index)
- return RORPIter.Signatures(dest_iter())
-
- def get_diffs(src_rpbase, sigiter):
- """Get diffs based on sigiter and files in src_rpbase
-
- This should be run on the local side.
-
- """
- for sig_rorp in sigiter:
- new_rp = src_rpbase.new_index(sig_rorp.index)
- yield RORPIter.diffonce(sig_rorp, new_rp)
-
- def patch(dest_rpbase, diffiter):
- """Process diffs in diffiter and update files in dest_rbpase.
-
- Run remotely.
-
- """
- for diff_rorp in diffiter:
- basisrp = dest_rpbase.new_index(diff_rorp.index)
- if basisrp.lstat(): Filelist.make_subdirs(basisrp)
- Log("Processing %s" % basisrp.path, 7)
- RORPIter.patchonce(dest_rpbase, basisrp, diff_rorp)
-
- def patch_and_increment(dest_rpbase, diffiter, inc_rpbase):
- """Apply diffs in diffiter to dest_rpbase, and increment to inc_rpbase
-
- Also to be run remotely.
-
- """
- for diff_rorp in diffiter:
- basisrp = dest_rpbase.new_index(diff_rorp.index)
- if diff_rorp.lstat(): Filelist.make_subdirs(basisrp)
- Log("Processing %s" % basisrp.path, 7)
- # XXX This isn't done yet...
-
- def make_subdirs(rpath):
- """Make sure that all the directories under the rpath exist
-
- This function doesn't try to get the permissions right on the
- underlying directories, just do the minimum to make sure the
- file can be created.
-
- """
- dirname = rpath.dirsplit()[0]
- if dirname == '.' or dirname == '': return
- dir_rp = RPath(rpath.conn, dirname)
- Filelist.make_subdirs(dir_rp)
- if not dir_rp.lstat(): dir_rp.mkdir()
-
-
-MakeStatic(Filelist)
diff --git a/rdiff-backup/src/header.py b/rdiff-backup/src/header.py
deleted file mode 100644
index f628a73..0000000
--- a/rdiff-backup/src/header.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-#
-# rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.8.0 released June 14, 2002
-# Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
-#
-# This program is licensed under the GNU General Public License (GPL).
-# Distributions of rdiff-backup usually include a copy of the GPL in a
-# file called COPYING. The GPL is also available online at
-# http://www.gnu.org/copyleft/gpl.html.
-#
-# See http://www.stanford.edu/~bescoto/rdiff-backup for more
-# information. Please send mail to me or the mailing list if you find
-# bugs or have any suggestions.
-
-from __future__ import nested_scopes, generators
-import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno, signal
-
-
diff --git a/rdiff-backup/src/highlevel.py b/rdiff-backup/src/highlevel.py
deleted file mode 100644
index 3aa6491..0000000
--- a/rdiff-backup/src/highlevel.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""High level functions for mirroring, mirror & inc, etc."""
-
-from __future__ import generators
-from static import *
-from log import *
-from rpath import *
-from robust import *
-from increment import *
-from destructive_stepping import *
-from rorpiter import *
-import Globals, Hardlink, MiscStats
-
-
-class SkipFileException(Exception):
- """Signal that the current file should be skipped but then continue
-
- This exception will often be raised when there is problem reading
- an individual file, but it makes sense for the rest of the backup
- to keep going.
-
- """
- pass
-
-
-class HighLevel:
- """High level static functions
-
- The design of some of these functions is represented on the
- accompanying diagram.
-
- """
- def Mirror(src_rpath, dest_rpath, inc_rpath = None, session_info = None):
- """Turn dest_rpath into a copy of src_rpath
-
- If inc_rpath is true, then this is the initial mirroring of an
- incremental backup, so checkpoint and write to data_dir.
- Otherwise only mirror and don't create any extra files.
-
- """
- SourceS = src_rpath.conn.HLSourceStruct
- DestS = dest_rpath.conn.HLDestinationStruct
-
- SourceS.set_session_info(session_info)
- DestS.set_session_info(session_info)
- src_init_dsiter = SourceS.split_initial_dsiter()
- dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
- diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
- if inc_rpath:
- DestS.patch_w_datadir_writes(dest_rpath, diffiter, inc_rpath)
- else: DestS.patch_and_finalize(dest_rpath, diffiter)
-
- dest_rpath.setdata()
-
- def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath,
- session_info = None):
- """Mirror + put increments in tree based at inc_rpath"""
- SourceS = src_rpath.conn.HLSourceStruct
- DestS = dest_rpath.conn.HLDestinationStruct
-
- SourceS.set_session_info(session_info)
- DestS.set_session_info(session_info)
- if not session_info: dest_rpath.conn.SaveState.touch_last_file()
- src_init_dsiter = SourceS.split_initial_dsiter()
- dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
- diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
- DestS.patch_increment_and_finalize(dest_rpath, diffiter, inc_rpath)
-
- dest_rpath.setdata()
- inc_rpath.setdata()
-
-MakeStatic(HighLevel)
-
-
-class HLSourceStruct:
- """Hold info used by HL on the source side"""
- _session_info = None # set to si if resuming
- def set_session_info(cls, session_info):
- cls._session_info = session_info
-
- def iterate_from(cls):
- """Supply more aruments to DestructiveStepping.Iterate_from"""
- if cls._session_info is None: Globals.select_source.set_iter()
- else: Globals.select_source.set_iter(cls._session_info.last_index, 1)
- return Globals.select_source
-
- def split_initial_dsiter(cls):
- """Set iterators of all dsrps from rpath, returning one"""
- dsiter = cls.iterate_from()
- initial_dsiter1, cls.initial_dsiter2 = Iter.multiplex(dsiter, 2)
- return initial_dsiter1
-
- def get_diffs_and_finalize(cls, sigiter):
- """Return diffs and finalize any dsrp changes remaining
-
- Return a rorpiterator with files included of signatures of
- dissimilar files. This is the last operation run on the local
- filestream, so finalize dsrp writes.
-
- """
- collated = RORPIter.CollateIterators(cls.initial_dsiter2, sigiter)
- finalizer = IterTreeReducer(DestructiveSteppingFinalizer, [])
- def error_handler(exc, dest_sig, dsrp):
- Log("Error %s producing a diff of %s" %
- (exc, dsrp and dsrp.path), 2)
- return None
-
- def diffs():
- for dsrp, dest_sig in collated:
- if dest_sig:
- if dest_sig.isplaceholder(): yield dest_sig
- else:
- diff = Robust.check_common_error(
- error_handler, RORPIter.diffonce, [dest_sig, dsrp])
- if diff: yield diff
- if dsrp: finalizer(dsrp.index, dsrp)
- finalizer.Finish()
- return diffs()
-
-MakeClass(HLSourceStruct)
-
-
-class HLDestinationStruct:
- """Hold info used by HL on the destination side"""
- _session_info = None # set to si if resuming
- def set_session_info(cls, session_info):
- cls._session_info = session_info
-
- def iterate_from(cls):
- """Return selection iterator to iterate all the mirror files"""
- if cls._session_info is None: Globals.select_mirror.set_iter()
- else: Globals.select_mirror.set_iter(cls._session_info.last_index)
- return Globals.select_mirror
-
- def split_initial_dsiter(cls):
- """Set initial_dsiters (iteration of all dsrps from rpath)"""
- result, cls.initial_dsiter2 = Iter.multiplex(cls.iterate_from(), 2)
- return result
-
- def get_dissimilar(cls, baserp, src_init_iter, dest_init_iter):
- """Get dissimilars
-
- Returns an iterator which enumerates the dsrps which are
- different on the source and destination ends. The dsrps do
- not necessarily exist on the destination end.
-
- Also, to prevent the system from getting backed up on the
- remote end, if we don't get enough dissimilars, stick in a
- placeholder every so often, like fiber. The more
- placeholders, the more bandwidth used, but if there aren't
- enough, lots of memory will be used because files will be
- accumulating on the source side. How much will accumulate
- will depend on the Globals.conn_bufsize value.
-
- """
- collated = RORPIter.CollateIterators(src_init_iter, dest_init_iter)
- def compare(src_rorp, dest_dsrp):
- """Return dest_dsrp if they are different, None if the same"""
- if not dest_dsrp:
- dest_dsrp = cls.get_dsrp(baserp, src_rorp.index)
- if dest_dsrp.lstat():
- Log("Warning: Found unexpected destination file %s, "
- "not processing it." % dest_dsrp.path, 2)
- return None
- elif (src_rorp and src_rorp == dest_dsrp and
- (not Globals.preserve_hardlinks or
- Hardlink.rorp_eq(src_rorp, dest_dsrp))):
- return None
- if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp):
- dest_dsrp.flaglinked()
- return dest_dsrp
-
- def generate_dissimilar():
- counter = 0
- for src_rorp, dest_dsrp in collated:
- if Globals.preserve_hardlinks:
- if src_rorp: Hardlink.add_rorp(src_rorp, 1)
- if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None)
- dsrp = compare(src_rorp, dest_dsrp)
- if dsrp:
- counter = 0
- yield dsrp
- elif counter == 20:
- placeholder = RORPath(src_rorp.index)
- placeholder.make_placeholder()
- counter = 0
- yield placeholder
- else: counter += 1
- return generate_dissimilar()
-
- def get_sigs(cls, baserp, src_init_iter):
- """Return signatures of all dissimilar files"""
- dest_iters1 = cls.split_initial_dsiter()
- dissimilars = cls.get_dissimilar(baserp, src_init_iter, dest_iters1)
- return RORPIter.Signatures(dissimilars)
-
- def get_dsrp(cls, dest_rpath, index):
- """Return initialized dsrp based on dest_rpath with given index"""
- dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index)
- if Globals.quoting_enabled: dsrp.quote_path()
- return dsrp
-
- def get_finalizer(cls):
- """Return finalizer, starting from session info if necessary"""
- old_finalizer = cls._session_info and cls._session_info.finalizer
- if old_finalizer: return old_finalizer
- else: return IterTreeReducer(DestructiveSteppingFinalizer, [])
-
- def get_ITR(cls, inc_rpath):
- """Return ITR, starting from state if necessary"""
- if cls._session_info and cls._session_info.ITR:
- return cls._session_info.ITR
- else:
- iitr = IterTreeReducer(IncrementITRB, [inc_rpath])
- iitr.root_branch.override_changed()
- Globals.ITRB = iitr.root_branch
- iitr.root_branch.Errors = 0
- return iitr
-
- def get_MirrorITR(cls, inc_rpath):
- """Return MirrorITR, starting from state if available"""
- if cls._session_info and cls._session_info.ITR:
- return cls._session_info.ITR
- ITR = IterTreeReducer(MirrorITRB, [inc_rpath])
- Globals.ITRB = ITR.root_branch
- ITR.root_branch.Errors = 0
- return ITR
-
- def patch_and_finalize(cls, dest_rpath, diffs):
- """Apply diffs and finalize"""
- collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
- finalizer = cls.get_finalizer()
- diff_rorp, dsrp = None, None
-
- def patch(diff_rorp, dsrp):
- if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
- if diff_rorp and not diff_rorp.isplaceholder():
- RORPIter.patchonce_action(None, dsrp, diff_rorp).execute()
- return dsrp
-
- def error_handler(exc, diff_rorp, dsrp):
- filename = dsrp and dsrp.path or os.path.join(*diff_rorp.index)
- Log("Error: %s processing file %s" % (exc, filename), 2)
-
- for indexed_tuple in collated:
- Log(lambda: "Processing %s" % str(indexed_tuple), 7)
- diff_rorp, dsrp = indexed_tuple
- dsrp = Robust.check_common_error(error_handler, patch,
- [diff_rorp, dsrp])
- finalizer(dsrp.index, dsrp)
- finalizer.Finish()
-
- def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath):
- """Apply diffs and finalize, with checkpointing and statistics"""
- collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
- finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
- MiscStats.open_dir_stats_file()
- dsrp, finished_dsrp = None, None
-
- try:
- for indexed_tuple in collated:
- Log(lambda: "Processing %s" % str(indexed_tuple), 7)
- diff_rorp, dsrp = indexed_tuple
- if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
- if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
- ITR(dsrp.index, diff_rorp, dsrp)
- finalizer(dsrp.index, dsrp)
- SaveState.checkpoint(ITR, finalizer, dsrp)
- finished_dsrp = dsrp
- ITR.Finish()
- finalizer.Finish()
- except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
-
- if Globals.preserve_hardlinks: Hardlink.final_writedata()
- MiscStats.close_dir_stats_file()
- MiscStats.write_session_statistics(ITR.root_branch)
- SaveState.checkpoint_remove()
-
- def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
- """Apply diffs, write increment if necessary, and finalize"""
- collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
- finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
- MiscStats.open_dir_stats_file()
- dsrp, finished_dsrp = None, None
-
- try:
- for indexed_tuple in collated:
- Log(lambda: "Processing %s" % str(indexed_tuple), 7)
- diff_rorp, dsrp = indexed_tuple
- index = indexed_tuple.index
- if not dsrp: dsrp = cls.get_dsrp(dest_rpath, index)
- if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
- ITR(index, diff_rorp, dsrp)
- finalizer(index, dsrp)
- SaveState.checkpoint(ITR, finalizer, dsrp)
- finished_dsrp = dsrp
- ITR.Finish()
- finalizer.Finish()
- except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
-
- if Globals.preserve_hardlinks: Hardlink.final_writedata()
- MiscStats.close_dir_stats_file()
- MiscStats.write_session_statistics(ITR.root_branch)
- SaveState.checkpoint_remove()
-
- def handle_last_error(cls, dsrp, finalizer, ITR):
- """If catch fatal error, try to checkpoint before exiting"""
- Log.exception(1, 2)
- TracebackArchive.log()
- SaveState.checkpoint(ITR, finalizer, dsrp, 1)
- if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir)
- SaveState.touch_last_file_definitive()
- raise
-
-MakeClass(HLDestinationStruct)
diff --git a/rdiff-backup/src/increment.py b/rdiff-backup/src/increment.py
deleted file mode 100644
index baba9e1..0000000
--- a/rdiff-backup/src/increment.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Provides Inc and *ITR classes, which relate to writing increment files"""
-
-import traceback
-from static import *
-from statistics import *
-from lazy import *
-
-class Inc:
- """Class containing increment functions"""
- def Increment_action(new, mirror, incpref):
- """Main file incrementing function, returns RobustAction
-
- new is the file on the active partition,
- mirror is the mirrored file from the last backup,
- incpref is the prefix of the increment file.
-
- This function basically moves the information about the mirror
- file to incpref.
-
- The returned RobustAction when executed should return the name
- of the incfile, or None if none was created.
-
- """
- if not (new and new.lstat() or mirror.lstat()):
- return Robust.null_action # Files deleted in meantime, do nothing
-
- Log("Incrementing mirror file " + mirror.path, 5)
- if ((new and new.isdir()) or mirror.isdir()) and not incpref.isdir():
- incpref.mkdir()
-
- if not mirror.lstat(): return Inc.makemissing_action(incpref)
- elif mirror.isdir(): return Inc.makedir_action(mirror, incpref)
- elif new.isreg() and mirror.isreg():
- return Inc.makediff_action(new, mirror, incpref)
- else: return Inc.makesnapshot_action(mirror, incpref)
-
- def Increment(new, mirror, incpref):
- return Inc.Increment_action(new, mirror, incpref).execute()
-
- def makemissing_action(incpref):
- """Signify that mirror file was missing"""
- def final(init_val):
- incrp = Inc.get_inc_ext(incpref, "missing")
- incrp.touch()
- return incrp
- return RobustAction(None, final, None)
-
- def makesnapshot_action(mirror, incpref):
- """Copy mirror to incfile, since new is quite different"""
- if (mirror.isreg() and Globals.compression and
- not Globals.no_compression_regexp.match(mirror.path)):
- snapshotrp = Inc.get_inc_ext(incpref, "snapshot.gz")
- return Robust.copy_with_attribs_action(mirror, snapshotrp, 1)
- else:
- snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
- return Robust.copy_with_attribs_action(mirror, snapshotrp, None)
-
- def makediff_action(new, mirror, incpref):
- """Make incfile which is a diff new -> mirror"""
- if (Globals.compression and
- not Globals.no_compression_regexp.match(mirror.path)):
- diff = Inc.get_inc_ext(incpref, "diff.gz")
- compress = 1
- else:
- diff = Inc.get_inc_ext(incpref, "diff")
- compress = None
-
- diff_tf = TempFileManager.new(diff)
- def init():
- Rdiff.write_delta(new, mirror, diff_tf, compress)
- RPath.copy_attribs(mirror, diff_tf)
- return diff
- return Robust.make_tf_robustaction(init, diff_tf, diff)
-
- def makedir_action(mirrordir, incpref):
- """Make file indicating directory mirrordir has changed"""
- dirsign = Inc.get_inc_ext(incpref, "dir")
- tf = TempFileManager.new(dirsign)
- def init():
- tf.touch()
- RPath.copy_attribs(mirrordir, tf)
- return dirsign
- return Robust.make_tf_robustaction(init, tf, dirsign)
-
- def get_inc(rp, time, typestr):
- """Return increment like rp but with time and typestr suffixes"""
- addtostr = lambda s: "%s.%s.%s" % (s, Time.timetostring(time), typestr)
- if rp.index:
- incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] +
- (addtostr(rp.index[-1]),))
- else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index)
- if Globals.quoting_enabled: incrp.quote_path()
- return incrp
-
- def get_inc_ext(rp, typestr):
- """Return increment with specified type and correct time
-
- If the file exists, then probably a previous backup has been
- aborted. We then keep asking FindTime to get a time later
- than the one that already has an inc file.
-
- """
- inctime = 0
- while 1:
- inctime = Resume.FindTime(rp.index, inctime)
- incrp = Inc.get_inc(rp, inctime, typestr)
- if not incrp.lstat(): break
- return incrp
-
-MakeStatic(Inc)
-
-
-class IncrementITRB(StatsITRB):
- """Patch and increment mirror directory
-
- This has to be an ITR because directories that have files in them
- changed are flagged with an increment marker. There are four
- possibilities as to the order:
-
- 1. Normal file -> Normal file: right away
- 2. Directory -> Directory: wait until files in the directory
- are processed, as we won't know whether to add a marker
- until the end.
- 3. Normal file -> Directory: right away, so later files will
- have a directory to go into.
- 4. Directory -> Normal file: Wait until the end, so we can
- process all the files in the directory.
-
- Remember this object needs to be pickable.
-
- """
- # Iff true, mirror file was a directory
- mirror_isdirectory = None
- # If set, what the directory on the mirror side will be replaced with
- directory_replacement = None
- # True iff there has been some change at this level or lower (used
- # for marking directories to be flagged)
- changed = None
- # Holds the RPath of the created increment file, if any
- incrp = None
-
- def __init__(self, inc_rpath):
- """Set inc_rpath, an rpath of the base of the tree"""
- self.inc_rpath = inc_rpath
- StatsITRB.__init__(self)
-
- def start_process(self, index, diff_rorp, dsrp):
- """Initial processing of file
-
- diff_rorp is the RORPath of the diff from the remote side, and
- dsrp is the local file to be incremented
-
- """
- self.start_stats(dsrp)
- incpref = self.inc_rpath.new_index(index)
- if Globals.quoting_enabled: incpref.quote_path()
- if dsrp.isdir():
- self.init_dir(dsrp, diff_rorp, incpref)
- self.mirror_isdirectory = 1
- else: self.init_non_dir(dsrp, diff_rorp, incpref)
- self.setvals(diff_rorp, dsrp, incpref)
-
- def override_changed(self):
- """Set changed flag to true
-
- This is used only at the top level of a backup, to make sure
- that a marker is created recording every backup session.
-
- """
- self.changed = 1
-
- def setvals(self, diff_rorp, dsrp, incpref):
- """Record given values in state dict since in directory
-
- We don't do these earlier in case of a problem inside the
- init_* functions. Index isn't given because it is done by the
- superclass.
-
- """
- self.diff_rorp = diff_rorp
- self.dsrp = dsrp
- self.incpref = incpref
-
- def init_dir(self, dsrp, diff_rorp, incpref):
- """Process a directory (initial pass)
-
- If the directory is changing into a normal file, we need to
- save the normal file data in a temp file, and then create the
- real file once we are done with everything inside the
- directory.
-
- """
- if not (incpref.lstat() and incpref.isdir()): incpref.mkdir()
- if diff_rorp and diff_rorp.isreg() and diff_rorp.file:
- tf = TempFileManager.new(dsrp)
- def init():
- RPathStatic.copy_with_attribs(diff_rorp, tf)
- tf.set_attached_filetype(diff_rorp.get_attached_filetype())
- def error(exc, ran_init, init_val): tf.delete()
- RobustAction(init, None, error).execute()
- self.directory_replacement = tf
-
- def init_non_dir(self, dsrp, diff_rorp, incpref):
- """Process a non directory file (initial pass)"""
- if not diff_rorp: return # no diff, so no change necessary
- if diff_rorp.isreg() and (dsrp.isreg() or diff_rorp.isflaglinked()):
- # Write updated mirror to temp file so we can compute
- # reverse diff locally
- mirror_tf = TempFileManager.new(dsrp)
- old_dsrp_tf = TempFileManager.new(dsrp)
- def init_thunk():
- if diff_rorp.isflaglinked():
- Hardlink.link_rp(diff_rorp, mirror_tf, dsrp)
- else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
- mirror_tf).execute()
- self.incrp = Inc.Increment_action(mirror_tf, dsrp,
- incpref).execute()
- if dsrp.lstat(): RPathStatic.rename(dsrp, old_dsrp_tf)
- mirror_tf.rename(dsrp)
-
- def final(init_val): old_dsrp_tf.delete()
- def error(exc, ran_init, init_val):
- if ran_init: old_dsrp_tf.delete() # everything is fine
- else: # restore to previous state
- if old_dsrp_tf.lstat(): old_dsrp_tf.rename(dsrp)
- if self.incrp: self.incrp.delete()
- mirror_tf.delete()
-
- RobustAction(init_thunk, final, error).execute()
- else: self.incrp = Robust.chain(
- Inc.Increment_action(diff_rorp, dsrp, incpref),
- RORPIter.patchonce_action(None, dsrp, diff_rorp)).execute()[0]
-
- self.changed = 1
-
- def end_process(self):
- """Do final work when leaving a tree (directory)"""
- diff_rorp, dsrp, incpref = self.diff_rorp, self.dsrp, self.incpref
- if (self.mirror_isdirectory and (diff_rorp or self.changed)
- or self.directory_replacement):
- if self.directory_replacement:
- tf = self.directory_replacement
- self.incrp = Robust.chain(
- Inc.Increment_action(tf, dsrp, incpref),
- RORPIter.patchonce_action(None, dsrp, tf)).execute()[0]
- tf.delete()
- else:
- self.incrp = Inc.Increment(diff_rorp, dsrp, incpref)
- if diff_rorp:
- RORPIter.patchonce_action(None, dsrp, diff_rorp).execute()
-
- self.end_stats(diff_rorp, dsrp, self.incrp)
- if self.mirror_isdirectory or dsrp.isdir():
- MiscStats.write_dir_stats_line(self, dsrp.index)
-
- def can_fast_process(self, index, diff_rorp, dsrp):
- """True if there is no change in file and is just a leaf"""
- return not diff_rorp and dsrp.isreg()
-
- def fast_process(self, index, diff_rorp, dsrp):
- """Just update statistics"""
- StatsITRB.fast_process(self, dsrp)
-
- def branch_process(self, branch):
- """Update statistics, and the has_changed flag if change in branch"""
- if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio)
- if branch.changed: self.changed = 1
- self.add_file_stats(branch)
-
-
-class MirrorITRB(StatsITRB):
- """Like IncrementITR, but only patch mirror directory, don't increment"""
- # This is always None since no increments will be created
- incrp = None
- def __init__(self, inc_rpath):
- """Set inc_rpath, an rpath of the base of the inc tree"""
- self.inc_rpath = inc_rpath
- StatsITRB.__init__(self)
-
- def start_process(self, index, diff_rorp, mirror_dsrp):
- """Initialize statistics and do actual writing to mirror"""
- self.start_stats(mirror_dsrp)
- if diff_rorp and not diff_rorp.isplaceholder():
- RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute()
-
- self.incpref = self.inc_rpath.new_index(index)
- self.diff_rorp, self.mirror_dsrp = diff_rorp, mirror_dsrp
-
- def end_process(self):
- """Update statistics when leaving"""
- self.end_stats(self.diff_rorp, self.mirror_dsrp)
- if self.mirror_dsrp.isdir():
- MiscStats.write_dir_stats_line(self, self.mirror_dsrp.index)
-
- def can_fast_process(self, index, diff_rorp, mirror_dsrp):
- """True if there is no change in file and it is just a leaf"""
- return not diff_rorp and mirror_dsrp.isreg()
-
- def fast_process(self, index, diff_rorp, mirror_dsrp):
- """Just update statistics"""
- StatsITRB.fast_process(self, mirror_dsrp)
-
- def branch_process(self, branch):
- """Update statistics with subdirectory results"""
- if Globals.sleep_ratio is not None: Time.sleep(Globals.sleep_ratio)
- self.add_file_stats(branch)
-
-
-from log import *
-from rpath import *
-from robust import *
-from rorpiter import *
-import Globals, Time, MiscStats
-
diff --git a/rdiff-backup/src/iterfile.py b/rdiff-backup/src/iterfile.py
deleted file mode 100644
index f95b4e8..0000000
--- a/rdiff-backup/src/iterfile.py
+++ /dev/null
@@ -1,257 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Convert an iterator to a file object and vice-versa"""
-
-import cPickle, array
-import Globals, C
-
-
-class IterFileException(Exception): pass
-
-class UnwrapFile:
- """Contains some basic methods for parsing a file containing an iter"""
- def __init__(self, file):
- self.file = file
-
- def _s2l_old(self, s):
- """Convert string to long int"""
- assert len(s) == 7
- l = 0L
- for i in range(7): l = l*256 + ord(s[i])
- return l
-
- def _get(self):
- """Return pair (type, data) next in line on the file
-
- type is a single character which is either "o" for object, "f"
- for file, "c" for a continution of a file, or None if no more
- data can be read. Data is either the file's data, if type is
- "c" or "f", or the actual object if the type is "o".
-
- """
- header = self.file.read(8)
- if not header: return None, None
- if len(header) != 8:
- assert None, "Header %s is only %d bytes" % (header, len(header))
- type, length = header[0], C.str2long(header[1:])
- buf = self.file.read(length)
- if type == "o": return type, cPickle.loads(buf)
- else: return type, buf
-
-
-class IterWrappingFile(UnwrapFile):
- """An iterator generated from a file.
-
- Initialize with a file type object, and then it will return the
- elements of the file in order.
-
- """
- def __init__(self, file):
- UnwrapFile.__init__(self, file)
- self.currently_in_file = None
-
- def __iter__(self): return self
-
- def next(self):
- if self.currently_in_file:
- self.currently_in_file.close() # no error checking by this point
- type, data = self._get()
- if not type: raise StopIteration
- if type == "o": return data
- elif type == "f":
- file = IterVirtualFile(self, data)
- if data: self.currently_in_file = file
- else: self.currently_in_file = None
- return file
- else: raise IterFileException("Bad file type %s" % type)
-
-
-class IterVirtualFile(UnwrapFile):
- """Another version of a pretend file
-
- This is returned by IterWrappingFile when a file is embedded in
- the main file that the IterWrappingFile is based around.
-
- """
- def __init__(self, iwf, initial_data):
- """Initializer
-
- initial_data is the data from the first block of the file.
- iwf is the iter wrapping file that spawned this
- IterVirtualFile.
-
- """
- UnwrapFile.__init__(self, iwf.file)
- self.iwf = iwf
- self.buffer = initial_data
- self.closed = None
-
- def read(self, length = -1):
- """Read length bytes from the file, updating buffers as necessary"""
- assert not self.closed
- if self.iwf.currently_in_file:
- if length >= 0:
- while length >= len(self.buffer):
- if not self.addtobuffer(): break
- real_len = min(length, len(self.buffer))
- else:
- while 1:
- if not self.addtobuffer(): break
- real_len = len(self.buffer)
- else: real_len = min(length, len(self.buffer))
-
- return_val = self.buffer[:real_len]
- self.buffer = self.buffer[real_len:]
- return return_val
-
- def addtobuffer(self):
- """Read a chunk from the file and add it to the buffer"""
- assert self.iwf.currently_in_file
- type, data = self._get()
- assert type == "c", "Type is %s instead of c" % type
- if data:
- self.buffer += data
- return 1
- else:
- self.iwf.currently_in_file = None
- return None
-
- def close(self):
- """Currently just reads whats left and discards it"""
- while self.iwf.currently_in_file:
- self.addtobuffer()
- self.buffer = ""
- self.closed = 1
-
-
-class FileWrappingIter:
- """A file interface wrapping around an iterator
-
- This is initialized with an iterator, and then converts it into a
- stream of characters. The object will evaluate as little of the
- iterator as is necessary to provide the requested bytes.
-
- The actual file is a sequence of marshaled objects, each preceded
- by 8 bytes which identifies the following the type of object, and
- specifies its length. File objects are not marshalled, but the
- data is written in chunks of Globals.blocksize, and the following
- blocks can identify themselves as continuations.
-
- """
- def __init__(self, iter):
- """Initialize with iter"""
- self.iter = iter
- self.array_buf = array.array('c')
- self.currently_in_file = None
- self.closed = None
-
- def read(self, length):
- """Return next length bytes in file"""
- assert not self.closed
- while len(self.array_buf) < length:
- if not self.addtobuffer(): break
-
- result = self.array_buf[:length].tostring()
- del self.array_buf[:length]
- return result
-
- def addtobuffer(self):
- """Updates self.buffer, adding a chunk from the iterator.
-
- Returns None if we have reached the end of the iterator,
- otherwise return true.
-
- """
- array_buf = self.array_buf
- if self.currently_in_file:
- array_buf.fromstring("c")
- array_buf.fromstring(self.addfromfile())
- else:
- try: currentobj = self.iter.next()
- except StopIteration: return None
- if hasattr(currentobj, "read") and hasattr(currentobj, "close"):
- self.currently_in_file = currentobj
- array_buf.fromstring("f")
- array_buf.fromstring(self.addfromfile())
- else:
- pickle = cPickle.dumps(currentobj, 1)
- array_buf.fromstring("o")
- array_buf.fromstring(C.long2str(long(len(pickle))))
- array_buf.fromstring(pickle)
- return 1
-
- def addfromfile(self):
- """Read a chunk from the current file and return it"""
- # Check file read for errors, buf = "" if find one
- buf = Robust.check_common_error(self.read_error_handler,
- self.currently_in_file.read,
- [Globals.blocksize])
- if not buf:
- assert not self.currently_in_file.close()
- self.currently_in_file = None
- return C.long2str(long(len(buf))) + buf
-
- def read_error_handler(self, exc, blocksize):
- """Log error when reading from file"""
- Log("Error '%s' reading from fileobj, truncating" % (str(exc),), 2)
- return ""
-
- def _l2s_old(self, l):
- """Convert long int to string of 7 characters"""
- s = ""
- for i in range(7):
- l, remainder = divmod(l, 256)
- s = chr(remainder) + s
- assert remainder == 0
- return s
-
- def close(self): self.closed = 1
-
-
-class BufferedRead:
- """Buffer the .read() calls to the given file
-
- This is used to lessen overhead and latency when a file is sent
- over a connection. Profiling said that arrays were faster than
- strings here.
-
- """
- def __init__(self, file):
- self.file = file
- self.array_buf = array.array('c')
- self.bufsize = Globals.conn_bufsize
-
- def read(self, l = -1):
- array_buf = self.array_buf
- if l < 0: # Read as much as possible
- result = array_buf.tostring() + self.file.read()
- del array_buf[:]
- return result
-
- if len(array_buf) < l: # Try to make buffer at least as long as l
- array_buf.fromstring(self.file.read(max(self.bufsize, l)))
- result = array_buf[:l].tostring()
- del array_buf[:l]
- return result
-
- def close(self): return self.file.close()
-
-from log import *
-from robust import *
diff --git a/rdiff-backup/src/lazy.py b/rdiff-backup/src/lazy.py
deleted file mode 100644
index 6a076e7..0000000
--- a/rdiff-backup/src/lazy.py
+++ /dev/null
@@ -1,368 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Define some lazy data structures and functions acting on them"""
-
-from __future__ import generators
-import os, stat, types
-from static import *
-import psyco
-
-class Iter:
- """Hold static methods for the manipulation of lazy iterators"""
-
- def filter(predicate, iterator):
- """Like filter in a lazy functional programming language"""
- for i in iterator:
- if predicate(i): yield i
-
- def map(function, iterator):
- """Like map in a lazy functional programming language"""
- for i in iterator: yield function(i)
-
- def foreach(function, iterator):
- """Run function on each element in iterator"""
- for i in iterator: function(i)
-
- def cat(*iters):
- """Lazily concatenate iterators"""
- for iter in iters:
- for i in iter: yield i
-
- def cat2(iter_of_iters):
- """Lazily concatenate iterators, iterated by big iterator"""
- for iter in iter_of_iters:
- for i in iter: yield i
-
- def empty(iter):
- """True if iterator has length 0"""
- for i in iter: return None
- return 1
-
- def equal(iter1, iter2, verbose = None, operator = lambda x, y: x == y):
- """True if iterator 1 has same elements as iterator 2
-
- Use equality operator, or == if it is unspecified.
-
- """
- for i1 in iter1:
- try: i2 = iter2.next()
- except StopIteration:
- if verbose: print "End when i1 = %s" % (i1,)
- return None
- if not operator(i1, i2):
- if verbose: print "%s not equal to %s" % (i1, i2)
- return None
- try: i2 = iter2.next()
- except StopIteration: return 1
- if verbose: print "End when i2 = %s" % (i2,)
- return None
-
- def Or(iter):
- """True if any element in iterator is true. Short circuiting"""
- i = None
- for i in iter:
- if i: return i
- return i
-
- def And(iter):
- """True if all elements in iterator are true. Short circuiting"""
- i = 1
- for i in iter:
- if not i: return i
- return i
-
- def len(iter):
- """Return length of iterator"""
- i = 0
- while 1:
- try: iter.next()
- except StopIteration: return i
- i = i+1
-
- def foldr(f, default, iter):
- """foldr the "fundamental list recursion operator"?"""
- try: next = iter.next()
- except StopIteration: return default
- return f(next, Iter.foldr(f, default, iter))
-
- def foldl(f, default, iter):
- """the fundamental list iteration operator.."""
- while 1:
- try: next = iter.next()
- except StopIteration: return default
- default = f(default, next)
-
- def multiplex(iter, num_of_forks, final_func = None, closing_func = None):
- """Split a single iterater into a number of streams
-
- The return val will be a list with length num_of_forks, each
- of which will be an iterator like iter. final_func is the
- function that will be called on each element in iter just as
- it is being removed from the buffer. closing_func is called
- when all the streams are finished.
-
- """
- if num_of_forks == 2 and not final_func and not closing_func:
- im2 = IterMultiplex2(iter)
- return (im2.yielda(), im2.yieldb())
- if not final_func: final_func = lambda i: None
- if not closing_func: closing_func = lambda: None
-
- # buffer is a list of elements that some iterators need and others
- # don't
- buffer = []
-
- # buffer[forkposition[i]] is the next element yieled by iterator
- # i. If it is -1, yield from the original iter
- starting_forkposition = [-1] * num_of_forks
- forkposition = starting_forkposition[:]
- called_closing_func = [None]
-
- def get_next(fork_num):
- """Return the next element requested by fork_num"""
- if forkposition[fork_num] == -1:
- try: buffer.insert(0, iter.next())
- except StopIteration:
- # call closing_func if necessary
- if (forkposition == starting_forkposition and
- not called_closing_func[0]):
- closing_func()
- called_closing_func[0] = None
- raise StopIteration
- for i in range(num_of_forks): forkposition[i] += 1
-
- return_val = buffer[forkposition[fork_num]]
- forkposition[fork_num] -= 1
-
- blen = len(buffer)
- if not (blen-1) in forkposition:
- # Last position in buffer no longer needed
- assert forkposition[fork_num] == blen-2
- final_func(buffer[blen-1])
- del buffer[blen-1]
- return return_val
-
- def make_iterator(fork_num):
- while(1): yield get_next(fork_num)
-
- return tuple(map(make_iterator, range(num_of_forks)))
-
-MakeStatic(Iter)
-
-
-class IterMultiplex2:
- """Multiplex an iterator into 2 parts
-
- This is a special optimized case of the Iter.multiplex function,
- used when there is no closing_func or final_func, and we only want
- to split it into 2. By profiling, this is a time sensitive class.
-
- """
- def __init__(self, iter):
- self.a_leading_by = 0 # How many places a is ahead of b
- self.buffer = []
- self.iter = iter
-
- def yielda(self):
- """Return first iterator"""
- buf, iter = self.buffer, self.iter
- while(1):
- if self.a_leading_by >= 0: # a is in front, add new element
- elem = iter.next() # exception will be passed
- buf.append(elem)
- else: elem = buf.pop(0) # b is in front, subtract an element
- self.a_leading_by += 1
- yield elem
-
- def yieldb(self):
- """Return second iterator"""
- buf, iter = self.buffer, self.iter
- while(1):
- if self.a_leading_by <= 0: # b is in front, add new element
- elem = iter.next() # exception will be passed
- buf.append(elem)
- else: elem = buf.pop(0) # a is in front, subtract an element
- self.a_leading_by -= 1
- yield elem
-
-
-class IterTreeReducer:
- """Tree style reducer object for iterator
-
- The indicies of a RORPIter form a tree type structure. This class
- can be used on each element of an iter in sequence and the result
- will be as if the corresponding tree was reduced. This tries to
- bridge the gap between the tree nature of directories, and the
- iterator nature of the connection between hosts and the temporal
- order in which the files are processed.
-
- """
- def __init__(self, branch_class, branch_args):
- """ITR initializer"""
- self.branch_class = branch_class
- self.branch_args = branch_args
- self.index = None
- self.root_branch = branch_class(*branch_args)
- self.branches = [self.root_branch]
-
- def finish_branches(self, index):
- """Run Finish() on all branches index has passed
-
- When we pass out of a branch, delete it and process it with
- the parent. The innermost branches will be the last in the
- list. Return None if we are out of the entire tree, and 1
- otherwise.
-
- """
- branches = self.branches
- while 1:
- to_be_finished = branches[-1]
- base_index = to_be_finished.base_index
- if base_index != index[:len(base_index)]:
- # out of the tree, finish with to_be_finished
- to_be_finished.call_end_proc()
- del branches[-1]
- if not branches: return None
- branches[-1].branch_process(to_be_finished)
- else: return 1
-
- def add_branch(self, index):
- """Return branch of type self.branch_class, add to branch list"""
- branch = self.branch_class(*self.branch_args)
- branch.base_index = index
- self.branches.append(branch)
- return branch
-
- def process_w_branch(self, branch, args):
- """Run start_process on latest branch"""
- Robust.check_common_error(branch.on_error,
- branch.start_process, args)
- if not branch.caught_exception: branch.start_successful = 1
-
- def Finish(self):
- """Call at end of sequence to tie everything up"""
- while 1:
- to_be_finished = self.branches.pop()
- to_be_finished.call_end_proc()
- if not self.branches: break
- self.branches[-1].branch_process(to_be_finished)
-
- def __call__(self, *args):
- """Process args, where args[0] is current position in iterator
-
- Returns true if args successfully processed, false if index is
- not in the current tree and thus the final result is
- available.
-
- Also note below we set self.index after doing the necessary
- start processing, in case there is a crash in the middle.
-
- """
- index = args[0]
- if self.index is None:
- self.root_branch.base_index = index
- self.process_w_branch(self.root_branch, args)
- self.index = index
- return 1
-
- if index <= self.index:
- Log("Warning: oldindex %s >= newindex %s" % (self.index, index), 2)
- return 1
-
- if self.finish_branches(index) is None:
- return None # We are no longer in the main tree
- last_branch = self.branches[-1]
- if last_branch.start_successful:
- if last_branch.can_fast_process(*args):
- last_branch.fast_process(*args)
- else:
- branch = self.add_branch(index)
- self.process_w_branch(branch, args)
- else: last_branch.log_prev_error(index)
-
- self.index = index
- return 1
-
-psyco.bind(IterTreeReducer)
-
-
-class ITRBranch:
- """Helper class for IterTreeReducer below
-
- There are five stub functions below: start_process, end_process,
- branch_process, can_fast_process, and fast_process. A class that
- subclasses this one will probably fill in these functions to do
- more.
-
- It is important that this class be pickable, so keep that in mind
- when subclassing (this is used to resume failed sessions).
-
- """
- base_index = index = None
- finished = None
- caught_exception = start_successful = None
-
- def call_end_proc(self):
- """Runs the end_process on self, checking for errors"""
- if self.finished or not self.start_successful:
- self.caught_exception = 1
- if self.caught_exception: self.log_prev_error(self.base_index)
- else: Robust.check_common_error(self.on_error, self.end_process)
- self.finished = 1
-
- def start_process(self, *args):
- """Do some initial processing (stub)"""
- pass
-
- def end_process(self):
- """Do any final processing before leaving branch (stub)"""
- pass
-
- def branch_process(self, branch):
- """Process a branch right after it is finished (stub)"""
- assert branch.finished
- pass
-
- def can_fast_process(self, *args):
- """True if object can be processed without new branch (stub)"""
- return None
-
- def fast_process(self, *args):
- """Process args without new child branch (stub)"""
- pass
-
- def on_error(self, exc, *args):
- """This is run on any exception in start/end-process"""
- self.caught_exception = 1
- if args and args[0] and isinstance(args[0], tuple):
- filename = os.path.join(*args[0])
- elif self.index: filename = os.path.join(*self.index)
- else: filename = "."
- Log("Error '%s' processing %s" % (exc, filename), 2)
-
- def log_prev_error(self, index):
- """Call function if no pending exception"""
- Log("Skipping %s because of previous error" %
- (os.path.join(*index),), 2)
-
-
-# Put at bottom to prevent (viciously) circular module dependencies
-from robust import *
-from log import *
diff --git a/rdiff-backup/src/librsync.py b/rdiff-backup/src/librsync.py
deleted file mode 100644
index eb5e235..0000000
--- a/rdiff-backup/src/librsync.py
+++ /dev/null
@@ -1,195 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Provides a high-level interface to some librsync functions
-
-This is a python wrapper around the lower-level _librsync module,
-which is written in C. The goal was to use C as little as possible...
-
-"""
-
-import _librsync, types, array
-
-blocksize = _librsync.RS_JOB_BLOCKSIZE
-
-class librsyncError(Exception):
- """Signifies error in internal librsync processing (bad signature, etc.)
-
- underlying _librsync.librsyncError's are regenerated using this
- class because the C-created exceptions are by default
- unPickleable. There is probably a way to fix this in _librsync,
- but this scheme was easier.
-
- """
- pass
-
-
-class LikeFile:
- """File-like object used by SigFile, DeltaFile, and PatchFile"""
- mode = "rb"
-
- # This will be replaced in subclasses by an object with
- # appropriate cycle() method
- maker = None
-
- def __init__(self, infile, need_seek = None):
- """LikeFile initializer - zero buffers, set eofs off"""
- self.check_file(infile, need_seek)
- self.infile = infile
- self.closed = self.infile_closed = None
- self.inbuf = ""
- self.outbuf = array.array('c')
- self.eof = self.infile_eof = None
-
- def check_file(self, file, need_seek = None):
- """Raise type error if file doesn't have necessary attributes"""
- if not hasattr(file, "read"):
- raise TypeError("Basis file must have a read() method")
- if not hasattr(file, "close"):
- raise TypeError("Basis file must have a close() method")
- if need_seek and not hasattr(file, "seek"):
- raise TypeError("Basis file must have a seek() method")
-
- def read(self, length = -1):
- """Build up self.outbuf, return first length bytes"""
- if length == -1:
- while not self.eof: self._add_to_outbuf_once()
- real_len = len(self.outbuf)
- else:
- while not self.eof and len(self.outbuf) < length:
- self._add_to_outbuf_once()
- real_len = min(length, len(self.outbuf))
-
- return_val = self.outbuf[:real_len].tostring()
- del self.outbuf[:real_len]
- return return_val
-
- def _add_to_outbuf_once(self):
- """Add one cycle's worth of output to self.outbuf"""
- if not self.infile_eof: self._add_to_inbuf()
- try: self.eof, len_inbuf_read, cycle_out = self.maker.cycle(self.inbuf)
- except _librsync.librsyncError, e: raise librsyncError(str(e))
- self.inbuf = self.inbuf[len_inbuf_read:]
- self.outbuf.fromstring(cycle_out)
-
- def _add_to_inbuf(self):
- """Make sure len(self.inbuf) >= blocksize"""
- assert not self.infile_eof
- while len(self.inbuf) < blocksize:
- new_in = self.infile.read(blocksize)
- if not new_in:
- self.infile_eof = 1
- assert not self.infile.close()
- self.infile_closed = 1
- break
- self.inbuf += new_in
-
- def close(self):
- """Close infile"""
- if not self.infile_closed: assert not self.infile.close()
- self.closed = 1
-
-
-class SigFile(LikeFile):
- """File-like object which incrementally generates a librsync signature"""
- def __init__(self, infile):
- """SigFile initializer - takes basis file
-
- basis file only needs to have read() and close() methods. It
- will be closed when we come to the end of the signature.
-
- """
- LikeFile.__init__(self, infile)
- try: self.maker = _librsync.new_sigmaker()
- except _librsync.librsyncError, e: raise librsyncError(str(e))
-
-class DeltaFile(LikeFile):
- """File-like object which incrementally generates a librsync delta"""
- def __init__(self, signature, new_file):
- """DeltaFile initializer - call with signature and new file
-
- Signature can either be a string or a file with read() and
- close() methods. New_file also only needs to have read() and
- close() methods. It will be closed when self is closed.
-
- """
- LikeFile.__init__(self, new_file)
- if type(signature) is types.StringType: sig_string = signature
- else:
- self.check_file(signature)
- sig_string = signature.read()
- assert not signature.close()
- try: self.maker = _librsync.new_deltamaker(sig_string)
- except _librsync.librsyncError, e: raise librsyncError(str(e))
-
-
-class PatchedFile(LikeFile):
- """File-like object which applies a librsync delta incrementally"""
- def __init__(self, basis_file, delta_file):
- """PatchedFile initializer - call with basis delta
-
- Here basis_file must be a true Python file, because we may
- need to seek() around in it a lot, and this is done in C.
- delta_file only needs read() and close() methods.
-
- """
- LikeFile.__init__(self, delta_file)
- if type(basis_file) is not types.FileType:
- raise TypeError("basis_file must be a (true) file")
- try: self.maker = _librsync.new_patchmaker(basis_file)
- except _librsync.librsyncError, e: raise librsyncError(str(e))
-
-
-class SigGenerator:
- """Calculate signature.
-
- Input and output is same as SigFile, but the interface is like md5
- module, not filelike object
-
- """
- def __init__(self):
- """Return new signature instance"""
- try: self.sig_maker = _librsync.new_sigmaker()
- except _librsync.librsyncError, e: raise librsyncError(str(e))
- self.gotsig = None
- self.buffer = ""
- self.sig_string = ""
-
- def update(self, buf):
- """Add buf to data that signature will be calculated over"""
- if self.gotsig:
- raise librsyncError("SigGenerator already provided signature")
- self.buffer += buf
- while len(self.buffer) >= blocksize:
- if self.process_buffer():
- raise librsyncError("Premature EOF received from sig_maker")
-
- def process_buffer(self):
- """Run self.buffer through sig_maker, add to self.sig_string"""
- try: eof, len_buf_read, cycle_out = self.sig_maker.cycle(self.buffer)
- except _librsync.librsyncError, e: raise librsyncError(str(e))
- self.buffer = self.buffer[len_buf_read:]
- self.sig_string += cycle_out
- return eof
-
- def getsig(self):
- """Return signature over given data"""
- while not self.process_buffer(): pass # keep running until eof
- return self.sig_string
-
diff --git a/rdiff-backup/src/librsync_memoryleak2.py b/rdiff-backup/src/librsync_memoryleak2.py
deleted file mode 100644
index 72400b2..0000000
--- a/rdiff-backup/src/librsync_memoryleak2.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env python
-
-"""Demonstrate a memory leak in pysync/librsync"""
-
-import os, _librsync
-from librsync import *
-
-os.chdir("/tmp")
-
-# Write 2 1 byte files
-afile = open("a", "wb")
-afile.write("a")
-afile.close()
-
-efile = open("e", "wb")
-efile.write("e")
-efile.close()
-
-def copy(infileobj, outpath):
- outfile = open(outpath, "wb")
- while 1:
- buf = infileobj.read(32768)
- if not buf: break
- outfile.write(buf)
- assert not outfile.close()
- assert not infileobj.close()
-
-def test_cycle():
- for i in xrange(100000):
- sm = _librsync.new_sigmaker()
- sm.cycle("a")
-
-def main_test():
- for i in xrange(100000):
- # Write signature file
- afile = open("a", "rb")
- copy(SigFile(afile), "sig")
-
- # Write delta file
- efile = open("e", "r")
- sigfile = open("sig", "rb")
- copy(DeltaFile(sigfile, efile), "delta")
-
- # Write patched file
- afile = open("e", "rb")
- deltafile = open("delta", "rb")
- copy(PatchedFile(afile, deltafile), "a.out")
-
-main_test()
diff --git a/rdiff-backup/src/log.py b/rdiff-backup/src/log.py
deleted file mode 100644
index 5c03b27..0000000
--- a/rdiff-backup/src/log.py
+++ /dev/null
@@ -1,183 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Manage logging, displaying and recording messages with required verbosity"""
-
-import time, sys, traceback, types
-
-
-class LoggerError(Exception): pass
-
-class Logger:
- """All functions which deal with logging"""
- def __init__(self):
- self.log_file_open = None
- self.log_file_local = None
- self.verbosity = self.term_verbosity = 3
- # termverbset is true if the term_verbosity has been explicity set
- self.termverbset = None
-
- def setverbosity(self, verbosity_string):
- """Set verbosity levels. Takes a number string"""
- try: self.verbosity = int(verbosity_string)
- except ValueError:
- Log.FatalError("Verbosity must be a number, received '%s' "
- "instead." % verbosity_string)
- if not self.termverbset: self.term_verbosity = self.verbosity
-
- def setterm_verbosity(self, termverb_string):
- """Set verbosity to terminal. Takes a number string"""
- try: self.term_verbosity = int(termverb_string)
- except ValueError:
- Log.FatalError("Terminal verbosity must be a number, received "
- "'%s' instead." % termverb_string)
- self.termverbset = 1
-
- def open_logfile(self, rpath):
- """Inform all connections of an open logfile.
-
- rpath.conn will write to the file, and the others will pass
- write commands off to it.
-
- """
- assert not self.log_file_open
- rpath.conn.Log.open_logfile_local(rpath)
- for conn in Globals.connections:
- conn.Log.open_logfile_allconn(rpath.conn)
-
- def open_logfile_allconn(self, log_file_conn):
- """Run on all connections to signal log file is open"""
- self.log_file_open = 1
- self.log_file_conn = log_file_conn
-
- def open_logfile_local(self, rpath):
- """Open logfile locally - should only be run on one connection"""
- assert rpath.conn is Globals.local_connection
- try: self.logfp = rpath.open("a")
- except (OSError, IOError), e:
- raise LoggerError("Unable to open logfile %s: %s"
- % (rpath.path, e))
- self.log_file_local = 1
- self.logrp = rpath
-
- def close_logfile(self):
- """Close logfile and inform all connections"""
- if self.log_file_open:
- for conn in Globals.connections:
- conn.Log.close_logfile_allconn()
- self.log_file_conn.Log.close_logfile_local()
-
- def close_logfile_allconn(self):
- """Run on every connection"""
- self.log_file_open = None
-
- def close_logfile_local(self):
- """Run by logging connection - close logfile"""
- assert self.log_file_conn is Globals.local_connection
- assert not self.logfp.close()
- self.log_file_local = None
-
- def format(self, message, verbosity):
- """Format the message, possibly adding date information"""
- if verbosity < 9: return message + "\n"
- else: return "%s %s\n" % (time.asctime(time.localtime(time.time())),
- message)
-
- def __call__(self, message, verbosity):
- """Log message that has verbosity importance
-
- message can be a string, which is logged as-is, or a function,
- which is then called and should return the string to be
- logged. We do it this way in case producing the string would
- take a significant amount of CPU.
-
- """
- if verbosity > self.verbosity and verbosity > self.term_verbosity:
- return
-
- if not type(message) is types.StringType:
- assert type(message) is types.FunctionType
- message = message()
-
- if verbosity <= self.verbosity: self.log_to_file(message)
- if verbosity <= self.term_verbosity:
- self.log_to_term(message, verbosity)
-
- def log_to_file(self, message):
- """Write the message to the log file, if possible"""
- if self.log_file_open:
- if self.log_file_local:
- self.logfp.write(self.format(message, self.verbosity))
- else: self.log_file_conn.Log.log_to_file(message)
-
- def log_to_term(self, message, verbosity):
- """Write message to stdout/stderr"""
- if verbosity <= 2 or Globals.server: termfp = sys.stderr
- else: termfp = sys.stdout
- termfp.write(self.format(message, self.term_verbosity))
-
- def conn(self, direction, result, req_num):
- """Log some data on the connection
-
- The main worry with this function is that something in here
- will create more network traffic, which will spiral to
- infinite regress. So, for instance, logging must only be done
- to the terminal, because otherwise the log file may be remote.
-
- """
- if self.term_verbosity < 9: return
- if type(result) is types.StringType: result_repr = repr(result)
- else: result_repr = str(result)
- if Globals.server: conn_str = "Server"
- else: conn_str = "Client"
- self.log_to_term("%s %s (%d): %s" %
- (conn_str, direction, req_num, result_repr), 9)
-
- def FatalError(self, message):
- self("Fatal Error: " + message, 1)
- Main.cleanup()
- sys.exit(1)
-
- def exception_to_string(self, arglist = []):
- """Return string version of current exception plus what's in arglist"""
- type, value, tb = sys.exc_info()
- s = ("Exception '%s' raised of class '%s':\n%s" %
- (value, type, "".join(traceback.format_tb(tb))))
- if arglist:
- s += "__Arguments:\n" + "\n".join(map(str, arglist))
- return s
-
- def exception(self, only_terminal = 0, verbosity = 5):
- """Log an exception and traceback
-
- If only_terminal is None, log normally. If it is 1, then only
- log to disk if log file is local (self.log_file_open = 1). If
- it is 2, don't log to disk at all.
-
- """
- assert only_terminal in (0, 1, 2)
- if (only_terminal == 0 or
- (only_terminal == 1 and self.log_file_open)):
- logging_func = self.__call__
- else: logging_func = self.log_to_term
-
- logging_func(self.exception_to_string(), verbosity)
-
-Log = Logger()
-import Globals, Main
diff --git a/rdiff-backup/src/manage.py b/rdiff-backup/src/manage.py
deleted file mode 100644
index 2e1d7b6..0000000
--- a/rdiff-backup/src/manage.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""list, delete, and otherwise manage increments"""
-
-from __future__ import generators
-from static import *
-from log import *
-import Globals, Time
-
-
-class ManageException(Exception): pass
-
-class Manage:
- def get_file_type(rp):
- """Returns one of "regular", "directory", "missing", or "special"."""
- if not rp.lstat(): return "missing"
- elif rp.isdir(): return "directory"
- elif rp.isreg(): return "regular"
- else: return "special"
-
- def get_inc_type(inc):
- """Return file type increment represents"""
- assert inc.isincfile()
- type = inc.getinctype()
- if type == "dir": return "directory"
- elif type == "diff": return "regular"
- elif type == "missing": return "missing"
- elif type == "snapshot": return Manage.get_file_type(inc)
- else: assert None, "Unknown type %s" % (type,)
-
- def describe_incs_parsable(incs, mirror_time, mirrorrp):
- """Return a string parsable by computer describing the increments
-
- Each line is a time in seconds of the increment, and then the
- type of the file. It will be sorted oldest to newest. For example:
-
- 10000 regular
- 20000 directory
- 30000 special
- 40000 missing
- 50000 regular <- last will be the current mirror
-
- """
- incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
- incpairs.sort()
- result = ["%s %s" % (time, Manage.get_inc_type(inc))
- for time, inc in incpairs]
- result.append("%s %s" % (mirror_time, Manage.get_file_type(mirrorrp)))
- return "\n".join(result)
-
- def describe_incs_human(incs, mirror_time, mirrorrp):
- """Return a string describing all the the root increments"""
- incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
- incpairs.sort()
-
- result = ["Found %d increments:" % len(incpairs)]
- for time, inc in incpairs:
- result.append(" %s %s" %
- (inc.dirsplit()[1], Time.timetopretty(time)))
- result.append("Current mirror: %s" % Time.timetopretty(mirror_time))
- return "\n".join(result)
-
- def delete_earlier_than(baserp, time):
- """Deleting increments older than time in directory baserp
-
- time is in seconds. It will then delete any empty directories
- in the tree. To process the entire backup area, the
- rdiff-backup-data directory should be the root of the tree.
-
- """
- baserp.conn.Manage.delete_earlier_than_local(baserp, time)
-
- def delete_earlier_than_local(baserp, time):
- """Like delete_earlier_than, but run on local connection for speed"""
- assert baserp.conn is Globals.local_connection
- def yield_files(rp):
- yield rp
- if rp.isdir():
- for filename in rp.listdir():
- for sub_rp in yield_files(rp.append(filename)):
- yield sub_rp
-
- for rp in yield_files(baserp):
- if ((rp.isincfile() and
- Time.stringtotime(rp.getinctime()) < time) or
- (rp.isdir() and not rp.listdir())):
- Log("Deleting increment file %s" % rp.path, 5)
- rp.delete()
-
-MakeStatic(Manage)
-
-
-class IncObj:
- """Increment object - represent a completed increment"""
- def __init__(self, incrp):
- """IncObj initializer
-
- incrp is an RPath of a path like increments.TIMESTR.dir
- standing for the root of the increment.
-
- """
- if not incrp.isincfile():
- raise ManageException("%s is not an inc file" % incrp.path)
- self.incrp = incrp
- self.time = Time.stringtotime(incrp.getinctime())
-
- def getbaserp(self):
- """Return rp of the incrp without extensions"""
- return self.incrp.getincbase()
-
- def pretty_time(self):
- """Return a formatted version of inc's time"""
- return Time.timetopretty(self.time)
-
- def full_description(self):
- """Return string describing increment"""
- s = ["Increment file %s" % self.incrp.path,
- "Date: %s" % self.pretty_time()]
- return "\n".join(s)
diff --git a/rdiff-backup/src/memoryleak.c b/rdiff-backup/src/memoryleak.c
deleted file mode 100644
index 9fc698b..0000000
--- a/rdiff-backup/src/memoryleak.c
+++ /dev/null
@@ -1,23 +0,0 @@
-#include <stdio.h>
-#include <rsync.h>
-
-main()
-{
- FILE *basis_file, *sig_file;
- char filename[50];
- rs_stats_t stats;
- rs_result result;
- long i;
-
- for(i=0; i<=100000; i++) {
- basis_file = fopen("a", "r");
- sig_file = fopen("sig", "w");
-
- result = rs_sig_file(basis_file, sig_file,
- RS_DEFAULT_BLOCK_LEN, RS_DEFAULT_STRONG_LEN,
- &stats);
- if (result != RS_DONE) exit(result);
- fclose(basis_file);
- fclose(sig_file);
- }
-}
diff --git a/rdiff-backup/src/myrdiff.py b/rdiff-backup/src/myrdiff.py
deleted file mode 100755
index 48485f7..0000000
--- a/rdiff-backup/src/myrdiff.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env python
-
-"""Like rdiff, but written in python and uses librsync module.
-
-Useful for benchmarking and testing of librsync and _librsync.
-
-"""
-
-import librsync, sys
-blocksize = 32768
-
-def makesig(inpath, outpath):
- """Write a signature of inpath at outpath"""
- sf = librsync.SigFile(open(inpath, "rb"))
- fout = open(outpath, "wb")
- while 1:
- buf = sf.read(blocksize)
- if not buf: break
- fout.write(buf)
- assert not sf.close()
- assert not fout.close()
-
-def makedelta(sigpath, newpath, deltapath):
- """Write delta at deltapath using signature at sigpath"""
- df = librsync.DeltaFile(open(sigpath, "rb"), open(newpath, "rb"))
- fout = open(deltapath, "wb")
- while 1:
- buf = df.read(blocksize)
- if not buf: break
- fout.write(buf)
- assert not df.close()
- assert not fout.close()
-
-def makepatch(basis_path, delta_path, new_path):
- """Write new given basis and delta"""
- pf = librsync.PatchedFile(open(basis_path, "rb"), open(delta_path, "rb"))
- fout = open(new_path, "wb")
- while 1:
- buf = pf.read(blocksize)
- if not buf: break
- fout.write(buf)
- assert not pf.close()
- assert not fout.close()
-
-if sys.argv[1] == "signature":
- makesig(sys.argv[2], sys.argv[3])
-elif sys.argv[1] == "delta":
- makedelta(sys.argv[2], sys.argv[3], sys.argv[4])
-elif sys.argv[1] == "patch":
- makepatch(sys.argv[2], sys.argv[3], sys.argv[4])
-else: assert 0, "Bad mode argument %s" % (sys.argv[1],)
diff --git a/rdiff-backup/src/profiled_rdb.py b/rdiff-backup/src/profiled_rdb.py
deleted file mode 100755
index 7412847..0000000
--- a/rdiff-backup/src/profiled_rdb.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-
-"""Run rdiff-backup with profiling on
-
-Same as rdiff-backup but runs profiler, and prints profiling
-statistics afterwards.
-
-"""
-
-__no_execute__ = 1
-import sys, rdiff_backup.Main, profile, pstats
-profile.run("rdiff_backup.Main.Main(%s)" % repr(sys.argv[1:]),
- "profile-output")
-p = pstats.Stats("profile-output")
-p.sort_stats('time')
-p.print_stats(40)
-#p.print_callers(20)
-
-
diff --git a/rdiff-backup/src/restore.py b/rdiff-backup/src/restore.py
deleted file mode 100644
index e361512..0000000
--- a/rdiff-backup/src/restore.py
+++ /dev/null
@@ -1,392 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Read increment files and restore to original"""
-
-from __future__ import generators
-import tempfile
-from static import *
-
-
-class RestoreError(Exception): pass
-
-class Restore:
- def Restore(inc_rpath, mirror, target, rest_time):
- """Recursively restore inc_rpath and mirror to target at rest_time
-
- Like restore_recusive below, but with a more friendly
- interface (it converts to DSRPaths if necessary, finds the inc
- files with the appropriate base, and makes rid).
-
- rest_time is the time in seconds to restore to;
-
- inc_rpath should not be the name of an increment file, but the
- increment file shorn of its suffixes and thus should have the
- same index as mirror.
-
- """
- if not isinstance(mirror, DSRPath): mirror = DSRPath(1, mirror)
- if not isinstance(target, DSRPath): target = DSRPath(None, target)
-
- mirror_time = Restore.get_mirror_time()
- rest_time = Restore.get_rest_time(rest_time, mirror_time)
- inc_list = Restore.get_inclist(inc_rpath)
- rid = RestoreIncrementData(inc_rpath.index, inc_rpath, inc_list)
- rid.sortincseq(rest_time, mirror_time)
- Restore.check_hardlinks(rest_time)
- Restore.restore_recursive(inc_rpath.index, mirror, rid, target,
- rest_time, mirror_time)
-
- def get_mirror_time():
- """Return the time (in seconds) of latest mirror"""
- current_mirror_incs = \
- Restore.get_inclist(Globals.rbdir.append("current_mirror"))
- if not current_mirror_incs:
- Log.FatalError("Could not get time of current mirror")
- elif len(current_mirror_incs) > 1:
- Log("Warning, two different dates for current mirror found", 2)
- return Time.stringtotime(current_mirror_incs[0].getinctime())
-
- def get_rest_time(old_rest_time, mirror_time):
- """If old_rest_time is between two increments, return older time
-
- There is a slightly tricky reason for doing this: The rest of
- the code just ignores increments that are older than
- rest_time. But sometimes we want to consider the very next
- increment older than rest time, because rest_time will be
- between two increments, and what was actually on the mirror
- side will correspond to the older one.
-
- So here we assume all rdiff-backup events were recorded in
- "increments" increments, and if its in-between we pick the
- older one here.
-
- """
- base_incs = Restore.get_inclist(Globals.rbdir.append("increments"))
- if not base_incs: return old_rest_time
- inctimes = [Time.stringtotime(inc.getinctime()) for inc in base_incs]
- inctimes.append(mirror_time)
- older_times = filter(lambda time: time <= old_rest_time, inctimes)
- if older_times: return max(older_times)
- else: # restore time older than oldest increment, just return that
- return min(inctimes)
-
- def get_inclist(inc_rpath):
- """Returns increments with given base"""
- dirname, basename = inc_rpath.dirsplit()
- parent_dir = RPath(inc_rpath.conn, dirname, ())
- if not parent_dir.isdir(): return [] # inc directory not created yet
- index = inc_rpath.index
-
- if index:
- get_inc_ext = lambda filename: \
- RPath(inc_rpath.conn, inc_rpath.base,
- inc_rpath.index[:-1] + (filename,))
- else: get_inc_ext = lambda filename: \
- RPath(inc_rpath.conn, os.path.join(dirname, filename))
-
- inc_list = []
- for filename in parent_dir.listdir():
- inc = get_inc_ext(filename)
- if inc.isincfile() and inc.getincbase_str() == basename:
- inc_list.append(inc)
- return inc_list
-
- def check_hardlinks(rest_time):
- """Check for hard links and enable hard link support if found"""
- if (Globals.preserve_hardlinks != 0 and
- Hardlink.retrieve_final(rest_time)):
- Log("Hard link information found, attempting to preserve "
- "hard links.", 5)
- SetConnections.UpdateGlobal('preserve_hardlinks', 1)
- else: SetConnections.UpdateGlobal('preserve_hardlinks', None)
-
- def restore_recursive(index, mirror, rid, target, time, mirror_time):
- """Recursive restore function.
-
- rid is a RestoreIncrementData object whose inclist is already
- sortedincseq'd, and target is the dsrp to restore to.
-
- Note that target may have a different index than mirror and
- rid, because we may be restoring a file whose index is, say
- ('foo','bar') to a target whose path does not contain
- "foo/bar".
-
- """
- assert isinstance(mirror, DSRPath) and isinstance(target, DSRPath)
- assert mirror.index == rid.index
-
- mirror_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ())
- target_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ())
- for rcd in Restore.yield_rcds(rid.index, mirror, rid,
- target, time, mirror_time):
- rcd.RestoreFile()
- if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror)
- target_finalizer(rcd.target.index, rcd.target)
- target_finalizer.Finish()
- mirror_finalizer.Finish()
-
- def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time):
- """Iterate RestoreCombinedData objects starting with given args
-
- rid is a RestoreCombinedData object. target is an rpath where
- the created file should go.
-
- In this case the "mirror" directory is treated as the source,
- and we are actually copying stuff onto what Select considers
- the source directory.
-
- """
- select_result = Globals.select_mirror.Select(target)
- if select_result == 0: return
-
- if mirrorrp and not Globals.select_source.Select(mirrorrp):
- mirrorrp = None
- rcd = RestoreCombinedData(rid, mirrorrp, target)
-
- if mirrorrp and mirrorrp.isdir() or \
- rid and rid.inc_rpath and rid.inc_rpath.isdir():
- sub_rcds = Restore.yield_sub_rcds(index, mirrorrp, rid,
- target, rest_time, mirror_time)
- else: sub_rcds = None
-
- if select_result == 1:
- yield rcd
- if sub_rcds:
- for sub_rcd in sub_rcds: yield sub_rcd
- elif select_result == 2:
- if sub_rcds:
- try: first = sub_rcds.next()
- except StopIteration: return # no tuples found inside, skip
- yield rcd
- yield first
- for sub_rcd in sub_rcds: yield sub_rcd
-
- def yield_sub_rcds(index, mirrorrp, rid, target, rest_time, mirror_time):
- """Yield collated tuples from inside given args"""
- if not Restore.check_dir_exists(mirrorrp, rid): return
- mirror_iter = Restore.yield_mirrorrps(mirrorrp)
- rid_iter = Restore.yield_rids(rid, rest_time, mirror_time)
-
- for indexed_tup in RORPIter.CollateIterators(mirror_iter, rid_iter):
- index = indexed_tup.index
- new_mirrorrp, new_rid = indexed_tup
- for rcd in Restore.yield_rcds(index, new_mirrorrp,
- new_rid, target.append(index[-1]), rest_time, mirror_time):
- yield rcd
-
- def check_dir_exists(mirrorrp, rid):
- """Return true if target should be a directory"""
- if rid and rid.inc_list:
- # Incs say dir if last (earliest) one is a dir increment
- return rid.inc_list[-1].getinctype() == "dir"
- elif mirrorrp: return mirrorrp.isdir() # if no incs, copy mirror
- else: return None
-
- def yield_mirrorrps(mirrorrp):
- """Yield mirrorrps underneath given mirrorrp"""
- if mirrorrp and mirrorrp.isdir():
- if Globals.quoting_enabled:
- for rp in FilenameMapping.get_quoted_dir_children(mirrorrp):
- yield rp
- else:
- dirlist = mirrorrp.listdir()
- dirlist.sort()
- for filename in dirlist: yield mirrorrp.append(filename)
-
- def yield_rids(rid, rest_time, mirror_time):
- """Yield RestoreIncrementData objects within given rid dir
-
- If the rid doesn't correspond to a directory, don't yield any
- elements. If there are increments whose corresponding base
- doesn't exist, the first element will be None. All the rpaths
- involved correspond to files in the increment directory.
-
- """
- if not rid or not rid.inc_rpath or not rid.inc_rpath.isdir(): return
- rid_dict = {} # dictionary of basenames:rids
- dirlist = rid.inc_rpath.listdir()
- if Globals.quoting_enabled:
- dirlist = [FilenameMapping.unquote(fn) for fn in dirlist]
-
- def affirm_dict_indexed(basename):
- """Make sure the rid dictionary has given basename as key"""
- if not rid_dict.has_key(basename):
- rid_dict[basename] = RestoreIncrementData(
- rid.index + (basename,), None, []) # init with empty rid
-
- def add_to_dict(filename):
- """Add filename to the inc tuple dictionary"""
- rp = rid.inc_rpath.append(filename)
- if Globals.quoting_enabled: rp.quote_path()
- if rp.isincfile() and rp.getinctype() != 'data':
- basename = rp.getincbase_str()
- affirm_dict_indexed(basename)
- rid_dict[basename].inc_list.append(rp)
- elif rp.isdir():
- affirm_dict_indexed(filename)
- rid_dict[filename].inc_rpath = rp
-
- for filename in dirlist: add_to_dict(filename)
- keys = rid_dict.keys()
- keys.sort()
-
- # sortincseq now to avoid descending .missing directories later
- for key in keys:
- rid = rid_dict[key]
- if rid.inc_rpath or rid.inc_list:
- rid.sortincseq(rest_time, mirror_time)
- yield rid
-
-MakeStatic(Restore)
-
-
-class RestoreIncrementData:
- """Contains information about a specific index from the increments dir
-
- This is just a container class, used because it would be easier to
- work with than an IndexedTuple.
-
- """
- def __init__(self, index, inc_rpath, inc_list):
- self.index = index
- self.inc_rpath = inc_rpath
- self.inc_list = inc_list
-
- def sortincseq(self, rest_time, mirror_time):
- """Sort self.inc_list sequence, throwing away irrelevant increments"""
- if not self.inc_list or rest_time >= mirror_time:
- self.inc_list = []
- return
-
- newer_incs = self.get_newer_incs(rest_time, mirror_time)
- i = 0
- while(i < len(newer_incs)):
- # Only diff type increments require later versions
- if newer_incs[i].getinctype() != "diff": break
- i = i+1
- self.inc_list = newer_incs[:i+1]
- self.inc_list.reverse() # return in reversed order (latest first)
-
- def get_newer_incs(self, rest_time, mirror_time):
- """Return list of newer incs sorted by time (increasing)
-
- Also discard increments older than rest_time (rest_time we are
- assuming is the exact time rdiff-backup was run, so no need to
- consider the next oldest increment or any of that)
-
- """
- incpairs = []
- for inc in self.inc_list:
- time = Time.stringtotime(inc.getinctime())
- if time >= rest_time: incpairs.append((time, inc))
- incpairs.sort()
- return [pair[1] for pair in incpairs]
-
-
-class RestoreCombinedData:
- """Combine index information from increment and mirror directories
-
- This is similar to RestoreIncrementData but has mirror information
- also.
-
- """
- def __init__(self, rid, mirror, target):
- """Init - set values from one or both if they exist
-
- mirror and target are DSRPaths of the corresponding files in
- the mirror and target directory respectively. rid is a
- RestoreIncrementData as defined above
-
- """
- if rid:
- self.index = rid.index
- self.inc_rpath = rid.inc_rpath
- self.inc_list = rid.inc_list
- if mirror:
- self.mirror = mirror
- assert mirror.index == self.index
- else: self.mirror = None
- elif mirror:
- self.index = mirror.index
- self.mirror = mirror
- self.inc_list = []
- self.inc_rpath = None
- else: assert None, "neither rid nor mirror given"
- self.target = target
-
- def RestoreFile(self):
- """Non-recursive restore function """
- if not self.inc_list and not (self.mirror and self.mirror.lstat()):
- return # no increments were applicable
- self.log()
-
- if self.restore_hardlink(): return
-
- if not self.inc_list or self.inc_list[0].getinctype() == "diff":
- assert self.mirror and self.mirror.lstat(), \
- "No base to go with incs for %s" % self.target.path
- RPath.copy_with_attribs(self.mirror, self.target)
- for inc in self.inc_list: self.applyinc(inc, self.target)
-
- def log(self):
- """Log current restore action"""
- inc_string = ','.join([inc.path for inc in self.inc_list])
- Log("Restoring %s with increments %s to %s" %
- (self.mirror and self.mirror.path,
- inc_string, self.target.path), 5)
-
- def restore_hardlink(self):
- """Hard link target and return true if hard linking appropriate"""
- if (Globals.preserve_hardlinks and
- Hardlink.restore_link(self.index, self.target)):
- RPath.copy_attribs(self.inc_list and self.inc_list[-1] or
- self.mirror, self.target)
- return 1
- return None
-
- def applyinc(self, inc, target):
- """Apply increment rp inc to targetrp target"""
- Log("Applying increment %s to %s" % (inc.path, target.path), 6)
- inctype = inc.getinctype()
- if inctype == "diff":
- if not target.lstat():
- raise RestoreError("Bad increment sequence at " + inc.path)
- Rdiff.patch_action(target, inc,
- delta_compressed = inc.isinccompressed()
- ).execute()
- elif inctype == "dir":
- if not target.isdir():
- if target.lstat():
- raise RestoreError("File %s already exists" % target.path)
- target.mkdir()
- elif inctype == "missing": return
- elif inctype == "snapshot":
- if inc.isinccompressed():
- target.write_from_fileobj(inc.open("rb", compress = 1))
- else: RPath.copy(inc, target)
- else: raise RestoreError("Unknown inctype %s" % inctype)
- RPath.copy_attribs(inc, target)
-
-
-from log import *
-from destructive_stepping import *
-from rpath import *
-from rorpiter import *
-import Globals, Time, Rdiff, Hardlink, FilenameMapping, SetConnections
diff --git a/rdiff-backup/src/rlist.py b/rdiff-backup/src/rlist.py
deleted file mode 100644
index c0f8ee9..0000000
--- a/rdiff-backup/src/rlist.py
+++ /dev/null
@@ -1,240 +0,0 @@
-from __future__ import generators
-import marshal, sha, types
-execfile("iterfile.py")
-
-#######################################################################
-#
-# rlist - Define the CachingIter, and sig/diff/patch ops on iterators
-#
-
-class CachingIter:
- """Cache parts of an iter using a list
-
- Turn an iter into something that you can prepend elements into,
- and also read from without apparently changing the state.
-
- """
- def __init__(self, iter_or_list):
- if type(iter_or_list) is types.ListType:
- self.iter = iter(iter_or_list)
- else: self.iter = iter_or_list
- self.next = self.iter.next
- self.head = []
-
- def __iter__(self): return self
-
- def _next(self):
- """Take elements from the head list
-
- When there are elements waiting before the main iterator, this
- is the next function. If not, iter.next returns to being next.
-
- """
- head = self.head
- a = head[0]
- del head[0]
- if not head: self.next = self.iter.next
- return a
-
- def nextrange(self, m):
- """Return next m elements in list"""
- l = head[:m]
- del head[:m]
- for i in xrange(m - len(l)): l.append(self.iter.next())
- return l
-
- def peek(self):
- """Return next element without removing it from iterator"""
- n = self.next()
- self.push(n)
- return n
-
- def push(self, elem):
- """Insert an element into the iterator at the beginning"""
- if not self.head: self.next = self._next
- self.head.insert(0, elem)
-
- def pushrange(self, elem_list):
- """Insert list of multiple elements at the beginning"""
- if not self.head: self.next = self._next
- self.head[:0] = elem_list
-
- def cache(self, m):
- """Move next m elements from iter to internal list
-
- If m is None, append the entire rest of the iterator.
-
- """
- h, it = self.head, self.iter
- if m is None:
- for i in it: h.append(i)
- else:
- for i in xrange(m): h.append(it.next())
-
- def __getitem__(self, key):
- """Support a[i:j] style notation. Non destructive"""
- if type(key) is types.SliceType:
- if key.stop > len(self.head): self.cache(key.stop - len(self.head))
- return self.head[key.start, key.stop]
- else:
- if key >= len(self.head): self.cache(key + 1 - len(self.head))
- return self.head[key]
-
-
-
-class RListDelta:
- """Note a difference from one iterator (A) to another (B)
-
- The min, max pairs are indicies which stand for the half-open
- interval (min, max], and elemlist is a list of all the elements in
- A which fall within this interval.
-
- These are produced by the function RList.Deltas(...)
-
- """
- def __init__(self, (min, max), elemlist):
- self.min, self.max = min, max
- self.elemlist = elemlist
-
-
-
-class RList:
- """Tools for signatures, diffing, and patching an iterator
-
- This class requires that the iterators involved are yielding
- objects that have .index and .data attributes. Two objects with
- the same .data attribute are supposed to be equivalent. The
- iterator must also yield the objects in increasing order with
- respect to the .index attribute.
-
- """
- blocksize = 100
-
- def Signatures(iter):
- """Return iterator of signatures from stream of pairs
-
- Each signature is an ordered pair (last index sig applies to,
- SHA digest of data)
-
- """
- i, s = 0, sha.new()
- for iter_elem in iter:
- s.update(marshal.dumps(iter_elem.data))
- i = i+1
- if i == RList.blocksize:
- yield (iter_elem.index, s.digest())
- i, s = 0, sha.new()
- if i != 0: yield (iter_elem.index, s.digest())
-
- def sig_one_block(iter_or_list):
- """Return the digest portion of a signature on given list"""
- s = sha.new()
- for iter_elem in iter_or_list: s.update(marshal.dumps(iter_elem.data))
- return s.digest()
-
- def Deltas(remote_sigs, iter):
- """Return iterator of Delta objects that bring iter to remote"""
- def get_before(index, iter):
- """Return elements in iter whose index is before or equal index
- iter needs to be pushable
- """
- l = []
- while 1:
- try: iter_elem = iter.next()
- except StopIteration: return l
- if iter_elem.index > index: break
- l.append(iter_elem)
- iter.push(iter_elem)
- return l
-
- if not isinstance(iter, CachingIter): iter = CachingIter(iter)
- oldindex = None
- for (rs_index, rs_digest) in remote_sigs:
- l = get_before(rs_index, iter)
- if rs_digest != RList.sig_one_block(l):
- yield RListDelta((oldindex, rs_index), l)
- oldindex = rs_index
-
- def patch_once(basis, delta):
- """Apply one delta to basis to return original iterator
-
- This returns original iterator up to and including the max range
- of delta, then stop. basis should be pushable.
-
- """
- # Return elements of basis until start of delta range
- for basis_elem in basis:
- if basis_elem.index > delta.min:
- basis.push(basis_elem)
- break
- yield basis_elem
-
- # Yield elements of delta...
- for elem in delta.elemlist: yield elem
-
- # Finally, discard basis until end of delta range
- for basis_elem in basis:
- if basis_elem.index > delta.max:
- basis.push(basis_elem)
- break
-
- def Patch(basis, deltas):
- """Apply a delta stream to basis iterator, yielding original"""
- if not isinstance(basis, CachingIter): basis = CachingIter(basis)
- for d in deltas:
- for elem in RList.patch_once(basis, d): yield elem
- for elem in basis: yield elem
-
- def get_difference_once(basis, delta):
- """From one delta, find differences from basis
-
- Will return pairs (basis_elem, new_elem) where basis_elem is
- the element from the basis iterator and new_elem is the
- element from the other iterator. If either is missing None
- will take its place. If both are present iff two have the
- same index.
-
- """
- # Discard any elements of basis before delta starts
- for basis_elem in basis:
- if basis_elem.index > delta.min:
- basis.push(basis_elem)
- break
-
- # In range compare each one by one
- di, boverflow, doverflow = 0, None, None
- while 1:
- # Set indicies and data, or mark if at end of range already
- try:
- basis_elem = basis.next()
- if basis_elem.index > delta.max:
- basis.push(basis_elem)
- boverflow = 1
- except StopIteration: boverflow = 1
- if di >= len(delta.elemlist): doverflow = 1
- else: delta_elem = delta.elemlist[di]
-
- if boverflow and doverflow: break
- elif boverflow:
- yield (None, delta_elem)
- di = di+1
- elif doverflow: yield (basis_elem, None)
-
- # Now can assume that everything is in range
- elif basis_elem.index > delta_elem.index:
- yield (None, delta_elem)
- basis.push(basis_elem)
- di = di+1
- elif basis_elem.index == delta_elem.index:
- if basis_elem.data != delta_elem.data:
- yield (basis_elem, delta_elem)
- di = di+1
- else: yield (basis_elem, None)
-
- def Dissimilar(basis, deltas):
- """Return iter of differences from delta iter and basis iter"""
- if not isinstance(basis, CachingIter): basis = CachingIter(basis)
- for d in deltas:
- for triple in RList.get_difference_once(basis, d): yield triple
-
-MakeStatic(RList)
diff --git a/rdiff-backup/src/robust.py b/rdiff-backup/src/robust.py
deleted file mode 100644
index be7f1e8..0000000
--- a/rdiff-backup/src/robust.py
+++ /dev/null
@@ -1,698 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Prevent mirror from being corrupted; handle errors
-
-Ideally no matter an instance of rdiff-backup gets aborted, no
-information should get lost. The target directory should be left in a
-coherent state, and later instances of rdiff-backup should clean
-things up so there is no sign that anything ever got aborted or
-failed.
-
-Thus, files should be updated in an atomic way as possible. Each file
-should be updated (and the corresponding diff files written) or not,
-and it should be clear which happened. In general, I don't think this
-is possible, since the creation of the diff files and the changing of
-updated files cannot be guarateed to happen together. It is possible,
-I think, to record various information to files which would allow a
-later process to figure out what the last operation was, but this
-would add several file operations to the processing of each file, and
-I don't think, would be a good tradeoff.
-
-The compromise reached here is that diff files should be created just
-before the mirror files are updated, and each file update should be
-done with a rename operation on a file in the same directory.
-Furthermore, every once in a while, rdiff-backup will record which
-file it just finished processing. If any fatal errors are caught, it
-will also record the last processed file. Future instances may not
-know exactly when the previous instance was aborted, but they will be
-able to narrow down the possibilities.
-
-"""
-
-import tempfile, errno, signal, cPickle, C
-from static import *
-
-class RobustAction:
- """Represents a file operation to be accomplished later"""
- def __init__(self, init_thunk, final_func, error_handler):
- """RobustAction initializer
-
- All the thunks are functions whose return value will be
- ignored. init_thunk should not make any irreversible changes
- but prepare for the writing of the important data. final_func
- should be as short as possible and do the real work.
- error_handler is run if there is an error in init_thunk or
- final_func. Errors in init_thunk should be corrected by
- error_handler as if nothing had been run in the first place.
-
- init_thunk takes no arguments.
-
- final_thunk takes the return value of init_thunk as its
- argument, and its return value is returned by execute().
-
- error_handler takes three arguments: the exception, a value
- which is true just in case self.init_thunk ran correctly, and
- a value which will be the return value of init_thunk if it ran
- correctly.
-
- """
- self.init_thunk = init_thunk or self.default_init_thunk
- self.final_func = final_func or self.default_final_func
- self.error_handler = error_handler or self.default_error_handler
-
- def execute(self):
- """Actually run the operation"""
- ran_init_thunk = None
- try:
- init_val = self.init_thunk()
- ran_init_thunk = 1
- return self.final_func(init_val)
- except Exception, exc: # Catch all errors
- Log.exception()
- TracebackArchive.add()
- if ran_init_thunk: self.error_handler(exc, 1, init_val)
- else: self.error_handler(exc, None, None)
- raise exc
-
- def default_init_thunk(self): return None
- def default_final_func(self, init_val): return init_val
- def default_error_handler(self, exc, ran_init, init_val): pass
-
-
-class Robust:
- """Contains various methods designed to make things safer"""
- null_action = RobustAction(None, None, None)
- def chain(*robust_action_list):
- """Return chain tying together a number of robust actions
-
- The whole chain will be aborted if some error occurs in
- initialization stage of any of the component actions.
-
- """
- ras_with_started_inits, init_return_vals = [], []
- def init():
- for ra in robust_action_list:
- ras_with_started_inits.append(ra)
- init_return_vals.append(ra.init_thunk())
- return init_return_vals
- def final(init_return_vals):
- final_vals = []
- for ra, init_val in zip(robust_action_list, init_return_vals):
- final_vals.append(ra.final_func(init_val))
- return final_vals
- def error(exc, ran_init, init_val):
- for ra, init_val in zip(ras_with_started_inits, init_return_vals):
- ra.error_handler(exc, 1, init_val)
- for ra in ras_with_started_inits[len(init_return_vals):]:
- ra.error_handler(exc, None, None)
- return RobustAction(init, final, error)
-
- def chain_nested(*robust_action_list):
- """Like chain but final actions performed in reverse order"""
- ras_with_started_inits, init_vals = [], []
- def init():
- for ra in robust_action_list:
- ras_with_started_inits.append(ra)
- init_vals.append(ra.init_thunk())
- return init_vals
- def final(init_vals):
- ras_and_inits = zip(robust_action_list, init_vals)
- ras_and_inits.reverse()
- final_vals = []
- for ra, init_val in ras_and_inits:
- final_vals.append(ra.final_func(init_val))
- return final_vals
- def error(exc, ran_init, init_val):
- for ra, init_val in zip(ras_with_started_inits, init_vals):
- ra.error_handler(exc, 1, init_val)
- for ra in ras_with_started_inits[len(init_vals):]:
- ra.error_handler(exc, None, None)
- return RobustAction(init, final, error)
-
- def make_tf_robustaction(init_thunk, tempfiles, final_renames = None):
- """Shortcut RobustAction creator when only tempfiles involved
-
- Often the robust action will just consist of some initial
- stage, renaming tempfiles in the final stage, and deleting
- them if there is an error. This function makes it easier to
- create RobustActions of that type.
-
- """
- if isinstance(tempfiles, TempFile): tempfiles = (tempfiles,)
- if isinstance(final_renames, RPath): final_renames = (final_renames,)
- if final_renames is None: final_renames = [None] * len(tempfiles)
- assert len(tempfiles) == len(final_renames)
-
- def final(init_val): # rename tempfiles to final positions
- for tempfile, destination in zip(tempfiles, final_renames):
- if destination:
- if destination.isdir(): # Cannot rename over directory
- destination.delete()
- tempfile.rename(destination)
- return init_val
- def error(exc, ran_init, init_val):
- for tf in tempfiles: tf.delete()
- return RobustAction(init_thunk, final, error)
-
- def copy_action(rorpin, rpout):
- """Return robust action copying rorpin to rpout
-
- The source can be a rorp or an rpath. Does not recurse. If
- directories copied, then just exit (output directory not
- overwritten).
-
- """
- tfl = [None] # Need some mutable state to hold tf value
- def init():
- if not (rorpin.isdir() and rpout.isdir()): # already a dir
- tfl[0] = tf = TempFileManager.new(rpout)
- if rorpin.isreg(): tf.write_from_fileobj(rorpin.open("rb"))
- else: RPath.copy(rorpin, tf)
- return tf
- else: return None
- def final(tf):
- if tf and tf.lstat():
- if rpout.isdir(): rpout.delete()
- tf.rename(rpout)
- return rpout
- def error(exc, ran_init, init_val):
- if tfl[0]: tfl[0].delete()
- return RobustAction(init, final, error)
-
- def copy_with_attribs_action(rorpin, rpout, compress = None):
- """Like copy_action but also copy attributes"""
- tfl = [None] # Need some mutable state for error handler
- def init():
- if not (rorpin.isdir() and rpout.isdir()): # already a dir
- tfl[0] = tf = TempFileManager.new(rpout)
- if rorpin.isreg():
- tf.write_from_fileobj(rorpin.open("rb"), compress)
- else: RPath.copy(rorpin, tf)
- if tf.lstat(): # Some files, like sockets, won't be created
- RPathStatic.copy_attribs(rorpin, tf)
- return tf
- else: return None
- def final(tf):
- if rorpin.isdir() and rpout.isdir():
- RPath.copy_attribs(rorpin, rpout)
- elif tf and tf.lstat():
- if rpout.isdir(): rpout.delete() # can't rename over dir
- tf.rename(rpout)
- return rpout
- def error(exc, ran_init, init_val):
- if tfl[0]: tfl[0].delete()
- return RobustAction(init, final, error)
-
- def copy_attribs_action(rorpin, rpout):
- """Return action which just copies attributes
-
- Copying attributes is already pretty atomic, so just run
- normal sequence.
-
- """
- def final(init_val):
- RPath.copy_attribs(rorpin, rpout)
- return rpout
- return RobustAction(None, final, None)
-
- def symlink_action(rpath, linktext):
- """Return symlink action by moving one file over another"""
- tf = TempFileManager.new(rpath)
- def init(): tf.symlink(linktext)
- return Robust.make_tf_robustaction(init, tf, rpath)
-
- def destructive_write_action(rp, s):
- """Return action writing string s to rpath rp in robust way
-
- This will overwrite any data currently in rp.
-
- """
- tf = TempFileManager.new(rp)
- def init():
- fp = tf.open("wb")
- fp.write(s)
- fp.close()
- tf.setdata()
- return Robust.make_tf_robustaction(init, tf, rp)
-
- def check_common_error(error_handler, function, args = []):
- """Apply function to args, if error, run error_handler on exception
-
- This uses the catch_error predicate below to only catch
- certain exceptions which seems innocent enough.
-
- """
- try: return function(*args)
- except Exception, exc:
- TracebackArchive.add([function] + list(args))
- if Robust.catch_error(exc):
- Log.exception()
- conn = Globals.backup_writer
- if conn is not None: # increment error count
- ITRB_exists = conn.Globals.is_not_None('ITRB')
- if ITRB_exists: conn.Globals.ITRB.increment_stat('Errors')
- if error_handler: return error_handler(exc, *args)
- else: return
- Log.exception(1, 2)
- raise
-
- def catch_error(exc):
- """Return true if exception exc should be caught"""
- for exception_class in (SkipFileException, DSRPPermError,
- RPathException, Rdiff.RdiffException,
- librsync.librsyncError,
- C.UnknownFileTypeError):
- if isinstance(exc, exception_class): return 1
- if (isinstance(exc, EnvironmentError) and
- errno.errorcode[exc[0]] in ('EPERM', 'ENOENT', 'EACCES', 'EBUSY',
- 'EEXIST', 'ENOTDIR', 'ENAMETOOLONG',
- 'EINTR', 'ENOTEMPTY', 'EIO', 'ETXTBSY',
- 'ESRCH', 'EINVAL')):
- return 1
- return 0
-
- def listrp(rp):
- """Like rp.listdir() but return [] if error, and sort results"""
- def error_handler(exc):
- Log("Error listing directory %s" % rp.path, 2)
- return []
- dir_listing = Robust.check_common_error(error_handler, rp.listdir)
- dir_listing.sort()
- return dir_listing
-
- def signal_handler(signum, frame):
- """This is called when signal signum is caught"""
- raise SignalException(signum)
-
- def install_signal_handlers():
- """Install signal handlers on current connection"""
- for signum in [signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM]:
- signal.signal(signum, Robust.signal_handler)
-
-MakeStatic(Robust)
-
-
-class SignalException(Exception):
- """SignalException(signum) means signal signum has been received"""
- pass
-
-
-class TracebackArchive:
- """Save last 10 caught exceptions, so they can be printed if fatal"""
- _traceback_strings = []
- def add(cls, extra_args = []):
- """Add most recent exception to archived list
-
- If extra_args are present, convert to strings and add them as
- extra information to same traceback archive.
-
- """
- cls._traceback_strings.append(Log.exception_to_string(extra_args))
- if len(cls._traceback_strings) > 10:
- cls._traceback_strings = cls._traceback_strings[:10]
-
- def log(cls):
- """Print all exception information to log file"""
- if cls._traceback_strings:
- Log("------------ Old traceback info -----------\n%s\n"
- "-------------------------------------------" %
- ("\n".join(cls._traceback_strings),), 3)
-
-MakeClass(TracebackArchive)
-
-
-class TempFileManager:
- """Manage temp files"""
-
- # This is a connection-specific list of temp files, to be cleaned
- # up before rdiff-backup exits.
- _tempfiles = []
-
- # To make collisions less likely, this gets put in the file name
- # and incremented whenever a new file is requested.
- _tfindex = 0
-
- def new(cls, rp_base, same_dir = 1):
- """Return new tempfile that isn't in use.
-
- If same_dir, tempfile will be in same directory as rp_base.
- Otherwise, use tempfile module to get filename.
-
- """
- conn = rp_base.conn
- if conn is not Globals.local_connection:
- return conn.TempFileManager.new(rp_base, same_dir)
-
- def find_unused(conn, dir):
- """Find an unused tempfile with connection conn in directory dir"""
- while 1:
- if cls._tfindex > 100000000:
- Log("Resetting index", 2)
- cls._tfindex = 0
- tf = TempFile(conn, os.path.join(dir,
- "rdiff-backup.tmp.%d" % cls._tfindex))
- cls._tfindex = cls._tfindex+1
- if not tf.lstat(): return tf
-
- if same_dir: tf = find_unused(conn, rp_base.dirsplit()[0])
- else: tf = TempFile(conn, tempfile.mktemp())
- cls._tempfiles.append(tf)
- return tf
-
- def remove_listing(cls, tempfile):
- """Remove listing of tempfile"""
- if Globals.local_connection is not tempfile.conn:
- tempfile.conn.TempFileManager.remove_listing(tempfile)
- elif tempfile in cls._tempfiles: cls._tempfiles.remove(tempfile)
-
- def delete_all(cls):
- """Delete all remaining tempfiles"""
- for tf in cls._tempfiles[:]: tf.delete()
-
-MakeClass(TempFileManager)
-
-
-from rpath import *
-
-class TempFile(RPath):
- """Like an RPath, but keep track of which ones are still here"""
- def rename(self, rp_dest):
- """Rename temp file to permanent location, possibly overwriting"""
- if self.isdir() and not rp_dest.isdir():
- # Cannot move a directory directly over another file
- rp_dest.delete()
- if (isinstance(rp_dest, DSRPath) and rp_dest.delay_perms
- and not self.hasfullperms()):
- # If we are moving to a delayed perm directory, delay
- # permission change on destination.
- rp_dest.chmod(self.getperms())
- self.chmod(0700)
- RPathStatic.rename(self, rp_dest)
-
- # Sometimes this just seems to fail silently, as in one
- # hardlinked twin is moved over the other. So check to make
- # sure below.
- self.setdata()
- if self.lstat():
- rp_dest.delete()
- RPathStatic.rename(self, rp_dest)
- self.setdata()
- if self.lstat(): raise OSError("Cannot rename tmp file correctly")
- TempFileManager.remove_listing(self)
-
- def delete(self):
- RPath.delete(self)
- TempFileManager.remove_listing(self)
-
-
-class SaveState:
- """Save state in the middle of backups for resuming later"""
- _last_file_sym = None # RPath of sym pointing to last file processed
- _last_file_definitive_rp = None # Touch this if last file is really last
- _last_checkpoint_time = 0 # time in seconds of last checkpoint
- _checkpoint_rp = None # RPath of checkpoint data pickle
-
- def init_filenames(cls):
- """Set rpaths of markers. Assume rbdir already set."""
- if not Globals.isbackup_writer:
- return Globals.backup_writer.SaveState.init_filenames()
-
- assert Globals.local_connection is Globals.rbdir.conn, \
- (Globals.rbdir.conn, Globals.backup_writer)
-
- cls._last_file_sym = Globals.rbdir.append(
- "last-file-incremented.%s.data" % Time.curtimestr)
- cls._checkpoint_rp = Globals.rbdir.append(
- "checkpoint-data.%s.data" % Time.curtimestr)
- cls._last_file_definitive_rp = Globals.rbdir.append(
- "last-file-definitive.%s.data" % Time.curtimestr)
-
- def touch_last_file(cls):
- """Touch last file marker, indicating backup has begun"""
- if not cls._last_file_sym.lstat(): cls._last_file_sym.touch()
-
- def touch_last_file_definitive(cls):
- """Create last-file-definitive marker
-
- When a backup gets aborted, there may be time to indicate the
- last file successfully processed, and this should be touched.
- Sometimes when the abort is hard, there may be a last file
- indicated, but further files since then have been processed,
- in which case this shouldn't be touched.
-
- """
- cls._last_file_definitive_rp.touch()
-
- def record_last_file_action(cls, last_file_rorp):
- """Action recording last file to be processed as symlink in rbdir
-
- last_file_rorp is None means that no file is known to have
- been processed.
-
- """
- if last_file_rorp:
- symtext = apply(os.path.join,
- ('increments',) + last_file_rorp.index)
- return Robust.symlink_action(cls._last_file_sym, symtext)
- else: return RobustAction(None, lambda init_val: cls.touch_last_file(),
- None)
-
- def checkpoint(cls, ITR, finalizer, last_file_rorp, override = None):
- """Save states of tree reducer and finalizer during inc backup
-
- If override is true, checkpoint even if one isn't due.
-
- """
- if not override and not cls.checkpoint_needed(): return
- assert cls._checkpoint_rp, "_checkpoint_rp not set yet"
-
- cls._last_checkpoint_time = time.time()
- Log("Writing checkpoint time %s" % cls._last_checkpoint_time, 7)
- state_string = cPickle.dumps((ITR, finalizer))
- Robust.chain(Robust.destructive_write_action(cls._checkpoint_rp,
- state_string),
- cls.record_last_file_action(last_file_rorp)).execute()
-
- def checkpoint_needed(cls):
- """Returns true if another checkpoint is called for"""
- return (time.time() > cls._last_checkpoint_time +
- Globals.checkpoint_interval)
-
- def checkpoint_remove(cls):
- """Remove all checkpointing data after successful operation"""
- for rp in Resume.get_relevant_rps(): rp.delete()
- if Globals.preserve_hardlinks: Hardlink.remove_all_checkpoints()
-
-MakeClass(SaveState)
-
-
-class ResumeException(Exception):
- """Indicates some error has been encountered while trying to resume"""
- pass
-
-class Resume:
- """Check for old aborted backups and resume if necessary"""
- _session_info_list = None # List of ResumeSessionInfo's, sorted by time
- def FindTime(cls, index, later_than = 0):
- """For a given index, find the appropriate time to use for inc
-
- If it is clear which time to use (because it is determined by
- definitive records, or there are no aborted backup, etc.) then
- just return the appropriate time. Otherwise, if an aborted
- backup was last checkpointed before the index, assume that it
- didn't get there, and go for the older time. If an inc file
- is already present, the function will be rerun with later time
- specified.
-
- """
- assert Globals.isbackup_writer
- if Time.prevtime > later_than: return Time.prevtime # usual case
-
- for si in cls.get_sis_covering_index(index):
- if si.time > later_than: return si.time
- raise SkipFileException("Index %s already covered, skipping" %
- str(index))
-
- def get_sis_covering_index(cls, index):
- """Return sorted list of SessionInfos which may cover index
-
- Aborted backup may be relevant unless index is lower and we
- are sure that it didn't go further.
-
- """
- return filter(lambda session_info:
- not ((session_info.last_index is None or
- session_info.last_index < index) and
- session_info.last_definitive),
- cls._session_info_list)
-
- def SetSessionInfo(cls):
- """Read data directory and initialize _session_info"""
- assert Globals.isbackup_writer
- silist = []
- rp_quad_dict = cls.group_rps_by_time(cls.get_relevant_rps())
- times = rp_quad_dict.keys()
- times.sort()
- for time in times:
- try: silist.append(cls.quad_to_si(time, rp_quad_dict[time]))
- except ResumeException:
- Log("Bad resume information found, skipping", 2)
- cls._session_info_list = silist
-
- def get_relevant_rps(cls):
- """Return list of relevant rpaths in rbdata directory"""
- relevant_bases = ['last-file-incremented', 'last-file-mirrored',
- 'checkpoint-data', 'last-file-definitive']
- rps = map(Globals.rbdir.append, Globals.rbdir.listdir())
- return filter(lambda rp: rp.isincfile()
- and rp.getincbase_str() in relevant_bases, rps)
-
- def group_rps_by_time(cls, rplist):
- """Take list of rps return time dict {time: quadlist}
-
- Times in seconds are the keys, values are triples of rps
- [last-file-incremented, last-file-mirrored, checkpoint-data,
- last-is-definitive].
-
- """
- result = {}
- for rp in rplist:
- time = Time.stringtotime(rp.getinctime())
- if result.has_key(time): quadlist = result[time]
- else: quadlist = [None, None, None, None]
- base_string = rp.getincbase_str()
- if base_string == 'last-file-incremented': quadlist[0] = rp
- elif base_string == 'last-file-mirrored': quadlist[1] = rp
- elif base_string == 'last-file-definitive': quadlist[3] = 1
- else:
- assert base_string == 'checkpoint-data'
- quadlist[2] = rp
- result[time] = quadlist
- return result
-
- def quad_to_si(cls, time, quad):
- """Take time, quadlist, return associated ResumeSessionInfo"""
- increment_sym, mirror_sym, checkpoint_rp, last_definitive = quad
- if increment_sym and mirror_sym:
- raise ResumeException("both mirror and inc sym shouldn't exist")
- ITR, finalizer = None, None
- if increment_sym:
- mirror = None
- last_index = cls.sym_to_index(increment_sym)
- if checkpoint_rp:
- ITR, finalizer = cls.unpickle_checkpoint(checkpoint_rp)
- elif mirror_sym:
- mirror = 1
- last_index = cls.sym_to_index(mirror_sym)
- if checkpoint_rp:
- finalizer = cls.unpickle_checkpoint(checkpoint_rp)
- else: raise ResumeException("Missing increment or mirror sym")
- return ResumeSessionInfo(mirror, time, last_index, last_definitive,
- finalizer, ITR)
-
- def sym_to_index(cls, sym_rp):
- """Read last file sym rp, return last file index
-
- If sym_rp is not a sym at all, return None, indicating that no
- file index was ever conclusively processed.
-
- """
- if not sym_rp.issym(): return None
- link_components = sym_rp.readlink().split("/")
- assert link_components[0] == 'increments'
- return tuple(link_components[1:])
-
- def unpickle_checkpoint(cls, checkpoint_rp):
- """Read data from checkpoint_rp and return unpickled data
-
- Return value is pair (patch increment ITR, finalizer state).
-
- """
- fp = checkpoint_rp.open("rb")
- data = fp.read()
- fp.close()
- try: result = cPickle.loads(data)
- except Exception, exc:
- raise ResumeException("Bad pickle at %s: %s" %
- (checkpoint_rp.path, exc))
- return result
-
- def ResumeCheck(cls):
- """Return relevant ResumeSessionInfo if there's one we should resume
-
- Also if find RSI to resume, reset current time to old resume
- time.
-
- """
- cls.SetSessionInfo()
- if not cls._session_info_list:
- if Globals.resume == 1:
- Log.FatalError("User specified resume, but no data on "
- "previous backup found.")
- else: return None
- else:
- si = cls._session_info_list[-1]
- if (Globals.resume == 1 or
- (time.time() <= (si.time + Globals.resume_window) and
- not Globals.resume == 0)):
- Log("Resuming aborted backup dated %s" %
- Time.timetopretty(si.time), 2)
- Time.setcurtime(si.time)
- if Globals.preserve_hardlinks:
- if (not si.last_definitive or not
- Hardlink.retrieve_checkpoint(Globals.rbdir, si.time)):
- Log("Hardlink information not successfully "
- "recovered.", 2)
- return si
- else:
- Log("Last backup dated %s was aborted, but we aren't "
- "resuming it." % Time.timetopretty(si.time), 2)
- return None
- assert None
-
-MakeClass(Resume)
-
-
-class ResumeSessionInfo:
- """Hold information about a previously aborted session"""
- def __init__(self, mirror, time, last_index,
- last_definitive, finalizer = None, ITR = None):
- """Class initializer
-
- time - starting time in seconds of backup
- mirror - true if backup was a mirror, false if increment
- last_index - Last confirmed index processed by backup, or None
- last_definitive - True is we know last_index is really last
- finalizer - the dsrp finalizer if available
- ITR - For increment, ITM reducer (assume mirror if NA)
-
- """
- self.time = time
- self.mirror = mirror
- self.last_index = last_index
- self.last_definitive = last_definitive
- self.ITR, self.finalizer, = ITR, finalizer
-
-
-from log import *
-from destructive_stepping import *
-import Time, Rdiff, librsync
-from highlevel import *
-
diff --git a/rdiff-backup/src/rorpiter.py b/rdiff-backup/src/rorpiter.py
deleted file mode 100644
index 2e9bd06..0000000
--- a/rdiff-backup/src/rorpiter.py
+++ /dev/null
@@ -1,301 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Operations on Iterators of Read Only Remote Paths"""
-
-from __future__ import generators
-import tempfile, UserList, types, librsync
-from static import *
-from log import *
-from rpath import *
-from robust import *
-from iterfile import *
-import Globals, Rdiff, Hardlink
-
-class RORPIterException(Exception): pass
-
-class RORPIter:
- """Functions relating to iterators of Read Only RPaths
-
- The main structure will be an iterator that yields RORPaths.
- Every RORPath has a "raw" form that makes it more amenable to
- being turned into a file. The raw form of the iterator yields
- each RORPath in the form of the tuple (index, data_dictionary,
- files), where files is the number of files attached (usually 1 or
- 0). After that, if a file is attached, it yields that file.
-
- """
- def ToRaw(rorp_iter):
- """Convert a rorp iterator to raw form"""
- for rorp in rorp_iter:
- if rorp.file:
- yield (rorp.index, rorp.data, 1)
- yield rorp.file
- else: yield (rorp.index, rorp.data, 0)
-
- def FromRaw(raw_iter):
- """Convert raw rorp iter back to standard form"""
- for index, data, num_files in raw_iter:
- rorp = RORPath(index, data)
- if num_files:
- assert num_files == 1, "Only one file accepted right now"
- rorp.setfile(RORPIter.getnext(raw_iter))
- yield rorp
-
- def ToFile(rorp_iter):
- """Return file version of iterator"""
- return FileWrappingIter(RORPIter.ToRaw(rorp_iter))
-
- def FromFile(fileobj):
- """Recover rorp iterator from file interface"""
- return RORPIter.FromRaw(IterWrappingFile(fileobj))
-
- def IterateRPaths(base_rp):
- """Return an iterator yielding RPaths with given base rp"""
- yield base_rp
- if base_rp.isdir():
- dirlisting = base_rp.listdir()
- dirlisting.sort()
- for filename in dirlisting:
- for rp in RORPIter.IterateRPaths(base_rp.append(filename)):
- yield rp
-
- def Signatures(rp_iter):
- """Yield signatures of rpaths in given rp_iter"""
- def error_handler(exc, rp):
- Log("Error generating signature for %s" % rp.path)
- return None
-
- for rp in rp_iter:
- if rp.isplaceholder(): yield rp
- else:
- rorp = rp.getRORPath()
- if rp.isreg():
- if rp.isflaglinked(): rorp.flaglinked()
- else:
- fp = Robust.check_common_error(
- error_handler, Rdiff.get_signature, (rp,))
- if fp: rorp.setfile(fp)
- else: continue
- yield rorp
-
- def GetSignatureIter(base_rp):
- """Return a signature iterator recurring over the base_rp"""
- return RORPIter.Signatures(RORPIter.IterateRPaths(base_rp))
-
- def CollateIterators(*rorp_iters):
- """Collate RORPath iterators by index
-
- So it takes two or more iterators of rorps and returns an
- iterator yielding tuples like (rorp1, rorp2) with the same
- index. If one or the other lacks that index, it will be None
-
- """
- # overflow[i] means that iter[i] has been exhausted
- # rorps[i] is None means that it is time to replenish it.
- iter_num = len(rorp_iters)
- if iter_num == 2:
- return RORPIter.Collate2Iters(rorp_iters[0], rorp_iters[1])
- overflow = [None] * iter_num
- rorps = overflow[:]
-
- def setrorps(overflow, rorps):
- """Set the overflow and rorps list"""
- for i in range(iter_num):
- if not overflow[i] and rorps[i] is None:
- try: rorps[i] = rorp_iters[i].next()
- except StopIteration:
- overflow[i] = 1
- rorps[i] = None
-
- def getleastindex(rorps):
- """Return the first index in rorps, assuming rorps isn't empty"""
- return min(map(lambda rorp: rorp.index,
- filter(lambda x: x, rorps)))
-
- def yield_tuples(iter_num, overflow, rorps):
- while 1:
- setrorps(overflow, rorps)
- if not None in overflow: break
-
- index = getleastindex(rorps)
- yieldval = []
- for i in range(iter_num):
- if rorps[i] and rorps[i].index == index:
- yieldval.append(rorps[i])
- rorps[i] = None
- else: yieldval.append(None)
- yield IndexedTuple(index, yieldval)
- return yield_tuples(iter_num, overflow, rorps)
-
- def Collate2Iters(riter1, riter2):
- """Special case of CollateIterators with 2 arguments
-
- This does the same thing but is faster because it doesn't have
- to consider the >2 iterator case. Profiler says speed is
- important here.
-
- """
- relem1, relem2 = None, None
- while 1:
- if not relem1:
- try: relem1 = riter1.next()
- except StopIteration:
- if relem2: yield IndexedTuple(index2, (None, relem2))
- for relem2 in riter2:
- yield IndexedTuple(relem2.index, (None, relem2))
- break
- index1 = relem1.index
- if not relem2:
- try: relem2 = riter2.next()
- except StopIteration:
- if relem1: yield IndexedTuple(index1, (relem1, None))
- for relem1 in riter1:
- yield IndexedTuple(relem1.index, (relem1, None))
- break
- index2 = relem2.index
-
- if index1 < index2:
- yield IndexedTuple(index1, (relem1, None))
- relem1 = None
- elif index1 == index2:
- yield IndexedTuple(index1, (relem1, relem2))
- relem1, relem2 = None, None
- else: # index2 is less
- yield IndexedTuple(index2, (None, relem2))
- relem2 = None
-
- def getnext(iter):
- """Return the next element of an iterator, raising error if none"""
- try: next = iter.next()
- except StopIteration: raise RORPIterException("Unexpected end to iter")
- return next
-
- def GetDiffIter(sig_iter, new_iter):
- """Return delta iterator from sig_iter to new_iter
-
- The accompanying file for each will be a delta as produced by
- rdiff, unless the destination file does not exist, in which
- case it will be the file in its entirety.
-
- sig_iter may be composed of rorps, but new_iter should have
- full RPaths.
-
- """
- collated_iter = RORPIter.CollateIterators(sig_iter, new_iter)
- for rorp, rp in collated_iter: yield RORPIter.diffonce(rorp, rp)
-
- def diffonce(sig_rorp, new_rp):
- """Return one diff rorp, based from signature rorp and orig rp"""
- if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked():
- if new_rp: diff_rorp = new_rp.getRORPath()
- else: diff_rorp = RORPath(sig_rorp.index)
- diff_rorp.flaglinked()
- return diff_rorp
- elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg():
- diff_rorp = new_rp.getRORPath()
- #fp = sig_rorp.open("rb")
- #print "---------------------", fp
- #tmp_sig_rp = RPath(Globals.local_connection, "/tmp/sig")
- #tmp_sig_rp.delete()
- #tmp_sig_rp.write_from_fileobj(fp)
- #diff_rorp.setfile(Rdiff.get_delta_sigfileobj(tmp_sig_rp.open("rb"),
- # new_rp))
- diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"),
- new_rp))
- diff_rorp.set_attached_filetype('diff')
- return diff_rorp
- else:
- # Just send over originial if diff isn't appropriate
- if sig_rorp: sig_rorp.close_if_necessary()
- if not new_rp: return RORPath(sig_rorp.index)
- elif new_rp.isreg():
- diff_rorp = new_rp.getRORPath(1)
- diff_rorp.set_attached_filetype('snapshot')
- return diff_rorp
- else: return new_rp.getRORPath()
-
- def PatchIter(base_rp, diff_iter):
- """Patch the appropriate rps in basis_iter using diff_iter"""
- basis_iter = RORPIter.IterateRPaths(base_rp)
- collated_iter = RORPIter.CollateIterators(basis_iter, diff_iter)
- for basisrp, diff_rorp in collated_iter:
- RORPIter.patchonce_action(base_rp, basisrp, diff_rorp).execute()
-
- def patchonce_action(base_rp, basisrp, diff_rorp):
- """Return action patching basisrp using diff_rorp"""
- assert diff_rorp, "Missing diff index %s" % basisrp.index
- if not diff_rorp.lstat():
- return RobustAction(None, lambda init_val: basisrp.delete(), None)
-
- if Globals.preserve_hardlinks and diff_rorp.isflaglinked():
- if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
- tf = TempFileManager.new(basisrp)
- def init(): Hardlink.link_rp(diff_rorp, tf, basisrp)
- return Robust.make_tf_robustaction(init, tf, basisrp)
- elif basisrp and basisrp.isreg() and diff_rorp.isreg():
- if diff_rorp.get_attached_filetype() != 'diff':
- raise RPathException("File %s appears to have changed during"
- " processing, skipping" % (basisrp.path,))
- return Rdiff.patch_with_attribs_action(basisrp, diff_rorp)
- else: # Diff contains whole file, just copy it over
- if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
- return Robust.copy_with_attribs_action(diff_rorp, basisrp)
-
-MakeStatic(RORPIter)
-
-
-class IndexedTuple(UserList.UserList):
- """Like a tuple, but has .index
-
- This is used by CollateIterator above, and can be passed to the
- IterTreeReducer.
-
- """
- def __init__(self, index, sequence):
- self.index = index
- self.data = tuple(sequence)
-
- def __len__(self): return len(self.data)
-
- def __getitem__(self, key):
- """This only works for numerical keys (easier this way)"""
- return self.data[key]
-
- def __lt__(self, other): return self.__cmp__(other) == -1
- def __le__(self, other): return self.__cmp__(other) != 1
- def __ne__(self, other): return not self.__eq__(other)
- def __gt__(self, other): return self.__cmp__(other) == 1
- def __ge__(self, other): return self.__cmp__(other) != -1
-
- def __cmp__(self, other):
- assert isinstance(other, IndexedTuple)
- if self.index < other.index: return -1
- elif self.index == other.index: return 0
- else: return 1
-
- def __eq__(self, other):
- if isinstance(other, IndexedTuple):
- return self.index == other.index and self.data == other.data
- elif type(other) is types.TupleType:
- return self.data == other
- else: return None
-
- def __str__(self):
- return "(%s).%s" % (", ".join(map(str, self.data)), self.index)
diff --git a/rdiff-backup/src/rpath.py b/rdiff-backup/src/rpath.py
deleted file mode 100644
index e82fbc7..0000000
--- a/rdiff-backup/src/rpath.py
+++ /dev/null
@@ -1,875 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Wrapper class around a real path like "/usr/bin/env"
-
-The RPath (short for Remote Path) and associated classes make some
-function calls more convenient and also make working with files on
-remote systems transparent.
-
-For instance, suppose
-
-rp = RPath(connection_object, "/usr/bin/env")
-
-Then rp.getperms() returns the permissions of that file, and
-rp.delete() deletes that file. Both of these will work the same even
-if "usr/bin/env" is on a different computer. So many rdiff-backup
-functions use rpaths so they don't have to know whether the files they
-are dealing with are local or remote.
-
-"""
-
-import os, stat, re, sys, shutil, gzip, socket
-from static import *
-
-
-class RPathException(Exception): pass
-
-class RPathStatic:
- """Contains static methods for use with RPaths"""
- def copyfileobj(inputfp, outputfp):
- """Copies file inputfp to outputfp in blocksize intervals"""
- blocksize = Globals.blocksize
- while 1:
- inbuf = inputfp.read(blocksize)
- if not inbuf: break
- outputfp.write(inbuf)
-
- def cmpfileobj(fp1, fp2):
- """True if file objects fp1 and fp2 contain same data"""
- blocksize = Globals.blocksize
- while 1:
- buf1 = fp1.read(blocksize)
- buf2 = fp2.read(blocksize)
- if buf1 != buf2: return None
- elif not buf1: return 1
-
- def check_for_files(*rps):
- """Make sure that all the rps exist, raise error if not"""
- for rp in rps:
- if not rp.lstat():
- raise RPathException("File %s does not exist" % rp.path)
-
- def move(rpin, rpout):
- """Move rpin to rpout, renaming if possible"""
- try: RPath.rename(rpin, rpout)
- except os.error:
- RPath.copy(rpin, rpout)
- rpin.delete()
-
- def copy(rpin, rpout):
- """Copy RPath rpin to rpout. Works for symlinks, dirs, etc."""
- Log("Regular copying %s to %s" % (rpin.index, rpout.path), 6)
- if not rpin.lstat():
- raise RPathException, ("File %s does not exist" % rpin.index)
-
- if rpout.lstat():
- if rpin.isreg() or not RPath.cmp(rpin, rpout):
- rpout.delete() # easier to write that compare
- else: return
-
- if rpin.isreg(): RPath.copy_reg_file(rpin, rpout)
- elif rpin.isdir(): rpout.mkdir()
- elif rpin.issym(): rpout.symlink(rpin.readlink())
- elif rpin.ischardev():
- major, minor = rpin.getdevnums()
- rpout.makedev("c", major, minor)
- elif rpin.isblkdev():
- major, minor = rpin.getdevnums()
- rpout.makedev("b", major, minor)
- elif rpin.isfifo(): rpout.mkfifo()
- elif rpin.issock(): rpout.mksock()
- else: raise RPathException("File %s has unknown type" % rpin.path)
-
- def copy_reg_file(rpin, rpout):
- """Copy regular file rpin to rpout, possibly avoiding connection"""
- try:
- if rpout.conn is rpin.conn:
- rpout.conn.shutil.copyfile(rpin.path, rpout.path)
- rpout.setdata()
- return
- except AttributeError: pass
- rpout.write_from_fileobj(rpin.open("rb"))
-
- def cmp(rpin, rpout):
- """True if rpin has the same data as rpout
-
- cmp does not compare file ownership, permissions, or times, or
- examine the contents of a directory.
-
- """
- RPath.check_for_files(rpin, rpout)
- if rpin.isreg():
- if not rpout.isreg(): return None
- fp1, fp2 = rpin.open("rb"), rpout.open("rb")
- result = RPathStatic.cmpfileobj(fp1, fp2)
- if fp1.close() or fp2.close():
- raise RPathException("Error closing file")
- return result
- elif rpin.isdir(): return rpout.isdir()
- elif rpin.issym():
- return rpout.issym() and (rpin.readlink() == rpout.readlink())
- elif rpin.ischardev():
- return rpout.ischardev() and \
- (rpin.getdevnums() == rpout.getdevnums())
- elif rpin.isblkdev():
- return rpout.isblkdev() and \
- (rpin.getdevnums() == rpout.getdevnums())
- elif rpin.isfifo(): return rpout.isfifo()
- elif rpin.issock(): return rpout.issock()
- else: raise RPathException("File %s has unknown type" % rpin.path)
-
- def copy_attribs(rpin, rpout):
- """Change file attributes of rpout to match rpin
-
- Only changes the chmoddable bits, uid/gid ownership, and
- timestamps, so both must already exist.
-
- """
- Log("Copying attributes from %s to %s" % (rpin.index, rpout.path), 7)
- RPath.check_for_files(rpin, rpout)
- if rpin.issym(): return # symlinks have no valid attributes
- if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid())
- rpout.chmod(rpin.getperms())
- if not rpin.isdev(): rpout.setmtime(rpin.getmtime())
-
- def cmp_attribs(rp1, rp2):
- """True if rp1 has the same file attributes as rp2
-
- Does not compare file access times. If not changing
- ownership, do not check user/group id.
-
- """
- RPath.check_for_files(rp1, rp2)
- if Globals.change_ownership and rp1.getuidgid() != rp2.getuidgid():
- result = None
- elif rp1.getperms() != rp2.getperms(): result = None
- elif rp1.issym() and rp2.issym(): # Don't check times for some types
- result = 1
- elif rp1.isblkdev() and rp2.isblkdev(): result = 1
- elif rp1.ischardev() and rp2.ischardev(): result = 1
- else: result = (rp1.getmtime() == rp2.getmtime())
- Log("Compare attribs %s and %s: %s" % (rp1.path, rp2.path, result), 7)
- return result
-
- def copy_with_attribs(rpin, rpout):
- """Copy file and then copy over attributes"""
- RPath.copy(rpin, rpout)
- RPath.copy_attribs(rpin, rpout)
-
- def quick_cmp_with_attribs(rp1, rp2):
- """Quicker version of cmp_with_attribs
-
- Instead of reading all of each file, assume that regular files
- are the same if the attributes compare.
-
- """
- if not RPath.cmp_attribs(rp1, rp2): return None
- if rp1.isreg() and rp2.isreg() and (rp1.getlen() == rp2.getlen()):
- return 1
- return RPath.cmp(rp1, rp2)
-
- def cmp_with_attribs(rp1, rp2):
- """Combine cmp and cmp_attribs"""
- return RPath.cmp_attribs(rp1, rp2) and RPath.cmp(rp1, rp2)
-
- def rename(rp_source, rp_dest):
- """Rename rp_source to rp_dest"""
- assert rp_source.conn is rp_dest.conn
- Log(lambda: "Renaming %s to %s" % (rp_source.path, rp_dest.path), 7)
- rp_source.conn.os.rename(rp_source.path, rp_dest.path)
- rp_dest.data = rp_source.data
- rp_source.data = {'type': None}
-
- # If we are moving to a DSRPath, assume that the current times
- # are the intended ones. We need to save them now in case
- # they are changed later.
- if isinstance(rp_dest, DSRPath):
- if rp_dest.delay_mtime:
- if 'mtime' in rp_dest.data:
- rp_dest.setmtime(rp_dest.data['mtime'])
- if rp_dest.delay_atime:
- if 'atime' in rp_dest.data:
- rp_dest.setatime(rp_dest.data['atime'])
-
- def tupled_lstat(filename):
- """Like os.lstat, but return only a tuple, or None if os.error
-
- Later versions of os.lstat return a special lstat object,
- which can confuse the pickler and cause errors in remote
- operations. This has been fixed in Python 2.2.1.
-
- """
- try: return tuple(os.lstat(filename))
- except os.error: return None
-
- def make_socket_local(rpath):
- """Make a local socket at the given path
-
- This takes an rpath so that it will be checked by Security.
- (Miscellaneous strings will not be.)
-
- """
- assert rpath.conn is Globals.local_connection
- s = socket.socket(socket.AF_UNIX)
- try: s.bind(rpath.path)
- except socket.error, exc:
- raise SkipFileException("Socket error: " + str(exc))
-
- def gzip_open_local_read(rpath):
- """Return open GzipFile. See security note directly above"""
- assert rpath.conn is Globals.local_connection
- return gzip.GzipFile(rpath.path, "rb")
-
- def open_local_read(rpath):
- """Return open file (provided for security reasons)"""
- assert rpath.conn is Globals.local_connection
- return open(rpath.path, "rb")
-
-MakeStatic(RPathStatic)
-
-
-class RORPath(RPathStatic):
- """Read Only RPath - carry information about a path
-
- These contain information about a file, and possible the file's
- data, but do not have a connection and cannot be written to or
- changed. The advantage of these objects is that they can be
- communicated by encoding their index and data dictionary.
-
- """
- def __init__(self, index, data = None):
- self.index = index
- if data: self.data = data
- else: self.data = {'type':None} # signify empty file
- self.file = None
-
- def __eq__(self, other):
- """True iff the two rorpaths are equivalent"""
- if self.index != other.index: return None
-
- for key in self.data.keys(): # compare dicts key by key
- if ((key == 'uid' or key == 'gid') and
- (not Globals.change_ownership or self.issym())):
- # Don't compare gid/uid for symlinks or if not change_ownership
- pass
- elif key == 'atime' and not Globals.preserve_atime: pass
- elif key == 'devloc' or key == 'inode' or key == 'nlink': pass
- elif key == 'size' and self.isdir(): pass
- elif (not other.data.has_key(key) or
- self.data[key] != other.data[key]): return None
- return 1
-
- def __ne__(self, other): return not self.__eq__(other)
-
- def __str__(self):
- """Pretty print file statistics"""
- return "Index: %s\nData: %s" % (self.index, self.data)
-
- def __getstate__(self):
- """Return picklable state
-
- This is necessary in case the RORPath is carrying around a
- file object, which can't/shouldn't be pickled.
-
- """
- return (self.index, self.data)
-
- def __setstate__(self, rorp_state):
- """Reproduce RORPath from __getstate__ output"""
- self.index, self.data = rorp_state
-
- def make_placeholder(self):
- """Make rorp into a placeholder
-
- This object doesn't contain any information about the file,
- but, when passed along, may show where the previous stages are
- in their processing. It is the RORPath equivalent of fiber.
- This placeholder size, in conjunction with the placeholder
- threshold in Highlevel .. generate_dissimilar seem to yield an
- OK tradeoff between unnecessary placeholders and lots of
- memory usage, but I'm not sure exactly why.
-
- """
- self.data = {'placeholder': " "*500}
-
- def isplaceholder(self):
- """True if the object is a placeholder"""
- return self.data.has_key('placeholder')
-
- def lstat(self):
- """Returns type of file
-
- The allowable types are None if the file doesn't exist, 'reg'
- for a regular file, 'dir' for a directory, 'dev' for a device
- file, 'fifo' for a fifo, 'sock' for a socket, and 'sym' for a
- symlink.
-
- """
- return self.data['type']
- gettype = lstat
-
- def isdir(self):
- """True if self is a dir"""
- return self.data['type'] == 'dir'
-
- def isreg(self):
- """True if self is a regular file"""
- return self.data['type'] == 'reg'
-
- def issym(self):
- """True if path is of a symlink"""
- return self.data['type'] == 'sym'
-
- def isfifo(self):
- """True if path is a fifo"""
- return self.data['type'] == 'fifo'
-
- def ischardev(self):
- """True if path is a character device file"""
- return self.data['type'] == 'dev' and self.data['devnums'][0] == 'c'
-
- def isblkdev(self):
- """True if path is a block device file"""
- return self.data['type'] == 'dev' and self.data['devnums'][0] == 'b'
-
- def isdev(self):
- """True if path is a device file"""
- return self.data['type'] == 'dev'
-
- def issock(self):
- """True if path is a socket"""
- return self.data['type'] == 'sock'
-
- def getperms(self):
- """Return permission block of file"""
- return self.data['perms']
-
- def getsize(self):
- """Return length of file in bytes"""
- return self.data['size']
-
- def getuidgid(self):
- """Return userid/groupid of file"""
- return self.data['uid'], self.data['gid']
-
- def getatime(self):
- """Return access time in seconds"""
- return self.data['atime']
-
- def getmtime(self):
- """Return modification time in seconds"""
- return self.data['mtime']
-
- def getinode(self):
- """Return inode number of file"""
- return self.data['inode']
-
- def getdevloc(self):
- """Device number file resides on"""
- return self.data['devloc']
-
- def getnumlinks(self):
- """Number of places inode is linked to"""
- return self.data['nlink']
-
- def readlink(self):
- """Wrapper around os.readlink()"""
- return self.data['linkname']
-
- def getdevnums(self):
- """Return a devices major/minor numbers from dictionary"""
- return self.data['devnums'][1:]
-
- def setfile(self, file):
- """Right now just set self.file to be the already opened file"""
- assert file and not self.file
- def closing_hook(): self.file_already_open = None
- self.file = RPathFileHook(file, closing_hook)
- self.file_already_open = None
-
- def get_attached_filetype(self):
- """If there is a file attached, say what it is
-
- Currently the choices are 'snapshot' meaning an exact copy of
- something, and 'diff' for an rdiff style diff.
-
- """
- return self.data['filetype']
-
- def set_attached_filetype(self, type):
- """Set the type of the attached file"""
- self.data['filetype'] = type
-
- def isflaglinked(self):
- """True if rorp is a signature/diff for a hardlink file
-
- This indicates that a file's data need not be transferred
- because it is hardlinked on the remote side.
-
- """
- return self.data.has_key('linked')
-
- def flaglinked(self):
- """Signal that rorp is a signature/diff for a hardlink file"""
- self.data['linked'] = 1
-
- def open(self, mode):
- """Return file type object if any was given using self.setfile"""
- if mode != "rb": raise RPathException("Bad mode %s" % mode)
- if self.file_already_open:
- raise RPathException("Attempt to open same file twice")
- self.file_already_open = 1
- return self.file
-
- def close_if_necessary(self):
- """If file is present, discard data and close"""
- if self.file:
- while self.file.read(Globals.blocksize): pass
- assert not self.file.close(), \
- "Error closing file\ndata = %s\nindex = %s\n" % (self.data,
- self.index)
- self.file_already_open = None
-
-
-class RPath(RORPath):
- """Remote Path class - wrapper around a possibly non-local pathname
-
- This class contains a dictionary called "data" which should
- contain all the information about the file sufficient for
- identification (i.e. if two files have the the same (==) data
- dictionary, they are the same file).
-
- """
- regex_chars_to_quote = re.compile("[\\\\\\\"\\$`]")
-
- def __init__(self, connection, base, index = (), data = None):
- """RPath constructor
-
- connection = self.conn is the Connection the RPath will use to
- make system calls, and index is the name of the rpath used for
- comparison, and should be a tuple consisting of the parts of
- the rpath after the base split up. For instance ("foo",
- "bar") for "foo/bar" (no base), and ("local", "bin") for
- "/usr/local/bin" if the base is "/usr".
-
- For the root directory "/", the index is empty and the base is
- "/".
-
- """
- self.conn = connection
- self.index = index
- self.base = base
- if base is not None:
- if base == "/": self.path = "/" + "/".join(index)
- else: self.path = "/".join((base,) + index)
- self.file = None
- if data or base is None: self.data = data
- else: self.data = self.conn.C.make_file_dict(self.path)
-
- def __str__(self):
- return "Path: %s\nIndex: %s\nData: %s" % (self.path, self.index,
- self.data)
-
- def __getstate__(self):
- """Return picklable state
-
- The connection must be local because we can't pickle a
- connection. Data and any attached file also won't be saved.
-
- """
- assert self.conn is Globals.local_connection
- return (self.index, self.base, self.data)
-
- def __setstate__(self, rpath_state):
- """Reproduce RPath from __getstate__ output"""
- self.conn = Globals.local_connection
- self.index, self.base, self.data = rpath_state
- self.path = "/".join((self.base,) + self.index)
-
- def setdata(self):
- """Set data dictionary using C extension"""
- self.data = self.conn.C.make_file_dict(self.path)
-
- def make_file_dict_old(self):
- """Create the data dictionary"""
- statblock = self.conn.RPathStatic.tupled_lstat(self.path)
- if statblock is None:
- return {'type':None}
- data = {}
- mode = statblock[stat.ST_MODE]
-
- if stat.S_ISREG(mode): type = 'reg'
- elif stat.S_ISDIR(mode): type = 'dir'
- elif stat.S_ISCHR(mode):
- type = 'dev'
- data['devnums'] = ('c',) + self._getdevnums()
- elif stat.S_ISBLK(mode):
- type = 'dev'
- data['devnums'] = ('b',) + self._getdevnums()
- elif stat.S_ISFIFO(mode): type = 'fifo'
- elif stat.S_ISLNK(mode):
- type = 'sym'
- data['linkname'] = self.conn.os.readlink(self.path)
- elif stat.S_ISSOCK(mode): type = 'sock'
- else: raise C.UnknownFileError(self.path)
- data['type'] = type
- data['size'] = statblock[stat.ST_SIZE]
- data['perms'] = stat.S_IMODE(mode)
- data['uid'] = statblock[stat.ST_UID]
- data['gid'] = statblock[stat.ST_GID]
- data['inode'] = statblock[stat.ST_INO]
- data['devloc'] = statblock[stat.ST_DEV]
- data['nlink'] = statblock[stat.ST_NLINK]
-
- if not (type == 'sym' or type == 'dev'):
- # mtimes on symlinks and dev files don't work consistently
- data['mtime'] = long(statblock[stat.ST_MTIME])
- data['atime'] = long(statblock[stat.ST_ATIME])
- return data
-
- def check_consistency(self):
- """Raise an error if consistency of rp broken
-
- This is useful for debugging when the cache and disk get out
- of sync and you need to find out where it happened.
-
- """
- temptype = self.data['type']
- self.setdata()
- assert temptype == self.data['type'], \
- "\nName: %s\nOld: %s --> New: %s\n" % \
- (self.path, temptype, self.data['type'])
-
- def _getdevnums(self):
- """Return tuple for special file (major, minor)"""
- s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path)
- return (s >> 8, s & 0xff)
-
- def quote_path(self):
- """Set path from quoted version of index"""
- quoted_list = [FilenameMapping.quote(path) for path in self.index]
- self.path = "/".join([self.base] + quoted_list)
- self.setdata()
-
- def chmod(self, permissions):
- """Wrapper around os.chmod"""
- self.conn.os.chmod(self.path, permissions)
- self.data['perms'] = permissions
-
- def settime(self, accesstime, modtime):
- """Change file modification times"""
- Log("Setting time of %s to %d" % (self.path, modtime), 7)
- self.conn.os.utime(self.path, (accesstime, modtime))
- self.data['atime'] = accesstime
- self.data['mtime'] = modtime
-
- def setmtime(self, modtime):
- """Set only modtime (access time to present)"""
- Log(lambda: "Setting time of %s to %d" % (self.path, modtime), 7)
- self.conn.os.utime(self.path, (time.time(), modtime))
- self.data['mtime'] = modtime
-
- def chown(self, uid, gid):
- """Set file's uid and gid"""
- self.conn.os.chown(self.path, uid, gid)
- self.data['uid'] = uid
- self.data['gid'] = gid
-
- def mkdir(self):
- Log("Making directory " + self.path, 6)
- self.conn.os.mkdir(self.path)
- self.setdata()
-
- def rmdir(self):
- Log("Removing directory " + self.path, 6)
- self.conn.os.rmdir(self.path)
- self.data = {'type': None}
-
- def listdir(self):
- """Return list of string paths returned by os.listdir"""
- return self.conn.os.listdir(self.path)
-
- def symlink(self, linktext):
- """Make symlink at self.path pointing to linktext"""
- self.conn.os.symlink(linktext, self.path)
- self.setdata()
- assert self.issym()
-
- def hardlink(self, linkpath):
- """Make self into a hardlink joined to linkpath"""
- self.conn.os.link(linkpath, self.path)
- self.setdata()
-
- def mkfifo(self):
- """Make a fifo at self.path"""
- self.conn.os.mkfifo(self.path)
- self.setdata()
- assert self.isfifo()
-
- def mksock(self):
- """Make a socket at self.path"""
- self.conn.RPathStatic.make_socket_local(self)
- self.setdata()
- assert self.issock()
-
- def touch(self):
- """Make sure file at self.path exists"""
- Log("Touching " + self.path, 7)
- self.conn.open(self.path, "w").close()
- self.setdata()
- assert self.isreg()
-
- def hasfullperms(self):
- """Return true if current process has full permissions on the file"""
- if self.isowner(): return self.getperms() % 01000 >= 0700
- elif self.isgroup(): return self.getperms() % 0100 >= 070
- else: return self.getperms() % 010 >= 07
-
- def readable(self):
- """Return true if current process has read permissions on the file"""
- if self.isowner(): return self.getperms() % 01000 >= 0400
- elif self.isgroup(): return self.getperms() % 0100 >= 040
- else: return self.getperms() % 010 >= 04
-
- def executable(self):
- """Return true if current process has execute permissions"""
- if self.isowner(): return self.getperms() % 0200 >= 0100
- elif self.isgroup(): return self.getperms() % 020 >= 010
- else: return self.getperms() % 02 >= 01
-
- def isowner(self):
- """Return true if current process is owner of rp or root"""
- uid = self.conn.os.getuid()
- return uid == 0 or uid == self.data['uid']
-
- def isgroup(self):
- """Return true if current process is in group of rp"""
- return self.conn.Globals.get('process_gid') == self.data['gid']
-
- def delete(self):
- """Delete file at self.path
-
- The destructive stepping allows this function to delete
- directories even if they have files and we lack permissions.
-
- """
- Log("Deleting %s" % self.path, 7)
- self.setdata()
- if not self.lstat(): return # must have been deleted in meantime
- elif self.isdir():
- itm = IterTreeReducer(RpathDeleter, [])
- for dsrp in Select(DSRPath(None, self)).set_iter():
- itm(dsrp.index, dsrp)
- itm.Finish()
- else: self.conn.os.unlink(self.path)
- self.setdata()
-
- def quote(self):
- """Return quoted self.path for use with os.system()"""
- return '"%s"' % self.regex_chars_to_quote.sub(
- lambda m: "\\"+m.group(0), self.path)
-
- def normalize(self):
- """Return RPath canonical version of self.path
-
- This just means that redundant /'s will be removed, including
- the trailing one, even for directories. ".." components will
- be retained.
-
- """
- newpath = "/".join(filter(lambda x: x and x != ".",
- self.path.split("/")))
- if self.path[0] == "/": newpath = "/" + newpath
- elif not newpath: newpath = "."
- return self.newpath(newpath)
-
- def dirsplit(self):
- """Returns a tuple of strings (dirname, basename)
-
- Basename is never '' unless self is root, so it is unlike
- os.path.basename. If path is just above root (so dirname is
- root), then dirname is ''. In all other cases dirname is not
- the empty string. Also, dirsplit depends on the format of
- self, so basename could be ".." and dirname could be a
- subdirectory. For an atomic relative path, dirname will be
- '.'.
-
- """
- normed = self.normalize()
- if normed.path.find("/") == -1: return (".", normed.path)
- comps = normed.path.split("/")
- return "/".join(comps[:-1]), comps[-1]
-
- def newpath(self, newpath, index = ()):
- """Return new RPath with the same connection but different path"""
- return self.__class__(self.conn, newpath, index)
-
- def append(self, ext):
- """Return new RPath with same connection by adjoing ext"""
- return self.__class__(self.conn, self.base, self.index + (ext,))
-
- def append_path(self, ext, new_index = ()):
- """Like append, but add ext to path instead of to index"""
- assert not self.index # doesn't make sense if index isn't ()
- return self.__class__(self.conn, "/".join((self.base, ext)), new_index)
-
- def new_index(self, index):
- """Return similar RPath but with new index"""
- return self.__class__(self.conn, self.base, index)
-
- def open(self, mode, compress = None):
- """Return open file. Supports modes "w" and "r".
-
- If compress is true, data written/read will be gzip
- compressed/decompressed on the fly. The extra complications
- below are for security reasons - try to make the extent of the
- risk apparent from the remote call.
-
- """
- if self.conn is Globals.local_connection:
- if compress: return gzip.GzipFile(self.path, mode)
- else: return open(self.path, mode)
-
- if compress:
- if mode == "r" or mode == "rb":
- return self.conn.RPathStatic.gzip_open_local_read(self)
- else: return self.conn.gzip.GzipFile(self.path, mode)
- else:
- if mode == "r" or mode == "rb":
- return self.conn.RPathStatic.open_local_read(self)
- else: return self.conn.open(self.path, mode)
-
- def write_from_fileobj(self, fp, compress = None):
- """Reads fp and writes to self.path. Closes both when done
-
- If compress is true, fp will be gzip compressed before being
- written to self.
-
- """
- Log("Writing file object to " + self.path, 7)
- assert not self.lstat(), "File %s already exists" % self.path
- outfp = self.open("wb", compress = compress)
- RPath.copyfileobj(fp, outfp)
- if fp.close() or outfp.close():
- raise RPathException("Error closing file")
- self.setdata()
-
- def isincfile(self):
- """Return true if path looks like an increment file
-
- Also sets various inc information used by the *inc* functions.
-
- """
- if self.index: dotsplit = self.index[-1].split(".")
- else: dotsplit = self.base.split(".")
- if dotsplit[-1] == "gz":
- compressed = 1
- if len(dotsplit) < 4: return None
- timestring, ext = dotsplit[-3:-1]
- else:
- compressed = None
- if len(dotsplit) < 3: return None
- timestring, ext = dotsplit[-2:]
- if Time.stringtotime(timestring) is None: return None
- if not (ext == "snapshot" or ext == "dir" or
- ext == "missing" or ext == "diff" or ext == "data"):
- return None
- self.inc_timestr = timestring
- self.inc_compressed = compressed
- self.inc_type = ext
- if compressed: self.inc_basestr = ".".join(dotsplit[:-3])
- else: self.inc_basestr = ".".join(dotsplit[:-2])
- return 1
-
- def isinccompressed(self):
- """Return true if inc file is compressed"""
- return self.inc_compressed
-
- def getinctype(self):
- """Return type of an increment file"""
- return self.inc_type
-
- def getinctime(self):
- """Return timestring of an increment file"""
- return self.inc_timestr
-
- def getincbase(self):
- """Return the base filename of an increment file in rp form"""
- if self.index:
- return self.__class__(self.conn, self.base, self.index[:-1] +
- (self.inc_basestr,))
- else: return self.__class__(self.conn, self.inc_basestr)
-
- def getincbase_str(self):
- """Return the base filename string of an increment file"""
- return self.getincbase().dirsplit()[1]
-
- def makedev(self, type, major, minor):
- """Make a special file with specified type, and major/minor nums"""
- cmdlist = ['mknod', self.path, type, str(major), str(minor)]
- if self.conn.os.spawnvp(os.P_WAIT, 'mknod', cmdlist) != 0:
- raise RPathException("Error running %s" % cmdlist)
- if type == 'c': datatype = 'chr'
- elif type == 'b': datatype = 'blk'
- else: raise RPathException
- self.setdata()
-
- def getRORPath(self, include_contents = None):
- """Return read only version of self"""
- rorp = RORPath(self.index, self.data)
- if include_contents: rorp.setfile(self.open("rb"))
- return rorp
-
-
-class RPathFileHook:
- """Look like a file, but add closing hook"""
- def __init__(self, file, closing_thunk):
- self.file = file
- self.closing_thunk = closing_thunk
-
- def read(self, length = -1): return self.file.read(length)
- def write(self, buf): return self.file.write(buf)
-
- def close(self):
- """Close file and then run closing thunk"""
- result = self.file.close()
- self.closing_thunk()
- return result
-
-
-# Import these late to avoid circular dependencies
-import FilenameMapping
-from lazy import *
-from selection import *
-from destructive_stepping import *
-from highlevel import *
-
-class RpathDeleter(ITRBranch):
- """Delete a directory. Called by RPath.delete()"""
- def start_process(self, index, dsrp):
- self.dsrp = dsrp
-
- def end_process(self):
- if self.dsrp.isdir(): self.dsrp.rmdir()
- else: self.dsrp.delete()
-
- def can_fast_process(self, index, dsrp): return not dsrp.isdir()
- def fast_process(self, index, dsrp): dsrp.delete()
-
diff --git a/rdiff-backup/src/selection.py b/rdiff-backup/src/selection.py
deleted file mode 100644
index 9fb43fc..0000000
--- a/rdiff-backup/src/selection.py
+++ /dev/null
@@ -1,650 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Iterate exactly the requested files in a directory
-
-Parses includes and excludes to yield correct files. More
-documentation on what this code does can be found on the man page.
-
-"""
-
-from __future__ import generators
-import re
-from log import *
-from robust import *
-from destructive_stepping import *
-import FilenameMapping
-
-
-class SelectError(Exception):
- """Some error dealing with the Select class"""
- pass
-
-class FilePrefixError(SelectError):
- """Signals that a specified file doesn't start with correct prefix"""
- pass
-
-class GlobbingError(SelectError):
- """Something has gone wrong when parsing a glob string"""
- pass
-
-
-class Select:
- """Iterate appropriate DSRPaths in given directory
-
- This class acts as an iterator on account of its next() method.
- Basically, it just goes through all the files in a directory in
- order (depth-first) and subjects each file to a bunch of tests
- (selection functions) in order. The first test that includes or
- excludes the file means that the file gets included (iterated) or
- excluded. The default is include, so with no tests we would just
- iterate all the files in the directory in order.
-
- The one complication to this is that sometimes we don't know
- whether or not to include a directory until we examine its
- contents. For instance, if we want to include all the **.py
- files. If /home/ben/foo.py exists, we should also include /home
- and /home/ben, but if these directories contain no **.py files,
- they shouldn't be included. For this reason, a test may not
- include or exclude a directory, but merely "scan" it. If later a
- file in the directory gets included, so does the directory.
-
- As mentioned above, each test takes the form of a selection
- function. The selection function takes a dsrp, and returns:
-
- None - means the test has nothing to say about the related file
- 0 - the file is excluded by the test
- 1 - the file is included
- 2 - the test says the file (must be directory) should be scanned
-
- Also, a selection function f has a variable f.exclude which should
- be true iff f could potentially exclude some file. This is used
- to signal an error if the last function only includes, which would
- be redundant and presumably isn't what the user intends.
-
- """
- # This re should not match normal filenames, but usually just globs
- glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S)
-
- def __init__(self, dsrpath, quoted_filenames = None):
- """DSRPIterator initializer. dsrp is the root directory
-
- When files have quoted characters in them, quoted_filenames
- should be true. Then RPath's index will be the unquoted
- version.
-
- """
- assert isinstance(dsrpath, DSRPath)
- self.selection_functions = []
- self.dsrpath = dsrpath
- self.prefix = self.dsrpath.path
- self.quoting_on = Globals.quoting_enabled and quoted_filenames
-
- def set_iter(self, starting_index = None, iterate_parents = None,
- sel_func = None):
- """Initialize more variables, get ready to iterate
-
- Will iterate indicies greater than starting_index. If
- iterate_parents is true, will also include parents of
- starting_index in iteration. Selection function sel_func is
- called on each dsrp and is usually self.Select. Returns self
- just for convenience.
-
- """
- if not sel_func: sel_func = self.Select
- self.dsrpath.setdata() # this may have changed since Select init
- if starting_index is not None:
- self.starting_index = starting_index
- self.iter = self.iterate_starting_from(self.dsrpath,
- self.iterate_starting_from, sel_func)
- elif self.quoting_on:
- self.iter = self.Iterate(self.dsrpath, self.Iterate, sel_func)
- else: self.iter = self.Iterate_fast(self.dsrpath, sel_func)
-
- # only iterate parents if we are not starting from beginning
- self.iterate_parents = starting_index is not None and iterate_parents
- self.next = self.iter.next
- self.__iter__ = lambda: self
- return self
-
- def Iterate_fast(self, dsrpath, sel_func):
- """Like Iterate, but don't recur, saving time
-
- Only handles standard case (quoting off, starting from
- beginning).
-
- """
- def error_handler(exc, filename):
- Log("Error initializing file %s/%s" % (dsrpath.path, filename), 2)
- return None
-
- def diryield(dsrpath):
- """Generate relevant files in directory dsrpath
-
- Returns (dsrp, num) where num == 0 means dsrp should be
- generated normally, num == 1 means the dsrp is a directory
- and should be included iff something inside is included.
-
- """
- for filename in Robust.listrp(dsrpath):
- new_dsrp = Robust.check_common_error(error_handler,
- dsrpath.append, (filename,))
- if new_dsrp:
- s = sel_func(new_dsrp)
- if s == 1: yield (new_dsrp, 0)
- elif s == 2 and new_dsrp.isdir(): yield (new_dsrp, 1)
-
- yield dsrpath
- diryield_stack = [diryield(dsrpath)]
- delayed_dsrp_stack = []
-
- while diryield_stack:
- try: dsrp, val = diryield_stack[-1].next()
- except StopIteration:
- diryield_stack.pop()
- if delayed_dsrp_stack: delayed_dsrp_stack.pop()
- continue
- if val == 0:
- if delayed_dsrp_stack:
- for delayed_dsrp in delayed_dsrp_stack: yield delayed_dsrp
- del delayed_dsrp_stack[:]
- yield dsrp
- if dsrp.isdir(): diryield_stack.append(diryield(dsrp))
- elif val == 1:
- delayed_dsrp_stack.append(dsrp)
- diryield_stack.append(diryield(dsrp))
-
- def Iterate(self, dsrpath, rec_func, sel_func):
- """Return iterator yielding dsrps in dsrpath
-
- rec_func is usually the same as this function and is what
- Iterate uses to find files in subdirectories. It is used in
- iterate_starting_from.
-
- sel_func is the selection function to use on the dsrps. It is
- usually self.Select.
-
- """
- s = sel_func(dsrpath)
- if s == 0: return
- elif s == 1: # File is included
- yield dsrpath
- if dsrpath.isdir():
- for dsrp in self.iterate_in_dir(dsrpath, rec_func, sel_func):
- yield dsrp
- elif s == 2:
- if dsrpath.isdir(): # Directory is merely scanned
- iid = self.iterate_in_dir(dsrpath, rec_func, sel_func)
- try: first = iid.next()
- except StopIteration: return # no files inside; skip dsrp
- yield dsrpath
- yield first
- for dsrp in iid: yield dsrp
- else: assert 0, "Invalid selection result %s" % (str(s),)
-
- def iterate_in_dir(self, dsrpath, rec_func, sel_func):
- """Iterate the dsrps in directory dsrpath."""
- def error_handler(exc, filename):
- Log("Error initializing file %s/%s" % (dsrpath.path, filename), 2)
- return None
-
- if self.quoting_on:
- for subdir in FilenameMapping.get_quoted_dir_children(dsrpath):
- for dsrp in rec_func(subdir, rec_func, sel_func):
- yield dsrp
- else:
- for filename in Robust.listrp(dsrpath):
- new_dsrp = Robust.check_common_error(
- error_handler, dsrpath.append, [filename])
- if new_dsrp:
- for dsrp in rec_func(new_dsrp, rec_func, sel_func):
- yield dsrp
-
- def iterate_starting_from(self, dsrpath, rec_func, sel_func):
- """Like Iterate, but only yield indicies > self.starting_index"""
- if dsrpath.index > self.starting_index: # past starting_index
- for dsrp in self.Iterate(dsrpath, self.Iterate, sel_func):
- yield dsrp
- elif (dsrpath.index == self.starting_index[:len(dsrpath.index)]
- and dsrpath.isdir()):
- # May encounter starting index on this branch
- if self.iterate_parents: yield dsrpath
- for dsrp in self.iterate_in_dir(dsrpath,
- self.iterate_starting_from,
- sel_func): yield dsrp
-
- def iterate_with_finalizer(self):
- """Like Iterate, but missing some options, and add finalizer"""
- finalize = IterTreeReducer(DestructiveSteppingFinalizer, ())
- for dsrp in self:
- yield dsrp
- finalize(dsrp.index, dsrp)
- finalize.Finish()
-
- def Select(self, dsrp):
- """Run through the selection functions and return dominant val 0/1/2"""
- for sf in self.selection_functions:
- result = sf(dsrp)
- if result is not None: return result
- return 1
-
- def ParseArgs(self, argtuples, filelists):
- """Create selection functions based on list of tuples
-
- The tuples have the form (option string, additional argument)
- and are created when the initial commandline arguments are
- read. The reason for the extra level of processing is that
- the filelists may only be openable by the main connection, but
- the selection functions need to be on the backup reader or
- writer side. When the initial arguments are parsed the right
- information is sent over the link.
-
- """
- filelists_index = 0
- try:
- for opt, arg in argtuples:
- if opt == "--exclude":
- self.add_selection_func(self.glob_get_sf(arg, 0))
- elif opt == "--exclude-device-files":
- self.add_selection_func(self.devfiles_get_sf(0))
- elif opt == "--exclude-filelist":
- self.add_selection_func(self.filelist_get_sf(
- filelists[filelists_index], 0, arg))
- filelists_index += 1
- elif opt == "--exclude-globbing-filelist":
- map(self.add_selection_func,
- self.filelist_globbing_get_sfs(
- filelists[filelists_index], 0, arg))
- filelists_index += 1
- elif opt == "--exclude-other-filesystems":
- self.add_selection_func(self.other_filesystems_get_sf(0))
- elif opt == "--exclude-regexp":
- self.add_selection_func(self.regexp_get_sf(arg, 0))
- elif opt == "--exclude-special-files":
- self.add_selection_func(self.special_get_sf(0))
- elif opt == "--include":
- self.add_selection_func(self.glob_get_sf(arg, 1))
- elif opt == "--include-filelist":
- self.add_selection_func(self.filelist_get_sf(
- filelists[filelists_index], 1, arg))
- filelists_index += 1
- elif opt == "--include-globbing-filelist":
- map(self.add_selection_func,
- self.filelist_globbing_get_sfs(
- filelists[filelists_index], 1, arg))
- filelists_index += 1
- elif opt == "--include-regexp":
- self.add_selection_func(self.regexp_get_sf(arg, 1))
- else: assert 0, "Bad selection option %s" % opt
- except IOError: pass#SelectError, e: self.parse_catch_error(e)
- assert filelists_index == len(filelists)
-
- self.parse_last_excludes()
- self.parse_rbdir_exclude()
-
- def parse_catch_error(self, exc):
- """Deal with selection error exc"""
- if isinstance(exc, FilePrefixError):
- Log.FatalError(
-"""Fatal Error: The file specification
-' %s'
-cannot match any files in the base directory
-' %s'
-Useful file specifications begin with the base directory or some
-pattern (such as '**') which matches the base directory.""" %
- (exc, self.prefix))
- elif isinstance(e, GlobbingError):
- Log.FatalError("Fatal Error while processing expression\n"
- "%s" % exc)
- else: raise
-
- def parse_rbdir_exclude(self):
- """Add exclusion of rdiff-backup-data dir to front of list"""
- self.add_selection_func(
- self.glob_get_tuple_sf(("rdiff-backup-data",), 0), 1)
-
- def parse_last_excludes(self):
- """Exit with error if last selection function isn't an exclude"""
- if (self.selection_functions and
- not self.selection_functions[-1].exclude):
- Log.FatalError(
-"""Last selection expression:
- %s
-only specifies that files be included. Because the default is to
-include all files, the expression is redundant. Exiting because this
-probably isn't what you meant.""" %
- (self.selection_functions[-1].name,))
-
- def add_selection_func(self, sel_func, add_to_start = None):
- """Add another selection function at the end or beginning"""
- if add_to_start: self.selection_functions.insert(0, sel_func)
- else: self.selection_functions.append(sel_func)
-
- def filelist_get_sf(self, filelist_fp, inc_default, filelist_name):
- """Return selection function by reading list of files
-
- The format of the filelist is documented in the man page.
- filelist_fp should be an (open) file object.
- inc_default should be true if this is an include list,
- false for an exclude list.
- filelist_name is just a string used for logging.
-
- """
- Log("Reading filelist %s" % filelist_name, 4)
- tuple_list, something_excluded = \
- self.filelist_read(filelist_fp, inc_default, filelist_name)
- Log("Sorting filelist %s" % filelist_name, 4)
- tuple_list.sort()
- i = [0] # We have to put index in list because of stupid scoping rules
-
- def selection_function(dsrp):
- while 1:
- if i[0] >= len(tuple_list): return None
- include, move_on = \
- self.filelist_pair_match(dsrp, tuple_list[i[0]])
- if move_on:
- i[0] += 1
- if include is None: continue # later line may match
- return include
-
- selection_function.exclude = something_excluded or inc_default == 0
- selection_function.name = "Filelist: " + filelist_name
- return selection_function
-
- def filelist_read(self, filelist_fp, include, filelist_name):
- """Read filelist from fp, return (tuplelist, something_excluded)"""
- prefix_warnings = [0]
- def incr_warnings(exc):
- """Warn if prefix is incorrect"""
- prefix_warnings[0] += 1
- if prefix_warnings[0] < 6:
- Log("Warning: file specification '%s' in filelist %s\n"
- "doesn't start with correct prefix %s. Ignoring." %
- (exc, filelist_name, self.prefix), 2)
- if prefix_warnings[0] == 5:
- Log("Future prefix errors will not be logged.", 2)
-
- something_excluded, tuple_list = None, []
- separator = Globals.null_separator and "\0" or "\n"
- for line in filelist_fp.read().split(separator):
- if not line: continue # skip blanks
- try: tuple = self.filelist_parse_line(line, include)
- except FilePrefixError, exc:
- incr_warnings(exc)
- continue
- tuple_list.append(tuple)
- if not tuple[1]: something_excluded = 1
- if filelist_fp.close():
- Log("Error closing filelist %s" % filelist_name, 2)
- return (tuple_list, something_excluded)
-
- def filelist_parse_line(self, line, include):
- """Parse a single line of a filelist, returning a pair
-
- pair will be of form (index, include), where index is another
- tuple, and include is 1 if the line specifies that we are
- including a file. The default is given as an argument.
- prefix is the string that the index is relative to.
-
- """
- if line[:2] == "+ ": # Check for "+ "/"- " syntax
- include = 1
- line = line[2:]
- elif line[:2] == "- ":
- include = 0
- line = line[2:]
-
- if not line.startswith(self.prefix): raise FilePrefixError(line)
- line = line[len(self.prefix):] # Discard prefix
- index = tuple(filter(lambda x: x, line.split("/"))) # remove empties
- return (index, include)
-
- def filelist_pair_match(self, dsrp, pair):
- """Matches a filelist tuple against a dsrp
-
- Returns a pair (include, move_on). include is None if the
- tuple doesn't match either way, and 0/1 if the tuple excludes
- or includes the dsrp.
-
- move_on is true if the tuple cannot match a later index, and
- so we should move on to the next tuple in the index.
-
- """
- index, include = pair
- if include == 1:
- if index < dsrp.index: return (None, 1)
- if index == dsrp.index: return (1, 1)
- elif index[:len(dsrp.index)] == dsrp.index:
- return (1, None) # /foo/bar implicitly includes /foo
- else: return (None, None) # dsrp greater, not initial sequence
- elif include == 0:
- if dsrp.index[:len(index)] == index:
- return (0, None) # /foo implicitly excludes /foo/bar
- elif index < dsrp.index: return (None, 1)
- else: return (None, None) # dsrp greater, not initial sequence
- else: assert 0, "Include is %s, should be 0 or 1" % (include,)
-
- def filelist_globbing_get_sfs(self, filelist_fp, inc_default, list_name):
- """Return list of selection functions by reading fileobj
-
- filelist_fp should be an open file object
- inc_default is true iff this is an include list
- list_name is just the name of the list, used for logging
- See the man page on --[include/exclude]-globbing-filelist
-
- """
- Log("Reading globbing filelist %s" % list_name, 4)
- separator = Globals.null_separator and "\0" or "\n"
- for line in filelist_fp.read().split(separator):
- if not line: continue # skip blanks
- if line[:2] == "+ ": yield self.glob_get_sf(line[2:], 1)
- elif line[:2] == "- ": yield self.glob_get_sf(line[2:], 0)
- else: yield self.glob_get_sf(line, inc_default)
-
- def other_filesystems_get_sf(self, include):
- """Return selection function matching files on other filesystems"""
- assert include == 0 or include == 1
- root_devloc = self.dsrpath.getdevloc()
- def sel_func(dsrp):
- if dsrp.getdevloc() == root_devloc: return None
- else: return include
- sel_func.exclude = not include
- sel_func.name = "Match other filesystems"
- return sel_func
-
- def regexp_get_sf(self, regexp_string, include):
- """Return selection function given by regexp_string"""
- assert include == 0 or include == 1
- try: regexp = re.compile(regexp_string)
- except:
- Log("Error compiling regular expression %s" % regexp_string, 1)
- raise
-
- def sel_func(dsrp):
- if regexp.search(dsrp.path): return include
- else: return None
-
- sel_func.exclude = not include
- sel_func.name = "Regular expression: %s" % regexp_string
- return sel_func
-
- def devfiles_get_sf(self, include):
- """Return a selection function matching all dev files"""
- if self.selection_functions:
- Log("Warning: exclude-device-files is not the first "
- "selector.\nThis may not be what you intended", 3)
- def sel_func(dsrp):
- if dsrp.isdev(): return include
- else: return None
- sel_func.exclude = not include
- sel_func.name = (include and "include" or "exclude") + " device files"
- return sel_func
-
- def special_get_sf(self, include):
- """Return sel function matching sockets, symlinks, sockets, devs"""
- if self.selection_functions:
- Log("Warning: exclude-special-files is not the first "
- "selector.\nThis may not be what you intended", 3)
- def sel_func(dsrp):
- if dsrp.issym() or dsrp.issock() or dsrp.isfifo() or dsrp.isdev():
- return include
- else: return None
- sel_func.exclude = not include
- sel_func.name = (include and "include" or "exclude") + " special files"
- return sel_func
-
- def glob_get_sf(self, glob_str, include):
- """Return selection function given by glob string"""
- assert include == 0 or include == 1
- if glob_str == "**": sel_func = lambda dsrp: include
- elif not self.glob_re.match(glob_str): # normal file
- sel_func = self.glob_get_filename_sf(glob_str, include)
- else: sel_func = self.glob_get_normal_sf(glob_str, include)
-
- sel_func.exclude = not include
- sel_func.name = "Command-line %s glob: %s" % \
- (include and "include" or "exclude", glob_str)
- return sel_func
-
- def glob_get_filename_sf(self, filename, include):
- """Get a selection function given a normal filename
-
- Some of the parsing is better explained in
- filelist_parse_line. The reason this is split from normal
- globbing is things are a lot less complicated if no special
- globbing characters are used.
-
- """
- if not filename.startswith(self.prefix):
- raise FilePrefixError(filename)
- index = tuple(filter(lambda x: x,
- filename[len(self.prefix):].split("/")))
- return self.glob_get_tuple_sf(index, include)
-
- def glob_get_tuple_sf(self, tuple, include):
- """Return selection function based on tuple"""
- def include_sel_func(dsrp):
- if (dsrp.index == tuple[:len(dsrp.index)] or
- dsrp.index[:len(tuple)] == tuple):
- return 1 # /foo/bar implicitly matches /foo, vice-versa
- else: return None
-
- def exclude_sel_func(dsrp):
- if dsrp.index[:len(tuple)] == tuple:
- return 0 # /foo excludes /foo/bar, not vice-versa
- else: return None
-
- if include == 1: sel_func = include_sel_func
- elif include == 0: sel_func = exclude_sel_func
- sel_func.exclude = not include
- sel_func.name = "Tuple select %s" % (tuple,)
- return sel_func
-
- def glob_get_normal_sf(self, glob_str, include):
- """Return selection function based on glob_str
-
- The basic idea is to turn glob_str into a regular expression,
- and just use the normal regular expression. There is a
- complication because the selection function should return '2'
- (scan) for directories which may contain a file which matches
- the glob_str. So we break up the glob string into parts, and
- any file which matches an initial sequence of glob parts gets
- scanned.
-
- Thanks to Donovan Baarda who provided some code which did some
- things similar to this.
-
- """
- if glob_str.lower().startswith("ignorecase:"):
- re_comp = lambda r: re.compile(r, re.I | re.S)
- glob_str = glob_str[len("ignorecase:"):]
- else: re_comp = lambda r: re.compile(r, re.S)
-
- # matches what glob matches and any files in directory
- glob_comp_re = re_comp("^%s($|/)" % self.glob_to_re(glob_str))
-
- if glob_str.find("**") != -1:
- glob_str = glob_str[:glob_str.find("**")+2] # truncate after **
-
- scan_comp_re = re_comp("^(%s)$" %
- "|".join(self.glob_get_prefix_res(glob_str)))
-
- def include_sel_func(dsrp):
- if glob_comp_re.match(dsrp.path): return 1
- elif scan_comp_re.match(dsrp.path): return 2
- else: return None
-
- def exclude_sel_func(dsrp):
- if glob_comp_re.match(dsrp.path): return 0
- else: return None
-
- # Check to make sure prefix is ok
- if not include_sel_func(self.dsrpath): raise FilePrefixError(glob_str)
-
- if include: return include_sel_func
- else: return exclude_sel_func
-
- def glob_get_prefix_res(self, glob_str):
- """Return list of regexps equivalent to prefixes of glob_str"""
- glob_parts = glob_str.split("/")
- if "" in glob_parts[1:-1]: # "" OK if comes first or last, as in /foo/
- raise GlobbingError("Consecutive '/'s found in globbing string "
- + glob_str)
-
- prefixes = map(lambda i: "/".join(glob_parts[:i+1]),
- range(len(glob_parts)))
- # we must make exception for root "/", only dir to end in slash
- if prefixes[0] == "": prefixes[0] = "/"
- return map(self.glob_to_re, prefixes)
-
- def glob_to_re(self, pat):
- """Returned regular expression equivalent to shell glob pat
-
- Currently only the ?, *, [], and ** expressions are supported.
- Ranges like [a-z] are also currently unsupported. There is no
- way to quote these special characters.
-
- This function taken with minor modifications from efnmatch.py
- by Donovan Baarda.
-
- """
- i, n, res = 0, len(pat), ''
- while i < n:
- c, s = pat[i], pat[i:i+2]
- i = i+1
- if s == '**':
- res = res + '.*'
- i = i + 1
- elif c == '*': res = res + '[^/]*'
- elif c == '?': res = res + '[^/]'
- elif c == '[':
- j = i
- if j < n and pat[j] in '!^': j = j+1
- if j < n and pat[j] == ']': j = j+1
- while j < n and pat[j] != ']': j = j+1
- if j >= n: res = res + '\\[' # interpret the [ literally
- else: # Deal with inside of [..]
- stuff = pat[i:j].replace('\\','\\\\')
- i = j+1
- if stuff[0] in '!^': stuff = '^' + stuff[1:]
- res = res + '[' + stuff + ']'
- else: res = res + re.escape(c)
- return res
-
-
diff --git a/rdiff-backup/src/static.py b/rdiff-backup/src/static.py
deleted file mode 100644
index 6b99bd5..0000000
--- a/rdiff-backup/src/static.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""MakeStatic and MakeClass
-
-These functions are used to make all the instance methods in a class
-into static or class methods.
-
-"""
-
-class StaticMethodsError(Exception): pass
-
-def MakeStatic(cls):
- """turn instance methods into static ones
-
- The methods (that don't begin with _) of any class that
- subclasses this will be turned into static methods.
-
- """
- for name in dir(cls):
- if name[0] != "_":
- cls.__dict__[name] = staticmethod(cls.__dict__[name])
-
-def MakeClass(cls):
- """Turn instance methods into classmethods. Ignore _ like above"""
- for name in dir(cls):
- if name[0] != "_":
- cls.__dict__[name] = classmethod(cls.__dict__[name])
diff --git a/rdiff-backup/src/statistics.py b/rdiff-backup/src/statistics.py
deleted file mode 100644
index 261b2f3..0000000
--- a/rdiff-backup/src/statistics.py
+++ /dev/null
@@ -1,345 +0,0 @@
-# Copyright 2002 Ben Escoto
-#
-# This file is part of rdiff-backup.
-#
-# rdiff-backup is free software; you can redistribute it and/or modify
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation; either version 2 of the License, or (at your
-# option) any later version.
-#
-# rdiff-backup is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with rdiff-backup; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-# USA
-
-"""Generate and process aggregated backup information"""
-
-from lazy import *
-import re
-
-
-class StatsException(Exception): pass
-
-class StatsObj:
- """Contains various statistics, provide string conversion functions"""
- # used when quoting files in get_stats_line
- space_regex = re.compile(" ")
-
- stat_file_attrs = ('SourceFiles', 'SourceFileSize',
- 'MirrorFiles', 'MirrorFileSize',
- 'NewFiles', 'NewFileSize',
- 'DeletedFiles', 'DeletedFileSize',
- 'ChangedFiles',
- 'ChangedSourceSize', 'ChangedMirrorSize',
- 'IncrementFiles', 'IncrementFileSize')
- stat_misc_attrs = ('Errors', 'TotalDestinationSizeChange')
- stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime')
- stat_attrs = (('Filename',) + stat_time_attrs +
- stat_misc_attrs + stat_file_attrs)
-
- # Below, the second value in each pair is true iff the value
- # indicates a number of bytes
- stat_file_pairs = (('SourceFiles', None), ('SourceFileSize', 1),
- ('MirrorFiles', None), ('MirrorFileSize', 1),
- ('NewFiles', None), ('NewFileSize', 1),
- ('DeletedFiles', None), ('DeletedFileSize', 1),
- ('ChangedFiles', None),
- ('ChangedSourceSize', 1), ('ChangedMirrorSize', 1),
- ('IncrementFiles', None), ('IncrementFileSize', 1))
-
- # This is used in get_byte_summary_string below
- byte_abbrev_list = ((1024*1024*1024*1024, "TB"),
- (1024*1024*1024, "GB"),
- (1024*1024, "MB"),
- (1024, "KB"))
-
- def __init__(self):
- """Set attributes to None"""
- for attr in self.stat_attrs: self.__dict__[attr] = None
-
- def get_stat(self, attribute):
- """Get a statistic"""
- return self.__dict__[attribute]
-
- def set_stat(self, attr, value):
- """Set attribute to given value"""
- self.__dict__[attr] = value
-
- def increment_stat(self, attr):
- """Add 1 to value of attribute"""
- self.__dict__[attr] += 1
-
- def get_total_dest_size_change(self):
- """Return total destination size change
-
- This represents the total change in the size of the
- rdiff-backup destination directory.
-
- """
- addvals = [self.NewFileSize, self.ChangedSourceSize,
- self.IncrementFileSize]
- subtractvals = [self.DeletedFileSize, self.ChangedMirrorSize]
- for val in addvals + subtractvals:
- if val is None:
- result = None
- break
- else:
- def addlist(l): return reduce(lambda x,y: x+y, l)
- result = addlist(addvals) - addlist(subtractvals)
- self.TotalDestinationSizeChange = result
- return result
-
- def get_stats_line(self, index, use_repr = 1):
- """Return one line abbreviated version of full stats string"""
- file_attrs = map(lambda attr: str(self.get_stat(attr)),
- self.stat_file_attrs)
- if not index: filename = "."
- else:
- filename = apply(os.path.join, index)
- if use_repr:
- # use repr to quote newlines in relative filename, then
- # take of leading and trailing quote and quote spaces.
- filename = self.space_regex.sub("\\x20", repr(filename)[1:-1])
- return " ".join([filename,] + file_attrs)
-
- def set_stats_from_line(self, line):
- """Set statistics from given line"""
- def error(): raise StatsException("Bad line '%s'" % line)
- if line[-1] == "\n": line = line[:-1]
- lineparts = line.split(" ")
- if len(lineparts) < len(stat_file_attrs): error()
- for attr, val_string in zip(stat_file_attrs,
- lineparts[-len(stat_file_attrs):]):
- try: val = long(val_string)
- except ValueError:
- try: val = float(val_string)
- except ValueError: error()
- self.set_stat(attr, val)
- return self
-
- def get_stats_string(self):
- """Return extended string printing out statistics"""
- return "%s%s%s" % (self.get_timestats_string(),
- self.get_filestats_string(),
- self.get_miscstats_string())
-
- def get_timestats_string(self):
- """Return portion of statistics string dealing with time"""
- timelist = []
- if self.StartTime is not None:
- timelist.append("StartTime %.2f (%s)\n" %
- (self.StartTime, Time.timetopretty(self.StartTime)))
- if self.EndTime is not None:
- timelist.append("EndTime %.2f (%s)\n" %
- (self.EndTime, Time.timetopretty(self.EndTime)))
- if self.ElapsedTime or (self.StartTime is not None and
- self.EndTime is not None):
- if self.ElapsedTime is None:
- self.ElapsedTime = self.EndTime - self.StartTime
- timelist.append("ElapsedTime %.2f (%s)\n" %
- (self.ElapsedTime, Time.inttopretty(self.ElapsedTime)))
- return "".join(timelist)
-
- def get_filestats_string(self):
- """Return portion of statistics string about files and bytes"""
- def fileline(stat_file_pair):
- """Return zero or one line of the string"""
- attr, in_bytes = stat_file_pair
- val = self.get_stat(attr)
- if val is None: return ""
- if in_bytes:
- return "%s %s (%s)\n" % (attr, val,
- self.get_byte_summary_string(val))
- else: return "%s %s\n" % (attr, val)
-
- return "".join(map(fileline, self.stat_file_pairs))
-
- def get_miscstats_string(self):
- """Return portion of extended stat string about misc attributes"""
- misc_string = ""
- tdsc = self.get_total_dest_size_change()
- if tdsc is not None:
- misc_string += ("TotalDestinationSizeChange %s (%s)\n" %
- (tdsc, self.get_byte_summary_string(tdsc)))
- if self.Errors is not None: misc_string += "Errors %d\n" % self.Errors
- return misc_string
-
- def get_byte_summary_string(self, byte_count):
- """Turn byte count into human readable string like "7.23GB" """
- if byte_count < 0:
- sign = "-"
- byte_count = -byte_count
- else: sign = ""
-
- for abbrev_bytes, abbrev_string in self.byte_abbrev_list:
- if byte_count >= abbrev_bytes:
- # Now get 3 significant figures
- abbrev_count = float(byte_count)/abbrev_bytes
- if abbrev_count >= 100: precision = 0
- elif abbrev_count >= 10: precision = 1
- else: precision = 2
- return "%s%%.%df %s" % (sign, precision, abbrev_string) \
- % (abbrev_count,)
- byte_count = round(byte_count)
- if byte_count == 1: return sign + "1 byte"
- else: return "%s%d bytes" % (sign, byte_count)
-
- def get_stats_logstring(self, title):
- """Like get_stats_string, but add header and footer"""
- header = "--------------[ %s ]--------------" % title
- footer = "-" * len(header)
- return "%s\n%s%s\n" % (header, self.get_stats_string(), footer)
-
- def set_stats_from_string(self, s):
- """Initialize attributes from string, return self for convenience"""
- def error(line): raise StatsException("Bad line '%s'" % line)
-
- for line in s.split("\n"):
- if not line: continue
- line_parts = line.split()
- if len(line_parts) < 2: error(line)
- attr, value_string = line_parts[:2]
- if not attr in self.stat_attrs: error(line)
- try:
- try: val1 = long(value_string)
- except ValueError: val1 = None
- val2 = float(value_string)
- if val1 == val2: self.set_stat(attr, val1) # use integer val
- else: self.set_stat(attr, val2) # use float
- except ValueError: error(line)
- return self
-
- def write_stats_to_rp(self, rp):
- """Write statistics string to given rpath"""
- tf = TempFileManager.new(rp)
- def init_thunk():
- fp = tf.open("w")
- fp.write(self.get_stats_string())
- fp.close()
- Robust.make_tf_robustaction(init_thunk, (tf,), (rp,)).execute()
-
- def read_stats_from_rp(self, rp):
- """Set statistics from rpath, return self for convenience"""
- fp = rp.open("r")
- self.set_stats_from_string(fp.read())
- fp.close()
- return self
-
- def stats_equal(self, s):
- """Return true if s has same statistics as self"""
- assert isinstance(s, StatsObj)
- for attr in self.stat_file_attrs:
- if self.get_stat(attr) != s.get_stat(attr): return None
- return 1
-
- def set_to_average(self, statobj_list):
- """Set self's attributes to average of those in statobj_list"""
- for attr in self.stat_attrs: self.set_stat(attr, 0)
- for statobj in statobj_list:
- for attr in self.stat_attrs:
- if statobj.get_stat(attr) is None: self.set_stat(attr, None)
- elif self.get_stat(attr) is not None:
- self.set_stat(attr, statobj.get_stat(attr) +
- self.get_stat(attr))
-
- # Don't compute average starting/stopping time
- self.StartTime = None
- self.EndTime = None
-
- for attr in self.stat_attrs:
- if self.get_stat(attr) is not None:
- self.set_stat(attr,
- self.get_stat(attr)/float(len(statobj_list)))
- return self
-
- def get_statsobj_copy(self):
- """Return new StatsObj object with same stats as self"""
- s = StatObj()
- for attr in self.stat_attrs: s.set_stat(attr, self.get_stat(attr))
- return s
-
-
-class StatsITRB(ITRBranch, StatsObj):
- """Keep track of per directory statistics
-
- This is subclassed by the mirroring and incrementing ITRs.
-
- """
- def __init__(self):
- """StatsITR initializer - zero out statistics"""
- attr_dict = self.__dict__
- for attr in StatsObj.stat_file_attrs: attr_dict[attr] = 0
- self.ElapsedTime = self.Filename = None
-
- def start_stats(self, mirror_dsrp):
- """Record status of mirror dsrp
-
- This is called before the mirror is processed so we remember
- the old state.
-
- """
- if mirror_dsrp.lstat():
- self.mirror_base_exists = 1
- self.mirror_base_size = self.stats_getsize(mirror_dsrp)
- else: self.mirror_base_exists = None
-
- def stats_getsize(self, rp):
- """Return size of rp, with error checking"""
- try: return rp.getsize()
- except KeyError: return 0
-
- def end_stats(self, diff_rorp, mirror_dsrp, inc_rp = None):
- """Set various statistics after mirror processed"""
- if mirror_dsrp.lstat():
- source_size = self.stats_getsize(mirror_dsrp)
- self.SourceFiles += 1
- self.SourceFileSize += source_size
- if self.mirror_base_exists:
- self.MirrorFiles += 1
- self.MirrorFileSize += self.mirror_base_size
- if diff_rorp: # otherwise no change
- self.ChangedFiles += 1
- self.ChangedSourceSize += source_size
- self.ChangedMirrorSize += self.mirror_base_size
- self.stats_incr_incfiles(inc_rp)
- else: # new file was created
- self.NewFiles += 1
- self.NewFileSize += source_size
- self.stats_incr_incfiles(inc_rp)
- else:
- if self.mirror_base_exists: # file was deleted from mirror
- self.MirrorFiles += 1
- self.MirrorFileSize += self.mirror_base_size
- self.DeletedFiles += 1
- self.DeletedFileSize += self.mirror_base_size
- self.stats_incr_incfiles(inc_rp)
-
- def fast_process(self, mirror_rorp):
- """Use when there is no change from source to mirror"""
- source_size = self.stats_getsize(mirror_rorp)
- self.SourceFiles += 1
- self.MirrorFiles += 1
- self.SourceFileSize += source_size
- self.MirrorFileSize += source_size
-
- def stats_incr_incfiles(self, inc_rp):
- """Increment IncrementFile statistics"""
- if inc_rp:
- self.IncrementFiles += 1
- self.IncrementFileSize += self.stats_getsize(inc_rp)
-
- def add_file_stats(self, branch):
- """Add all file statistics from branch to current totals"""
- for attr in self.stat_file_attrs:
- self.__dict__[attr] += branch.__dict__[attr]
-
-
-from log import *
-from increment import *
-from robust import *
-import Globals
diff --git a/rdiff-backup/src/temp-stats b/rdiff-backup/src/temp-stats
deleted file mode 100644
index 0efea64..0000000
--- a/rdiff-backup/src/temp-stats
+++ /dev/null
@@ -1,241 +0,0 @@
-No changes in 10000 files:
-
-Wed May 15 23:54:53 2002 profile-output
-
- 1805774 function calls (1480846 primitive calls) in 45.710 CPU seconds
-
- Ordered by: internal time
- List reduced from 259 to 20 due to restriction <20>
-
- ncalls tottime percall cumtime percall filename:lineno(function)
-151824/30615 5.620 0.000 18.810 0.001 lazy.py:251(__call__)
-91212/30612 4.060 0.000 17.300 0.001 lazy.py:222(process_w_subinstance)
- 31006 2.630 0.000 4.650 0.000 rpath.py:448(setdata)
-101629/20412 1.890 0.000 14.690 0.001 selection.py:96(Iterate)
-61218/20410 1.880 0.000 14.370 0.001 selection.py:122(iterate_in_dir)
- 10206 1.400 0.000 44.620 0.004 highlevel.py:245(error_checked)
-30618/10206 1.350 0.000 27.650 0.003 rorpiter.py:124(Collate2Iters)
- 486 1.280 0.003 19.690 0.041 highlevel.py:165(generate_dissimilar)
- 30748 1.270 0.000 6.640 0.000 rpath.py:408(__init__)
- 31006 1.260 0.000 1.260 0.000 rpath.py:182(tupled_lstat)
- 121209 1.140 0.000 1.140 0.000 lazy.py:214(intree)
- 20411 1.110 0.000 9.010 0.000 destructive_stepping.py:33(__init__)
- 10205 1.050 0.000 1.050 0.000 rpath.py:430(__str__)
- 30612 0.860 0.000 1.410 0.000 lazy.py:218(set_subinstance)
- 20411 0.840 0.000 2.830 0.000 destructive_stepping.py:77(set_init_perms)
- 486 0.830 0.002 26.900 0.055 highlevel.py:98(diffs)
- 30771 0.780 0.000 0.780 0.000 /usr/lib/python2.2/posixpath.py:44(join)
- 10205 0.720 0.000 3.870 0.000 increment.py:138(start_process)
- 91845 0.710 0.000 0.710 0.000 rorpiter.py:246(__getitem__)
- 20616 0.690 0.000 0.930 0.000 rpath.py:593(isowner)
-
-
-Wed May 15 23:54:53 2002 profile-output
-
- 1805774 function calls (1480846 primitive calls) in 45.710 CPU seconds
-
- Ordered by: cumulative time
- List reduced from 259 to 20 due to restriction <20>
-
- ncalls tottime percall cumtime percall filename:lineno(function)
- 1 0.010 0.010 45.710 45.710 profile:0(Globals.Main.Main(['--no-resume', 'manyfiles/', 'out']))
- 1 0.000 0.000 45.700 45.700 main.py:188(Main)
- 1 0.000 0.000 45.700 45.700 <string>:1(?)
- 1 0.000 0.000 45.670 45.670 main.py:169(take_action)
- 1 0.000 0.000 45.670 45.670 main.py:221(Backup)
- 1 0.000 0.000 45.650 45.650 highlevel.py:50(Mirror_and_increment)
- 1 0.240 0.240 45.650 45.650 highlevel.py:239(patch_increment_and_finalize)
- 10208 0.270 0.000 44.890 0.004 highlevel.py:268(check_skip_error)
- 10206 1.400 0.000 44.620 0.004 highlevel.py:245(error_checked)
-30618/10206 1.350 0.000 27.650 0.003 rorpiter.py:124(Collate2Iters)
- 486 0.830 0.002 26.900 0.055 highlevel.py:98(diffs)
- 486 0.020 0.000 19.720 0.041 rorpiter.py:58(Signatures)
- 486 1.280 0.003 19.690 0.041 highlevel.py:165(generate_dissimilar)
-151824/30615 5.620 0.000 18.810 0.001 lazy.py:251(__call__)
-91212/30612 4.060 0.000 17.300 0.001 lazy.py:222(process_w_subinstance)
- 20412 0.520 0.000 14.900 0.001 lazy.py:166(yielda)
-101629/20412 1.890 0.000 14.690 0.001 selection.py:96(Iterate)
-61218/20410 1.880 0.000 14.370 0.001 selection.py:122(iterate_in_dir)
- 41229 0.660 0.000 11.180 0.000 robust.py:194(check_common_error)
- 20409 0.310 0.000 9.810 0.000 selection.py:130(<lambda>)
-
-----------------------------------------------------------------------------
-
-Writing 10000 new small files
-
-Thu May 16 00:05:36 2002 profile-output
-
- 2452569 function calls (2249461 primitive calls) in 75.680 CPU seconds
-
- Ordered by: internal time
- List reduced from 236 to 20 due to restriction <20>
-
- ncalls tottime percall cumtime percall filename:lineno(function)
- 71752 4.580 0.000 4.580 0.000 rpath.py:182(tupled_lstat)
- 71752 4.170 0.000 9.860 0.000 rpath.py:448(setdata)
-101216/20410 3.550 0.000 11.070 0.001 lazy.py:251(__call__)
-60808/20408 2.690 0.000 9.980 0.000 lazy.py:222(process_w_subinstance)
- 10206 2.460 0.000 74.630 0.007 highlevel.py:217(error_checked)
- 20012 2.080 0.000 2.820 0.000 rpath.py:674(open)
- 40819 2.050 0.000 13.440 0.000 destructive_stepping.py:33(__init__)
- 10228 2.050 0.000 5.030 0.000 rpath.py:625(normalize)
- 10000 1.980 0.000 8.060 0.001 rpath.py:684(write_from_fileobj)
- 51086 1.970 0.000 9.210 0.000 rpath.py:408(__init__)
- 143692 1.640 0.000 1.640 0.000 connection.py:40(__getattr__)
- 10206 1.610 0.000 8.920 0.001 highlevel.py:165(generate_dissimilar)
-30618/10206 1.550 0.000 29.540 0.003 rorpiter.py:124(Collate2Iters)
- 10205 1.520 0.000 1.520 0.000 rpath.py:227(__str__)
- 20410 1.450 0.000 1.710 0.000 rpath.py:525(setmtime)
- 10226 1.380 0.000 1.670 0.000 rpath.py:163(rename)
- 20000 1.380 0.000 1.380 0.000 rpath.py:772(read)
- 10206 1.350 0.000 3.490 0.000 rpath.py:109(copy_attribs)
- 40819 1.290 0.000 3.100 0.000 destructive_stepping.py:77(set_init_perms)
-50817/10208 1.250 0.000 7.890 0.001 selection.py:96(Iterate)
-
-
-Thu May 16 00:05:36 2002 profile-output
-
- 2452569 function calls (2249461 primitive calls) in 75.680 CPU seconds
-
- Ordered by: cumulative time
- List reduced from 236 to 20 due to restriction <20>
-
- ncalls tottime percall cumtime percall filename:lineno(function)
- 1 0.000 0.000 75.680 75.680 profile:0(Globals.Main.Main(['manyfiles/', 'out']))
- 1 0.000 0.000 75.680 75.680 <string>:1(?)
- 1 0.000 0.000 75.680 75.680 main.py:188(Main)
- 1 0.000 0.000 75.670 75.670 main.py:221(Backup)
- 1 0.000 0.000 75.670 75.670 main.py:169(take_action)
- 1 0.000 0.000 75.650 75.650 highlevel.py:27(Mirror)
- 1 0.450 0.450 75.650 75.650 highlevel.py:210(patch_and_finalize)
- 10206 0.190 0.000 74.820 0.007 highlevel.py:268(check_skip_error)
- 10206 2.460 0.000 74.630 0.007 highlevel.py:217(error_checked)
-30618/10206 1.550 0.000 29.540 0.003 rorpiter.py:124(Collate2Iters)
- 10206 1.060 0.000 29.020 0.003 highlevel.py:98(diffs)
- 10216 0.270 0.000 28.800 0.003 robust.py:54(execute)
- 10205 1.140 0.000 22.760 0.002 robust.py:148(init)
- 40819 2.050 0.000 13.440 0.000 destructive_stepping.py:33(__init__)
-101216/20410 3.550 0.000 11.070 0.001 lazy.py:251(__call__)
-60808/20408 2.690 0.000 9.980 0.000 lazy.py:222(process_w_subinstance)
- 10206 0.570 0.000 9.860 0.001 rorpiter.py:58(Signatures)
- 71752 4.170 0.000 9.860 0.000 rpath.py:448(setdata)
- 10226 0.540 0.000 9.770 0.001 robust.py:240(new)
- 51086 1.970 0.000 9.210 0.000 rpath.py:408(__init__)
-
-----------------------------------------------------------------------------
-Nothing changed, source side:
-
- ncalls tottime percall cumtime percall filename:lineno(function)
-50608/10205 1.970 0.000 5.420 0.001 lazy.py:251(__call__)
- 10208/1 1.260 0.000 17.740 17.740 iterfile.py:166(addtobuffer)
-30404/10204 1.240 0.000 5.010 0.000 lazy.py:222(process_w_subinstance)
- 1 1.180 1.180 17.740 17.740 highlevel.py:98(diffs)
- 10221 1.050 0.000 1.930 0.000 rpath.py:448(setdata)
-50814/10206 0.880 0.000 7.610 0.001 selection.py:96(Iterate)
-30609/10205 0.780 0.000 7.420 0.001 selection.py:122(iterate_in_dir)
- 10205 0.600 0.000 0.600 0.000 iterfile.py:197(_l2s)
- 10204 0.600 0.000 0.700 0.000 lazy.py:218(set_subinstance)
- 10215 0.550 0.000 2.760 0.000 rpath.py:408(__init__)
- 10206 0.470 0.000 4.900 0.000 destructive_stepping.py:33(__init__)
- 10206 0.430 0.000 1.570 0.000 destructive_stepping.py:77(set_init_perms)
- 10411 0.430 0.000 0.610 0.000 rpath.py:593(isowner)
- 10206 0.390 0.000 0.390 0.000 lazy.py:177(yieldb)
- 10206 0.380 0.000 10.860 0.001 rorpiter.py:124(Collate2Iters)
- 10208 0.370 0.000 0.370 0.000 rpath.py:182(tupled_lstat)
- 20614 0.360 0.000 6.010 0.000 robust.py:194(check_common_error)
- 40403 0.330 0.000 0.330 0.000 lazy.py:214(intree)
- 10205 0.320 0.000 0.980 0.000 rpath.py:581(readable)
- 10205 0.320 0.000 0.790 0.000 destructive_stepping.py:206(end_process)
-
-
-Thu May 16 00:21:24 2002 profile-output
-
- 646330 function calls (503964 primitive calls) in 17.820 CPU seconds
-
- Ordered by: cumulative time
- List reduced from 213 to 20 due to restriction <20>
-
- ncalls tottime percall cumtime percall filename:lineno(function)
- 1 0.000 0.000 17.820 17.820 <string>:1(?)
- 1 0.000 0.000 17.820 17.820 profile:0(Globals.Main.Main(['--remote-schema', '%s --server', 'manyfiles/', './rdiff-backup::out']))
- 1 0.000 0.000 17.820 17.820 main.py:188(Main)
- 1 0.000 0.000 17.770 17.770 main.py:221(Backup)
- 1 0.000 0.000 17.770 17.770 main.py:169(take_action)
- 47/46 0.000 0.000 17.750 0.386 connection.py:425(__call__)
- 1 0.000 0.000 17.750 17.750 highlevel.py:50(Mirror_and_increment)
- 47/46 0.000 0.000 17.750 0.386 connection.py:352(reval)
- 22/1 0.000 0.000 17.740 17.740 connection.py:448(readfromid)
- 10208/1 1.260 0.000 17.740 17.740 iterfile.py:166(addtobuffer)
- 22/1 0.140 0.006 17.740 17.740 iterfile.py:153(read)
- 22/1 0.030 0.001 17.740 17.740 connection.py:326(answer_request)
- 47/46 0.000 0.000 17.740 0.386 connection.py:307(get_response)
- 10207/1 0.130 0.000 17.740 17.740 rorpiter.py:23(ToRaw)
- 1 1.180 1.180 17.740 17.740 highlevel.py:98(diffs)
- 10206 0.380 0.000 10.860 0.001 rorpiter.py:124(Collate2Iters)
- 1 0.000 0.000 10.050 10.050 connection.py:480(read)
- 1 0.000 0.000 10.050 10.050 iterfile.py:221(read)
- 1 0.000 0.000 10.050 10.050 rorpiter.py:31(FromRaw)
- 1 0.000 0.000 10.050 10.050 iterfile.py:23(_get)
-
-----------------------------------------------------------------------------
-Nothing changed, destination side:
-
-Thu May 16 00:27:58 2002 profile-output
-
- 2769004 function calls (2586001 primitive calls) in 113.720 CPU seconds
-
- Ordered by: internal time
- List reduced from 325 to 20 due to restriction <20>
-
- ncalls tottime percall cumtime percall filename:lineno(function)
-80820/60409 12.080 0.000 26.640 0.000 iterfile.py:221(read)
- 92424 9.240 0.000 9.240 0.000 rpath.py:182(tupled_lstat)
- 92426 6.060 0.000 16.750 0.000 rpath.py:448(setdata)
- 10243 5.360 0.001 10.890 0.001 rpath.py:625(normalize)
-40411/30205 4.010 0.000 31.240 0.001 iterfile.py:23(_get)
-101216/20410 3.810 0.000 73.120 0.004 lazy.py:251(__call__)
-60808/20408 3.300 0.000 72.140 0.004 lazy.py:222(process_w_subinstance)
- 61337 2.770 0.000 15.330 0.000 rpath.py:408(__init__)
- 40409 2.630 0.000 2.630 0.000 iterfile.py:16(_s2l)
- 10000 2.220 0.000 13.140 0.001 rpath.py:684(write_from_fileobj)
- 10206 2.200 0.000 112.620 0.011 highlevel.py:245(error_checked)
- 20411 1.940 0.000 2.260 0.000 rpath.py:525(setmtime)
- 154184 1.910 0.000 1.910 0.000 connection.py:40(__getattr__)
- 10204 1.890 0.000 57.280 0.006 increment.py:211(init_non_dir)
- 10207 1.820 0.000 4.090 0.000 rpath.py:568(touch)
- 30615 1.730 0.000 13.490 0.000 destructive_stepping.py:33(__init__)
- 71585 1.690 0.000 1.690 0.000 /usr/lib/python2.2/posixpath.py:44(join)
- 10233 1.440 0.000 1.730 0.000 rpath.py:163(rename)
- 10204 1.400 0.000 1.400 0.000 rpath.py:227(__str__)
- 10206 1.330 0.000 3.900 0.000 rpath.py:109(copy_attribs)
-
-
-Thu May 16 00:27:58 2002 profile-output
-
- 2769004 function calls (2586001 primitive calls) in 113.720 CPU seconds
-
- Ordered by: cumulative time
- List reduced from 325 to 20 due to restriction <20>
-
- ncalls tottime percall cumtime percall filename:lineno(function)
- 1 0.000 0.000 113.720 113.720 <string>:1(?)
- 1 0.000 0.000 113.720 113.720 profile:0(Globals.Main.Main(['--remote-schema', '%s --server', './rdiff-backup::manyfiles/', 'out']))
- 1 0.000 0.000 113.720 113.720 main.py:188(Main)
- 1 0.000 0.000 113.690 113.690 main.py:169(take_action)
- 1 0.000 0.000 113.690 113.690 main.py:221(Backup)
- 1 0.010 0.010 113.660 113.660 highlevel.py:50(Mirror_and_increment)
- 1 0.300 0.300 113.650 113.650 highlevel.py:239(patch_increment_and_finalize)
- 10208 0.300 0.000 112.940 0.011 highlevel.py:268(check_skip_error)
- 10206 2.200 0.000 112.620 0.011 highlevel.py:245(error_checked)
-101216/20410 3.810 0.000 73.120 0.004 lazy.py:251(__call__)
-60808/20408 3.300 0.000 72.140 0.004 lazy.py:222(process_w_subinstance)
- 10205 0.780 0.000 61.080 0.006 increment.py:138(start_process)
- 10204 1.890 0.000 57.280 0.006 increment.py:211(init_non_dir)
- 10220 0.520 0.000 45.290 0.004 robust.py:54(execute)
- 10218 0.420 0.000 34.170 0.003 robust.py:76(init)
- 10204 1.090 0.000 33.700 0.003 robust.py:148(init)
-40411/30205 4.010 0.000 31.240 0.001 iterfile.py:23(_get)
-20412/10206 0.800 0.000 30.020 0.003 rorpiter.py:124(Collate2Iters)
-20411/10205 1.300 0.000 29.480 0.003 rorpiter.py:31(FromRaw)
-30411/20205 0.890 0.000 27.950 0.001 iterfile.py:54(next)
-